From 43a25d93ebdabea52f99b05e15b06250cd8f07d7 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 17 May 2023 16:05:49 +0000 Subject: Add latest changes from gitlab-org/gitlab@16-0-stable-ee --- doc/.vale/gitlab/CIConfigFile.yml | 6 +- doc/.vale/gitlab/CodeblockFences.yml | 4 +- doc/.vale/gitlab/ElementDescriptors.yml | 2 +- doc/.vale/gitlab/HeadingDepth.yml | 4 +- doc/.vale/gitlab/Substitutions.yml | 1 + doc/.vale/gitlab/Uppercase.yml | 1 + doc/.vale/gitlab/spelling-exceptions.txt | 22 +- doc/administration/audit_event_streaming.md | 64 +- doc/administration/audit_events.md | 311 +- doc/administration/auth/atlassian.md | 2 +- doc/administration/auth/authentiq.md | 12 - doc/administration/auth/cognito.md | 2 +- doc/administration/auth/crowd.md | 4 +- doc/administration/auth/index.md | 5 +- doc/administration/auth/jwt.md | 2 +- doc/administration/auth/ldap/google_secure_ldap.md | 6 +- doc/administration/auth/ldap/index.md | 21 +- .../auth/ldap/ldap-troubleshooting.md | 6 +- .../auth/ldap/ldap_synchronization.md | 150 +- doc/administration/auth/oidc.md | 469 +- doc/administration/auth/test_oidc_oauth.md | 57 + doc/administration/clusters/kas.md | 75 +- doc/administration/compliance.md | 100 +- doc/administration/configure.md | 71 +- doc/administration/dedicated/index.md | 127 +- doc/administration/docs_self_host.md | 43 +- doc/administration/environment_variables.md | 2 +- doc/administration/external_pipeline_validation.md | 3 +- doc/administration/feature_flags.md | 10 +- doc/administration/file_hooks.md | 2 +- .../disaster_recovery/background_verification.md | 24 +- .../geo/disaster_recovery/planned_failover.md | 4 +- .../runbooks/planned_failover_multi_node.md | 2 +- .../runbooks/planned_failover_single_node.md | 2 +- doc/administration/geo/index.md | 5 +- .../geo/replication/configuration.md | 27 +- .../geo/replication/container_registry.md | 16 +- doc/administration/geo/replication/datatypes.md | 22 +- .../geo/replication/geo_validation_tests.md | 3 +- .../replication/img/adding_a_secondary_v13_3.png | Bin 20195 -> 0 bytes .../replication/img/adding_a_secondary_v15_8.png | Bin 0 -> 14698 bytes .../geo/replication/single_sign_on.md | 9 + .../geo/replication/troubleshooting.md | 29 +- .../geo/replication/upgrading_the_geo_sites.md | 1 + doc/administration/geo/secondary_proxy/index.md | 1 + doc/administration/geo/setup/database.md | 57 +- doc/administration/geo/setup/external_database.md | 18 +- doc/administration/geo/setup/index.md | 24 +- doc/administration/get_started.md | 8 +- doc/administration/git_protocol.md | 4 +- doc/administration/gitaly/configure_gitaly.md | 560 ++- .../gitaly/gitaly_geo_capabilities.md | 41 + doc/administration/gitaly/index.md | 28 +- doc/administration/gitaly/monitoring.md | 15 +- doc/administration/gitaly/praefect.md | 558 ++- doc/administration/gitaly/recovery.md | 72 +- doc/administration/gitaly/reference.md | 42 +- doc/administration/gitaly/troubleshooting.md | 58 +- doc/administration/housekeeping.md | 31 + doc/administration/incoming_email.md | 3 +- doc/administration/index.md | 239 +- doc/administration/instance_limits.md | 120 +- doc/administration/integration/kroki.md | 15 +- doc/administration/integration/plantuml.md | 15 +- doc/administration/integration/terminal.md | 6 +- doc/administration/job_artifacts.md | 33 +- doc/administration/lfs/index.md | 19 +- doc/administration/load_balancer.md | 10 +- doc/administration/logs/index.md | 78 +- doc/administration/logs/log_parsing.md | 20 +- doc/administration/maintenance_mode/index.md | 28 +- doc/administration/merge_request_diffs.md | 4 +- doc/administration/monitoring/github_imports.md | 4 +- .../img/self_monitoring_overview_dashboard.png | Bin 51508 -> 0 bytes .../gitlab_self_monitoring_project/index.md | 119 +- doc/administration/monitoring/index.md | 3 - .../performance/grafana_configuration.md | 31 + doc/administration/monitoring/performance/index.md | 4 +- .../monitoring/prometheus/gitlab_metrics.md | 141 +- doc/administration/monitoring/prometheus/index.md | 7 +- .../monitoring/prometheus/node_exporter.md | 4 +- .../monitoring/prometheus/pgbouncer_exporter.md | 4 +- .../monitoring/prometheus/postgres_exporter.md | 4 +- .../monitoring/prometheus/redis_exporter.md | 4 +- .../monitoring/prometheus/registry_exporter.md | 4 +- doc/administration/nfs.md | 2 +- doc/administration/object_storage.md | 1105 +++-- doc/administration/operations/gitlab_sshd.md | 19 +- doc/administration/operations/index.md | 40 +- .../operations/moving_repositories.md | 16 +- doc/administration/operations/puma.md | 4 +- doc/administration/operations/rails_console.md | 2 +- doc/administration/package_information/defaults.md | 8 +- .../package_information/supported_os.md | 18 +- doc/administration/packages/container_registry.md | 172 +- doc/administration/packages/dependency_proxy.md | 2 +- doc/administration/packages/index.md | 2 +- doc/administration/pages/index.md | 456 +- doc/administration/pages/source.md | 46 +- doc/administration/pages/troubleshooting.md | 304 ++ .../postgresql/database_load_balancing.md | 5 + doc/administration/postgresql/external.md | 22 +- .../postgresql/multiple_databases.md | 91 +- doc/administration/postgresql/pgbouncer.md | 21 +- .../postgresql/replication_and_failover.md | 2 - doc/administration/raketasks/github_import.md | 8 +- doc/administration/raketasks/maintenance.md | 6 +- .../raketasks/project_import_export.md | 144 +- doc/administration/raketasks/service_desk_email.md | 2 +- doc/administration/raketasks/uploads/sanitize.md | 2 +- doc/administration/read_only_gitlab.md | 44 +- .../redis/replication_and_failover.md | 43 + .../redis/replication_and_failover_external.md | 22 +- doc/administration/redis/troubleshooting.md | 8 +- .../reference_architectures/10k_users.md | 288 +- .../reference_architectures/1k_users.md | 4 +- .../reference_architectures/25k_users.md | 290 +- .../reference_architectures/2k_users.md | 117 +- .../reference_architectures/3k_users.md | 252 +- .../reference_architectures/50k_users.md | 290 +- .../reference_architectures/5k_users.md | 248 +- .../reference_architectures/index.md | 242 +- doc/administration/reply_by_email.md | 6 +- doc/administration/repository_checks.md | 35 +- doc/administration/restart_gitlab.md | 14 + doc/administration/server_hooks.md | 72 +- .../sidekiq/extra_sidekiq_routing.md | 11 - doc/administration/sidekiq/index.md | 43 +- .../sidekiq/processing_specific_job_classes.md | 27 +- .../sidekiq/sidekiq_memory_killer.md | 4 +- doc/administration/silent_mode/index.md | 64 + .../static_objects_external_storage.md | 2 +- doc/administration/system_hooks.md | 2 +- doc/administration/terraform_state.md | 30 +- doc/administration/troubleshooting/index.md | 25 + doc/administration/uploads.md | 11 +- doc/administration/wikis/index.md | 4 +- doc/api/api_resources.md | 11 +- doc/api/audit_events.md | 12 +- doc/api/award_emoji.md | 60 +- doc/api/branches.md | 8 +- doc/api/bulk_imports.md | 74 +- doc/api/cluster_agents.md | 8 +- doc/api/commits.md | 8 + doc/api/dependencies.md | 4 +- doc/api/deploy_keys.md | 31 +- doc/api/deploy_tokens.md | 10 +- doc/api/deployments.md | 12 +- doc/api/discussions.md | 453 +- doc/api/dora/metrics.md | 6 +- doc/api/draft_notes.md | 57 + doc/api/environments.md | 11 +- doc/api/epic_issues.md | 2 +- doc/api/error_tracking.md | 41 +- doc/api/feature_flag_specs.md | 14 - doc/api/feature_flag_user_lists.md | 8 +- doc/api/feature_flags.md | 6 +- doc/api/features.md | 7 +- doc/api/freeze_periods.md | 4 +- doc/api/geo_nodes.md | 33 + doc/api/geo_sites.md | 1111 +++++ doc/api/graphql/branch_rules.md | 2 +- doc/api/graphql/custom_emoji.md | 4 +- doc/api/graphql/getting_started.md | 29 +- doc/api/graphql/index.md | 2 +- doc/api/graphql/reference/index.md | 2590 +++++++++-- doc/api/graphql/removed_items.md | 24 +- doc/api/group_access_tokens.md | 28 + doc/api/group_badges.md | 6 +- doc/api/group_boards.md | 4 +- doc/api/group_clusters.md | 6 +- doc/api/group_import_export.md | 33 +- doc/api/group_iterations.md | 6 +- doc/api/group_level_variables.md | 2 +- doc/api/group_milestones.md | 6 +- doc/api/group_protected_branches.md | 469 ++ doc/api/group_protected_environments.md | 4 +- doc/api/group_relations_export.md | 4 +- doc/api/group_releases.md | 4 +- doc/api/group_repository_storage_moves.md | 2 +- doc/api/group_wikis.md | 4 +- doc/api/groups.md | 66 +- doc/api/import.md | 46 +- doc/api/index.md | 9 +- doc/api/instance_clusters.md | 6 +- doc/api/instance_level_ci_variables.md | 2 +- doc/api/integrations.md | 126 +- doc/api/issues.md | 38 +- doc/api/iterations.md | 6 +- doc/api/job_artifacts.md | 10 +- doc/api/jobs.md | 9 +- doc/api/keys.md | 46 +- doc/api/license.md | 32 + doc/api/lint.md | 208 +- doc/api/managed_licenses.md | 147 +- doc/api/member_roles.md | 2 + doc/api/members.md | 5 +- doc/api/merge_request_approvals.md | 72 +- doc/api/merge_requests.md | 192 +- doc/api/merge_trains.md | 73 +- doc/api/metadata.md | 4 +- doc/api/milestones.md | 14 +- doc/api/namespaces.md | 11 +- doc/api/notes.md | 64 +- doc/api/oauth2.md | 4 +- doc/api/openapi/openapi.yaml | 2 +- doc/api/openapi/openapi_interactive.md | 2 +- doc/api/packages.md | 3 + doc/api/packages/composer.md | 2 +- doc/api/packages/conan.md | 38 +- doc/api/packages/debian.md | 26 +- doc/api/packages/debian_group_distributions.md | 12 +- doc/api/packages/go_proxy.md | 2 +- doc/api/packages/helm.md | 2 +- doc/api/packages/maven.md | 2 +- doc/api/packages/npm.md | 2 +- doc/api/packages/nuget.md | 2 +- doc/api/packages/pypi.md | 2 +- doc/api/packages/rubygems.md | 2 +- doc/api/packages/terraform-modules.md | 22 +- doc/api/pages.md | 12 +- doc/api/pages_domains.md | 10 +- doc/api/personal_access_tokens.md | 32 +- doc/api/pipeline_triggers.md | 2 +- doc/api/pipelines.md | 31 +- doc/api/plan_limits.md | 3 - doc/api/product_analytics.md | 4 +- doc/api/project_access_tokens.md | 28 + doc/api/project_badges.md | 2 +- doc/api/project_clusters.md | 6 +- doc/api/project_import_export.md | 133 +- doc/api/project_level_variables.md | 2 +- doc/api/project_relations_export.md | 2 +- doc/api/project_snippets.md | 108 +- doc/api/project_vulnerabilities.md | 4 +- doc/api/projects.md | 250 +- doc/api/protected_branches.md | 72 +- doc/api/protected_environments.md | 4 +- doc/api/protected_tags.md | 72 +- doc/api/releases/index.md | 27 +- doc/api/releases/links.md | 45 +- doc/api/remote_mirrors.md | 9 +- doc/api/repositories.md | 39 +- doc/api/resource_groups.md | 4 +- doc/api/rest/deprecations.md | 112 + doc/api/rest/index.md | 122 +- doc/api/runners.md | 46 +- doc/api/search.md | 15 +- doc/api/secure_files.md | 19 +- doc/api/settings.md | 60 +- doc/api/status_checks.md | 174 +- doc/api/suggestions.md | 16 +- doc/api/system_hooks.md | 6 +- doc/api/templates/dockerfiles.md | 2 +- doc/api/topics.md | 4 +- doc/api/usage_data.md | 3 +- doc/api/users.md | 116 +- doc/api/version.md | 4 +- doc/api/visual_review_discussions.md | 12 +- doc/api/vulnerability_exports.md | 5 - doc/api/wikis.md | 4 +- doc/architecture/blueprints/_template.md | 16 + .../blueprints/cells/cells-feature-admin-area.md | 59 + .../cells/cells-feature-agent-for-kubernetes.md | 30 + .../blueprints/cells/cells-feature-backups.md | 62 + .../blueprints/cells/cells-feature-ci-runners.md | 170 + .../cells/cells-feature-container-registry.md | 132 + .../cells/cells-feature-contributions-forks.md | 121 + .../blueprints/cells/cells-feature-dashboard.md | 30 + .../cells/cells-feature-data-migration.md | 131 + .../cells/cells-feature-database-sequences.md | 95 + .../blueprints/cells/cells-feature-git-access.md | 164 + .../blueprints/cells/cells-feature-gitlab-pages.md | 30 + .../cells/cells-feature-global-search.md | 48 + .../blueprints/cells/cells-feature-graphql.md | 95 + .../cells/cells-feature-organizations.md | 59 + .../cells/cells-feature-personal-namespaces.md | 30 + ...ells-feature-router-endpoints-classification.md | 47 + .../cells/cells-feature-schema-changes.md | 56 + .../blueprints/cells/cells-feature-secrets.md | 49 + .../blueprints/cells/cells-feature-snippets.md | 30 + .../blueprints/cells/cells-feature-template.md | 30 + .../blueprints/cells/cells-feature-uploads.md | 30 + doc/architecture/blueprints/cells/glossary.md | 106 + doc/architecture/blueprints/cells/goals.md | 59 + .../cells/images/pods-and-fulfillment.png | Bin 0 -> 20899 bytes .../blueprints/cells/images/term-cell.png | Bin 0 -> 26613 bytes .../blueprints/cells/images/term-cluster.png | Bin 0 -> 91814 bytes .../blueprints/cells/images/term-organization.png | Bin 0 -> 29527 bytes .../cells/images/term-top-level-group.png | Bin 0 -> 15122 bytes doc/architecture/blueprints/cells/impact.md | 58 + doc/architecture/blueprints/cells/index.md | 360 ++ ...sal-stateless-router-with-buffering-requests.md | 649 +++ ...oposal-stateless-router-with-routes-learning.md | 673 +++ doc/architecture/blueprints/ci_data_decay/index.md | 6 +- .../ci_data_decay/pipeline_partitioning.md | 19 +- .../ci_pipeline_components/dev_workflow.md | 154 + .../ci_pipeline_components/img/new_release.png | Bin 0 -> 13622 bytes .../ci_pipeline_components/img/pipeline_main.png | Bin 0 -> 6644 bytes .../ci_pipeline_components/img/pipeline_tag.png | Bin 0 -> 8697 bytes .../blueprints/ci_pipeline_components/index.md | 231 +- doc/architecture/blueprints/ci_scale/index.md | 4 +- .../clickhouse_dbwriter.png | Bin 0 -> 46544 bytes .../clickhouse_ingestion_pipeline/index.md | 289 ++ .../clickhouse_read_abstraction_layer/index.md | 318 ++ .../blueprints/clickhouse_usage/index.md | 58 + .../self_managed_costs_and_requirements/index.md | 65 + .../blueprints/code_search_with_zoekt/index.md | 307 ++ .../index.md | 94 +- .../consolidating_groups_and_projects/index.md | 155 +- .../container_registry_metadata_database/index.md | 2 + .../database/scalability/patterns/index.md | 1 - .../database/scalability/patterns/read_mostly.md | 1 - .../database/scalability/patterns/time_decay.md | 1 - .../blueprints/database_scaling/size-limits.md | 1 - .../blueprints/database_testing/index.md | 13 +- .../blueprints/gitlab_agent_deployments/index.md | 4 + .../blueprints/gitlab_ci_events/index.md | 63 + .../proposal-1-using-the-gitlab-ci-file.md | 60 + .../proposal-2-using-the-rules-keyword.md | 38 + ...proposal-3-using-the-gitlab-ci-events-folder.md | 64 + .../proposal-4-creating-events-via-ci-files.md | 73 + .../blueprints/gitlab_ml_experiments/index.md | 170 + .../gitlab_observability_backend/metrics/index.md | 6 +- doc/architecture/blueprints/graphql_api/index.md | 2 + doc/architecture/blueprints/object_pools/index.md | 495 +++ .../blueprints/object_storage/index.md | 6 +- doc/architecture/blueprints/organization/index.md | 175 + .../iteration0-organizations-introduction.png | Bin 67160 -> 0 bytes .../pods/images/pods-and-fulfillment.png | Bin 20899 -> 0 bytes .../blueprints/pods/images/term-cluster.png | Bin 63268 -> 0 bytes .../blueprints/pods/images/term-organization.png | Bin 7150 -> 0 bytes .../blueprints/pods/images/term-pod.png | Bin 16104 -> 0 bytes .../pods/images/term-top-level-namespace.png | Bin 11451 -> 0 bytes doc/architecture/blueprints/pods/index.md | 359 +- .../blueprints/pods/pods-feature-admin-area.md | 61 +- .../pods/pods-feature-agent-for-kubernetes.md | 32 +- .../blueprints/pods/pods-feature-backups.md | 64 +- .../blueprints/pods/pods-feature-ci-runners.md | 172 +- .../pods/pods-feature-container-registry.md | 134 +- .../pods/pods-feature-contributions-forks.md | 123 +- .../blueprints/pods/pods-feature-dashboard.md | 32 +- .../blueprints/pods/pods-feature-data-migration.md | 133 +- .../pods/pods-feature-database-sequences.md | 97 +- .../blueprints/pods/pods-feature-git-access.md | 166 +- .../blueprints/pods/pods-feature-gitlab-pages.md | 32 +- .../blueprints/pods/pods-feature-global-search.md | 50 +- .../blueprints/pods/pods-feature-graphql.md | 97 +- .../blueprints/pods/pods-feature-organizations.md | 61 +- .../pods/pods-feature-personal-namespaces.md | 32 +- ...pods-feature-router-endpoints-classification.md | 49 +- .../blueprints/pods/pods-feature-schema-changes.md | 58 +- .../blueprints/pods/pods-feature-secrets.md | 51 +- .../blueprints/pods/pods-feature-snippets.md | 32 +- .../blueprints/pods/pods-feature-template.md | 32 +- .../blueprints/pods/pods-feature-uploads.md | 32 +- ...sal-stateless-router-with-buffering-requests.md | 651 +-- ...oposal-stateless-router-with-routes-learning.md | 675 +-- doc/architecture/blueprints/rate_limiting/index.md | 4 +- .../remote_development/img/remote_dev_15_7.png | Bin 108160 -> 112261 bytes .../blueprints/remote_development/index.md | 226 +- .../blueprints/runner_scaling/index.md | 8 +- doc/architecture/blueprints/runner_tokens/index.md | 56 +- .../blueprints/search/code_search_with_zoekt.md | 305 -- .../blueprints/secret_detection/index.md | 81 +- doc/architecture/blueprints/work_items/index.md | 2 + doc/architecture/index.md | 1 - doc/ci/caching/index.md | 69 +- doc/ci/chatops/img/gitlab-chatops-icon-small.png | Bin 2922 -> 0 bytes doc/ci/chatops/index.md | 129 +- doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md | 4 +- doc/ci/cloud_deployment/heroku.md | 4 +- doc/ci/cloud_deployment/index.md | 4 +- doc/ci/cloud_services/aws/index.md | 13 +- doc/ci/cloud_services/azure/index.md | 6 +- doc/ci/cloud_services/google_cloud/index.md | 9 +- doc/ci/cloud_services/index.md | 72 +- doc/ci/components/index.md | 201 + doc/ci/directed_acyclic_graph/index.md | 2 +- doc/ci/docker/authenticate_registry.md | 144 + doc/ci/docker/docker_layer_caching.md | 61 + doc/ci/docker/index.md | 1 - doc/ci/docker/using_docker_build.md | 361 +- doc/ci/docker/using_kaniko.md | 4 +- doc/ci/environments/deployment_approvals.md | 18 +- doc/ci/environments/deployment_safety.md | 8 +- doc/ci/environments/environments_dashboard.md | 4 +- doc/ci/environments/external_deployment_tools.md | 78 +- .../img/environment_auto_stop_v13_10.png | Bin 16265 -> 0 bytes .../environments/img/environments_project_home.png | Bin 70687 -> 23937 bytes doc/ci/environments/incremental_rollouts.md | 4 +- doc/ci/environments/index.md | 242 +- doc/ci/environments/protected_environments.md | 23 +- .../authenticating-with-hashicorp-vault/index.md | 87 +- doc/ci/examples/deployment/composer-npm-deploy.md | 4 +- doc/ci/examples/deployment/index.md | 6 +- .../end_to_end_testing_webdriverio/index.md | 2 +- doc/ci/examples/index.md | 3 +- .../laravel_with_gitlab_and_envoy/index.md | 3 +- doc/ci/examples/semantic-release.md | 17 +- doc/ci/git_submodules.md | 53 +- doc/ci/index.md | 15 +- doc/ci/interactive_web_terminal/index.md | 6 +- doc/ci/introduction/index.md | 2 +- doc/ci/jobs/ci_job_token.md | 65 +- doc/ci/jobs/index.md | 6 +- doc/ci/jobs/job_artifacts.md | 358 ++ doc/ci/jobs/job_artifacts_troubleshooting.md | 149 + doc/ci/jobs/job_control.md | 6 +- doc/ci/large_repositories/index.md | 2 +- doc/ci/lint.md | 4 +- doc/ci/migration/circleci.md | 11 +- doc/ci/migration/jenkins.md | 3 +- doc/ci/mobile_devops.md | 428 +- doc/ci/pipeline_editor/index.md | 13 +- doc/ci/pipelines/cicd_minutes.md | 71 +- doc/ci/pipelines/downstream_pipelines.md | 80 +- doc/ci/pipelines/img/code_coverage_graph_v13_1.png | Bin 29299 -> 0 bytes .../img/coverage_check_approval_rule_14_1.png | Bin 34393 -> 0 bytes .../img/job_artifacts_browser_button_v13_11.png | Bin 9633 -> 0 bytes .../pipelines/img/job_artifacts_browser_v13_11.png | Bin 10083 -> 0 bytes .../img/job_artifacts_jobs_page_v13_11.png | Bin 22599 -> 0 bytes .../img/job_artifacts_merge_request_v13_11.png | Bin 13256 -> 0 bytes .../img/job_artifacts_pipelines_page_v13_11.png | Bin 47332 -> 0 bytes .../pipelines/img/job_latest_artifacts_browser.png | Bin 10551 -> 0 bytes .../img/multi_project_pipeline_graph_v14_3.png | Bin 30119 -> 0 bytes doc/ci/pipelines/img/pipeline_mini_graph_v15_0.png | Bin 6061 -> 0 bytes .../img/pipelines_test_coverage_build.png | Bin 4481 -> 0 bytes .../img/pipelines_test_coverage_mr_widget.png | Bin 6375 -> 0 bytes doc/ci/pipelines/index.md | 7 +- doc/ci/pipelines/job_artifacts.md | 526 +-- doc/ci/pipelines/merge_request_pipelines.md | 25 +- doc/ci/pipelines/merge_trains.md | 4 + doc/ci/pipelines/merged_results_pipelines.md | 7 + doc/ci/pipelines/pipeline_architectures.md | 2 +- doc/ci/pipelines/pipeline_artifacts.md | 4 +- doc/ci/pipelines/pipeline_efficiency.md | 2 +- doc/ci/pipelines/schedules.md | 5 +- doc/ci/pipelines/settings.md | 115 +- doc/ci/quick_start/index.md | 7 +- doc/ci/quick_start/tutorial.md | 504 +++ doc/ci/resource_groups/index.md | 79 +- doc/ci/review_apps/index.md | 21 +- doc/ci/runners/configure_runners.md | 56 +- doc/ci/runners/index.md | 10 +- doc/ci/runners/register_runner.md | 117 + doc/ci/runners/runners_scope.md | 6 +- doc/ci/runners/saas/linux_saas_runner.md | 20 +- doc/ci/runners/saas/macos/codesigning.md | 103 +- doc/ci/runners/saas/macos/environment.md | 4 +- doc/ci/runners/saas/macos_saas_runner.md | 4 +- doc/ci/runners/saas/windows_saas_runner.md | 6 +- doc/ci/secrets/id_token_authentication.md | 130 +- doc/ci/secrets/index.md | 15 +- doc/ci/secure_files/index.md | 13 +- doc/ci/services/index.md | 7 +- doc/ci/ssh_keys/index.md | 2 +- doc/ci/test_cases/index.md | 18 +- doc/ci/testing/accessibility_testing.md | 15 +- doc/ci/testing/browser_performance_testing.md | 7 +- doc/ci/testing/code_coverage.md | 129 + doc/ci/testing/code_quality.md | 17 +- doc/ci/testing/fail_fast_testing.md | 4 +- doc/ci/testing/img/code_coverage_graph_v13_1.png | Bin 0 -> 29299 bytes doc/ci/testing/img/code_coverage_group_report.png | Bin 0 -> 71793 bytes doc/ci/testing/img/code_quality_summary_15_9.png | Bin 0 -> 71968 bytes .../img/coverage_check_approval_rule_14_1.png | Bin 0 -> 34393 bytes .../testing/img/pipelines_test_coverage_build.png | Bin 0 -> 4481 bytes .../img/pipelines_test_coverage_mr_widget.png | Bin 0 -> 6375 bytes doc/ci/testing/index.md | 5 +- doc/ci/testing/load_performance_testing.md | 13 +- doc/ci/testing/metrics_reports.md | 2 +- doc/ci/testing/test_coverage_visualization.md | 58 +- doc/ci/testing/unit_test_report_examples.md | 23 +- doc/ci/testing/unit_test_reports.md | 2 +- doc/ci/triggers/index.md | 19 +- doc/ci/troubleshooting.md | 10 +- doc/ci/variables/index.md | 37 +- doc/ci/variables/predefined_variables.md | 12 +- doc/ci/variables/where_variables_can_be_used.md | 2 +- doc/ci/yaml/artifacts_reports.md | 20 +- doc/ci/yaml/includes.md | 153 +- doc/ci/yaml/index.md | 253 +- doc/ci/yaml/yaml_optimization.md | 4 +- doc/cloud_seed/index.md | 2 +- doc/development/advanced_search.md | 336 ++ doc/development/ai_architecture.md | 108 + doc/development/ai_features.md | 453 ++ doc/development/api_graphql_styleguide.md | 27 +- doc/development/api_styleguide.md | 65 + doc/development/application_secrets.md | 2 +- doc/development/application_slis/index.md | 2 +- doc/development/application_slis/rails_request.md | 282 ++ .../application_slis/rails_request_apdex.md | 253 -- doc/development/approval_rules.md | 11 - doc/development/architecture.md | 39 +- doc/development/audit_event_guide/index.md | 2 +- doc/development/auto_devops.md | 6 +- .../create_source_code_be/gitaly_touch_points.md | 2 +- .../backend/create_source_code_be/index.md | 8 +- doc/development/bulk_import.md | 13 +- doc/development/caching.md | 6 +- doc/development/cascading_settings.md | 36 +- doc/development/changelog.md | 2 +- doc/development/chatops_on_gitlabcom.md | 4 +- doc/development/cicd/cicd_tables.md | 121 + doc/development/cicd/index.md | 3 +- doc/development/cicd/templates.md | 1 - doc/development/code_intelligence/index.md | 2 +- doc/development/code_owners/index.md | 135 + doc/development/code_review.md | 25 +- doc/development/contributing/community_roles.md | 11 - doc/development/contributing/design.md | 6 +- doc/development/contributing/first_contribution.md | 340 ++ doc/development/contributing/img/bot_ready.png | Bin 0 -> 9367 bytes doc/development/contributing/img/changes_tab.png | Bin 0 -> 49899 bytes doc/development/contributing/img/gdk_home.png | Bin 0 -> 16636 bytes doc/development/contributing/img/mr_button.png | Bin 0 -> 14900 bytes .../contributing/img/new_merge_request.png | Bin 0 -> 9302 bytes doc/development/contributing/img/ui_text_after.png | Bin 0 -> 4446 bytes .../contributing/img/ui_text_before.png | Bin 0 -> 5243 bytes doc/development/contributing/index.md | 203 +- doc/development/contributing/issue_workflow.md | 324 +- .../contributing/merge_request_workflow.md | 126 +- doc/development/contributing/style_guides.md | 24 +- doc/development/dangerbot.md | 28 +- .../database/add_foreign_key_to_existing_column.md | 19 +- .../database/adding_database_indexes.md | 34 +- .../database/avoiding_downtime_in_migrations.md | 2 +- .../database/batched_background_migrations.md | 253 +- doc/development/database/ci_mirrored_tables.md | 18 +- doc/development/database/clickhouse/index.md | 62 + .../database/clickhouse/merge_request_analytics.md | 355 ++ .../database/clickhouse/tiered_storage.md | 138 + doc/development/database/creating_enums.md | 12 +- doc/development/database/database_dictionary.md | 16 +- doc/development/database/database_lab.md | 96 +- .../database/database_migration_pipeline.md | 42 +- .../database/database_reviewer_guidelines.md | 5 +- .../database/efficient_in_operator_queries.md | 2 +- doc/development/database/index.md | 5 +- .../database/iterating_tables_in_batches.md | 62 +- doc/development/database/load_balancing.md | 59 + doc/development/database/loose_foreign_keys.md | 2 +- doc/development/database/multiple_databases.md | 22 +- doc/development/database/query_performance.md | 4 +- doc/development/database/required_stops.md | 56 + .../database/strings_and_the_text_data_type.md | 39 +- doc/development/database/table_partitioning.md | 192 +- doc/development/database/transaction_guidelines.md | 2 +- .../database/understanding_explain_plans.md | 31 +- doc/development/database_review.md | 29 +- doc/development/deprecation_guidelines/index.md | 4 +- doc/development/diffs.md | 11 - doc/development/directory_structure.md | 11 - doc/development/distributed_tracing.md | 61 +- doc/development/distribution/index.md | 35 + doc/development/documentation/alpha_beta.md | 49 + doc/development/documentation/contribute.md | 83 + doc/development/documentation/feature_flags.md | 74 +- doc/development/documentation/index.md | 76 +- doc/development/documentation/redirects.md | 29 +- .../documentation/restful_api_styleguide.md | 2 +- .../documentation/site_architecture/global_nav.md | 3 +- .../documentation/site_architecture/index.md | 2 +- doc/development/documentation/styleguide/index.md | 53 +- .../documentation/styleguide/word_list.md | 245 +- doc/development/documentation/testing.md | 16 +- .../documentation/topic_types/concept.md | 14 +- .../documentation/topic_types/glossary.md | 70 + doc/development/documentation/topic_types/index.md | 59 +- doc/development/documentation/topic_types/task.md | 18 +- .../documentation/topic_types/tutorial.md | 31 +- doc/development/documentation/versions.md | 7 +- doc/development/documentation/workflow.md | 168 +- doc/development/ee_features.md | 110 +- doc/development/elasticsearch.md | 625 --- doc/development/emails.md | 33 +- doc/development/experiment_guide/index.md | 2 +- doc/development/export_csv.md | 2 +- doc/development/fe_guide/accessibility.md | 51 +- doc/development/fe_guide/content_editor.md | 6 +- .../fe_guide/customizable_dashboards.md | 2 +- doc/development/fe_guide/emojis.md | 2 +- doc/development/fe_guide/frontend_faq.md | 4 +- doc/development/fe_guide/graphql.md | 661 +-- doc/development/fe_guide/index.md | 4 +- .../fe_guide/merge_request_widget_extensions.md | 2 - doc/development/fe_guide/performance.md | 26 +- doc/development/fe_guide/registry_architecture.md | 2 +- doc/development/fe_guide/source_editor.md | 20 +- doc/development/fe_guide/storybook.md | 28 +- doc/development/fe_guide/style/html.md | 2 +- doc/development/fe_guide/style/index.md | 2 +- doc/development/fe_guide/style/javascript.md | 24 +- doc/development/fe_guide/style/scss.md | 3 +- doc/development/fe_guide/style/vue.md | 292 +- doc/development/fe_guide/vue.md | 130 +- doc/development/fe_guide/vuex.md | 16 +- doc/development/fe_guide/widgets.md | 4 +- doc/development/feature_categorization/index.md | 5 + doc/development/feature_development.md | 12 +- doc/development/feature_flags/controls.md | 14 +- doc/development/feature_flags/index.md | 14 +- doc/development/features_inside_dot_gitlab.md | 6 +- doc/development/fips_compliance.md | 91 +- doc/development/gemfile.md | 38 +- doc/development/geo.md | 4 +- doc/development/git_object_deduplication.md | 20 +- doc/development/gitaly.md | 27 +- doc/development/github_importer.md | 53 +- doc/development/gitlab_flavored_markdown/index.md | 4 +- .../specification_guide/index.md | 6 +- doc/development/gitlab_shell/index.md | 8 +- doc/development/gitpod_internals.md | 2 +- doc/development/go_guide/go_upgrade.md | 3 +- doc/development/go_guide/index.md | 16 +- doc/development/graphql_guide/graphql_pro.md | 2 +- doc/development/graphql_guide/index.md | 2 +- doc/development/graphql_guide/monitoring.md | 2 +- doc/development/graphql_guide/pagination.md | 2 +- doc/development/graphql_guide/reviewing.md | 96 + doc/development/i18n/externalization.md | 15 +- doc/development/i18n/index.md | 2 +- doc/development/i18n/merging_translations.md | 2 +- doc/development/i18n/proofreader.md | 7 +- doc/development/i18n/translation.md | 2 +- doc/development/image_scaling.md | 4 +- doc/development/img/feature-flag-metrics.png | Bin 88110 -> 27814 bytes doc/development/import_export.md | 2 +- doc/development/import_project.md | 146 +- doc/development/index.md | 9 +- doc/development/integrations/codesandbox.md | 155 - doc/development/integrations/index.md | 77 +- doc/development/integrations/jenkins.md | 6 +- doc/development/integrations/jira_connect.md | 66 +- doc/development/integrations/secure.md | 21 +- .../integrations/secure_partner_integration.md | 6 +- doc/development/interacting_components.md | 2 +- doc/development/internal_api/index.md | 135 +- doc/development/internal_users.md | 9 +- doc/development/issue_types.md | 2 +- doc/development/json.md | 2 +- doc/development/kubernetes.md | 6 +- doc/development/labels/index.md | 347 ++ doc/development/lfs.md | 138 +- doc/development/logging.md | 48 +- ...equest_application_and_rate_limit_guidelines.md | 11 - .../merge_request_concepts/approval_rules.md | 2 +- .../merge_request_concepts/diffs/frontend.md | 38 +- .../merge_request_concepts/diffs/index.md | 2 +- doc/development/merge_request_concepts/index.md | 4 +- .../merge_request_concepts/performance.md | 6 +- doc/development/merge_request_diffs.md | 11 - .../merge_request_performance_guidelines.md | 11 - doc/development/migration_style_guide.md | 148 +- doc/development/navigation_sidebar.md | 56 + doc/development/organization/index.md | 85 + doc/development/packages/debian_repository.md | 4 +- doc/development/packages/index.md | 2 +- doc/development/pages/index.md | 6 +- doc/development/performance.md | 19 + doc/development/permissions.md | 125 +- doc/development/pipelines/index.md | 227 +- doc/development/pipelines/internals.md | 7 +- doc/development/pipelines/performance.md | 2 +- .../product_qualified_lead_guide/index.md | 6 +- doc/development/project_templates.md | 164 +- doc/development/projections.md | 4 +- doc/development/prometheus_metrics.md | 14 +- doc/development/python_guide/index.md | 2 +- doc/development/rails_update.md | 52 +- doc/development/rake_tasks.md | 56 +- doc/development/real_time.md | 538 ++- doc/development/redis.md | 28 +- doc/development/redis/new_redis_instance.md | 66 +- doc/development/repository_mirroring.md | 2 +- doc/development/rubocop_development_guide.md | 6 +- doc/development/scalability.md | 2 +- .../search/advanced_search_migration_styleguide.md | 311 ++ doc/development/sec/CycloneDX_property_taxonomy.md | 72 + doc/development/sec/analyzer_development_guide.md | 168 +- doc/development/sec/index.md | 6 +- .../sec/security_report_ingestion_overview.md | 103 +- doc/development/sec/token_revocation_api.md | 4 +- doc/development/secure_coding_guidelines.md | 123 +- doc/development/service_ping/implement.md | 79 +- doc/development/service_ping/index.md | 48 +- doc/development/service_ping/metrics_dictionary.md | 8 +- .../service_ping/metrics_instrumentation.md | 8 +- doc/development/service_ping/metrics_lifecycle.md | 64 +- .../service_ping/performance_indicator_metrics.md | 2 +- doc/development/service_ping/review_guidelines.md | 37 +- doc/development/service_ping/troubleshooting.md | 2 +- doc/development/service_ping/usage_data.md | 2 +- doc/development/shell_commands.md | 2 +- .../sidekiq/compatibility_across_updates.md | 151 +- doc/development/sidekiq/index.md | 2 +- doc/development/sidekiq/worker_attributes.md | 20 +- doc/development/snowplow/event_dictionary_guide.md | 13 +- doc/development/snowplow/implementation.md | 189 +- doc/development/snowplow/index.md | 4 +- doc/development/snowplow/infrastructure.md | 2 +- doc/development/snowplow/review_guidelines.md | 17 +- doc/development/snowplow/schemas.md | 2 +- doc/development/snowplow/troubleshooting.md | 2 +- .../exploratory_testing.md | 1 + .../model_and_services.md | 6 +- .../dashboards/error_budget_detail.md | 25 - .../img/error_budget_detail_sli_detail.png | Bin 97895 -> 0 bytes .../img/error_budgets_kibana_dashboard_v15_10.png | Bin 0 -> 142096 bytes doc/development/stage_group_observability/index.md | 42 +- doc/development/testing_guide/best_practices.md | 75 +- doc/development/testing_guide/contract/index.md | 4 +- .../testing_guide/end_to_end/beginners_guide.md | 14 +- .../testing_guide/end_to_end/best_practices.md | 6 +- .../end_to_end/execution_context_selection.md | 12 +- .../testing_guide/end_to_end/resources.md | 2 +- .../end_to_end/rspec_metadata_tests.md | 4 +- .../running_tests_that_require_special_setup.md | 95 +- doc/development/testing_guide/flaky_tests.md | 54 +- doc/development/testing_guide/frontend_testing.md | 311 +- doc/development/testing_guide/review_apps.md | 8 +- doc/development/testing_guide/testing_levels.md | 12 +- .../testing_guide/testing_migrations_guide.md | 2 +- doc/development/uploads/index.md | 2 +- doc/development/uploads/working_with_uploads.md | 2 +- doc/development/ux/index.md | 26 + doc/development/value_stream_analytics.md | 44 +- doc/development/vs_code_debugging.md | 69 + doc/development/wikis.md | 10 +- doc/development/windows.md | 2 +- doc/development/work_items_widgets.md | 24 + doc/development/workhorse/configuration.md | 2 +- doc/development/workhorse/index.md | 11 +- doc/development/workspace/index.md | 162 +- doc/drawers/advanced_search_syntax.md | 1 + doc/gitlab-basics/add-file.md | 178 +- doc/gitlab-basics/command-line-commands.md | 126 +- doc/gitlab-basics/feature_branch_workflow.md | 1 - doc/gitlab-basics/start-using-git.md | 14 +- doc/index.md | 9 +- doc/install/aws/eks_clusters_aws.md | 2 +- doc/install/aws/gitlab_hybrid_on_aws.md | 2 +- doc/install/aws/gitlab_sre_for_aws.md | 1 - doc/install/aws/index.md | 3 +- doc/install/aws/manual_install_aws.md | 16 +- doc/install/azure/index.md | 2 +- doc/install/cloud_providers.md | 17 + doc/install/docker.md | 39 +- doc/install/google_cloud_platform/index.md | 2 +- doc/install/index.md | 14 +- doc/install/install_methods.md | 16 +- doc/install/installation.md | 96 +- doc/install/next_steps.md | 2 +- doc/install/openshift_and_gitlab/index.md | 2 +- doc/install/requirements.md | 7 +- doc/integration/advanced_search/elasticsearch.md | 167 +- .../elasticsearch_troubleshooting.md | 16 +- doc/integration/alicloud.md | 2 +- doc/integration/arkose.md | 2 +- doc/integration/auth0.md | 2 +- doc/integration/azure.md | 136 +- doc/integration/cas.md | 79 +- doc/integration/datadog.md | 6 +- doc/integration/ding_talk.md | 5 +- doc/integration/external-issue-tracker.md | 4 +- doc/integration/facebook.md | 4 +- doc/integration/github.md | 2 +- doc/integration/gitlab.md | 2 +- doc/integration/gitpod.md | 4 +- doc/integration/glab/index.md | 36 +- doc/integration/gmail_action_buttons_for_gitlab.md | 5 +- doc/integration/google.md | 4 +- doc/integration/index.md | 7 +- doc/integration/jenkins.md | 31 +- doc/integration/jira/configure.md | 78 +- doc/integration/jira/connect-app.md | 255 +- doc/integration/jira/development_panel.md | 135 +- doc/integration/jira/dvcs/index.md | 196 +- doc/integration/jira/dvcs/troubleshooting.md | 20 +- .../jira/img/jira-upload-app-success_v13_11.png | Bin 11440 -> 0 bytes .../jira/img/jira-upload-app_v13_11.png | Bin 20667 -> 0 bytes .../jira/img/jira_added_user_to_group.png | Bin 21646 -> 0 bytes doc/integration/jira/img/jira_create_new_group.png | Bin 70535 -> 0 bytes .../jira/img/jira_dev_panel_jira_setup_3.png | Bin 80136 -> 0 bytes .../jira/img/jira_dev_panel_jira_setup_4.png | Bin 21592 -> 0 bytes .../jira/img/jira_dev_panel_jira_setup_5.png | Bin 10336 -> 0 bytes .../jira/img/jira_dev_panel_setup_com_1.png | Bin 15392 -> 0 bytes .../jira/img/jira_dev_panel_setup_com_2.png | Bin 22370 -> 0 bytes .../jira/img/jira_dev_panel_setup_com_3_v13_9.png | Bin 28042 -> 0 bytes .../jira/img/jira_dev_panel_setup_com_4_v13_9.png | Bin 23360 -> 0 bytes doc/integration/jira/img/jira_group_access.png | Bin 20934 -> 0 bytes .../jira/img/jira_issue_detail_view_v13.10.png | Bin 49442 -> 0 bytes doc/integration/jira/img/jira_issue_reference.png | Bin 19583 -> 0 bytes doc/integration/jira/img/jira_project_settings.png | Bin 14149 -> 0 bytes .../jira/img/jira_service_close_issue.png | Bin 29632 -> 0 bytes .../jira/img/open_jira_issues_list_v14_6.png | Bin 89984 -> 0 bytes doc/integration/jira/index.md | 84 +- doc/integration/jira/issues.md | 121 +- doc/integration/jira/jira_cloud_configuration.md | 26 +- doc/integration/jira/jira_server_configuration.md | 82 +- doc/integration/jira/troubleshooting.md | 12 +- doc/integration/kerberos.md | 37 +- doc/integration/mattermost/index.md | 10 +- doc/integration/oauth2_generic.md | 64 +- doc/integration/omniauth.md | 12 +- doc/integration/openid_connect_provider.md | 2 +- doc/integration/partner_marketplace.md | 116 +- doc/integration/salesforce.md | 2 +- doc/integration/saml.md | 48 +- doc/integration/security_partners/index.md | 4 +- doc/integration/slash_commands.md | 2 +- doc/integration/sourcegraph.md | 19 +- doc/integration/trello_power_up.md | 10 +- doc/integration/twitter.md | 2 +- doc/integration/vault.md | 4 +- doc/legal/index.md | 1 - doc/operations/error_tracking.md | 272 +- doc/operations/feature_flags.md | 31 +- doc/operations/img/copy-group-id.png | Bin 112686 -> 0 bytes doc/operations/img/create-gitlab-application.png | Bin 521206 -> 0 bytes .../img/error_tracking_setting_dsn_v14_4.png | Bin 39249 -> 0 bytes .../img/error_tracking_setting_v14_3.png | Bin 27537 -> 0 bytes doc/operations/img/listing_groups.png | Bin 40621 -> 0 bytes doc/operations/incident_management/alerts.md | 25 + .../incident_timeline_events.md | 21 +- doc/operations/incident_management/incidents.md | 15 +- .../incident_management/manage_incidents.md | 38 +- doc/operations/incident_management/slack.md | 11 +- doc/operations/incident_management/status_page.md | 4 +- doc/operations/index.md | 30 +- doc/operations/metrics/alerts.md | 80 +- doc/operations/metrics/dashboards/default.md | 41 +- doc/operations/metrics/dashboards/develop.md | 37 +- .../img/actions_menu_create_add_panel_v13_3.png | Bin 9982 -> 0 bytes .../actions_menu_create_new_dashboard_v13_3.png | Bin 14957 -> 0 bytes .../img/dashboard_external_link_v13_1.png | Bin 12708 -> 0 bytes .../img/dashboard_local_timezone_v13_1.png | Bin 65094 -> 0 bytes .../dashboards/img/external_dashboard_link.png | Bin 20192 -> 0 bytes .../img/heatmap_chart_too_much_data_v_13_2.png | Bin 7310 -> 0 bytes .../metrics/dashboards/img/heatmap_panel_type.png | Bin 8272 -> 0 bytes .../img/k8s_pod_health_dashboard_v13_3.png | Bin 60933 -> 0 bytes .../img/metrics_dashboard_annotations_ui_v13.0.png | Bin 31654 -> 0 bytes .../img/metrics_dashboard_panel_preview_v13_3.png | Bin 45857 -> 0 bytes .../metrics_dashboard_template_selection_v13_3.png | Bin 9033 -> 0 bytes ..._dashboard_template_selection_web_ide_v13_3.png | Bin 26123 -> 0 bytes .../dashboards/img/panel_context_menu_v14_0.png | Bin 11084 -> 0 bytes .../prometheus_dashboard_anomaly_panel_type.png | Bin 41015 -> 0 bytes .../prometheus_dashboard_area_panel_type_v12_8.png | Bin 53370 -> 0 bytes ...theus_dashboard_bar_chart_panel_type_v12.10.png | Bin 4761 -> 0 bytes .../img/prometheus_dashboard_column_panel_type.png | Bin 13219 -> 0 bytes ...prometheus_dashboard_gauge_panel_type_v13_3.png | Bin 17303 -> 0 bytes ...ometheus_dashboard_label_variable_shorthand.png | Bin 3897 -> 0 bytes .../img/prometheus_dashboard_label_variables.png | Bin 8076 -> 0 bytes .../img/prometheus_dashboard_repeated_label.png | Bin 3116 -> 0 bytes ...prometheus_dashboard_single_stat_panel_type.png | Bin 6871 -> 0 bytes ...s_dashboard_stacked_column_panel_type_v12_8.png | Bin 13898 -> 0 bytes .../prometheus_dashboard_yaml_validation_v13_1.png | Bin 27694 -> 0 bytes .../metrics/dashboards/img/related_links_v13_1.png | Bin 4086 -> 0 bytes doc/operations/metrics/dashboards/index.md | 266 +- doc/operations/metrics/dashboards/panel_types.md | 315 +- doc/operations/metrics/dashboards/settings.md | 56 +- .../metrics/dashboards/templating_variables.md | 131 +- doc/operations/metrics/dashboards/variables.md | 76 +- doc/operations/metrics/dashboards/yaml.md | 178 +- .../metrics/dashboards/yaml_number_format.md | 175 +- doc/operations/metrics/embed.md | 125 +- doc/operations/metrics/embed_grafana.md | 91 +- .../metrics/img/copy_link_to_chart_v12_10.png | Bin 21559 -> 0 bytes .../metrics/img/embed_metrics_issue_template.png | Bin 43908 -> 0 bytes .../img/embedded_metrics_rendered_v13_4.png | Bin 24072 -> 0 bytes .../metrics/img/example-dashboard_v13_3.png | Bin 64275 -> 0 bytes .../metrics/img/hide_embedded_metrics_v12_10.png | Bin 21312 -> 0 bytes .../metrics/img/prometheus_add_metric.png | Bin 24607 -> 0 bytes .../img/prometheus_cluster_health_embed_v12_9.png | Bin 50178 -> 0 bytes ...rometheus_dashboard_edit_metric_link_v_12_9.png | Bin 29178 -> 0 bytes .../img/prometheus_monitoring_dashboard_v13_3.png | Bin 64275 -> 0 bytes .../metrics/img/rendered_grafana_embed_v12_5.png | Bin 61194 -> 0 bytes .../metrics/img/view_embedded_metrics_v12_10.png | Bin 36717 -> 0 bytes doc/operations/metrics/index.md | 166 +- doc/operations/quickstart-guide.md | 229 - doc/policy/alpha-beta-support.md | 84 +- doc/policy/maintenance.md | 6 +- doc/raketasks/backup_gitlab.md | 26 +- doc/raketasks/backup_restore.md | 11 +- doc/raketasks/cleanup.md | 4 +- doc/raketasks/generate_sample_prometheus_data.md | 12 +- doc/raketasks/import.md | 171 +- doc/raketasks/index.md | 8 +- doc/raketasks/migrate_snippets.md | 2 +- doc/raketasks/restore_gitlab.md | 93 +- doc/raketasks/user_management.md | 2 +- doc/security/email_verification.md | 46 + doc/security/index.md | 3 +- doc/security/rate_limits.md | 18 +- doc/security/reset_user_password.md | 16 +- doc/security/token_overview.md | 15 +- doc/security/two_factor_authentication.md | 5 +- doc/security/user_email_confirmation.md | 2 +- doc/security/user_file_uploads.md | 59 +- doc/security/webhooks.md | 102 +- doc/subscriptions/bronze_starter.md | 8 +- doc/subscriptions/choosing_subscription.md | 61 + doc/subscriptions/community_programs.md | 80 + doc/subscriptions/customers_portal.md | 104 + doc/subscriptions/gitlab_com/index.md | 33 +- doc/subscriptions/gitlab_dedicated/index.md | 109 +- doc/subscriptions/index.md | 262 +- doc/subscriptions/quarterly_reconciliation.md | 2 +- doc/subscriptions/self_managed/index.md | 74 +- doc/topics/authentication/index.md | 8 +- doc/topics/autodevops/cicd_variables.md | 61 +- .../cloud_deployments/auto_devops_with_ec2.md | 4 +- .../cloud_deployments/auto_devops_with_ecs.md | 4 +- .../cloud_deployments/auto_devops_with_eks.md | 301 ++ .../cloud_deployments/auto_devops_with_gke.md | 8 +- doc/topics/autodevops/customize.md | 4 +- doc/topics/autodevops/index.md | 5 +- .../autodevops/multiple_clusters_auto_devops.md | 4 +- doc/topics/autodevops/prepare_deployment.md | 4 +- doc/topics/autodevops/requirements.md | 28 +- doc/topics/autodevops/stages.md | 9 +- doc/topics/autodevops/troubleshooting.md | 4 +- .../upgrading_auto_deploy_dependencies.md | 10 +- doc/topics/autodevops/upgrading_postgresql.md | 95 +- doc/topics/awesome_co.md | 143 - doc/topics/data_seeder.md | 331 ++ doc/topics/git/bisect.md | 79 +- doc/topics/git/cherry_picking.md | 1 - doc/topics/git/feature_branch_development.md | 11 - doc/topics/git/feature_branching.md | 34 +- doc/topics/git/getting_started.md | 11 - doc/topics/git/git_add.md | 1 - doc/topics/git/git_log.md | 63 +- doc/topics/git/git_rebase.md | 3 +- doc/topics/git/index.md | 15 +- doc/topics/git/lfs/index.md | 5 +- doc/topics/git/lfs/migrate_to_git_lfs.md | 4 +- doc/topics/git/merge_conflicts.md | 11 - doc/topics/git/partial_clone.md | 4 +- doc/topics/git/rollback_commits.md | 1 - doc/topics/git/stash.md | 1 - doc/topics/git/subtree.md | 53 +- doc/topics/git/tags.md | 44 +- doc/topics/git/terminology.md | 6 +- doc/topics/git/troubleshooting_git.md | 122 +- doc/topics/git/unstage.md | 15 +- doc/topics/git/useful_git_commands.md | 6 - doc/topics/gitlab_flow.md | 162 +- doc/topics/offline/quick_start_guide.md | 92 +- doc/topics/release_your_application.md | 1 + doc/topics/set_up_organization.md | 2 +- doc/topics/your_work.md | 21 +- doc/tutorials/agile_sprint.md | 104 +- doc/tutorials/agile_sprint/index.md | 101 + doc/tutorials/boards_for_teams/index.md | 202 + doc/tutorials/build_application.md | 32 + doc/tutorials/compliance_pipeline/index.md | 177 + .../convert_personal_namespace_into_group.md | 11 + .../convert_personal_namespace_to_group/index.md | 95 + doc/tutorials/create_compliance_pipeline.md | 11 + doc/tutorials/develop.md | 17 + doc/tutorials/fuzz_testing/index.md | 243 + doc/tutorials/fuzz_testing_tutorial.md | 11 + doc/tutorials/gitlab_navigation.md | 21 + doc/tutorials/img/branches_dropdown_v14_10.png | Bin 20781 -> 0 bytes doc/tutorials/img/clone_project_v14_9.png | Bin 30075 -> 0 bytes doc/tutorials/img/commit_message_v14_10.png | Bin 4616 -> 0 bytes doc/tutorials/index.md | 122 +- doc/tutorials/infrastructure.md | 15 + doc/tutorials/learn_git.md | 17 + .../left_sidebar/img/admin_area_v16_0.png | Bin 0 -> 7875 bytes doc/tutorials/left_sidebar/img/explore_v16_0.png | Bin 0 -> 13189 bytes doc/tutorials/left_sidebar/img/pin_v16_0.png | Bin 0 -> 2295 bytes doc/tutorials/left_sidebar/img/pinned_v16_0.png | Bin 0 -> 1369 bytes .../left_sidebar/img/project_selected_v16_0.png | Bin 0 -> 23651 bytes .../left_sidebar/img/search_projects_v16_0.png | Bin 0 -> 12721 bytes doc/tutorials/left_sidebar/img/shortcuts_v16_0.png | Bin 0 -> 1180 bytes doc/tutorials/left_sidebar/img/your_work_v16_0.png | Bin 0 -> 20880 bytes doc/tutorials/left_sidebar/index.md | 73 + .../img/branches_dropdown_v14_10.png | Bin 0 -> 20781 bytes .../img/clone_project_v14_9.png | Bin 0 -> 30075 bytes .../img/commit_message_v14_10.png | Bin 0 -> 4616 bytes doc/tutorials/make_first_git_commit/index.md | 271 ++ doc/tutorials/make_your_first_git_commit.md | 274 +- doc/tutorials/more_tutorials.md | 20 + doc/tutorials/move_personal_project_to_a_group.md | 92 +- .../move_personal_project_to_group/index.md | 89 + doc/tutorials/plan_and_track.md | 18 + doc/tutorials/scan_result_policy.md | 11 + doc/tutorials/scan_result_policy/index.md | 125 + doc/tutorials/secure_application.md | 17 + doc/update/background_migrations.md | 47 +- doc/update/deprecations.md | 4627 +++++++++++--------- doc/update/index.md | 596 ++- doc/update/mysql_to_postgresql.md | 11 - doc/update/package/convert_to_ee.md | 8 +- doc/update/package/index.md | 6 +- doc/update/patch_versions.md | 1 - doc/update/plan_your_upgrade.md | 14 +- doc/update/removals.md | 697 ++- doc/update/restore_after_failure.md | 12 - doc/update/upgrading_from_ce_to_ee.md | 1 - doc/update/upgrading_from_source.md | 112 +- doc/update/upgrading_postgresql_using_slony.md | 12 - doc/update/zero_downtime.md | 17 +- doc/user/admin_area/analytics/dev_ops_reports.md | 2 +- doc/user/admin_area/analytics/index.md | 2 +- doc/user/admin_area/appearance.md | 22 +- doc/user/admin_area/custom_project_templates.md | 25 +- doc/user/admin_area/external_users.md | 1 + doc/user/admin_area/index.md | 12 +- doc/user/admin_area/labels.md | 1 + doc/user/admin_area/license.md | 2 + doc/user/admin_area/license_file.md | 25 +- doc/user/admin_area/moderate_users.md | 8 +- .../admin_area/monitoring/background_migrations.md | 11 - doc/user/admin_area/monitoring/health_check.md | 2 - .../admin_area/reporting/git_abuse_rate_limit.md | 23 +- doc/user/admin_area/reporting/spamcheck.md | 12 +- doc/user/admin_area/review_abuse_reports.md | 4 +- .../settings/account_and_limit_settings.md | 42 +- .../admin_area/settings/continuous_integration.md | 48 +- .../settings/deprecated_api_rate_limits.md | 4 +- doc/user/admin_area/settings/email.md | 9 +- .../admin_area/settings/external_authorization.md | 33 +- .../admin_area/settings/files_api_rate_limits.md | 2 +- doc/user/admin_area/settings/help_page.md | 15 +- .../settings/import_export_rate_limits.md | 3 +- doc/user/admin_area/settings/index.md | 9 +- .../settings/project_integration_management.md | 2 +- .../settings/rate_limit_on_projects_api.md | 36 + .../admin_area/settings/security_and_compliance.md | 24 + .../admin_area/settings/sign_in_restrictions.md | 4 +- .../admin_area/settings/sign_up_restrictions.md | 48 +- doc/user/admin_area/settings/terraform_limits.md | 4 +- doc/user/admin_area/settings/third_party_offers.md | 4 +- doc/user/admin_area/settings/usage_statistics.md | 22 +- .../settings/visibility_and_access_controls.md | 18 +- doc/user/admin_area/user_cohorts.md | 16 +- doc/user/ai_features.md | 214 + doc/user/analytics/ci_cd_analytics.md | 12 +- doc/user/analytics/code_review_analytics.md | 4 +- doc/user/analytics/dora_metrics.md | 160 +- doc/user/analytics/index.md | 2 +- doc/user/analytics/value_stream_analytics.md | 247 +- doc/user/analytics/value_streams_dashboard.md | 96 +- doc/user/application_security/api_fuzzing/index.md | 90 +- .../api_security/api_discovery/index.md | 181 + .../application_security/api_security/index.md | 21 + .../breach_and_attack_simulation/index.md | 141 + .../application_security/configuration/index.md | 8 +- .../container_scanning/index.md | 76 +- .../application_security/coverage_fuzzing/index.md | 15 +- .../application_security/dast/authentication.md | 99 +- .../application_security/dast/browser_based.md | 45 +- doc/user/application_security/dast/checks/16.7.md | 4 +- .../dast/dast_troubleshooting.md | 2 +- doc/user/application_security/dast/index.md | 3 + doc/user/application_security/dast/proxy-based.md | 47 +- doc/user/application_security/dast_api/index.md | 110 +- .../application_security/dependency_list/index.md | 10 +- .../dependency_scanning/index.md | 173 +- .../application_security/iac_scanning/index.md | 9 +- doc/user/application_security/index.md | 11 +- .../offline_deployments/index.md | 2 +- .../policies/img/association_diagram.png | Bin 6624 -> 19149 bytes .../policies/img/policy_rule_mode_v14_9.png | Bin 34025 -> 0 bytes .../policies/img/policy_rule_mode_v15_9.png | Bin 0 -> 37866 bytes .../policies/img/policy_yaml_mode_v14_9.png | Bin 27424 -> 0 bytes .../policies/img/policy_yaml_mode_v15_9.png | Bin 0 -> 29904 bytes .../img/scan_execution_policy_rule_mode_v15_11.png | Bin 0 -> 38646 bytes .../img/scan_execution_policy_rule_mode_v15_5.png | Bin 23688 -> 0 bytes .../scheduled_scan_execution_policies_diagram.png | Bin 0 -> 12050 bytes doc/user/application_security/policies/index.md | 53 +- .../policies/scan-execution-policies.md | 24 +- .../policies/scan-result-policies.md | 37 +- doc/user/application_security/sast/analyzers.md | 4 +- .../sast/customize_rulesets.md | 18 +- doc/user/application_security/sast/index.md | 72 +- .../secret_detection/automatic_response.md | 242 + .../application_security/secret_detection/index.md | 153 +- .../secret_detection/post_processing.md | 100 +- .../secure_your_application.md | 2 +- .../img/security_center_dashboard_v13_4.png | Bin 29797 -> 0 bytes .../img/security_center_dashboard_v15_10.png | Bin 0 -> 22361 bytes .../security_dashboard/index.md | 12 +- doc/user/application_security/terminology/index.md | 70 +- .../application_security/vulnerabilities/index.md | 77 +- .../vulnerabilities/severities.md | 4 +- ...ject_security_dashboard_status_change_v14_2.png | Bin 37318 -> 0 bytes ...ject_security_dashboard_status_change_v16_0.png | Bin 0 -> 76118 bytes .../vulnerability_report/index.md | 16 +- doc/user/award_emojis.md | 34 +- doc/user/clusters/agent/ci_cd_workflow.md | 54 +- doc/user/clusters/agent/gitops.md | 8 +- .../agent/gitops/example_repository_structure.md | 78 + doc/user/clusters/agent/gitops/flux.md | 36 + doc/user/clusters/agent/gitops/flux_tutorial.md | 192 + doc/user/clusters/agent/gitops/helm.md | 10 +- .../clusters/agent/gitops/secrets_management.md | 10 +- doc/user/clusters/agent/index.md | 26 +- doc/user/clusters/agent/install/index.md | 25 +- doc/user/clusters/agent/troubleshooting.md | 117 +- doc/user/clusters/agent/work_with_agent.md | 4 +- doc/user/clusters/cost_management.md | 83 +- doc/user/clusters/create/index.md | 4 +- doc/user/clusters/environments.md | 6 +- doc/user/clusters/integrations.md | 102 +- doc/user/clusters/management_project.md | 6 +- doc/user/clusters/management_project_template.md | 4 +- .../migrating_from_gma_to_project_template.md | 6 +- doc/user/compliance/compliance_report/index.md | 289 +- doc/user/compliance/img/license-check_v13_4.png | Bin 25590 -> 0 bytes .../img/license_approval_policy_v15_9.png | Bin 33240 -> 30952 bytes .../img/policies_maintainer_add_v14_3.png | Bin 49418 -> 0 bytes .../img/policies_maintainer_edit_v14_3.png | Bin 26480 -> 0 bytes doc/user/compliance/img/policies_v13_0.png | Bin 22618 -> 0 bytes doc/user/compliance/index.md | 10 +- doc/user/compliance/license_approval_policies.md | 19 +- doc/user/compliance/license_check_rules.md | 89 +- doc/user/compliance/license_compliance/index.md | 152 +- .../license_scanning_of_cyclonedx_files/index.md | 55 +- doc/user/crm/index.md | 4 +- doc/user/discussions/img/index_notes_filters.png | Bin 21284 -> 0 bytes doc/user/discussions/index.md | 56 +- doc/user/enterprise_user/index.md | 82 +- doc/user/gitlab_com/index.md | 181 +- doc/user/group/access_and_permissions.md | 57 +- doc/user/group/clusters/index.md | 6 +- doc/user/group/compliance_frameworks.md | 101 +- .../contribution_analytics/img/group_stats_cal.png | Bin 2029 -> 0 bytes .../img/group_stats_table.png | Bin 22691 -> 0 bytes doc/user/group/custom_project_templates.md | 7 +- doc/user/group/devops_adoption/index.md | 2 +- doc/user/group/epics/epic_boards.md | 22 +- doc/user/group/epics/img/button_close_epic.png | Bin 13850 -> 0 bytes .../epics/img/epic_board_epic_create_v14_1.png | Bin 14584 -> 0 bytes .../epics/img/epic_board_epic_create_v15_10.png | Bin 0 -> 11786 bytes doc/user/group/epics/img/epic_board_v14_1.png | Bin 28691 -> 0 bytes doc/user/group/epics/img/epic_board_v15_10.png | Bin 0 -> 24915 bytes doc/user/group/epics/img/issue_list_v13_1.png | Bin 14825 -> 0 bytes doc/user/group/epics/img/issue_list_v15_11.png | Bin 0 -> 15451 bytes doc/user/group/epics/index.md | 2 +- doc/user/group/epics/manage_epics.md | 75 +- doc/user/group/import/index.md | 323 +- doc/user/group/index.md | 14 +- doc/user/group/insights/index.md | 2 +- doc/user/group/iterations/index.md | 2 + doc/user/group/manage.md | 145 +- doc/user/group/moderate_users.md | 48 + doc/user/group/reporting/git_abuse_rate_limit.md | 39 +- doc/user/group/repositories_analytics/index.md | 4 +- doc/user/group/saml_sso/example_saml_config.md | 2 + doc/user/group/saml_sso/group_sync.md | 33 +- .../saml_sso/img/Azure-manage-group-claims.png | Bin 84609 -> 24492 bytes .../img/group_saml_configuration_information.png | Bin 16489 -> 0 bytes .../saml_sso/img/group_saml_settings_v13_12.png | Bin 66055 -> 0 bytes doc/user/group/saml_sso/index.md | 638 +-- doc/user/group/saml_sso/scim_setup.md | 14 +- doc/user/group/saml_sso/troubleshooting.md | 16 +- doc/user/group/saml_sso/troubleshooting_scim.md | 26 +- doc/user/group/settings/group_access_tokens.md | 43 +- doc/user/group/settings/import_export.md | 11 - doc/user/group/subgroups/index.md | 6 +- .../img/object_hierarchy_example_V14_10.png | Bin 0 -> 20826 bytes doc/user/group/value_stream_analytics/index.md | 441 +- doc/user/img/explain_code_experiment.png | Bin 0 -> 86836 bytes doc/user/img/explain_this_vulnerability.png | Bin 0 -> 371791 bytes .../img/get_domain_verification_code_v16_0.png | Bin 0 -> 47328 bytes doc/user/img/retry_domain_verification_v16_0.png | Bin 0 -> 22593 bytes doc/user/img/todos_add_okrs_v16_0.png | Bin 0 -> 12705 bytes doc/user/img/todos_mark_done_okrs_v16_0.png | Bin 0 -> 13043 bytes doc/user/infrastructure/clusters/connect/index.md | 6 +- .../clusters/connect/new_civo_cluster.md | 28 +- .../clusters/connect/new_eks_cluster.md | 102 +- .../clusters/connect/new_gke_cluster.md | 30 +- .../clusters/deploy/inventory_object.md | 4 +- doc/user/infrastructure/clusters/index.md | 6 +- .../clusters/manage/clusters_health.md | 21 +- .../clusters/manage/img/k8s_cluster_monitoring.png | Bin 43141 -> 0 bytes .../management_project_applications/certmanager.md | 4 +- .../management_project_applications/ingress.md | 4 +- .../management_project_applications/vault.md | 4 +- .../clusters/migrate_to_gitlab_agent.md | 4 +- .../infrastructure/iac/gitlab_terraform_helpers.md | 4 +- doc/user/infrastructure/iac/index.md | 6 +- doc/user/infrastructure/iac/mr_integration.md | 4 +- doc/user/infrastructure/iac/terraform_state.md | 25 +- .../iac/terraform_template_recipes.md | 8 +- doc/user/infrastructure/iac/troubleshooting.md | 13 +- doc/user/infrastructure/index.md | 4 +- doc/user/instance/clusters/index.md | 2 +- doc/user/markdown.md | 179 +- doc/user/namespace/index.md | 4 +- doc/user/okrs.md | 79 +- doc/user/operations_dashboard/index.md | 4 +- doc/user/organization/index.md | 42 + doc/user/packages/conan_repository/index.md | 13 +- .../authenticate_with_container_registry.md | 2 +- doc/user/packages/container_registry/index.md | 2 +- .../reduce_container_registry_storage.md | 45 +- doc/user/packages/debian_repository/index.md | 130 +- doc/user/packages/dependency_proxy/index.md | 22 +- doc/user/packages/generic_packages/index.md | 2 + doc/user/packages/go_proxy/index.md | 2 +- doc/user/packages/gradle_repository/index.md | 375 +- .../packages/harbor_container_registry/index.md | 2 +- doc/user/packages/index.md | 10 +- doc/user/packages/infrastructure_registry/index.md | 105 +- doc/user/packages/maven_repository/index.md | 367 +- doc/user/packages/npm_registry/index.md | 142 +- doc/user/packages/nuget_repository/index.md | 4 +- doc/user/packages/package_registry/index.md | 34 +- .../reduce_package_registry_storage.md | 3 + .../package_registry/supported_functionality.md | 227 +- .../package_registry/supported_hash_types.md | 11 - .../package_registry/supported_package_managers.md | 32 +- doc/user/packages/pypi_repository/index.md | 2 + .../packages/terraform_module_registry/index.md | 184 +- doc/user/packages/yarn_repository/index.md | 349 +- doc/user/permissions.md | 80 +- doc/user/product_analytics/index.md | 167 +- doc/user/profile/account/create_accounts.md | 28 +- doc/user/profile/account/delete_account.md | 3 +- .../profile/account/two_factor_authentication.md | 225 +- doc/user/profile/achievements.md | 236 + doc/user/profile/active_sessions.md | 4 +- doc/user/profile/comment_templates.md | 69 + doc/user/profile/contributions_calendar.md | 6 +- .../img/busy_indicator_note_header_v13_9.png | Bin 24006 -> 0 bytes .../profile/img/busy_indicator_notes_v13_9.png | Bin 41947 -> 0 bytes .../img/busy_indicator_profile_page_v13_6.png | Bin 25119 -> 0 bytes .../img/busy_indicator_settings_menu_v13_6.png | Bin 26843 -> 0 bytes .../img/busy_indicator_sidebar_collapsed_v13_9.png | Bin 6190 -> 0 bytes .../profile/img/busy_indicator_sidebar_v13_9.png | Bin 21914 -> 0 bytes .../img/busy_indicator_user_popovers_v13_6.png | Bin 32158 -> 0 bytes .../img/profile-preferences-syntax-themes.png | Bin 44844 -> 0 bytes .../profile-preferences-syntax-themes_v15_11.png | Bin 0 -> 56643 bytes .../profile/img/saved_replies_dropdown_v15_10.png | Bin 0 -> 23623 bytes .../img/user_profile_achievements_v15_11.png | Bin 0 -> 24613 bytes doc/user/profile/index.md | 139 +- doc/user/profile/notifications.md | 16 +- doc/user/profile/personal_access_tokens.md | 27 +- doc/user/profile/preferences.md | 51 +- doc/user/profile/saved_replies.md | 11 + doc/user/project/badges.md | 14 +- doc/user/project/canary_deployments.md | 8 +- doc/user/project/changelogs.md | 6 +- doc/user/project/clusters/add_eks_clusters.md | 12 +- doc/user/project/clusters/add_existing_cluster.md | 6 +- doc/user/project/clusters/add_gke_clusters.md | 8 +- doc/user/project/clusters/add_remove_clusters.md | 6 +- doc/user/project/clusters/cluster_access.md | 6 +- doc/user/project/clusters/deploy_to_cluster.md | 6 +- .../project/clusters/gitlab_managed_clusters.md | 6 +- doc/user/project/clusters/index.md | 6 +- .../clusters/multiple_kubernetes_clusters.md | 6 +- doc/user/project/clusters/runbooks/index.md | 6 +- doc/user/project/code_owners.md | 376 +- doc/user/project/codeowners/index.md | 381 ++ doc/user/project/codeowners/reference.md | 371 ++ doc/user/project/deploy_boards.md | 6 +- doc/user/project/deploy_keys/index.md | 31 +- doc/user/project/deploy_tokens/index.md | 4 +- doc/user/project/description_templates.md | 32 +- doc/user/project/file_lock.md | 9 +- doc/user/project/git_attributes.md | 37 +- doc/user/project/img/codeowners_in_UI_v15_10.png | Bin 0 -> 10529 bytes .../project/img/issue_board_system_notes_v13_6.png | Bin 8554 -> 0 bytes .../multi_approvals_code_owners_sections_v15_9.png | Bin 0 -> 18972 bytes doc/user/project/img/protected_tags_list_v12_3.png | Bin 4395 -> 0 bytes doc/user/project/img/protected_tags_page_v12_3.png | Bin 10431 -> 0 bytes .../protected_tags_permissions_dropdown_v12_3.png | Bin 4517 -> 0 bytes doc/user/project/import/bitbucket.md | 10 +- doc/user/project/import/bitbucket_server.md | 31 +- doc/user/project/import/clearcase.md | 3 +- doc/user/project/import/cvs.md | 6 +- doc/user/project/import/fogbugz.md | 15 +- doc/user/project/import/gitea.md | 31 +- doc/user/project/import/github.md | 178 +- doc/user/project/import/gitlab_com.md | 32 +- .../img/bitbucket_import_select_project_v12_3.png | Bin 31980 -> 0 bytes .../project/import/img/fogbugz_import_finished.png | Bin 17744 -> 0 bytes .../project/import/img/manifest_status_v13_3.png | Bin 31313 -> 0 bytes doc/user/project/import/index.md | 49 +- doc/user/project/import/jira.md | 18 +- doc/user/project/import/manifest.md | 13 +- doc/user/project/import/perforce.md | 5 +- doc/user/project/import/phabricator.md | 42 +- doc/user/project/import/repo_by_url.md | 17 +- doc/user/project/import/svn.md | 11 - doc/user/project/import/tfvc.md | 3 +- doc/user/project/index.md | 17 +- doc/user/project/integrations/apple_app_store.md | 34 +- doc/user/project/integrations/asana.md | 4 +- doc/user/project/integrations/bamboo.md | 4 +- doc/user/project/integrations/bugzilla.md | 4 +- .../project/integrations/custom_issue_tracker.md | 2 +- .../project/integrations/discord_notifications.md | 2 +- doc/user/project/integrations/emails_on_push.md | 2 +- doc/user/project/integrations/ewm.md | 4 +- doc/user/project/integrations/github.md | 4 +- .../integrations/gitlab_slack_application.md | 16 +- doc/user/project/integrations/google_play.md | 55 + doc/user/project/integrations/hangouts_chat.md | 8 +- doc/user/project/integrations/harbor.md | 8 +- doc/user/project/integrations/index.md | 22 +- doc/user/project/integrations/irker.md | 6 +- doc/user/project/integrations/mattermost.md | 2 +- .../integrations/mattermost_slash_commands.md | 2 +- doc/user/project/integrations/microsoft_teams.md | 4 +- doc/user/project/integrations/mlflow_client.md | 117 +- doc/user/project/integrations/mock_ci.md | 12 +- .../project/integrations/pipeline_status_emails.md | 2 +- doc/user/project/integrations/pivotal_tracker.md | 4 +- doc/user/project/integrations/prometheus.md | 128 +- .../integrations/prometheus_library/cloudwatch.md | 47 +- .../integrations/prometheus_library/haproxy.md | 33 +- .../integrations/prometheus_library/index.md | 42 +- .../integrations/prometheus_library/kubernetes.md | 65 +- .../integrations/prometheus_library/nginx.md | 41 +- .../prometheus_library/nginx_ingress.md | 47 +- .../prometheus_library/nginx_ingress_vts.md | 45 +- doc/user/project/integrations/pumble.md | 2 +- doc/user/project/integrations/redmine.md | 5 +- doc/user/project/integrations/servicenow.md | 4 +- doc/user/project/integrations/shimo.md | 4 +- doc/user/project/integrations/slack.md | 18 +- .../project/integrations/slack_slash_commands.md | 4 +- doc/user/project/integrations/squash_tm.md | 37 + doc/user/project/integrations/unify_circuit.md | 4 +- doc/user/project/integrations/webex_teams.md | 2 +- doc/user/project/integrations/webhook_events.md | 28 +- doc/user/project/integrations/webhooks.md | 42 +- doc/user/project/integrations/youtrack.md | 2 +- doc/user/project/integrations/zentao.md | 4 +- doc/user/project/issue_board.md | 112 +- doc/user/project/issues/associate_zoom_meeting.md | 2 +- doc/user/project/issues/confidential_issues.md | 15 +- doc/user/project/issues/create_issues.md | 9 +- doc/user/project/issues/crosslinking_issues.md | 4 +- doc/user/project/issues/csv_import.md | 6 +- doc/user/project/issues/design_management.md | 13 + doc/user/project/issues/due_dates.md | 3 +- doc/user/project/issues/issue_weight.md | 1 - doc/user/project/issues/managing_issues.md | 24 +- doc/user/project/issues/sorting_issue_lists.md | 2 +- doc/user/project/labels.md | 4 +- doc/user/project/members/index.md | 4 +- .../project/members/share_project_with_groups.md | 34 +- .../project/merge_requests/allow_collaboration.md | 4 +- doc/user/project/merge_requests/approvals/index.md | 24 +- doc/user/project/merge_requests/approvals/rules.md | 48 +- .../project/merge_requests/approvals/settings.md | 17 +- .../project/merge_requests/cherry_pick_changes.md | 12 +- doc/user/project/merge_requests/commits.md | 62 +- doc/user/project/merge_requests/confidential.md | 2 +- doc/user/project/merge_requests/conflicts.md | 5 +- .../merge_requests/creating_merge_requests.md | 28 +- doc/user/project/merge_requests/csv_export.md | 2 +- doc/user/project/merge_requests/dependencies.md | 2 +- doc/user/project/merge_requests/drafts.md | 3 +- doc/user/project/merge_requests/getting_started.md | 11 - .../merge_requests/img/commit_nav_v16_0.png | Bin 0 -> 10423 bytes .../img/previously_merged_commits_v16_0.png | Bin 0 -> 15992 bytes .../img/remove_source_branch_status.png | Bin 32586 -> 0 bytes doc/user/project/merge_requests/index.md | 77 +- .../merge_requests/merge_when_pipeline_succeeds.md | 15 +- doc/user/project/merge_requests/methods/index.md | 1 - doc/user/project/merge_requests/revert_changes.md | 3 +- .../project/merge_requests/reviews/data_usage.md | 2 +- .../img/add_another_suggestion_to_batch_v13_1.jpg | Bin 23078 -> 0 bytes .../img/add_first_suggestion_to_batch_v13_1.jpg | Bin 24694 -> 0 bytes .../img/apply_batch_of_suggestions_v13_1.jpg | Bin 26551 -> 0 bytes .../reviews/img/apply_suggestion_v13_9.png | Bin 37127 -> 0 bytes .../reviews/img/custom_commit_v13_9.png | Bin 41069 -> 0 bytes .../reviews/img/make_suggestion_v13_9.png | Bin 30463 -> 0 bytes .../reviews/img/multi-line-suggestion-syntax.png | Bin 8831 -> 0 bytes .../img/remove_suggestion_from_batch_v13_1.jpg | Bin 24101 -> 0 bytes .../reviews/img/suggestion_button_v13_9.png | Bin 27319 -> 0 bytes .../suggestions_custom_commit_messages_v14_7.png | Bin 14774 -> 0 bytes doc/user/project/merge_requests/reviews/index.md | 22 +- .../project/merge_requests/reviews/suggestions.md | 204 +- .../project/merge_requests/squash_and_merge.md | 13 +- doc/user/project/merge_requests/status_checks.md | 16 +- .../milestones/burndown_and_burnup_charts.md | 20 +- doc/user/project/milestones/index.md | 11 +- .../experiment_tracking/img/candidate_v15_11.png | Bin 0 -> 24678 bytes .../ml/experiment_tracking/img/candidate_v15_7.png | Bin 35164 -> 0 bytes .../experiment_tracking/img/candidates_v15_11.png | Bin 0 -> 57878 bytes .../experiment_tracking/img/candidates_v15_7.png | Bin 47800 -> 0 bytes .../experiment_tracking/img/experiments_v15_11.png | Bin 0 -> 24811 bytes .../experiment_tracking/img/experiments_v15_7.png | Bin 23475 -> 0 bytes doc/user/project/ml/experiment_tracking/index.md | 95 +- doc/user/project/organize_work_with_projects.md | 4 +- .../dns_concepts.md | 11 +- .../custom_domains_ssl_tls_certification/index.md | 85 +- .../lets_encrypt_integration.md | 8 +- .../ssl_tls_concepts.md | 9 +- .../pages/getting_started/pages_ci_cd_template.md | 8 +- .../getting_started/pages_forked_sample_project.md | 12 +- .../pages/getting_started/pages_from_scratch.md | 19 +- .../getting_started/pages_new_project_template.md | 8 +- doc/user/project/pages/getting_started/pages_ui.md | 2 +- doc/user/project/pages/getting_started_part_one.md | 6 +- doc/user/project/pages/index.md | 11 +- doc/user/project/pages/introduction.md | 8 +- doc/user/project/pages/pages_access_control.md | 4 +- doc/user/project/pages/public_folder.md | 41 +- doc/user/project/pages/redirects.md | 6 +- doc/user/project/protected_branches.md | 104 +- doc/user/project/protected_tags.md | 63 +- doc/user/project/quick_actions.md | 203 +- doc/user/project/releases/index.md | 167 +- doc/user/project/releases/release_cicd_examples.md | 4 +- doc/user/project/releases/release_cli.md | 4 +- doc/user/project/releases/release_evidence.md | 140 + doc/user/project/releases/release_fields.md | 16 +- .../project/remote_development/connect_machine.md | 109 + doc/user/project/remote_development/index.md | 141 +- doc/user/project/repository/branches/default.md | 4 +- .../img/branch_filter_search_box_v13_12.png | Bin 15803 -> 0 bytes .../branches/img/compare_branches_v13_12.png | Bin 46536 -> 0 bytes .../img/repository_filter_search_box_v13_12.png | Bin 12634 -> 0 bytes .../branches/img/swap_revisions_after_v13_12.png | Bin 8949 -> 0 bytes .../branches/img/swap_revisions_before_v13_12.png | Bin 8935 -> 0 bytes .../img/view_branch_protections_v15_10.png | Bin 0 -> 28285 bytes doc/user/project/repository/branches/index.md | 300 +- doc/user/project/repository/code_suggestions.md | 182 + doc/user/project/repository/file_finder.md | 40 +- doc/user/project/repository/forking_workflow.md | 110 +- doc/user/project/repository/git_blame.md | 2 +- doc/user/project/repository/git_history.md | 2 +- .../project/repository/gpg_signed_commits/index.md | 2 +- .../project/repository/img/update-fork_v15_11.png | Bin 0 -> 8581 bytes doc/user/project/repository/index.md | 24 +- .../project/repository/jupyter_notebooks/index.md | 2 +- .../project/repository/mirror/bidirectional.md | 3 +- doc/user/project/repository/mirror/index.md | 33 +- doc/user/project/repository/mirror/pull.md | 6 +- doc/user/project/repository/mirror/push.md | 9 +- doc/user/project/repository/push_rules.md | 2 +- .../repository/reducing_the_repo_size_using_git.md | 25 +- .../repository/tags/img/tag-display_v15_9.png | Bin 0 -> 7320 bytes .../tags/img/tags_commits_view_v15_10.png | Bin 0 -> 7054 bytes doc/user/project/repository/tags/index.md | 120 + doc/user/project/repository/vscode.md | 7 +- doc/user/project/repository/web_editor.md | 32 +- .../repository/x509_signed_commits/index.md | 6 +- doc/user/project/requirements/index.md | 19 +- doc/user/project/service_desk.md | 293 +- doc/user/project/settings/import_export.md | 175 +- .../settings/import_export_troubleshooting.md | 93 +- doc/user/project/settings/index.md | 81 +- doc/user/project/settings/project_access_tokens.md | 43 +- doc/user/project/system_notes.md | 56 + doc/user/project/time_tracking.md | 1 - .../web_ide/img/admin_live_preview_v13_0.png | Bin 5508 -> 0 bytes .../project/web_ide/img/commit_changes_v13_11.png | Bin 116443 -> 0 bytes doc/user/project/web_ide/img/dark_theme_v13_0.png | Bin 99238 -> 0 bytes .../project/web_ide/img/fuzzy_finder_v15_7.png | Bin 0 -> 121069 bytes .../project/web_ide/img/live_preview_v13_0.png | Bin 29188 -> 0 bytes .../web_ide/img/solarized_dark_theme_v13_1.png | Bin 43014 -> 0 bytes doc/user/project/web_ide/img/terminal_status.png | Bin 3709 -> 0 bytes doc/user/project/web_ide/index.md | 517 +-- .../web_ide_beta/img/fuzzy_finder_v15_7.png | Bin 121069 -> 0 bytes doc/user/project/web_ide_beta/index.md | 109 +- doc/user/project/wiki/group.md | 16 +- doc/user/project/wiki/index.md | 4 +- doc/user/project/working_with_projects.md | 35 +- doc/user/public_access.md | 7 +- doc/user/report_abuse.md | 9 +- doc/user/reserved_names.md | 4 +- doc/user/search/advanced_search.md | 64 +- doc/user/search/exact_code_search.md | 14 +- .../search/global_search/advanced_search_syntax.md | 11 - doc/user/search/index.md | 23 +- doc/user/shortcuts.md | 11 +- doc/user/snippets.md | 4 +- doc/user/ssh.md | 18 +- doc/user/tasks.md | 29 +- doc/user/todos.md | 24 +- doc/user/usage_quotas.md | 42 +- doc/user/workspace/index.md | 172 +- 1486 files changed, 50645 insertions(+), 30290 deletions(-) delete mode 100644 doc/administration/auth/authentiq.md create mode 100644 doc/administration/auth/test_oidc_oauth.md delete mode 100644 doc/administration/geo/replication/img/adding_a_secondary_v13_3.png create mode 100644 doc/administration/geo/replication/img/adding_a_secondary_v15_8.png create mode 100644 doc/administration/gitaly/gitaly_geo_capabilities.md delete mode 100644 doc/administration/monitoring/gitlab_self_monitoring_project/img/self_monitoring_overview_dashboard.png create mode 100644 doc/administration/pages/troubleshooting.md delete mode 100644 doc/administration/sidekiq/extra_sidekiq_routing.md create mode 100644 doc/administration/silent_mode/index.md delete mode 100644 doc/api/feature_flag_specs.md create mode 100644 doc/api/geo_sites.md create mode 100644 doc/api/group_protected_branches.md create mode 100644 doc/api/rest/deprecations.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-admin-area.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-agent-for-kubernetes.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-backups.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-ci-runners.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-container-registry.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-contributions-forks.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-dashboard.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-data-migration.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-database-sequences.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-git-access.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-gitlab-pages.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-global-search.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-graphql.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-organizations.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-personal-namespaces.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-router-endpoints-classification.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-schema-changes.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-secrets.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-snippets.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-template.md create mode 100644 doc/architecture/blueprints/cells/cells-feature-uploads.md create mode 100644 doc/architecture/blueprints/cells/glossary.md create mode 100644 doc/architecture/blueprints/cells/goals.md create mode 100644 doc/architecture/blueprints/cells/images/pods-and-fulfillment.png create mode 100644 doc/architecture/blueprints/cells/images/term-cell.png create mode 100644 doc/architecture/blueprints/cells/images/term-cluster.png create mode 100644 doc/architecture/blueprints/cells/images/term-organization.png create mode 100644 doc/architecture/blueprints/cells/images/term-top-level-group.png create mode 100644 doc/architecture/blueprints/cells/impact.md create mode 100644 doc/architecture/blueprints/cells/index.md create mode 100644 doc/architecture/blueprints/cells/proposal-stateless-router-with-buffering-requests.md create mode 100644 doc/architecture/blueprints/cells/proposal-stateless-router-with-routes-learning.md create mode 100644 doc/architecture/blueprints/ci_pipeline_components/dev_workflow.md create mode 100644 doc/architecture/blueprints/ci_pipeline_components/img/new_release.png create mode 100644 doc/architecture/blueprints/ci_pipeline_components/img/pipeline_main.png create mode 100644 doc/architecture/blueprints/ci_pipeline_components/img/pipeline_tag.png create mode 100644 doc/architecture/blueprints/clickhouse_ingestion_pipeline/clickhouse_dbwriter.png create mode 100644 doc/architecture/blueprints/clickhouse_ingestion_pipeline/index.md create mode 100644 doc/architecture/blueprints/clickhouse_read_abstraction_layer/index.md create mode 100644 doc/architecture/blueprints/clickhouse_usage/index.md create mode 100644 doc/architecture/blueprints/clickhouse_usage/self_managed_costs_and_requirements/index.md create mode 100644 doc/architecture/blueprints/code_search_with_zoekt/index.md create mode 100644 doc/architecture/blueprints/gitlab_ci_events/index.md create mode 100644 doc/architecture/blueprints/gitlab_ci_events/proposal-1-using-the-gitlab-ci-file.md create mode 100644 doc/architecture/blueprints/gitlab_ci_events/proposal-2-using-the-rules-keyword.md create mode 100644 doc/architecture/blueprints/gitlab_ci_events/proposal-3-using-the-gitlab-ci-events-folder.md create mode 100644 doc/architecture/blueprints/gitlab_ci_events/proposal-4-creating-events-via-ci-files.md create mode 100644 doc/architecture/blueprints/gitlab_ml_experiments/index.md create mode 100644 doc/architecture/blueprints/object_pools/index.md create mode 100644 doc/architecture/blueprints/organization/index.md delete mode 100644 doc/architecture/blueprints/pods/images/iteration0-organizations-introduction.png delete mode 100644 doc/architecture/blueprints/pods/images/pods-and-fulfillment.png delete mode 100644 doc/architecture/blueprints/pods/images/term-cluster.png delete mode 100644 doc/architecture/blueprints/pods/images/term-organization.png delete mode 100644 doc/architecture/blueprints/pods/images/term-pod.png delete mode 100644 doc/architecture/blueprints/pods/images/term-top-level-namespace.png delete mode 100644 doc/architecture/blueprints/search/code_search_with_zoekt.md delete mode 100644 doc/ci/chatops/img/gitlab-chatops-icon-small.png create mode 100644 doc/ci/components/index.md create mode 100644 doc/ci/docker/authenticate_registry.md create mode 100644 doc/ci/docker/docker_layer_caching.md delete mode 100644 doc/ci/environments/img/environment_auto_stop_v13_10.png create mode 100644 doc/ci/jobs/job_artifacts.md create mode 100644 doc/ci/jobs/job_artifacts_troubleshooting.md delete mode 100644 doc/ci/pipelines/img/code_coverage_graph_v13_1.png delete mode 100644 doc/ci/pipelines/img/coverage_check_approval_rule_14_1.png delete mode 100644 doc/ci/pipelines/img/job_artifacts_browser_button_v13_11.png delete mode 100644 doc/ci/pipelines/img/job_artifacts_browser_v13_11.png delete mode 100644 doc/ci/pipelines/img/job_artifacts_jobs_page_v13_11.png delete mode 100644 doc/ci/pipelines/img/job_artifacts_merge_request_v13_11.png delete mode 100644 doc/ci/pipelines/img/job_artifacts_pipelines_page_v13_11.png delete mode 100644 doc/ci/pipelines/img/job_latest_artifacts_browser.png delete mode 100644 doc/ci/pipelines/img/multi_project_pipeline_graph_v14_3.png delete mode 100644 doc/ci/pipelines/img/pipeline_mini_graph_v15_0.png delete mode 100644 doc/ci/pipelines/img/pipelines_test_coverage_build.png delete mode 100644 doc/ci/pipelines/img/pipelines_test_coverage_mr_widget.png create mode 100644 doc/ci/quick_start/tutorial.md create mode 100644 doc/ci/runners/register_runner.md create mode 100644 doc/ci/testing/code_coverage.md create mode 100644 doc/ci/testing/img/code_coverage_graph_v13_1.png create mode 100644 doc/ci/testing/img/code_coverage_group_report.png create mode 100644 doc/ci/testing/img/code_quality_summary_15_9.png create mode 100644 doc/ci/testing/img/coverage_check_approval_rule_14_1.png create mode 100644 doc/ci/testing/img/pipelines_test_coverage_build.png create mode 100644 doc/ci/testing/img/pipelines_test_coverage_mr_widget.png create mode 100644 doc/development/advanced_search.md create mode 100644 doc/development/ai_architecture.md create mode 100644 doc/development/ai_features.md create mode 100644 doc/development/application_slis/rails_request.md delete mode 100644 doc/development/application_slis/rails_request_apdex.md delete mode 100644 doc/development/approval_rules.md create mode 100644 doc/development/cicd/cicd_tables.md create mode 100644 doc/development/code_owners/index.md delete mode 100644 doc/development/contributing/community_roles.md create mode 100644 doc/development/contributing/first_contribution.md create mode 100644 doc/development/contributing/img/bot_ready.png create mode 100644 doc/development/contributing/img/changes_tab.png create mode 100644 doc/development/contributing/img/gdk_home.png create mode 100644 doc/development/contributing/img/mr_button.png create mode 100644 doc/development/contributing/img/new_merge_request.png create mode 100644 doc/development/contributing/img/ui_text_after.png create mode 100644 doc/development/contributing/img/ui_text_before.png create mode 100644 doc/development/database/clickhouse/merge_request_analytics.md create mode 100644 doc/development/database/clickhouse/tiered_storage.md create mode 100644 doc/development/database/load_balancing.md delete mode 100644 doc/development/diffs.md delete mode 100644 doc/development/directory_structure.md create mode 100644 doc/development/distribution/index.md create mode 100644 doc/development/documentation/alpha_beta.md create mode 100644 doc/development/documentation/contribute.md create mode 100644 doc/development/documentation/topic_types/glossary.md delete mode 100644 doc/development/elasticsearch.md create mode 100644 doc/development/graphql_guide/reviewing.md delete mode 100644 doc/development/integrations/codesandbox.md create mode 100644 doc/development/labels/index.md delete mode 100644 doc/development/merge_request_application_and_rate_limit_guidelines.md delete mode 100644 doc/development/merge_request_diffs.md delete mode 100644 doc/development/merge_request_performance_guidelines.md create mode 100644 doc/development/navigation_sidebar.md create mode 100644 doc/development/organization/index.md create mode 100644 doc/development/search/advanced_search_migration_styleguide.md create mode 100644 doc/development/sec/CycloneDX_property_taxonomy.md delete mode 100644 doc/development/stage_group_observability/dashboards/img/error_budget_detail_sli_detail.png create mode 100644 doc/development/stage_group_observability/img/error_budgets_kibana_dashboard_v15_10.png create mode 100644 doc/development/ux/index.md create mode 100644 doc/development/vs_code_debugging.md create mode 100644 doc/install/cloud_providers.md delete mode 100644 doc/integration/jira/img/jira-upload-app-success_v13_11.png delete mode 100644 doc/integration/jira/img/jira-upload-app_v13_11.png delete mode 100644 doc/integration/jira/img/jira_added_user_to_group.png delete mode 100644 doc/integration/jira/img/jira_create_new_group.png delete mode 100644 doc/integration/jira/img/jira_dev_panel_jira_setup_3.png delete mode 100644 doc/integration/jira/img/jira_dev_panel_jira_setup_4.png delete mode 100644 doc/integration/jira/img/jira_dev_panel_jira_setup_5.png delete mode 100644 doc/integration/jira/img/jira_dev_panel_setup_com_1.png delete mode 100644 doc/integration/jira/img/jira_dev_panel_setup_com_2.png delete mode 100644 doc/integration/jira/img/jira_dev_panel_setup_com_3_v13_9.png delete mode 100644 doc/integration/jira/img/jira_dev_panel_setup_com_4_v13_9.png delete mode 100644 doc/integration/jira/img/jira_group_access.png delete mode 100644 doc/integration/jira/img/jira_issue_detail_view_v13.10.png delete mode 100644 doc/integration/jira/img/jira_issue_reference.png delete mode 100644 doc/integration/jira/img/jira_project_settings.png delete mode 100644 doc/integration/jira/img/jira_service_close_issue.png delete mode 100644 doc/integration/jira/img/open_jira_issues_list_v14_6.png delete mode 100644 doc/operations/img/copy-group-id.png delete mode 100644 doc/operations/img/create-gitlab-application.png delete mode 100644 doc/operations/img/error_tracking_setting_dsn_v14_4.png delete mode 100644 doc/operations/img/error_tracking_setting_v14_3.png delete mode 100644 doc/operations/img/listing_groups.png delete mode 100644 doc/operations/metrics/dashboards/img/actions_menu_create_add_panel_v13_3.png delete mode 100644 doc/operations/metrics/dashboards/img/actions_menu_create_new_dashboard_v13_3.png delete mode 100644 doc/operations/metrics/dashboards/img/dashboard_external_link_v13_1.png delete mode 100644 doc/operations/metrics/dashboards/img/dashboard_local_timezone_v13_1.png delete mode 100644 doc/operations/metrics/dashboards/img/external_dashboard_link.png delete mode 100644 doc/operations/metrics/dashboards/img/heatmap_chart_too_much_data_v_13_2.png delete mode 100644 doc/operations/metrics/dashboards/img/heatmap_panel_type.png delete mode 100644 doc/operations/metrics/dashboards/img/k8s_pod_health_dashboard_v13_3.png delete mode 100644 doc/operations/metrics/dashboards/img/metrics_dashboard_annotations_ui_v13.0.png delete mode 100644 doc/operations/metrics/dashboards/img/metrics_dashboard_panel_preview_v13_3.png delete mode 100644 doc/operations/metrics/dashboards/img/metrics_dashboard_template_selection_v13_3.png delete mode 100644 doc/operations/metrics/dashboards/img/metrics_dashboard_template_selection_web_ide_v13_3.png delete mode 100644 doc/operations/metrics/dashboards/img/panel_context_menu_v14_0.png delete mode 100644 doc/operations/metrics/dashboards/img/prometheus_dashboard_anomaly_panel_type.png delete mode 100644 doc/operations/metrics/dashboards/img/prometheus_dashboard_area_panel_type_v12_8.png delete mode 100644 doc/operations/metrics/dashboards/img/prometheus_dashboard_bar_chart_panel_type_v12.10.png delete mode 100644 doc/operations/metrics/dashboards/img/prometheus_dashboard_column_panel_type.png delete mode 100644 doc/operations/metrics/dashboards/img/prometheus_dashboard_gauge_panel_type_v13_3.png delete mode 100644 doc/operations/metrics/dashboards/img/prometheus_dashboard_label_variable_shorthand.png delete mode 100644 doc/operations/metrics/dashboards/img/prometheus_dashboard_label_variables.png delete mode 100644 doc/operations/metrics/dashboards/img/prometheus_dashboard_repeated_label.png delete mode 100644 doc/operations/metrics/dashboards/img/prometheus_dashboard_single_stat_panel_type.png delete mode 100644 doc/operations/metrics/dashboards/img/prometheus_dashboard_stacked_column_panel_type_v12_8.png delete mode 100644 doc/operations/metrics/dashboards/img/prometheus_dashboard_yaml_validation_v13_1.png delete mode 100644 doc/operations/metrics/dashboards/img/related_links_v13_1.png delete mode 100644 doc/operations/metrics/img/copy_link_to_chart_v12_10.png delete mode 100644 doc/operations/metrics/img/embed_metrics_issue_template.png delete mode 100644 doc/operations/metrics/img/embedded_metrics_rendered_v13_4.png delete mode 100644 doc/operations/metrics/img/example-dashboard_v13_3.png delete mode 100644 doc/operations/metrics/img/hide_embedded_metrics_v12_10.png delete mode 100644 doc/operations/metrics/img/prometheus_add_metric.png delete mode 100644 doc/operations/metrics/img/prometheus_cluster_health_embed_v12_9.png delete mode 100644 doc/operations/metrics/img/prometheus_dashboard_edit_metric_link_v_12_9.png delete mode 100644 doc/operations/metrics/img/prometheus_monitoring_dashboard_v13_3.png delete mode 100644 doc/operations/metrics/img/rendered_grafana_embed_v12_5.png delete mode 100644 doc/operations/metrics/img/view_embedded_metrics_v12_10.png delete mode 100644 doc/operations/quickstart-guide.md create mode 100644 doc/security/email_verification.md create mode 100644 doc/subscriptions/choosing_subscription.md create mode 100644 doc/subscriptions/community_programs.md create mode 100644 doc/subscriptions/customers_portal.md create mode 100644 doc/topics/autodevops/cloud_deployments/auto_devops_with_eks.md delete mode 100644 doc/topics/awesome_co.md create mode 100644 doc/topics/data_seeder.md delete mode 100644 doc/topics/git/feature_branch_development.md delete mode 100644 doc/topics/git/getting_started.md delete mode 100644 doc/topics/git/merge_conflicts.md create mode 100644 doc/tutorials/agile_sprint/index.md create mode 100644 doc/tutorials/boards_for_teams/index.md create mode 100644 doc/tutorials/build_application.md create mode 100644 doc/tutorials/compliance_pipeline/index.md create mode 100644 doc/tutorials/convert_personal_namespace_into_group.md create mode 100644 doc/tutorials/convert_personal_namespace_to_group/index.md create mode 100644 doc/tutorials/create_compliance_pipeline.md create mode 100644 doc/tutorials/develop.md create mode 100644 doc/tutorials/fuzz_testing/index.md create mode 100644 doc/tutorials/fuzz_testing_tutorial.md create mode 100644 doc/tutorials/gitlab_navigation.md delete mode 100644 doc/tutorials/img/branches_dropdown_v14_10.png delete mode 100644 doc/tutorials/img/clone_project_v14_9.png delete mode 100644 doc/tutorials/img/commit_message_v14_10.png create mode 100644 doc/tutorials/infrastructure.md create mode 100644 doc/tutorials/learn_git.md create mode 100644 doc/tutorials/left_sidebar/img/admin_area_v16_0.png create mode 100644 doc/tutorials/left_sidebar/img/explore_v16_0.png create mode 100644 doc/tutorials/left_sidebar/img/pin_v16_0.png create mode 100644 doc/tutorials/left_sidebar/img/pinned_v16_0.png create mode 100644 doc/tutorials/left_sidebar/img/project_selected_v16_0.png create mode 100644 doc/tutorials/left_sidebar/img/search_projects_v16_0.png create mode 100644 doc/tutorials/left_sidebar/img/shortcuts_v16_0.png create mode 100644 doc/tutorials/left_sidebar/img/your_work_v16_0.png create mode 100644 doc/tutorials/left_sidebar/index.md create mode 100644 doc/tutorials/make_first_git_commit/img/branches_dropdown_v14_10.png create mode 100644 doc/tutorials/make_first_git_commit/img/clone_project_v14_9.png create mode 100644 doc/tutorials/make_first_git_commit/img/commit_message_v14_10.png create mode 100644 doc/tutorials/make_first_git_commit/index.md create mode 100644 doc/tutorials/more_tutorials.md create mode 100644 doc/tutorials/move_personal_project_to_group/index.md create mode 100644 doc/tutorials/plan_and_track.md create mode 100644 doc/tutorials/scan_result_policy.md create mode 100644 doc/tutorials/scan_result_policy/index.md create mode 100644 doc/tutorials/secure_application.md delete mode 100644 doc/update/mysql_to_postgresql.md delete mode 100644 doc/update/restore_after_failure.md delete mode 100644 doc/update/upgrading_postgresql_using_slony.md delete mode 100644 doc/user/admin_area/monitoring/background_migrations.md create mode 100644 doc/user/admin_area/settings/rate_limit_on_projects_api.md create mode 100644 doc/user/admin_area/settings/security_and_compliance.md create mode 100644 doc/user/ai_features.md create mode 100644 doc/user/application_security/api_security/api_discovery/index.md create mode 100644 doc/user/application_security/api_security/index.md create mode 100644 doc/user/application_security/breach_and_attack_simulation/index.md delete mode 100644 doc/user/application_security/policies/img/policy_rule_mode_v14_9.png create mode 100644 doc/user/application_security/policies/img/policy_rule_mode_v15_9.png delete mode 100644 doc/user/application_security/policies/img/policy_yaml_mode_v14_9.png create mode 100644 doc/user/application_security/policies/img/policy_yaml_mode_v15_9.png create mode 100644 doc/user/application_security/policies/img/scan_execution_policy_rule_mode_v15_11.png delete mode 100644 doc/user/application_security/policies/img/scan_execution_policy_rule_mode_v15_5.png create mode 100644 doc/user/application_security/policies/img/scheduled_scan_execution_policies_diagram.png create mode 100644 doc/user/application_security/secret_detection/automatic_response.md delete mode 100644 doc/user/application_security/security_dashboard/img/security_center_dashboard_v13_4.png create mode 100644 doc/user/application_security/security_dashboard/img/security_center_dashboard_v15_10.png delete mode 100644 doc/user/application_security/vulnerability_report/img/project_security_dashboard_status_change_v14_2.png create mode 100644 doc/user/application_security/vulnerability_report/img/project_security_dashboard_status_change_v16_0.png create mode 100644 doc/user/clusters/agent/gitops/example_repository_structure.md create mode 100644 doc/user/clusters/agent/gitops/flux.md create mode 100644 doc/user/clusters/agent/gitops/flux_tutorial.md delete mode 100644 doc/user/compliance/img/license-check_v13_4.png delete mode 100644 doc/user/compliance/img/policies_maintainer_add_v14_3.png delete mode 100644 doc/user/compliance/img/policies_maintainer_edit_v14_3.png delete mode 100644 doc/user/compliance/img/policies_v13_0.png delete mode 100644 doc/user/discussions/img/index_notes_filters.png delete mode 100644 doc/user/group/contribution_analytics/img/group_stats_cal.png delete mode 100644 doc/user/group/contribution_analytics/img/group_stats_table.png delete mode 100644 doc/user/group/epics/img/button_close_epic.png delete mode 100644 doc/user/group/epics/img/epic_board_epic_create_v14_1.png create mode 100644 doc/user/group/epics/img/epic_board_epic_create_v15_10.png delete mode 100644 doc/user/group/epics/img/epic_board_v14_1.png create mode 100644 doc/user/group/epics/img/epic_board_v15_10.png delete mode 100644 doc/user/group/epics/img/issue_list_v13_1.png create mode 100644 doc/user/group/epics/img/issue_list_v15_11.png create mode 100644 doc/user/group/moderate_users.md delete mode 100644 doc/user/group/saml_sso/img/group_saml_configuration_information.png delete mode 100644 doc/user/group/saml_sso/img/group_saml_settings_v13_12.png delete mode 100644 doc/user/group/settings/import_export.md create mode 100644 doc/user/group/value_stream_analytics/img/object_hierarchy_example_V14_10.png create mode 100644 doc/user/img/explain_code_experiment.png create mode 100644 doc/user/img/explain_this_vulnerability.png create mode 100644 doc/user/img/get_domain_verification_code_v16_0.png create mode 100644 doc/user/img/retry_domain_verification_v16_0.png create mode 100644 doc/user/img/todos_add_okrs_v16_0.png create mode 100644 doc/user/img/todos_mark_done_okrs_v16_0.png delete mode 100644 doc/user/infrastructure/clusters/manage/img/k8s_cluster_monitoring.png create mode 100644 doc/user/organization/index.md delete mode 100644 doc/user/packages/package_registry/supported_hash_types.md create mode 100644 doc/user/profile/achievements.md create mode 100644 doc/user/profile/comment_templates.md delete mode 100644 doc/user/profile/img/busy_indicator_note_header_v13_9.png delete mode 100644 doc/user/profile/img/busy_indicator_notes_v13_9.png delete mode 100644 doc/user/profile/img/busy_indicator_profile_page_v13_6.png delete mode 100644 doc/user/profile/img/busy_indicator_settings_menu_v13_6.png delete mode 100644 doc/user/profile/img/busy_indicator_sidebar_collapsed_v13_9.png delete mode 100644 doc/user/profile/img/busy_indicator_sidebar_v13_9.png delete mode 100644 doc/user/profile/img/busy_indicator_user_popovers_v13_6.png delete mode 100644 doc/user/profile/img/profile-preferences-syntax-themes.png create mode 100644 doc/user/profile/img/profile-preferences-syntax-themes_v15_11.png create mode 100644 doc/user/profile/img/saved_replies_dropdown_v15_10.png create mode 100644 doc/user/profile/img/user_profile_achievements_v15_11.png create mode 100644 doc/user/profile/saved_replies.md create mode 100644 doc/user/project/codeowners/index.md create mode 100644 doc/user/project/codeowners/reference.md create mode 100644 doc/user/project/img/codeowners_in_UI_v15_10.png delete mode 100644 doc/user/project/img/issue_board_system_notes_v13_6.png create mode 100644 doc/user/project/img/multi_approvals_code_owners_sections_v15_9.png delete mode 100644 doc/user/project/img/protected_tags_list_v12_3.png delete mode 100644 doc/user/project/img/protected_tags_page_v12_3.png delete mode 100644 doc/user/project/img/protected_tags_permissions_dropdown_v12_3.png delete mode 100644 doc/user/project/import/img/bitbucket_import_select_project_v12_3.png delete mode 100644 doc/user/project/import/img/fogbugz_import_finished.png delete mode 100644 doc/user/project/import/img/manifest_status_v13_3.png delete mode 100644 doc/user/project/import/svn.md create mode 100644 doc/user/project/integrations/google_play.md create mode 100644 doc/user/project/integrations/squash_tm.md delete mode 100644 doc/user/project/merge_requests/getting_started.md create mode 100644 doc/user/project/merge_requests/img/commit_nav_v16_0.png create mode 100644 doc/user/project/merge_requests/img/previously_merged_commits_v16_0.png delete mode 100644 doc/user/project/merge_requests/img/remove_source_branch_status.png delete mode 100644 doc/user/project/merge_requests/reviews/img/add_another_suggestion_to_batch_v13_1.jpg delete mode 100644 doc/user/project/merge_requests/reviews/img/add_first_suggestion_to_batch_v13_1.jpg delete mode 100644 doc/user/project/merge_requests/reviews/img/apply_batch_of_suggestions_v13_1.jpg delete mode 100644 doc/user/project/merge_requests/reviews/img/apply_suggestion_v13_9.png delete mode 100644 doc/user/project/merge_requests/reviews/img/custom_commit_v13_9.png delete mode 100644 doc/user/project/merge_requests/reviews/img/make_suggestion_v13_9.png delete mode 100644 doc/user/project/merge_requests/reviews/img/multi-line-suggestion-syntax.png delete mode 100644 doc/user/project/merge_requests/reviews/img/remove_suggestion_from_batch_v13_1.jpg delete mode 100644 doc/user/project/merge_requests/reviews/img/suggestion_button_v13_9.png delete mode 100644 doc/user/project/merge_requests/reviews/img/suggestions_custom_commit_messages_v14_7.png create mode 100644 doc/user/project/ml/experiment_tracking/img/candidate_v15_11.png delete mode 100644 doc/user/project/ml/experiment_tracking/img/candidate_v15_7.png create mode 100644 doc/user/project/ml/experiment_tracking/img/candidates_v15_11.png delete mode 100644 doc/user/project/ml/experiment_tracking/img/candidates_v15_7.png create mode 100644 doc/user/project/ml/experiment_tracking/img/experiments_v15_11.png delete mode 100644 doc/user/project/ml/experiment_tracking/img/experiments_v15_7.png create mode 100644 doc/user/project/releases/release_evidence.md create mode 100644 doc/user/project/remote_development/connect_machine.md delete mode 100644 doc/user/project/repository/branches/img/branch_filter_search_box_v13_12.png delete mode 100644 doc/user/project/repository/branches/img/compare_branches_v13_12.png delete mode 100644 doc/user/project/repository/branches/img/repository_filter_search_box_v13_12.png delete mode 100644 doc/user/project/repository/branches/img/swap_revisions_after_v13_12.png delete mode 100644 doc/user/project/repository/branches/img/swap_revisions_before_v13_12.png create mode 100644 doc/user/project/repository/branches/img/view_branch_protections_v15_10.png create mode 100644 doc/user/project/repository/code_suggestions.md create mode 100644 doc/user/project/repository/img/update-fork_v15_11.png create mode 100644 doc/user/project/repository/tags/img/tag-display_v15_9.png create mode 100644 doc/user/project/repository/tags/img/tags_commits_view_v15_10.png create mode 100644 doc/user/project/repository/tags/index.md create mode 100644 doc/user/project/system_notes.md delete mode 100644 doc/user/project/web_ide/img/admin_live_preview_v13_0.png delete mode 100644 doc/user/project/web_ide/img/commit_changes_v13_11.png delete mode 100644 doc/user/project/web_ide/img/dark_theme_v13_0.png create mode 100644 doc/user/project/web_ide/img/fuzzy_finder_v15_7.png delete mode 100644 doc/user/project/web_ide/img/live_preview_v13_0.png delete mode 100644 doc/user/project/web_ide/img/solarized_dark_theme_v13_1.png delete mode 100644 doc/user/project/web_ide/img/terminal_status.png delete mode 100644 doc/user/project/web_ide_beta/img/fuzzy_finder_v15_7.png delete mode 100644 doc/user/search/global_search/advanced_search_syntax.md (limited to 'doc') diff --git a/doc/.vale/gitlab/CIConfigFile.yml b/doc/.vale/gitlab/CIConfigFile.yml index 4d2ba454410..5cbd02e799b 100644 --- a/doc/.vale/gitlab/CIConfigFile.yml +++ b/doc/.vale/gitlab/CIConfigFile.yml @@ -10,8 +10,4 @@ link: https://docs.gitlab.com/ee/development/documentation/versions.html level: error scope: raw raw: - - '(`gitlab-ci.yml`|' - - '`gitlabci.yml`|' - - '`gitlab.ci.yml`|' - - '`.gitlab.ci-yml`|' - - '`.gitlab-ci.yaml`)' + - '(?!`\.gitlab-ci\.yml`)`.?gitlab.?ci.?ya?ml`' diff --git a/doc/.vale/gitlab/CodeblockFences.yml b/doc/.vale/gitlab/CodeblockFences.yml index 9d5efe7f60a..27159f7e72e 100644 --- a/doc/.vale/gitlab/CodeblockFences.yml +++ b/doc/.vale/gitlab/CodeblockFences.yml @@ -5,9 +5,9 @@ # # For a list of all options, see https://vale.sh/docs/topics/styles/ extends: existence -message: "Instead of '%s' for the code block, use yaml, ruby, plaintext, markdown, javascript, shell, golang, python, dockerfile, or typescript." +message: "Instead of '%s' for the code block, use yaml, ruby, plaintext, markdown, javascript, shell, go, python, dockerfile, or typescript." link: https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#code-blocks level: error scope: raw raw: - - '\`\`\`(yml|rb|text|md|bash|sh\n|js\n|go\n|py\n|docker\n|ts)' + - '\`\`\`(yml|rb|text|md|bash|sh\n|js\n|golang\n|py\n|docker\n|ts)' diff --git a/doc/.vale/gitlab/ElementDescriptors.yml b/doc/.vale/gitlab/ElementDescriptors.yml index f806f5878e1..fd3acace744 100644 --- a/doc/.vale/gitlab/ElementDescriptors.yml +++ b/doc/.vale/gitlab/ElementDescriptors.yml @@ -11,4 +11,4 @@ level: warning ignorecase: true scope: raw raw: - - \*\*.+?\*\* button + - \*\*[^*]+\*\*\s+button diff --git a/doc/.vale/gitlab/HeadingDepth.yml b/doc/.vale/gitlab/HeadingDepth.yml index 5bbe667481c..000baf633d7 100644 --- a/doc/.vale/gitlab/HeadingDepth.yml +++ b/doc/.vale/gitlab/HeadingDepth.yml @@ -6,8 +6,8 @@ # For a list of all options, see https://vale.sh/docs/topics/styles/ extends: existence message: "Refactor the section or page to avoid headings greater than H5." -link: https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#headings-in-markdown +link: https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#heading-levels-in-markdown level: suggestion scope: raw raw: - - '(?<=\n)#{5,}\s.*' + - '(?<=\n)#{6,}\s.*' diff --git a/doc/.vale/gitlab/Substitutions.yml b/doc/.vale/gitlab/Substitutions.yml index 675abc6ef6d..225e4e74880 100644 --- a/doc/.vale/gitlab/Substitutions.yml +++ b/doc/.vale/gitlab/Substitutions.yml @@ -23,6 +23,7 @@ swap: params: parameters pg: PostgreSQL 'postgres$': PostgreSQL + golang: Go raketask: Rake task raketasks: Rake tasks rspec: RSpec diff --git a/doc/.vale/gitlab/Uppercase.yml b/doc/.vale/gitlab/Uppercase.yml index 19e0fec6622..1948842d026 100644 --- a/doc/.vale/gitlab/Uppercase.yml +++ b/doc/.vale/gitlab/Uppercase.yml @@ -147,6 +147,7 @@ exceptions: - NPM - NTP - OCI + - OIDC - OKD - OKR - ONLY diff --git a/doc/.vale/gitlab/spelling-exceptions.txt b/doc/.vale/gitlab/spelling-exceptions.txt index a2a23c03aa3..476b1eebf84 100644 --- a/doc/.vale/gitlab/spelling-exceptions.txt +++ b/doc/.vale/gitlab/spelling-exceptions.txt @@ -197,6 +197,7 @@ Coursier CPU CPUs CRAN +CRI-O cron crond cronjob @@ -223,6 +224,8 @@ cybersecurity CycloneDX Dangerfile DAST +Database Lab Engine +Database Lab Databricks Datadog datasource @@ -273,6 +276,8 @@ desugar desugars desynchronized Dev +devfile +devfiles DevOps Dhall dialogs @@ -290,6 +295,7 @@ Dockerfiles Dockerize Dockerized Dockerizing +Docusaurus dogfood dogfooding dogfoods @@ -325,6 +331,7 @@ ETag ETags Etsy Excon +exfiltrate exfiltration ExifTool expirable @@ -390,6 +397,8 @@ Gitter GLab globals globbing +globstar +globstars Gmail Godep Golang @@ -431,6 +440,7 @@ hotfixed hotfixes hotfixing hotspots +HTMLHint http https hyperparameter @@ -495,6 +505,7 @@ Kibana Kinesis Klar Knative +KPIs Kramdown Kroki kubeconfig @@ -574,7 +585,7 @@ mitigations mitmproxy mixin mixins -MLFlow +MLflow Mmap mockup mockups @@ -615,6 +626,7 @@ Nurtch NVMe nyc OAuth +OCP Octokit offboarded offboarding @@ -623,6 +635,7 @@ OIDs OKRs OKRs Okta +OLM OmniAuth onboarding OpenID @@ -666,6 +679,7 @@ Pipfile Pipfiles Piwik plaintext +podman Poedit polyfill polyfills @@ -833,8 +847,8 @@ sanitization SBOMs sbt SBT -scalers scalar's +scalers scatterplot scatterplots schedulable @@ -1049,6 +1063,7 @@ unencoded unencoder unencodes unencrypted +unescaped unfollow unfollowed unfollows @@ -1068,6 +1083,7 @@ unoptimize unoptimized unoptimizes unoptimizing +unparsable unpatched unpause unprioritized @@ -1182,7 +1198,7 @@ ZAProxy Zeitwerk Zendesk ZenTao +Zoekt zsh Zstandard Zuora -Zoekt diff --git a/doc/administration/audit_event_streaming.md b/doc/administration/audit_event_streaming.md index 92ccbe8b1a6..afc4926fec0 100644 --- a/doc/administration/audit_event_streaming.md +++ b/doc/administration/audit_event_streaming.md @@ -41,13 +41,13 @@ Streaming destinations receive **all** audit event data, which could include sen Users with the Owner role for a group can add streaming destinations for it: 1. On the top bar, select **Main menu > Groups** and find your group. -1. On the left sidebar, select **Security & Compliance > Audit events**. +1. On the left sidebar, select **Security and Compliance > Audit events**. 1. On the main area, select **Streams** tab. 1. Select **Add streaming destination** to show the section for adding destinations. 1. Enter the destination URL to add. 1. Optional. Locate the **Custom HTTP headers** table. -1. Ignore the **Active** checkbox because it isn't functional. To track progress on adding functionality to the **Active** checkbox, see the - [relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/361925). +1. Ignore the **Active** checkbox because it isn't functional. To track progress on adding functionality to the + **Active** checkbox, see [issue 367509](https://gitlab.com/gitlab-org/gitlab/-/issues/367509). 1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to 20 headers per streaming destination. 1. After all headers have been filled out, select **Add** to add the new streaming destination. @@ -119,7 +119,7 @@ Users with the Owner role for a group can list streaming destinations. To list the streaming destinations: 1. On the top bar, select **Main menu > Groups** and find your group. -1. On the left sidebar, select **Security & Compliance > Audit events**. +1. On the left sidebar, select **Security and Compliance > Audit events**. 1. On the main area, select **Streams** tab. 1. To the right of the item, select **Edit** (**{pencil}**) to see all the custom HTTP headers. @@ -164,13 +164,13 @@ Users with the Owner role for a group can update streaming destinations. To update a streaming destinations custom HTTP headers: 1. On the top bar, select **Main menu > Groups** and find your group. -1. On the left sidebar, select **Security & Compliance > Audit events**. +1. On the left sidebar, select **Security and Compliance > Audit events**. 1. On the main area, select **Streams** tab. 1. To the right of the item, select **Edit** (**{pencil}**). 1. Locate the **Custom HTTP headers** table. 1. Locate the header that you wish to update. -1. Ignore the **Active** checkbox because it isn't functional. To track progress on adding functionality to the **Active** checkbox, see the - [relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/361925). +1. Ignore the **Active** checkbox because it isn't functional. To track progress on adding functionality to the + **Active** checkbox, see [issue 367509](https://gitlab.com/gitlab-org/gitlab/-/issues/367509). 1. Select **Add header** to create a new name and value pair. Enter as many name and value pairs as required. You can add up to 20 headers per streaming destination. 1. Select **Save** to update the streaming destination. @@ -218,14 +218,14 @@ When the last destination is successfully deleted, streaming is disabled for the To delete a streaming destination: 1. On the top bar, select **Main menu > Groups** and find your group. -1. On the left sidebar, select **Security & Compliance > Audit events**. +1. On the left sidebar, select **Security and Compliance > Audit events**. 1. On the main area, select the **Streams** tab. 1. To the right of the item, select **Delete** (**{remove}**). To delete only the custom HTTP headers for a streaming destination: 1. On the top bar, select **Main menu > Groups** and find your group. -1. On the left sidebar, select **Security & Compliance > Audit events**. +1. On the left sidebar, select **Security and Compliance > Audit events**. 1. On the main area, select the **Streams** tab. 1. To the right of the item, **Edit** (**{pencil}**). 1. Locate the **Custom HTTP headers** table. @@ -282,7 +282,7 @@ the destination's value when [listing streaming destinations](#list-streaming-de Users with the Owner role for a group can list streaming destinations and see the verification tokens: 1. On the top bar, select **Main menu > Groups** and find your group. -1. On the left sidebar, select **Security & Compliance > Audit events**. +1. On the left sidebar, select **Security and Compliance > Audit events**. 1. On the main area, select the **Streams**. 1. View the verification token on the right side of each item. @@ -365,6 +365,50 @@ Streamed audit events have a predictable schema in the body of the response. | `target_id` | ID of the audit event's target | | | `target_type` | String representation of the target's type | | +### JSON payload schema + +```json +{ + "properties": { + "id": { + "type": "string" + }, + "author_id": { + "type": "integer" + }, + "author_name": { + "type": "string" + }, + "details": {}, + "ip_address": { + "type": "string" + }, + "entity_id": { + "type": "integer" + }, + "entity_path": { + "type": "string" + }, + "entity_type": { + "type": "string" + }, + "event_type": { + "type": "string" + }, + "target_id": { + "type": "integer" + }, + "target_type": { + "type": "string" + }, + "target_details": { + "type": "string" + }, + }, + "type": "object" +} +``` + ## Audit event streaming on Git operations > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332747) in GitLab 14.9 [with a flag](../administration/feature_flags.md) named `audit_event_streaming_git_operations`. Disabled by default. diff --git a/doc/administration/audit_events.md b/doc/administration/audit_events.md index c51b5fbb431..50bd943b8e4 100644 --- a/doc/administration/audit_events.md +++ b/doc/administration/audit_events.md @@ -49,7 +49,7 @@ You can view audit events scoped to a group or project. To view a group's audit events: 1. Go to the group. -1. On the left sidebar, select **Security & Compliance > Audit Events**. +1. On the left sidebar, select **Security and Compliance > Audit Events**. Group events do not include project audit events. Group events can also be accessed using the [Group Audit Events API](../api/audit_events.md#group-audit-events). Group event queries are limited to a maximum of 30 @@ -120,7 +120,7 @@ Successful sign-in events are the only audit events available at all tiers. To s 1. Select your avatar. 1. Select **Edit profile > Authentication log**. -After upgrading to a paid tier, you can see also see successful sign-in events on audit event pages. +After upgrading to a paid tier, you can also see successful sign-in events on audit event pages. ## Filter audit events @@ -159,11 +159,35 @@ You can view different events depending on the version of GitLab you have. The following actions on groups generate group audit events: +#### Group management + - Group name or path changed. - Group repository size limit changed. - Group created or deleted. - Group changed visibility. - User was added to group and with which [permissions](../user/permissions.md). +- Removed user from group. +- Project repository imported into group. +- [Project shared with group](../user/project/members/share_project_with_groups.md) and with which + [permissions](../user/permissions.md). +- Removal of a previously shared group with a project. +- LFS enabled or disabled. +- Membership lock enabled or disabled. +- Request access enabled or disabled. +- Roles allowed to create project changed. +- Group deploy token was successfully created, revoked, or deleted. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353452) in GitLab 14.9. +- Failed attempt to create a group deploy token. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353452) + in GitLab 14.9. +- [IP restrictions](../user/group/access_and_permissions.md#restrict-group-access-by-ip-address) changed. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/358986) in GitLab 15.0. +- Group had a security policy project linked, changed, or unlinked. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/377877) in GitLab 15.6. +- An environment is protected or unprotected. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216164) in GitLab 15.8. + +#### Authentication and authorization + - User sign-in using [Group SAML](../user/group/saml_sso/index.md). - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8071) in GitLab 14.5, changes to the following [group SAML](../user/group/saml_sso/index.md) configuration: @@ -177,32 +201,28 @@ The following actions on groups generate group audit events: - Default membership role. - SSO-SAML group sync configuration. - Permissions changes of a user assigned to a group. -- Removed user from group. -- Project repository imported into group. -- [Project shared with group](../user/project/members/share_project_with_groups.md) and with which - [permissions](../user/permissions.md). -- Removal of a previously shared group with a project. -- LFS enabled or disabled. -- Shared runners minutes limit changed. -- Membership lock enabled or disabled. -- Request access enabled or disabled. - 2FA enforcement or grace period changed. -- Roles allowed to create project changed. -- Group CI/CD variable added, removed, or protected status changed. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30857) in GitLab 13.3. + +#### Compliance and security + - Compliance framework created, updated, or deleted. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/340649) in GitLab 14.5. - Event streaming destination created, updated, or deleted. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/344664) in GitLab 14.6. +- Changes to push rules. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227629) in GitLab 15.0. +- Changes to streaming audit destination custom HTTP headers. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/366350) in GitLab 15.3. - Instance administrator started or stopped impersonation of a group member. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/300961) in GitLab 14.8. -- Group deploy token was successfully created, revoked, or deleted. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353452) in GitLab 14.9. -- Failed attempt to create a group deploy token. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353452) - in GitLab 14.9. -- [IP restrictions](../user/group/access_and_permissions.md#restrict-group-access-by-ip-address) changed. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/358986) in GitLab 15.0. -- Changes to push rules. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227629) in GitLab 15.0. + +#### CI/CD + +- Shared runners minutes limit changed. +- Group CI/CD variable added, removed, or protected status changed. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30857) in GitLab 13.3. + +#### Code collaboration + - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356152) in GitLab 15.1, changes to the following merge request approvals settings: - Prevent approval by author. @@ -210,145 +230,187 @@ The following actions on groups generate group audit events: - Prevent editing approval rules in projects and merge requests. - Require user password to approve. - Remove all approvals when commits are added to the source branch. -- Changes to streaming audit destination custom HTTP headers. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/366350) in GitLab 15.3. -- Group had a security policy project linked, changed, or unlinked. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/377877) in GitLab 15.6. -- An environment is protected or unprotected. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216164) in GitLab 15.8. +- Changes to Code Suggestions. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/405295) in GitLab 15.11. ### Project events The following actions on projects generate project audit events: -- Added or removed deploy keys -- Project created, deleted, renamed, moved (transferred), changed path -- Project changed visibility level -- User was added to project and with which [permissions](../user/permissions.md) -- Permission changes of a user assigned to a project -- User was removed from project -- Project export was downloaded -- Project repository was downloaded -- Project was archived -- Project was unarchived -- Added, removed, or updated protected branches -- Release was added to a project -- Release was updated -- Release was deleted. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/94793/) in GitLab 15.3. -- Release milestone associations changed -- Permission to approve merge requests by committers was updated. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9. -- Permission to approve merge requests by committers was updated. - - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9. - - Message for event [changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/72623/diffs) in GitLab 14.6. -- Permission to approve merge requests by authors was updated. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9. -- Number of required approvals was updated. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9. -- Added or removed users and groups from project approval groups. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213603) in GitLab 13.2. -- Project CI/CD variable added, removed, or protected status changed. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30857) in GitLab 13.4. -- Project access token was successfully created or revoked. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230007) in GitLab 13.9. -- Failed attempt to create or revoke a project access token. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230007) in GitLab 13.9. -- When default branch changes for a project. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/52339) in GitLab 13.9. -- Created, updated, or deleted DAST profiles, DAST scanner profiles, and DAST site profiles. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217872) in GitLab 14.1. -- Changed a project's compliance framework. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/329362) in GitLab 14.1. -- User password required for approvals was updated. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336211) in GitLab 14.2. -- Permission to modify merge requests approval rules in merge requests was updated. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336211) in GitLab 14.2. -- New approvals requirement when new commits are added to an MR was updated. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336211) in GitLab 14.2. -- When [strategies for feature flags](../operations/feature_flags.md#feature-flag-strategies) are changed. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68408) in GitLab 14.3. -- Allowing force push to protected branch changed. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/338873) in GitLab 14.3. -- Code owner approval requirement on merge requests targeting protected branch changed. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/338873) in GitLab 14.3. -- Users and groups allowed to merge and push to protected branch added or removed. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/338873) in GitLab 14.3. +#### Project management + +- Added or removed deploy keys. +- Project created, deleted, renamed, moved (transferred), changed path. +- Project changed visibility level. +- Project export was downloaded. +- Project repository was downloaded. +- Project was archived. +- Project was unarchived. +- Project had a security policy project linked, changed, or unlinked. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/377877) in GitLab 15.6. +- Project was scheduled for deletion due to inactivity. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85689) in GitLab 15.0. - Project deploy token was successfully created, revoked or deleted. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353451) in GitLab 14.9. - Failed attempt to create a project deploy token. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353451) in GitLab 14.9. -- When merge method is updated. +- When [strategies for feature flags](../operations/feature_flags.md#feature-flag-strategies) are changed. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68408) in GitLab 14.3. + +#### User management + +- User was added to project and with which [permissions](../user/permissions.md). +- Permission changes of a user assigned to a project. +- User was removed from project. +- Users and groups allowed to merge and push to protected branch added or removed. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/338873) in GitLab 14.3. + +#### Access control + +- Branch protection was added, removed, or updated. +- Failed attempt to create or revoke a project access token. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230007) in GitLab 13.9. +- Allowing force push to protected branch changed. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/338873) in GitLab 14.3. +- An environment is protected or unprotected. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216164) in GitLab 15.8. +- User password required for approvals was updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336211) in GitLab 14.2. +- Project access token was successfully created or revoked. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/230007) in GitLab 13.9. + +#### Code collaboration + +- Default description template for merge requests is updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355805) in GitLab 15.0. +- Merge commit message template is updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355805) in GitLab 15.0. +- Squash commit message template is updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355805) in GitLab 15.0. +- Delete source branch option by default enabled or disabled. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. -- Merged results pipelines enabled or disabled. +- Squash commits when merging is updated. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. -- Merge trains enabled or disabled. +- All discussions must be resolved enabled or disabled. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. +- Commit message suggestion is updated. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. - Automatically resolve merge request diff discussions enabled or disabled. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. - Show link to create or view a merge request when pushing from the command line enabled or disabled. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. -- Delete source branch option by default enabled or disabled. +- When merge method is updated. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. -- Squash commits when merging is updated. +- Merge trains enabled or disabled. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. +- Code owner approval requirement on merge requests targeting protected branch changed. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/338873) in GitLab 14.3. +- Permission to modify merge requests approval rules in merge requests was updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336211) in GitLab 14.2. +- New approvals requirement when new commits are added to an MR was updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336211) in GitLab 14.2. +- Added or removed users and groups from project approval groups. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213603) in GitLab 13.2. +- Permission to approve merge requests by committers was updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9. + - Message for event [changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/72623/diffs) in GitLab 14.6. +- Permission to approve merge requests by authors was updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9. +- Number of required approvals was updated. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7531) in GitLab 12.9. + +#### Release management + +- Release was added to a project. +- Release was updated. +- Release was deleted. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/94793/) in GitLab 15.3. +- Release milestone associations changed. + +#### CI/CD + +- Project CI/CD variable added, removed, or protected status changed. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30857) in GitLab 13.4. +- When default branch changes for a project. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/52339) in GitLab 13.9. - Pipelines must succeed enabled or disabled. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. - Skipped pipelines are considered successful enabled or disabled. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. -- All discussions must be resolved enabled or disabled. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. -- Commit message suggestion is updated. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. - Status check is added, edited, or deleted. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355805) in GitLab 15.0. -- Merge commit message template is updated. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355805) in GitLab 15.0. -- Squash commit message template is updated. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355805) in GitLab 15.0. -- Default description template for merge requests is updated. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355805) in GitLab 15.0. -- Project was scheduled for deletion due to inactivity. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85689) in GitLab 15.0. -- Project had a security policy project linked, changed, or unlinked. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/377877) in GitLab 15.6. -- An environment is protected or unprotected. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216164) in GitLab 15.8. +- Merged results pipelines enabled or disabled. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9. + +#### Compliance and security + +- Created, updated, or deleted DAST profiles, DAST scanner profiles, and DAST site profiles. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217872) in GitLab 14.1. +- Changed a project's compliance framework. + [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/329362) in GitLab 14.1. + +### GitLab agent for Kubernetes events + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/382133) in GitLab 15.10. + +GitLab generates audit events when a cluster agent token is created or revoked. ### Instance events **(PREMIUM SELF)** +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16826) in GitLab 13.5, audit events for failed second-factor authentication attempt. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276250) in GitLab 13.6, audit events for when a user is approved using the Admin Area. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276921) in GitLab 13.6, audit events for when a user's personal access token is successfully or unsuccessfully created or revoked. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/298783) in GitLab 13.9, audit events for when a user requests access to an instance or is rejected using the Admin Area. + The following user actions on a GitLab instance generate instance audit events: -- Sign-in events and the authentication type (such as standard, LDAP, or OmniAuth) -- Failed sign-ins -- Added SSH key -- Added or removed email -- Changed password -- Ask for password reset -- Grant OAuth access -- Started or stopped user impersonation -- Changed username. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7797) in GitLab 12.8. -- User was deleted. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/251) in GitLab 12.8. -- User was added. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/251) in GitLab 12.8. -- User requests access to an instance. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/298783) in GitLab 13.9. -- User was approved using the Admin Area. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276250) in GitLab 13.6. -- User was rejected using the Admin Area. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/298783) in GitLab 13.9. -- User was blocked using the Admin Area. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/251) in GitLab 12.8. -- User was blocked using the API. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25872) in GitLab 12.9. -- Failed second-factor authentication attempt. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16826) in - GitLab 13.5. -- A user's personal access token was successfully created or revoked. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276921) in GitLab 13.6. -- A failed attempt to create or revoke a user's personal access token. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276921) in GitLab 13.6. +#### Authentication + +- Sign-in events and the authentication type such as standard, LDAP, or OmniAuth. +- Failed sign-ins. +- Ask for password reset. +- Grant OAuth access. +- Failed second-factor authentication attempt. +- A user's personal access token was successfully or unsuccessfully created or revoked. +- A user's two-factor authentication was disabled. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/238177) in + GitLab 15.1. + +#### User management + +- Added SSH key. +- Added or removed email. +- Changed password. +- Started or stopped user impersonation. +- Changed username. +- User was added or deleted. +- User requests access to an instance. +- User was approved, rejected, or blocked using the Admin Area. +- User was blocked using the API. - Administrator added or removed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/323905) in GitLab 14.1. - Removed SSH key. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220127) in GitLab 14.1. - Added or removed GPG key. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220127) in GitLab 14.1. -- A user's two-factor authentication was disabled. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/238177) in - GitLab 15.1. - Enabled Admin Mode. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362101) in GitLab 15.7. +- All [group events](#group-events) and [project events](#project-events). +- User was unblocked using the Admin Area or API. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/115727) in GitLab 15.11. +- User was banned using the Admin Area or API. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/116103) in GitLab 15.11. +- User was unbanned using the Admin Area or API. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/116221) in GitLab 15.11. Instance events can also be accessed using the [Instance Audit Events API](../api/audit_events.md#instance-audit-events). +### GitLab Runner events + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335509) in GitLab 14.8, audit events for when a runner is registered. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/349540) in GitLab 14.9, audit events for when a runner is unregistered. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/349542) in GitLab 14.9, audit events for when a runner is assigned to or unassigned from a project. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355637) in GitLab 15.0, audit events for when a runner registration token is reset. + +GitLab generates audit events for the following GitLab Runner actions: + +- Instance, group, or project runner is registered. +- Instance, group, or project runner is unregistered. +- Runner is assigned to or unassigned from a project. +- Instance, group, or project runner registration token is reset. + [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/102579) in GitLab 15.6. + ## "Deleted User" events Audit events created after users are deleted are created for "Deleted User". For example, if a deleted user's access to @@ -364,6 +426,7 @@ Some events are not tracked in audit events. The following epics and issues prop - [Group settings and activity](https://gitlab.com/groups/gitlab-org/-/epics/475). - [Instance-level settings and activity](https://gitlab.com/groups/gitlab-org/-/epics/476). - [Deployment Approval activity](https://gitlab.com/gitlab-org/gitlab/-/issues/354782). +- [Approval rules processing by a non GitLab user](https://gitlab.com/gitlab-org/gitlab/-/issues/407384). If you don't see the event you want in any of the epics, you can either: diff --git a/doc/administration/auth/atlassian.md b/doc/administration/auth/atlassian.md index 58412078802..45617b9965c 100644 --- a/doc/administration/auth/atlassian.md +++ b/doc/administration/auth/atlassian.md @@ -41,7 +41,7 @@ To enable the Atlassian OmniAuth provider for passwordless authentication you mu sudo -u git -H editor /home/git/gitlab/config/gitlab.yml ``` -1. Edit the [common configuration file settings](../../integration/omniauth.md#configure-common-settings) +1. Configure the [common settings](../../integration/omniauth.md#configure-common-settings) to add `atlassian_oauth2` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. diff --git a/doc/administration/auth/authentiq.md b/doc/administration/auth/authentiq.md deleted file mode 100644 index a32d2a2cf94..00000000000 --- a/doc/administration/auth/authentiq.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -stage: Manage -group: Authentication and Authorization -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -remove_date: '2023-02-22' -redirect_to: '../../integration/omniauth.md' ---- - -# Authentiq OmniAuth Provider (removed) **(FREE SELF)** - -This feature was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/389452) in 15.9. -Use another [OmniAuth provider](../../integration/omniauth.md) instead. diff --git a/doc/administration/auth/cognito.md b/doc/administration/auth/cognito.md index a73595c731b..cfac958e297 100644 --- a/doc/administration/auth/cognito.md +++ b/doc/administration/auth/cognito.md @@ -41,7 +41,7 @@ To enable AWS Cognito as an authentication provider, complete the following step ## Configure GitLab -1. Edit the [common configuration file settings](../../integration/omniauth.md#configure-common-settings) +1. Configure the [common settings](../../integration/omniauth.md#configure-common-settings) to add `cognito` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. 1. On your GitLab server, open the configuration file. diff --git a/doc/administration/auth/crowd.md b/doc/administration/auth/crowd.md index 277e59b1a8a..f89e1a00928 100644 --- a/doc/administration/auth/crowd.md +++ b/doc/administration/auth/crowd.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w WARNING: This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/369117) in GitLab 15.3 and is planned for -removal in 16.0. +removal in 17.0. Authenticate to GitLab using the Atlassian Crowd OmniAuth provider. Enabling this provider also allows Crowd authentication for Git-over-https requests. @@ -40,7 +40,7 @@ this provider also allows Crowd authentication for Git-over-https requests. sudo -u git -H editor config/gitlab.yml ``` -1. Edit the [common configuration file settings](../../integration/omniauth.md#configure-common-settings) +1. Configure the [common settings](../../integration/omniauth.md#configure-common-settings) to add `crowd` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. diff --git a/doc/administration/auth/index.md b/doc/administration/auth/index.md index 2b978dd1f2c..4a8e230a944 100644 --- a/doc/administration/auth/index.md +++ b/doc/administration/auth/index.md @@ -1,5 +1,4 @@ --- -comments: false type: index stage: Manage group: Authentication and Authorization @@ -34,3 +33,7 @@ For more information, see the links shown on this page for each external provide | **User Removal** | SCIM (remove user from top-level group) | LDAP (remove user from groups and block from the instance)
SCIM | 1. Using Just-In-Time (JIT) provisioning, user accounts are created when the user first signs in. + +## Test OIDC/OAuth in GitLab + +See [Test OIDC/OAuth in GitLab](test_oidc_oauth.md) to learn how to test OIDC/OAuth authentication in your GitLab instance using your client application. diff --git a/doc/administration/auth/jwt.md b/doc/administration/auth/jwt.md index 8c9800aa792..9994b374038 100644 --- a/doc/administration/auth/jwt.md +++ b/doc/administration/auth/jwt.md @@ -25,7 +25,7 @@ JWT provides you with a secret key for you to use. sudo -u git -H editor config/gitlab.yml ``` -1. Edit the [common configuration file settings](../../integration/omniauth.md#configure-common-settings) +1. Configure the [common settings](../../integration/omniauth.md#configure-common-settings) to add `jwt` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. 1. Add the provider configuration. diff --git a/doc/administration/auth/ldap/google_secure_ldap.md b/doc/administration/auth/ldap/google_secure_ldap.md index e0612099221..042a65be500 100644 --- a/doc/administration/auth/ldap/google_secure_ldap.md +++ b/doc/administration/auth/ldap/google_secure_ldap.md @@ -18,6 +18,8 @@ The steps below cover: - Configuring the Secure LDAP Client in the Google administrator console. - Required GitLab configuration. +Secure LDAP is only available on specific Google Workspace editions. For more information, see the [Google Secure LDAP service documentation](https://support.google.com/a/answer/9048516). + ## Configuring Google LDAP client 1. Go to and sign in as a Google Workspace domain administrator. @@ -88,7 +90,7 @@ values obtained during the LDAP client configuration earlier: encryption: 'simple_tls' verify_certificates: true retry_empty_result_with_codes: [80] - + base: "DC=example,DC=com" tls_options: cert: | -----BEGIN CERTIFICATE----- @@ -152,7 +154,7 @@ values obtained during the LDAP client configuration earlier: servers: main: # 'main' is the GitLab 'provider ID' of this LDAP server label: 'Google Secure LDAP' - + base: "DC=example,DC=com" host: 'ldap.google.com' port: 636 uid: 'uid' diff --git a/doc/administration/auth/ldap/index.md b/doc/administration/auth/ldap/index.md index 0c7bd33c2c1..7687f7c9340 100644 --- a/doc/administration/auth/ldap/index.md +++ b/doc/administration/auth/ldap/index.md @@ -270,7 +270,7 @@ These LDAP sync configuration settings are available: | Setting | Description | Required | Examples | |-------------------|-------------|----------|----------| -| `group_base` | Base used to search for groups. | **{dotted-circle}** No | `'ou=groups,dc=gitlab,dc=example'` | +| `group_base` | Base used to search for groups. | **{dotted-circle}** No (required when `external_groups` is configured) | `'ou=groups,dc=gitlab,dc=example'` | | `admin_group` | The CN of a group containing GitLab administrators. Not `cn=administrators` or the full DN. | **{dotted-circle}** No | `'administrators'` | | `external_groups` | An array of CNs of groups containing users that should be considered external. Not `cn=interns` or the full DN. | **{dotted-circle}** No | `['interns', 'contractors']` | | `sync_ssh_keys` | The LDAP attribute containing a user's public SSH key. | **{dotted-circle}** No | `'sshPublicKey'` or false if not set | @@ -1031,6 +1031,25 @@ See [Google Secure LDAP](google_secure_ldap.md) for detailed configuration instr For more information on synchronizing users and groups between LDAP and GitLab, see [LDAP synchronization](ldap_synchronization.md). +## Move from LDAP to SAML + +1. [Configure SAML](../../../integration/saml.md). Add `auto_link_ldap_user` to: + - [`gitlab.rb` for Omnibus](../../../integration/saml.html?tab=Linux+package+%28Omnibus%29). + - [`values.yml` for Kubernetes](../../../integration/saml.html?tab=Helm+chart+%28Kubernetes%29). + For more information, see the [initial settings for all providers](../../../integration/omniauth.md#configure-initial-settings). + +1. Optional. [Disable the LDAP auth from the sign-in page](#disable-ldap-web-sign-in). + +1. Optional. To fix issues with linking users, you can first [remove those users' LDAP identities](ldap-troubleshooting.md#remove-the-identity-records-that-relate-to-the-removed-ldap-server). + +1. Confirm that users are able to sign in to their accounts. If a user cannot sign in, check if that user's LDAP is still there and remove it if necessary. If this issue persists, check the logs to identify the problem. + +1. In the configuration file, change: + - `omniauth_auto_link_user` to `saml` only. + - `omniauth_auto_link_ldap_user` to false. + - `ldap_enabled` to `false`. + You can also comment out the LDAP provider settings. + ## Troubleshooting See our [administrator guide to troubleshooting LDAP](ldap-troubleshooting.md). diff --git a/doc/administration/auth/ldap/ldap-troubleshooting.md b/doc/administration/auth/ldap/ldap-troubleshooting.md index 7e21d3c6655..909ab5245ca 100644 --- a/doc/administration/auth/ldap/ldap-troubleshooting.md +++ b/doc/administration/auth/ldap/ldap-troubleshooting.md @@ -170,10 +170,10 @@ We have a workaround, based on toggling the access level of affected users: 1. As an administrator, on the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Overview > Users**. 1. Select the name of the affected user. -1. In the upper right, select **Edit**. +1. In the upper-right corner, select **Edit**. 1. Change the user's access level from `Regular` to `Administrator` (or vice versa). 1. At the bottom of the page, select **Save changes**. -1. In the upper right, select **Edit** again. +1. In the upper-right corner, select **Edit** again. 1. Restore the user's original access level (`Regular` or `Administrator`) and select **Save changes** again. @@ -761,7 +761,7 @@ users, [see what to do when no users are found](#no-users-are-found). ### GitLab logs If a user account is blocked or unblocked due to the LDAP configuration, a -message is [logged to `application.log`](../../logs/index.md#applicationlog). +message is [logged to `application_json.log`](../../logs/index.md#application_jsonlog). If there is an unexpected error during an LDAP lookup (configuration error, timeout), the sign-in is rejected and a message is [logged to `production.log`](../../logs/index.md#productionlog). diff --git a/doc/administration/auth/ldap/ldap_synchronization.md b/doc/administration/auth/ldap/ldap_synchronization.md index cc300941470..f32a4af9e27 100644 --- a/doc/administration/auth/ldap/ldap_synchronization.md +++ b/doc/administration/auth/ldap/ldap_synchronization.md @@ -7,12 +7,16 @@ info: To determine the technical writer assigned to the Stage/Group associated w # LDAP synchronization **(PREMIUM SELF)** If you have [configured LDAP to work with GitLab](index.md), GitLab can automatically synchronize -users and groups. This process updates user and group information. +users and groups. + +LDAP synchronization updates user and group information for existing GitLab users that have an LDAP identity assigned. It does not create new GitLab users through LDAP. You can change when synchronization occurs. ## User sync +> Preventing LDAP username synchronization [introduced]() in GitLab 15.11. + Once per day, GitLab runs a worker to check and update GitLab users against LDAP. @@ -33,17 +37,132 @@ For more information, see [Bitmask Searches in LDAP](https://ctovswild.com/2009/ -The user is set to an `ldap_blocked` state in GitLab if the previous conditions -fail. This means the user cannot sign in or push or pull code. - The process also updates the following user information: - Name. Because of a [sync issue](https://gitlab.com/gitlab-org/gitlab/-/issues/342598), `name` is not synchronized if - [**Prevent users from changing their profile name**](../../../user/admin_area/settings/account_and_limit_settings.md#disable-user-profile-name-changes) is enabled. + [**Prevent users from changing their profile name**](../../../user/admin_area/settings/account_and_limit_settings.md#disable-user-profile-name-changes) is enabled or `sync_name` is set to `false`. - Email address. - SSH public keys if `sync_ssh_keys` is set. - Kerberos identity if Kerberos is enabled. +### Synchronize LDAP username + +By default, GitLab synchronizes the LDAP username field. + +To prevent this synchronization, you can set `sync_name` to `false`. + +::Tabs + +:::TabTitle Linux package (Omnibus) + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_rails['ldap_servers'] = { + 'main' => { + 'sync_name' => false, + } + } + ``` + +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +:::TabTitle Helm chart (Kubernetes) + +1. Export the Helm values: + + ```shell + helm get values gitlab > gitlab_values.yaml + ``` + +1. Edit `gitlab_values.yaml`: + + ```yaml + global: + appConfig: + ldap: + servers: + main: + sync_name: false + ``` + +1. Save the file and apply the new values: + + ```shell + helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab + ``` + +:::TabTitle Docker + +1. Edit `docker-compose.yml`: + + ```yaml + version: "3.6" + services: + gitlab: + environment: + GITLAB_OMNIBUS_CONFIG: | + gitlab_rails['ldap_servers'] = { + 'main' => { + 'sync_name' => false, + } + } + ``` + +1. Save the file and restart GitLab: + + ```shell + docker compose up -d + ``` + +:::TabTitle Self-compiled (source) + +1. Edit `/home/git/gitlab/config/gitlab.yml`: + + ```yaml + production: &base + ldap: + servers: + main: + sync_name: false + ``` + +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart + ``` + +::EndTabs + +### Blocked users + +A user is blocked if either the: + +- [Access check fails](#user-sync) and that user is set to an `ldap_blocked` state in GitLab. +- LDAP server is not available when that user signs in. + +If a user is blocked, that user cannot sign in or push or pull code. + +A blocked user is unblocked when they sign in with LDAP if all of the following are true: + +- All the access check conditions are true. +- The LDAP server is available when the user signs in. + +**All users** are blocked if the LDAP server is unavailable when an LDAP user synchronization is run. + +NOTE: +If all users are blocked due to the LDAP server not being available when an LDAP user synchronization is run, +a subsequent LDAP user synchronization does not automatically unblock those users. + ### Adjust LDAP user sync schedule By default, GitLab runs a worker once per day at 01:30 a.m. server time to @@ -387,6 +506,23 @@ To enable global group memberships lock: 1. Expand the **Visibility and access controls** section. 1. Ensure the **Lock memberships to LDAP synchronization** checkbox is selected. +### Change LDAP group synchronization settings management + +By default, group members with the Owner role can manage [LDAP group synchronization settings](../../../user/group/access_and_permissions.md#manage-group-memberships-via-ldap). + +GitLab administrators can remove this permission from group Owners: + +1. [Configure LDAP](index.md#configure-ldap). +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Settings > General**. +1. Expand **Visibility and access controls**. +1. Ensure the **Allow group owners to manage LDAP-related settings** checkbox is not checked. + +When **Allow group owners to manage LDAP-related settings** is disabled: + +- Group Owners cannot change LDAP synchronization settings for either top-level groups and subgroups. +- Instance administrators can manage LDAP group synchronization settings on all groups on an instance. + ### Adjust LDAP group sync schedule By default, GitLab runs a group sync process every hour, on the hour. @@ -411,7 +547,7 @@ sync to run once every two hours at the top of the hour. 1. Edit `/etc/gitlab/gitlab.rb`: ```ruby - gitlab_rails['ldap_group_sync_worker_cron'] = "0 */2 * * * *" + gitlab_rails['ldap_group_sync_worker_cron'] = "0 */2 * * *" ``` 1. Save the file and reconfigure GitLab: @@ -454,7 +590,7 @@ sync to run once every two hours at the top of the hour. gitlab: environment: GITLAB_OMNIBUS_CONFIG: | - gitlab_rails['ldap_group_sync_worker_cron'] = "0 */2 * * * *" + gitlab_rails['ldap_group_sync_worker_cron'] = "0 */2 * * *" ``` 1. Save the file and restart GitLab: diff --git a/doc/administration/auth/oidc.md b/doc/administration/auth/oidc.md index ea6922629d8..106cc6c23eb 100644 --- a/doc/administration/auth/oidc.md +++ b/doc/administration/auth/oidc.md @@ -29,7 +29,7 @@ The OpenID Connect provides you with a client's details and secret for you to us sudo -u git -H editor config/gitlab.yml ``` -1. Edit the [common configuration file settings](../../integration/omniauth.md#configure-common-settings) +1. Configure the [common settings](../../integration/omniauth.md#configure-common-settings) to add `openid_connect` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. @@ -40,7 +40,7 @@ The OpenID Connect provides you with a client's details and secret for you to us ```ruby gitlab_rails['omniauth_providers'] = [ { - name: "openid_connect", + name: "openid_connect", # do not change this parameter label: "Provider name", # optional label for login button, defaults to "Openid Connect" icon: "", args: { @@ -63,10 +63,55 @@ The OpenID Connect provides you with a client's details and secret for you to us ] ``` + For Omnibus GitLab with multiple identity providers: + + ```ruby + { 'name' => 'openid_connect', + 'label' => '...', + 'icon' => '...', + 'args' => { + 'name' => 'openid_connect', + 'strategy_class': 'OmniAuth::Strategies::OpenIDConnect', + 'scope' => ['openid', 'profile', 'email'], + 'discovery' => true, + 'response_type' => 'code', + 'issuer' => 'https://...', + 'client_auth_method' => 'query', + 'uid_field' => '...', + 'client_options' => { + `identifier`: "", + `secret`: "", + 'redirect_uri' => 'https://.../users/auth/openid_connect/callback' + } + } + }, + { 'name' => 'openid_connect_2fa', + 'label' => '...', + 'icon' => '...', + 'args' => { + 'name' => 'openid_connect_2fa', + 'strategy_class': 'OmniAuth::Strategies::OpenIDConnect', + 'scope' => ['openid', 'profile', 'email'], + 'discovery' => true, + 'response_type' => 'code', + 'issuer' => 'https://...', + 'client_auth_method' => 'query', + 'uid_field' => '...', + 'client_options' => { + ... + 'redirect_uri' => 'https://.../users/auth/openid_connect_2fa/callback' + } + } + } + ``` + + NOTE: + For more information on using multiple identity providers with OIDC, see [issue 5992](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5992). + For installation from source: ```yaml - - { name: 'openid_connect', + - { name: 'openid_connect', # do not change this parameter label: 'Provider name', # optional label for login button, defaults to "Openid Connect" icon: '', args: { @@ -89,10 +134,7 @@ The OpenID Connect provides you with a client's details and secret for you to us ``` NOTE: - For more information on each configuration option, refer to the: - - - [OmniAuth OpenID Connect usage documentation](https://github.com/m0n9oose/omniauth_openid_connect#usage). - - [OpenID Connect Core 1.0 specification](https://openid.net/specs/openid-connect-core-1_0.html). + For more information on each configuration option, refer to the [OmniAuth OpenID Connect usage documentation](https://github.com/omniauth/omniauth_openid_connect#usage) and [OpenID Connect Core 1.0 specification](https://openid.net/specs/openid-connect-core-1_0.html). 1. For the provider configuration, change the values for the provider to match your OpenID Connect client setup. Use the following as a guide: @@ -129,7 +171,7 @@ The OpenID Connect provides you with a client's details and secret for you to us - `client_options` are the OpenID Connect client-specific options. Specifically: - `identifier` is the client identifier as configured in the OpenID Connect service provider. - `secret` is the client secret as configured in the OpenID Connect service provider. For example, - [OmniAuth OpenIDConnect](https://github.com/omniauth/omniauth_openid_connect)) requires this. If the service provider doesn't require a secret, + [OmniAuth OpenID Connect](https://github.com/omniauth/omniauth_openid_connect) requires this. If the service provider doesn't require a secret, provide any value and it is ignored. - `redirect_uri` is the GitLab URL to redirect the user to after successful login (for example, `http://example.com/users/auth/openid_connect/callback`). @@ -165,7 +207,7 @@ for more details: ```ruby gitlab_rails['omniauth_providers'] = [ { - name: "openid_connect", + name: "openid_connect", # do not change this parameter label: "Google OpenID", # optional label for login button, defaults to "Openid Connect" args: { name: "openid_connect", @@ -203,7 +245,7 @@ Example Omnibus configuration block: ```ruby gitlab_rails['omniauth_providers'] = [ { - name: "openid_connect", + name: "openid_connect", # do not change this parameter label: "Azure OIDC", # optional label for login button, defaults to "Openid Connect" args: { name: "openid_connect", @@ -335,7 +377,7 @@ but `LocalAccounts` authenticates against local Active Directory accounts. Befor ```ruby gitlab_rails['omniauth_providers'] = [ { - name: "openid_connect", + name: "openid_connect", # do not change this parameter label: "Azure B2C OIDC", # optional label for login button, defaults to "Openid Connect" args: { name: "openid_connect", @@ -395,7 +437,7 @@ Example Omnibus configuration block: ```ruby gitlab_rails['omniauth_providers'] = [ { - name: "openid_connect", + name: "openid_connect", # do not change this parameter label: "Keycloak", # optional label for login button, defaults to "Openid Connect" args: { name: "openid_connect", @@ -475,7 +517,7 @@ To use symmetric key encryption: ```ruby gitlab_rails['omniauth_providers'] = [ { - name: "openid_connect", + name: "openid_connect", # do not change this parameter label: "Keycloak", # optional label for login button, defaults to "Openid Connect" args: { name: "openid_connect", @@ -519,7 +561,7 @@ Example Omnibus GitLab configuration (file path: `/etc/gitlab/gitlab.rb`): ```ruby gitlab_rails['omniauth_providers'] = [ { - name: "openid_connect", + name: "openid_connect", # do not change this parameter label: "Casdoor", # optional label for login button, defaults to "Openid Connect" args: { name: "openid_connect", @@ -542,7 +584,7 @@ gitlab_rails['omniauth_providers'] = [ Example installations from source configuration (file path: `config/gitlab.yml`): ```yaml - - { name: 'openid_connect', + - { name: 'openid_connect', # do not change this parameter label: 'Casdoor', # optional label for login button, defaults to "Openid Connect" args: { name: 'openid_connect', @@ -561,6 +603,401 @@ Example installations from source configuration (file path: `config/gitlab.yml`) } ``` +## Configure multiple OpenID Connect providers + +You can configure your application to use multiple OpenID Connect (OIDC) providers. You do this by explicitly setting the `strategy_class` in your configuration file. + +You should do this in either of the following scenarios: + +- [Migrating to the OpenID Connect protocol](../../integration/azure.md#migrate-to-the-openid-connect-protocol). +- Offering different levels of authentication. + +NOTE: +This is not compatible with [configuring users based on OIDC group membership](#configure-users-based-on-oidc-group-membership). For more information, see [issue 408248](https://gitlab.com/gitlab-org/gitlab/-/issues/408248). + +The following example configurations show how to offer different levels of authentication, one option with 2FA and one without 2FA. + +For Omnibus GitLab: + +```ruby +gitlab_rails['omniauth_providers'] = [ + { + name: "openid_connect", + label: "Provider name", # optional label for login button, defaults to "Openid Connect" + icon: "", + args: { + name: "openid_connect", + strategy_class: "OmniAuth::Strategies::OpenIDConnect", + scope: ["openid","profile","email"], + response_type: "code", + issuer: "", + discovery: true, + client_auth_method: "query", + uid_field: "", + send_scope_to_token_endpoint: "false", + pkce: true, + client_options: { + identifier: "", + secret: "", + redirect_uri: "/users/auth/openid_connect/callback" + } + } + }, + { + name: "openid_connect_2fa", + label: "Provider name 2FA", # optional label for login button, defaults to "Openid Connect" + icon: "", + args: { + name: "openid_connect_2fa", + strategy_class: "OmniAuth::Strategies::OpenIDConnect", + scope: ["openid","profile","email"], + response_type: "code", + issuer: "", + discovery: true, + client_auth_method: "query", + uid_field: "", + send_scope_to_token_endpoint: "false", + pkce: true, + client_options: { + identifier: "", + secret: "", + redirect_uri: "/users/auth/openid_connect_2fa/callback" + } + } + } +] +``` + +For installation from source: + +```yaml + - { name: 'openid_connect', + label: 'Provider name', # optional label for login button, defaults to "Openid Connect" + icon: '', + args: { + name: 'openid_connect', + strategy_class: "OmniAuth::Strategies::OpenIDConnect", + scope: ['openid','profile','email'], + response_type: 'code', + issuer: '', + discovery: true, + client_auth_method: 'query', + uid_field: '', + send_scope_to_token_endpoint: false, + pkce: true, + client_options: { + identifier: '', + secret: '', + redirect_uri: '/users/auth/openid_connect/callback' + } + } + } + - { name: 'openid_connect_2fa', + label: 'Provider name 2FA', # optional label for login button, defaults to "Openid Connect" + icon: '', + args: { + name: 'openid_connect_2fa', + strategy_class: "OmniAuth::Strategies::OpenIDConnect", + scope: ['openid','profile','email'], + response_type: 'code', + issuer: '', + discovery: true, + client_auth_method: 'query', + uid_field: '', + send_scope_to_token_endpoint: false, + pkce: true, + client_options: { + identifier: '', + secret: '', + redirect_uri: '/users/auth/openid_connect_2fa/callback' + } + } + } +``` + +In this use case, you might want to synchronize the `extern_uid` across the +different providers based on an existing known identifier in your +corporate directory. + +To do this, you set the `uid_field`. The following example code shows how to +do this: + +```python +def sync_missing_provider(self, user: User, extern_uid: str) + existing_identities = [] + for identity in user.identities: + existing_identities.append(identity.get("provider")) + + local_extern_uid = extern_uid.lower() + for provider in ("openid_connect_2fa", "openid_connect"): + identity = [ + identity + for identity in user.identities + if identity.get("provider") == provider + and identity.get("extern_uid").lower() != local_extern_uid + ] + if provider not in existing_identities or identity: + if identity and identity[0].get("extern_uid") != "": + logger.error(f"Found different identity for provider {provider} for user {user.id}") + continue + else: + logger.info(f"Add identity 'provider': {provider}, 'extern_uid': {extern_uid} for user {user.id}") + user.provider = provider + user.extern_uid = extern_uid + user = self.save_user(user) + return user +``` + +For more information, see the [GitLab API user method documentation](https://python-gitlab.readthedocs.io/en/stable/gl_objects/users.html#examples). + +## Configure users based on OIDC group membership **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/209898) in GitLab 15.10. + +You can configure OIDC group membership to: + +- Require users to be members of a certain group. +- Assign users [external roles](../../user/admin_area/external_users.md), or as + administrators based on group membership. + +GitLab checks these groups on each sign in and updates user attributes as necessary. +This feature **does not** allow you to automatically add users to GitLab +[groups](../../user/group/index.md). + +### Required groups + +Your identity provider (IdP) must pass group information to GitLab in the OIDC response. To use this +response to require users to be members of a certain group, configure GitLab to identify: + +- Where to look for the groups in the OIDC response, using the `groups_attribute` setting. +- Which group membership is required to sign in, using the `required_groups` setting. + +If you do not set `required_groups` or leave the setting empty, any user authenticated by the IdP through OIDC can use GitLab. + +For Omnibus GitLab: + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + name: "openid_connect", + label: "Provider name", + args: { + name: "openid_connect", + scope: ["openid","profile","email"], + response_type: "code", + issuer: "", + discovery: true, + client_auth_method: "query", + uid_field: "", + client_options: { + identifier: "", + secret: "", + redirect_uri: "/users/auth/openid_connect/callback", + gitlab: { + groups_attribute: "groups", + required_groups: ["Developer"] + } + } + } + } + ] + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect. + +For installation from source: + +1. Edit `/home/git/gitlab/config/gitlab.yml`: + + ```yaml + production: &base + omniauth: + providers: + - { name: 'openid_connect', + label: 'Provider name', + args: { + name: 'openid_connect', + scope: ['openid','profile','email'], + response_type: 'code', + issuer: '', + discovery: true, + client_auth_method: 'query', + uid_field: '', + client_options: { + identifier: '', + secret: '', + redirect_uri: '/users/auth/openid_connect/callback', + gitlab: { + groups_attribute: "groups", + required_groups: ["Developer"] + } + } + } + } + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#installations-from-source) + for the changes to take effect. + +### External groups + +Your IdP must pass group information to GitLab in the OIDC response. To use this +response to identify users as [external users](../../user/admin_area/external_users.md) +based on group membership, configure GitLab to identify: + +- Where to look for the groups in the OIDC response, using the `groups_attribute` setting. +- Which group memberships should identify a user as an + [external user](../../user/admin_area/external_users.md), using the + `external_groups` setting. + +For Omnibus GitLab: + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + name: "openid_connect", + label: "Provider name", + args: { + name: "openid_connect", + scope: ["openid","profile","email"], + response_type: "code", + issuer: "", + discovery: true, + client_auth_method: "query", + uid_field: "", + client_options: { + identifier: "", + secret: "", + redirect_uri: "/users/auth/openid_connect/callback", + gitlab: { + groups_attribute: "groups", + external_groups: ["Freelancer"] + } + } + } + } + ] + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect. + +For installation from source: + +1. Edit `/home/git/gitlab/config/gitlab.yml`: + + ```yaml + production: &base + omniauth: + providers: + - { name: 'openid_connect', + label: 'Provider name', + args: { + name: 'openid_connect', + scope: ['openid','profile','email'], + response_type: 'code', + issuer: '', + discovery: true, + client_auth_method: 'query', + uid_field: '', + client_options: { + identifier: '', + secret: '', + redirect_uri: '/users/auth/openid_connect/callback', + gitlab: { + groups_attribute: "groups", + external_groups: ["Freelancer"] + } + } + } + } + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#installations-from-source) + for the changes to take effect. + +### Administrator groups + +Your IdP must pass group information to GitLab in the OIDC response. To use this +response to assign users as administrator based on group membership, configure GitLab to identify: + +- Where to look for the groups in the OIDC response, using the `groups_attribute` setting. +- Which group memberships grant the user administrator access, using the + `admin_groups` setting. + +For Omnibus GitLab: + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + name: "openid_connect", + label: "Provider name", + args: { + name: "openid_connect", + scope: ["openid","profile","email"], + response_type: "code", + issuer: "", + discovery: true, + client_auth_method: "query", + uid_field: "", + client_options: { + identifier: "", + secret: "", + redirect_uri: "/users/auth/openid_connect/callback", + gitlab: { + groups_attribute: "groups", + admin_groups: ["Admin"] + } + } + } + } + ] + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect. + +For installation from source: + +1. Edit `/home/git/gitlab/config/gitlab.yml`: + + ```yaml + production: &base + omniauth: + providers: + - { name: 'openid_connect', + label: 'Provider name', + args: { + name: 'openid_connect', + scope: ['openid','profile','email'], + response_type: 'code', + issuer: '', + discovery: true, + client_auth_method: 'query', + uid_field: '', + client_options: { + identifier: '', + secret: '', + redirect_uri: '/users/auth/openid_connect/callback', + gitlab: { + groups_attribute: "groups", + admin_groups: ["Admin"] + } + } + } + } + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#installations-from-source) + for the changes to take effect. + ## Troubleshooting 1. Ensure `discovery` is set to `true`. If you set it to `false`, you must @@ -568,7 +1005,7 @@ Example installations from source configuration (file path: `config/gitlab.yml`) 1. Check your system clock to ensure the time is synchronized properly. -1. As mentioned in [the OmniAuth OpenID Connect documentation](https://github.com/m0n9oose/omniauth_openid_connect), +1. As mentioned in [the OmniAuth OpenID Connect documentation](https://github.com/omniauth/omniauth_openid_connect), make sure `issuer` corresponds to the base URL of the Discovery URL. For example, `https://accounts.google.com` is used for the URL `https://accounts.google.com/.well-known/openid-configuration`. diff --git a/doc/administration/auth/test_oidc_oauth.md b/doc/administration/auth/test_oidc_oauth.md new file mode 100644 index 00000000000..95cca1ced86 --- /dev/null +++ b/doc/administration/auth/test_oidc_oauth.md @@ -0,0 +1,57 @@ +--- +stage: Manage +group: Authentication and Authorization +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Test OIDC/OAuth in GitLab **(FREE)** + +To test OIDC/OAuth in GitLab, you must: + +1. [Enable OIDC/OAuth](#enable-oidcoauth-in-gitlab) +1. [Test OIDC/OAuth with your client application](#test-oidcoauth-with-your-client-application) +1. [Verify OIDC/OAuth authentication](#verify-oidcoauth-authentication) + +## Prerequisites + +Before you can test OIDC/OAuth on GitLab, you'll need the following: + +- Publicly accessible GitLab instance +- A client application that you want to use to test OIDC/OAuth +- A user account on the GitLab instance that you can use to log in and test OIDC/OAuth + +## Enable OIDC/OAuth in GitLab + +First, you must create OIDC/OAuth application on your GitLab instance. To do this: + +1. Sign in to GitLab as an administrator. +1. Select **Profile > Preferences > Applications**. +1. Fill in the details for your client application, including the name, redirect URI, and allowed scopes. +1. Make sure the `openid` scope is enabled. +1. Select **Save application** to create the new OAuth application. + +## Test OIDC/OAuth with your client application + +After you've created your OAuth application in GitLab, you can use it to test OIDC/OAuth: + +1. You can use as the OIDC/OAuth playground. +1. Sign out of GitLab. +1. Visit your client application and initiate the OIDC/OAuth flow, using the GitLab OAuth application you created in the previous step. +1. Follow the prompts to sign in to GitLab and authorize the client application to access your GitLab account. +1. After you've completed the OIDC/OAuth flow, your client application should have received an access token that it can use to authenticate with GitLab. + +## Verify OIDC/OAuth authentication + +To verify that OIDC/OAuth authentication is working correctly on GitLab, you can perform the following checks: + +1. Check that the access token you received in the previous step is valid and can be used to authenticate with GitLab. You can do this by making a test API request to GitLab, using the access token to authenticate. For example: + + ```shell + curl --header "Authorization: Bearer " https://mygitlabinstance.com/api/v4/user + ``` + + Replace `` with the actual access token you received in the previous step. If the API request succeeds and returns information about the authenticated user, then OIDC/OAuth authentication is working correctly. + +1. Check that the scopes you specified in your OAuth application are being enforced correctly. You can do this by making API requests that require the specific scopes and checking that they succeed or fail as expected. + +That's it! With these steps, you should be able to test OIDC/OAuth authentication on your GitLab instance using your client application. diff --git a/doc/administration/clusters/kas.md b/doc/administration/clusters/kas.md index a7f8f8e712b..6d6e8e5513c 100644 --- a/doc/administration/clusters/kas.md +++ b/doc/administration/clusters/kas.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -43,6 +43,33 @@ To enable the agent server on a single node: For additional configuration options, see the **Enable GitLab KAS** section of the [`gitlab.rb.template`](https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/master/files/gitlab-config-template/gitlab.rb.template). +##### Configure KAS to listen on a UNIX socket + +If you use GitLab behind a proxy, KAS might not work correctly. You can resolve this issue on a single-node installation, you can configure KAS to listen on a UNIX socket. + +To configure KAS to listen on a UNIX socket: + +1. Create a directory for the KAS sockets: + + ```shell + sudo mkdir -p /var/opt/gitlab/gitlab-kas/sockets/ + ``` + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_kas['internal_api_listen_network'] = 'unix' + gitlab_kas['internal_api_listen_address'] = '/var/opt/gitlab/gitlab-kas/sockets/internal-api.socket' + gitlab_kas['private_api_listen_network'] = 'unix' + gitlab_kas['private_api_listen_address'] = '/var/opt/gitlab/gitlab-kas/sockets/private-api.socket' + gitlab_kas['env'] = { + 'SSL_CERT_DIR' => "/opt/gitlab/embedded/ssl/certs/", + 'OWN_PRIVATE_API_URL' => 'unix:///var/opt/gitlab/gitlab-kas/sockets/private-api.socket' + } + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + #### Enable on multiple nodes To enable the agent server on multiple nodes: @@ -50,6 +77,8 @@ To enable the agent server on multiple nodes: 1. For each agent server node, edit `/etc/gitlab/gitlab.rb`: ```ruby + gitlab_kas_external_url 'wss://kas.gitlab.example.com/' + gitlab_kas['enable'] = true gitlab_kas['api_secret_key'] = '<32_bytes_long_base64_encoded_value>' gitlab_kas['private_api_secret_key'] = '<32_bytes_long_base64_encoded_value>' @@ -65,17 +94,23 @@ To enable the agent server on multiple nodes: gitlab_rails['gitlab_kas_external_k8s_proxy_url'] = 'https://gitlab.example.com/-/kubernetes-agent/k8s-proxy/' ``` - In this configuration: +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). - - `gitlab_kas['private_api_listen_address']` is the address the agent server listens on. You can set it to `0.0.0.0` or an IP address reachable by other nodes in the cluster. - - `OWN_PRIVATE_API_URL` is the environment variable used by the KAS process for service discovery. You can set it to a hostname or IP address of the node you're configuring. The node must be reachable by other nodes in the cluster. - - `gitlab_kas['api_secret_key']` is the shared secret used for authentication between KAS and GitLab. This value must be Base64-encoded and exactly 32 bytes long. - - `gitlab_kas['private_api_secret_key']` is the shared secret used for authentication between different KAS instances. This value must be Base64-encoded and exactly 32 bytes long. - - `gitlab_rails['gitlab_kas_external_url']` is the user-facing URL for the in-cluster `agentk`. - - `gitlab_rails['gitlab_kas_internal_url']` is the internal URL the GitLab backend uses to communicate with KAS. - - `gitlab_rails['gitlab_kas_external_k8s_proxy_url']` is the user-facing URL for Kubernetes API proxying. +##### Agent server node settings -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +| Setting | Description | +|---------|-------------| +| `gitlab_kas['private_api_listen_address']` | The address the agent server listens on. Set to `0.0.0.0` or to an IP address reachable by other nodes in the cluster. | +| `gitlab_kas['api_secret_key']` | The shared secret used for authentication between KAS and GitLab. The value must be Base64-encoded and exactly 32 bytes long. | +| `gitlab_kas['private_api_secret_key']` | The shared secret used for authentication between different KAS instances. The value must be Base64-encoded and exactly 32 bytes long. | +| `OWN_PRIVATE_API_URL` | The environment variable used by KAS for service discovery. Set to the hostname or IP address of the node you're configuring. The node must be reachable by other nodes in the cluster. | +| `gitlab_kas_external_url` | The user-facing URL for the in-cluster `agentk`. Can be a fully qualified domain or subdomain, **1** or a GitLab external URL. **2** If blank, defaults to a GitLab external URL. | +| `gitlab_rails['gitlab_kas_external_url']` | The user-facing URL for the in-cluster `agentk`. If blank, defaults to the `gitlab_kas_external_url`. | +| `gitlab_rails['gitlab_kas_external_k8s_proxy_url']` | The user-facing URL for Kubernetes API proxying. If blank, defaults to a URL based on `gitlab_kas_external_url`. | +| `gitlab_rails['gitlab_kas_internal_url']` | The internal URL the GitLab backend uses to communicate with KAS. | + +1. For example, `wss://kas.gitlab.example.com/`. +1. For example, `wss://gitlab.example.com/-/kubernetes-agent/`. ### For GitLab Helm Chart @@ -105,6 +140,24 @@ For GitLab [Helm Chart](https://docs.gitlab.com/charts/) installations: For details, see [how to use the GitLab-KAS chart](https://docs.gitlab.com/charts/charts/gitlab/kas/). +## Kubernetes API proxy cookie + +> Introduced in GitLab 15.10 [with feature flags](../feature_flags.md) named `kas_user_access` and `kas_user_access_project`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flags](../feature_flags.md) named `kas_user_access` and `kas_user_access_project`. + +KAS proxies Kubernetes API requests to the GitLab agent with either: + +- A [CI/CD job](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/kubernetes_ci_access.md). +- [GitLab user credentials](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/kubernetes_user_access.md). + +To authenticate with user credentials, Rails sets a cookie for the GitLab frontend. +This cookie is called `_gitlab_kas` and it contains an encrypted +session ID, like the [`_gitlab_session` cookie](../../user/profile/index.md#cookies-used-for-sign-in). +The `_gitlab_kas` cookie must be sent to the KAS proxy endpoint with every request +to authenticate and authorize the user. + ## Troubleshooting If you have issues while using the agent server for Kubernetes, view the diff --git a/doc/administration/compliance.md b/doc/administration/compliance.md index 592c110b446..9ae0e9d7790 100644 --- a/doc/administration/compliance.md +++ b/doc/administration/compliance.md @@ -18,25 +18,13 @@ standards or mandates from regulatory bodies. The following features help you define rules and policies to adhere to workflow requirements, separation of duties, and secure supply chain best practices: -- [**Credentials inventory**](../user/admin_area/credentials_inventory.md) (for - instances): With a credentials inventory, GitLab administrators can keep track - of the credentials used by all of the users in their GitLab instance. -- [**Granular user roles and flexible permissions**](../user/permissions.md) - (for instances, groups, and projects): Manage access and permissions with five - different user roles and settings for external users. Set permissions according - to people's role, rather than either read or write access to a repository. Don't - share the source code with people that only need access to the issue tracker. -- [**Merge request approvals**](../user/project/merge_requests/approvals/index.md) - (for instances, groups, and projects): Configure approvals required for - merge requests. -- [**Push rules**](../user/project/repository/push_rules.md) (for instances, groups, and - projects): Control pushes to your repositories. -- Separation of duties using [**protected branches**](../user/project/protected_branches.md#require-code-owner-approval-on-a-protected-branch) - and [**custom CI/CD configuration paths**](../ci/pipelines/settings.md#specify-a-custom-cicd-configuration-file) (for projects): Users can leverage the GitLab cross-project YAML configurations - to define deployers of code and developers of code. See how to use this setup - to define these roles in: - - The [Separation of Duties deploy project](https://gitlab.com/guided-explorations/separation-of-duties-deploy/blob/master/README.md). - - The [Separation of Duties project](https://gitlab.com/guided-explorations/separation-of-duties/blob/master/README.md). +| Feature | Instances | Groups | Projects | Description | +|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------|:-----------------------|:-----------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [Credentials inventory](../user/admin_area/credentials_inventory.md) | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Keep track of the credentials used by all of the users in a GitLab instance. | +| [Granular user roles
and flexible permissions](../user/permissions.md) | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Manage access and permissions with five different user roles and settings for external users. Set permissions according to people's role, rather than either read or write access to a repository. Don't share the source code with people that only need access to the issue tracker. | +| [Merge request approvals](../user/project/merge_requests/approvals/index.md) | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Configure approvals required for merge requests. | +| [Push rules](../user/project/repository/push_rules.md) | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Control pushes to your repositories. | +| Separation of duties using
[protected branches](../user/project/protected_branches.md#require-code-owner-approval-on-a-protected-branch) and
[custom CI/CD configuration paths](../ci/pipelines/settings.md#specify-a-custom-cicd-configuration-file) | **{dotted-circle}** No | **{dotted-circle}** No | **{check-circle}** Yes | Leverage the GitLab cross-project YAML configurations to define deployers of code and developers of code. See how to use this setup to define these roles in the [Separation of Duties deploy project](https://gitlab.com/guided-explorations/separation-of-duties-deploy/blob/master/README.md) and the [Separation of Duties project](https://gitlab.com/guided-explorations/separation-of-duties/blob/master/README.md). | ## Compliant workflow automation @@ -48,66 +36,42 @@ settings and automation to ensure that whatever a compliance team has configured stays configured and working correctly. These features can help you automate compliance: -- [**Compliance frameworks**](../user/group/compliance_frameworks.md) (for groups): Create a custom - compliance framework at the group level to describe the type of compliance requirements any child project needs to follow. -- [**Compliance pipelines**](../user/group/compliance_frameworks.md#compliance-pipelines) (for groups): Define a - pipeline configuration to run for any projects with a given compliance framework. +| Feature | Instances | Groups | Projects | Description | +|:------------------------------------------------------------------------------------|:-----------------------|:-----------------------|:-----------------------|:-------------------------------------------------------------------------------------------| +| [Compliance frameworks](../user/group/compliance_frameworks.md) | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Describe the type of compliance requirements projects must follow. | +| [Compliance pipelines](../user/group/compliance_frameworks.md#compliance-pipelines) | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Define a pipeline configuration to run for any projects with a given compliance framework. | ## Audit management An important part of any compliance program is being able to go back and understand -what happened, when it happened, and who was responsible. This is useful in audit +what happened, when it happened, and who was responsible. You can use this in audit situations as well as for understanding the root cause of issues when they occur. -It is useful to have both low-level, raw lists of audit data as well as high-level, + +It is helpful to have both low-level, raw lists of audit data as well as high-level, summary lists of audit data. Between these two, compliance teams can quickly identify if problems exist and then drill down into the specifics of those issues. These features can help provide visibility into GitLab and audit what is happening: -- [**Audit events**](audit_events.md) (for instances, groups, and projects): To - maintain the integrity of your code, audit events give administrators the - ability to view any modifications made within the GitLab server in an advanced - audit events system, so you can control, analyze, and track every change. -- [**Audit reports**](audit_reports.md) (for instances, groups, and projects): - Create and access reports based on the audit events that have occurred. Use - pre-built GitLab reports or the API to build your own. -- [**Auditor users**](auditor_users.md) (for instances): Auditor users are users - who are given read-only access to all projects, groups, and other resources on - the GitLab instance. -- [**Compliance report**](../user/compliance/compliance_report/index.md) (for - groups): Quickly get visibility into the compliance posture of your organization. +| Feature | Instances | Groups | Projects | Description | +|:-------------------------------------------------------------------|:-----------------------|:-----------------------|:-----------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [Audit events](audit_events.md) | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | To maintain the integrity of your code, audit events give administrators the ability to view any modifications made in the GitLab server in an advanced audit events system, so you can control, analyze, and track every change. | +| [Audit reports](audit_reports.md) | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Create and access reports based on the audit events that have occurred. Use pre-built GitLab reports or the API to build your own. | +| [Auditor users](auditor_users.md) | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Auditor users are users who are given read-only access to all projects, groups, and other resources on the GitLab instance. | +| [Compliance report](../user/compliance/compliance_report/index.md) | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Quickly get visibility into the compliance posture of your organization. | ## Other compliance features These features can also help with compliance requirements: -- [**Email all users of a project, group, or entire server**](../user/admin_area/email_from_gitlab.md) - (for instances): An administrator can email groups of users based on project - or group membership, or email everyone using the GitLab instance. These emails - are great for scheduled maintenance or upgrades. -- [**Enforce ToS acceptance**](../user/admin_area/settings/terms.md) (for - instances): Enforce your users accepting new terms of service by blocking GitLab - traffic. -- [**External Status Checks**](../user/project/merge_requests/status_checks.md) - (for projects): Interface with third-party systems you already use during - development to ensure you remain compliant. -- [**Generate reports on permission levels of users**](../user/admin_area/index.md#user-permission-export) - (for instances): Administrators can generate a report listing all users' access - permissions for groups and projects in the instance. -- [**License compliance**](../user/compliance/license_compliance/index.md) (for - projects): Search dependencies for their licenses. This lets you determine if - the licenses of your project's dependencies are compatible with your project's - license. -- [**Lock project membership to group**](../user/group/access_and_permissions.md#prevent-members-from-being-added-to-projects-in-a-group) - (for groups): Group owners can prevent new members from being added to projects - within a group. -- [**LDAP group sync**](auth/ldap/ldap_synchronization.md#group-sync) (for - instances): Gives administrators the ability to automatically sync groups and - manage SSH keys, permissions, and authentication, so you can focus on building - your product, not configuring your tools. -- [**LDAP group sync filters**](auth/ldap/ldap_synchronization.md#group-sync) - (for instances): Gives more flexibility to synchronize with LDAP based on - filters, meaning you can leverage LDAP attributes to map GitLab permissions. -- [**Omnibus GitLab package supports log forwarding**](https://docs.gitlab.com/omnibus/settings/logs.html#udp-log-forwarding) - (for instances): Forward your logs to a central system. -- [**Restrict SSH Keys**](../security/ssh_keys_restrictions.md) (for instances): - Control the technology and key length of SSH keys used to access GitLab. +| Feature | Instances | Groups | Projects | Description | +|:------------------------------------------------------------------------------------------------------------------------------------|:-----------------------|:-----------------------|:-----------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [Email all users of a project,
group, or entire server](../user/admin_area/email_from_gitlab.md) | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Email groups of users based on project or group membership, or email everyone using the GitLab instance. These emails are great for scheduled maintenance or upgrades. | +| [Enforce ToS acceptance](../user/admin_area/settings/terms.md) | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Enforce your users accepting new terms of service by blocking GitLab traffic. | +| [External Status Checks](../user/project/merge_requests/status_checks.md) | **{dotted-circle}** No | **{dotted-circle}** No | **{check-circle}** Yes | Interface with third-party systems you already use during development to ensure you remain compliant. | +| [Generate reports on permission
levels of users](../user/admin_area/index.md#user-permission-export) | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Generate a report listing all users' access permissions for groups and projects in the instance. | +| [License compliance](../user/compliance/license_compliance/index.md) | **{dotted-circle}** No | **{dotted-circle}** No | **{check-circle}** Yes | Search dependencies for their licenses. This lets you determine if the licenses of your project's dependencies are compatible with your project's license. | +| [Lock project membership to group](../user/group/access_and_permissions.md#prevent-members-from-being-added-to-projects-in-a-group) | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Group owners can prevent new members from being added to projects in a group. | +| [LDAP group sync](auth/ldap/ldap_synchronization.md#group-sync) | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Automatically synchronize groups and manage SSH keys, permissions, and authentication, so you can focus on building your product, not configuring your tools. | +| [LDAP group sync filters](auth/ldap/ldap_synchronization.md#group-sync) | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Gives more flexibility to synchronize with LDAP based on filters, meaning you can leverage LDAP attributes to map GitLab permissions. | +| [Omnibus GitLab package supports
log forwarding](https://docs.gitlab.com/omnibus/settings/logs.html#udp-log-forwarding) | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Forward your logs to a central system. | +| [Restrict SSH Keys](../security/ssh_keys_restrictions.md) | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Control the technology and key length of SSH keys used to access GitLab. | diff --git a/doc/administration/configure.md b/doc/administration/configure.md index bf618345a94..d68e81ebf69 100644 --- a/doc/administration/configure.md +++ b/doc/administration/configure.md @@ -7,44 +7,39 @@ type: reference # Configure your GitLab installation **(FREE SELF)** -Customize and configure your self-managed GitLab installation. Here are some quick links to get you started: +Customize and configure your self-managed GitLab installation. - [Authentication](auth/index.md) +- [CI/CD](../administration/cicd.md) - [Configuration](../user/admin_area/index.md) -- [Repository storage](repository_storage_paths.md) -- [Geo](geo/index.md) -- [Packages](packages/index.md) - -The following tables are intended to guide you to choose the right combination of capabilities based on your requirements. It is common to want the most -available, quickly recoverable, highly performant, and fully data resilient solution. However, there are tradeoffs. - -The tables lists features on the left and provides their capabilities to the right along with known trade-offs. - -## Gitaly Capabilities - -| | Availability | Recoverability | Data Resiliency | Performance | Risks/Trade-offs| -|-|--------------|----------------|-----------------|-------------|-----------------| -|Gitaly Cluster | Very high - tolerant of node failures | RTO for a single node of 10 s with no manual intervention | Data is stored on multiple nodes | Good - While writes may take slightly longer due to voting, read distribution improves read speeds | **Trade-off** - Slight decrease in write speed for redundant, strongly-consistent storage solution. **Risks** - [Does not support snapshot backups](gitaly/index.md#snapshot-backup-and-recovery-limitations), GitLab backup task can be slow for large data sets | -|Gitaly Shards | Single storage location is a single point of failure | Would need to restore only shards which failed | Single point of failure | Good - can allocate repositories to shards to spread load | **Trade-off** - Need to manually configure repositories into different shards to balance loads / storage space **Risks** - Single point of failure relies on recovery process when single-node failure occurs | -|Gitaly + NFS | Single storage location is a single point of failure | Single node failure requires restoration from backup | Single point of failure | Average - NFS is not ideally suited to large quantities of small reads / writes which can have a detrimental impact on performance | **Trade-off** - Familiar administration though NFS is not ideally suited to Git demands **Risks** - Many instances of NFS compatibility issues which provide very poor customer experiences | - -## Geo Capabilities - -If your availability needs to span multiple zones or multiple locations, read about [Geo](geo/index.md). - -| | Availability | Recoverability | Data Resiliency | Performance | Risks/Trade-offs| -|-|--------------|----------------|-----------------|-------------|-----------------| -|Geo| Depends on the architecture of the Geo site. It is possible to deploy secondaries in single and multiple node configurations. | Eventually consistent. Recovery point depends on replication lag, which depends on a number of factors such as network speeds. Geo supports failover from a primary to secondary site using manual commands that are scriptable. | Geo replicates 100% of planned data types and verifies 50%. See [limitations table](geo/replication/datatypes.md#limitations-on-replicationverification) for more detail. | Improves read/clone times for users of a secondary. | Geo is not intended to replace other backup/restore solutions. Because of replication lag and the possibility of replicating bad data from a primary, customers should also take regular backups of their primary site and test the restore process. | - -## Scenarios for failure modes and available mitigation paths - -The following table outlines failure modes and mitigation paths for the product offerings detailed in the tables above. Note - Gitaly Cluster install assumes an odd number replication factor of 3 or greater - -| Gitaly Mode | Loss of Single Gitaly Node | Application / Data Corruption | Regional Outage (Loss of Instance) | Notes | -| ----------- | -------------------------- | ----------------------------- | ---------------------------------- | ----- | -| Single Gitaly Node | Downtime - Must restore from backup | Downtime - Must restore from Backup | Downtime - Must wait for outage to end | | -| Single Gitaly Node + Geo Secondary | Downtime - Must restore from backup, can perform a manual failover to secondary | Downtime - Must restore from Backup, errors could have propagated to secondary | Manual intervention - failover to Geo secondary | | -| Sharded Gitaly Install | Partial Downtime - Only repositories on impacted node affected, must restore from backup | Partial Downtime - Only repositories on impacted node affected, must restore from backup | Downtime - Must wait for outage to end | | -| Sharded Gitaly Install + Geo Secondary | Partial Downtime - Only repositories on impacted node affected, must restore from backup, could perform manual failover to secondary for impacted repositories | Partial Downtime - Only repositories on impacted node affected, must restore from backup, errors could have propagated to secondary | Manual intervention - failover to Geo secondary | | -| Gitaly Cluster Install* | No Downtime - swaps repository primary to another node after 10 seconds | Not applicable; All writes are voted on by multiple Gitaly Cluster nodes | Downtime - Must wait for outage to end | Snapshot backups for Gitaly Cluster nodes not supported at this time | -| Gitaly Cluster Install* + Geo Secondary | No Downtime - swaps repository primary to another node after 10 seconds | Not applicable; All writes are voted on by multiple Gitaly Cluster nodes | Manual intervention - failover to Geo secondary | Snapshot backups for Gitaly Cluster nodes not supported at this time | +- [Consul](../administration/consul.md) +- [Environment variables](../administration/environment_variables.md) +- [File hooks](../administration/file_hooks.md) +- [Git protocol v2](../administration/git_protocol.md) +- [Incoming email](../administration/incoming_email.md) +- [Instance limits](../administration/instance_limits.md) +- [Instance Review](../administration/instance_review.md) +- [PostgreSQL](../administration/postgresql/index.md) +- [Load balancer](../administration/load_balancer.md) +- [NFS](../administration/nfs.md) +- [Postfix](../administration/reply_by_email_postfix_setup.md) +- [Redis](../administration/redis/index.md) +- [Sidekiq](../administration/sidekiq/index.md) +- [S/MIME signing](../administration/smime_signing_email.md) +- [Repository storage](../administration/repository_storage_paths.md) +- [Object storage](../administration/object_storage.md) +- [Merge request diffs storage](../administration/merge_request_diffs.md) +- [Static objects external storage](../administration/static_objects_external_storage.md) +- [Geo](../administration/geo/index.md) +- [Disaster recovery (Geo)](../administration/geo/disaster_recovery/index.md) +- [Agent server for Kubernetes](../administration/clusters/kas.md) +- [Server hooks](../administration/server_hooks.md) +- [Terraform state](../administration/terraform_state.md) +- [Terraform limits](../user/admin_area/settings/terraform_limits.md) +- [Packages](../administration/packages/index.md) +- [Web terminals](../administration/integration/terminal.md) +- [Wikis](../administration/wikis/index.md) +- [Invalidate Markdown cache](../administration/invalidate_markdown_cache.md) +- [Issue closing pattern](../administration/issue_closing_pattern.md) +- [Snippets](../administration/snippets/index.md) +- [Host the product documentation](../administration/docs_self_host.md) diff --git a/doc/administration/dedicated/index.md b/doc/administration/dedicated/index.md index 22b8cc13637..f2caa5fa368 100644 --- a/doc/administration/dedicated/index.md +++ b/doc/administration/dedicated/index.md @@ -26,11 +26,12 @@ To request the creation of a new GitLab Dedicated environment for your organizat - Desired backup region: An AWS region where the primary backups of your data are replicated. This can be the same as the primary or secondary region or different. - Desired instance subdomain: The main domain for GitLab Dedicated instances is `gitlab-dedicated.com`. You get to choose the subdomain name where your instance is accessible from (for example, `customer_name.gitlab-dedicated.com`). - Initial storage: Initial storage size for your repositories in GB. -- Availability Zone IDs for PrivateLink: If you plan to later add a PrivateLink connection (either [inbound](#inbound-private-link) or [outbound](#outbound-private-link)) to your environment, and you require the connections to be available in specific Availability Zones, you must provide up to two [Availability Zone IDs](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#az-ids) during onboarding. If not specified, GitLab will select two random Availability Zone IDs in which the connections will be available. +- Availability Zone IDs for PrivateLink: If you plan to later add a PrivateLink connection (either [inbound](#inbound-private-link) or [outbound](#outbound-private-link)) to your environment, and you require the connections to be available in specific Availability Zones, you must provide up to two [Availability Zone IDs](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#az-ids) during onboarding. If not specified, GitLab selects two random Availability Zone IDs where the connections are available. +- [KMS keys](https://docs.aws.amazon.com/kms/latest/developerguide/overview.html) for encrypted AWS services (if you are using that functionality). ### Maintenance window -When onboarding, you must also specify your preference for the weekly four-hour timeslot that GitLab uses to perform maintenance and upgrade operations on the tenant instance. +When onboarding, you must also specify your preference for the weekly four-hour time slot that GitLab uses to perform maintenance and upgrade operations on the tenant instance. - APAC (outside working hours): Wednesday 1pm-5pm UTC - EU (outside working hours): Tuesday 1am-5am UTC @@ -45,13 +46,118 @@ To change or update the configuration for your GitLab Dedicated instance, open a The turnaround time for processing configuration change requests is [documented in the GitLab handbook](https://about.gitlab.com/handbook/engineering/infrastructure/team/gitlab-dedicated/#handling-configuration-changes-for-tenant-environments). +### Encrypted Data At Rest (BYOK) + +If you want your GitLab data to be encrypted at rest, the KMS keys used must be accessible by GitLab services. KMS keys can be used in two modes for this purpose: + +1. Per-service KMS keys (Backup, EBS, RDS, S3), or +1. One KMS key for all services. + +If you use a key per service, all services must be encrypted at rest. Selective enablement of this feature is not supported. + +The keys provided have to reside in the same primary and secondary region specified during [onboarding](#onboarding). + +For instructions on how to create and manage KMS keys, visit [Managing keys](https://docs.aws.amazon.com/kms/latest/developerguide/getting-started.html) in the AWS KMS documentation. + +To create a KMS key using the AWS Console: + +1. In `Configure key`, select: + 1. Key type: **Symmetrical** + 1. Key usage: **Encrypt and decrypt** + 1. `Advanced options`: + 1. Key material origin: **KMS** + 1. Regionality: **Multi-Region key** +1. Enter your values for key alias, description, and tags. +1. Select Key administrators (optionally allow or deny key administrators to delete the key). +1. For Key usage permissions, add the GitLab AWS account using the **Other AWS accounts** dialog. + +The last page asks you to confirm the KMS key policy. It should look similar to the following example, populated with your account IDs and usernames: + +```json +{ + "Version": "2012-10-17", + "Id": "byok-key-policy", + "Statement": [ + { + "Sid": "Enable IAM User Permissions", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam:::root" + }, + "Action": "kms:*", + "Resource": "*" + }, + { + "Sid": "Allow access for Key Administrators", + "Effect": "Allow", + "Principal": { + "AWS": [ + "arn:aws:iam:::user/" + ] + }, + "Action": [ + "kms:Create*", + "kms:Describe*", + "kms:Enable*", + "kms:List*", + "kms:Put*", + "kms:Update*", + "kms:Revoke*", + "kms:Disable*", + "kms:Get*", + "kms:Delete*", + "kms:TagResource", + "kms:UntagResource", + "kms:ScheduleKeyDeletion", + "kms:CancelKeyDeletion", + "kms:ReplicateKey", + "kms:UpdatePrimaryRegion" + ], + "Resource": "*" + }, + { + "Sid": "Allow use of the key", + "Effect": "Allow", + "Principal": { + "AWS": [ + "arn:aws:iam:::root" + ] + }, + "Action": [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:DescribeKey" + ], + "Resource": "*" + }, + { + "Sid": "Allow attachment of persistent resources", + "Effect": "Allow", + "Principal": { + "AWS": [ + "arn:aws:iam:::root" + ] + }, + "Action": [ + "kms:CreateGrant", + "kms:ListGrants", + "kms:RevokeGrant" + ], + "Resource": "*" + } + ] +} +``` + ### Inbound Private Link [AWS Private Link](https://docs.aws.amazon.com/vpc/latest/privatelink/what-is-privatelink.html) allows users and applications in your VPC on AWS to securely connect to the GitLab Dedicated endpoint without network traffic going over the public internet. To enable the Inbound Private Link: -1. In the body of your [support ticket](#configuration-changes), include the IAM Principal for the AWS user or role in your own AWS Organization that will be establishing the VPC endpoint within your AWS account. GitLab Dedicated uses this IAM Principal for access-control. This IAM principal will be the only one able to set up an endpoint to the service. +1. In the body of your [support ticket](#configuration-changes), include the IAM Principal for the AWS user or role in your own AWS Organization that's establishing the VPC endpoint in your AWS account. GitLab Dedicated uses this IAM Principal for access-control. This IAM principal is the only one able to set up an endpoint to the service. 1. After your IAM Principal has been allowlisted, GitLab [creates the Endpoint Service](https://docs.aws.amazon.com/vpc/latest/privatelink/create-endpoint-service.html) and communicates the `Service Endpoint Name` on the support ticket. The service name is generated by AWS upon creation of the service endpoint. - GitLab handles the domain verification for the Private DNS name, so that DNS resolution of the tenant instance domain name in your VPC resolves to the PrivateLink endpoint. - GitLab makes the Endpoint Service available in the Availability Zones you specified during the initial onboarding. If you did not specify any Availability Zones, GitLab randomly selects the Availability Zones IDs. @@ -69,8 +175,8 @@ Outbound Private Links allow the GitLab Dedicated instance to securely communica To enable an Outbound Private Link: 1. In your [support ticket](#configuration-changes), GitLab provides you with an IAM role ARN that connects to your endpoint service. You can then add this ARN to the allowlist on your side to restrict connections to your endpoint service. -1. [Create the Endpoint service](https://docs.aws.amazon.com/vpc/latest/privatelink/create-endpoint-service.html) through which your internal service will be made available to GitLab Dedicated. Provide the associated `Service Endpoint Name` on the [support ticket](#configuration-changes). -1. When creating the Endpoint service, you must provide GitLab with a [verified Private DNS Name](https://docs.aws.amazon.com/vpc/latest/privatelink/manage-dns-names.html#verify-domain-ownership) for your service. Optionally, if you would like GitLab Dedicated to reach your service via other aliases, you have the ability to specify a list of Private Hosted Zone (PHZ) entries. With this option, GitLab sets up a Private Hosted Zone with DNS names aliased to the verified Private DNS name. To enable this functionality, you must provide GitLab a list of PHZ entries on your support ticket. After the PHZ has been created in the tenant environment, DNS resolution of any of the provided records will correctly resolve to the PrivateLink endpoint. +1. [Create the Endpoint service](https://docs.aws.amazon.com/vpc/latest/privatelink/create-endpoint-service.html) through which your internal service is available to GitLab Dedicated. Provide the associated `Service Endpoint Name` on the [support ticket](#configuration-changes). +1. When creating the Endpoint service, you must provide GitLab with a [verified Private DNS Name](https://docs.aws.amazon.com/vpc/latest/privatelink/manage-dns-names.html#verify-domain-ownership) for your service. Optionally, if you would like GitLab Dedicated to reach your service via other aliases, you have the ability to specify a list of Private Hosted Zone (PHZ) entries. With this option, GitLab sets up a Private Hosted Zone with DNS names aliased to the verified Private DNS name. To enable this functionality, you must provide GitLab a list of PHZ entries on your support ticket. After the PHZ is created in the tenant environment, DNS resolution of any of the provided records correctly resolves to the PrivateLink endpoint. 1. GitLab then configures the tenant instance to create the necessary Endpoint Interfaces based on the service names you provided. Any outbound calls made from the tenant GitLab instance are directed through the PrivateLink into your VPC. #### Custom certificates @@ -97,14 +203,13 @@ Prerequisites: To activate SAML for your GitLab Dedicated instance: -1. Read the [GitLab documentation about SAML](../../integration/saml.md#https://docs.gitlab.com/ee/integration/saml.html#configure-saml-on-your-idp) to gather all data your identity provider requires for configuration. You can also find some providers and their requirements in the [group SAML documentation](../../user/group/saml_sso/index.md#providers). -1. To make the necessary changes, include in your [support ticket](#configuration-changes) the desired [SAML configuration block](../../integration/saml.md#configure-saml-support-in-gitlab) that will be set on the GitLab application. At a minimum, GitLab needs the following information to enable SAML for your instance: +1. To make the necessary changes, include the desired [SAML configuration block](../../integration/saml.md#configure-saml-support-in-gitlab) for your GitLab application in your [support ticket](#configuration-changes). At a minimum, GitLab needs the following information to enable SAML for your instance: - Assertion consumer service URL - Certificate fingerprint or certificate - NameID format - SSO login button description -1. After GitLab deploys the SAML configuration to your instance, you will be notified on your support ticket. +1. After GitLab deploys the SAML configuration to your instance, you are notified on your support ticket. 1. To verify the SAML configuration is successful: - Check that the SSO login button description is displayed on your instance's login page. - Go to the metadata URL of your instance (`https://INSTANCE-URL/users/auth/saml/metadata`). This page can be used to simplify much of the configuration of the identity provider, as well as manually validate the settings. @@ -113,7 +218,7 @@ To activate SAML for your GitLab Dedicated instance: If [SAML request signing](../../integration/saml.md#sign-saml-authentication-requests-optional) is desired, a certificate must be obtained. This certificate can be self-signed which has the advantage of not having to prove ownership of an arbitrary Common Name (CN) to a public Certificate Authority (CA)). -To enable SAML request signing, indicate on your SAML [support ticket](#configuration-changes) that you would like request signing enabled. GitLab will then work with you on sending the Certificate Signing Request (CSR) for you to sign. Alternatively, the CSR can be signed with a public CA. After the certificate is signed, GitLab will add the certificate and its associated private key to the `security` section of the SAML configuration. Authentication requests from GitLab to your identity provider will then be signed. +To enable SAML request signing, indicate on your SAML [support ticket](#configuration-changes) that you want request signing enabled. GitLab works with you on sending the Certificate Signing Request (CSR) for you to sign. Alternatively, the CSR can be signed with a public CA. After the certificate is signed, GitLab adds the certificate and its associated private key to the `security` section of the SAML configuration. Authentication requests from GitLab to your identity provider can then be signed. #### SAML groups @@ -136,5 +241,5 @@ GitLab [application logs](../../administration/logs/index.md) are delivered to a To gain read only access to this bucket: -1. Open a [support ticket](#configuration-changes) with the title "Customer Log Access". In the body of the ticket, include a list of IAM Principal ARNs (users or roles) that will be fetching the logs from S3. -1. GitLab will then inform you of the name of the S3 bucket. Your nominated users/roles will then be able to list and get all objects in the S3 bucket. +1. Open a [support ticket](#configuration-changes) with the title "Customer Log Access". In the body of the ticket, include a list of IAM Principal ARNs (users or roles) that are fetching the logs from S3. +1. GitLab then informs you of the name of the S3 bucket. Your nominated users/roles can then able to list and get all objects in the S3 bucket. diff --git a/doc/administration/docs_self_host.md b/doc/administration/docs_self_host.md index 97eff35da91..d1ad36880dd 100644 --- a/doc/administration/docs_self_host.md +++ b/doc/administration/docs_self_host.md @@ -22,7 +22,7 @@ To host the GitLab product documentation, you can use: - GitLab Pages - Your own web server -After you create a website by using one of these methods, you redirect the UI links +After you create a website by using one of these methods, redirect the UI links in the product to point to your website. NOTE: @@ -41,7 +41,7 @@ In the following example, we expose this on the host under the same port. Make sure you either: - Allow port `4000` in your firewall. -- Use a different port. In following examples, replace the leftmost `4000` with the port different port. +- Use a different port. In following examples, replace the leftmost `4000` with a different port number. To run the GitLab product documentation website in a Docker container: @@ -74,9 +74,9 @@ To run the GitLab product documentation website in a Docker container: docker-compose up -d ``` -1. Visit `http://0.0.0.0:4000` to view the documentation website and verify +1. Visit `http://0.0.0.0:4000` to view the documentation website and verify that it works. -1. [Redirect the help links to the new Docs site](#redirect-the-help-links-to-the-new-docs-site). +1. [Redirect the help links to the new documentation site](#redirect-the-help-links-to-the-new-docs-site). ### Self-host the product documentation with GitLab Pages @@ -84,7 +84,7 @@ You can use GitLab Pages to host the GitLab product documentation. Prerequisite: -- Ensure the Pages site URL does not use a subfolder. Because of how the docs +- Ensure the Pages site URL does not use a subfolder. Because of the way the site is pre-compiled, the CSS and JavaScript files are relative to the main domain or subdomain. For example, URLs like `https://example.com/docs/` are not supported. @@ -114,13 +114,13 @@ To host the product documentation site with GitLab Pages: | [Project website](../user/project/pages/getting_started_part_one.md#project-website-examples) | Not supported | Supported | | [User or group website](../user/project/pages/getting_started_part_one.md#user-and-group-website-examples) | Supported | Supported | -1. [Redirect the help links to the new Docs site](#redirect-the-help-links-to-the-new-docs-site). +1. [Redirect the help links to the new documentation site](#redirect-the-help-links-to-the-new-docs-site). ### Self-host the product documentation on your own web server Because the product documentation site is static, you can take the contents of `/usr/share/nginx/html` from inside the container, and use your own web server to host -the docs wherever you want. +the documentation wherever you want. The `html` directory should be served as is and it has the following structure: @@ -135,7 +135,7 @@ In this example: - `index.html` is a simple HTML file that redirects to the directory containing the documentation. In this case, `14.5/`. -To extract the HTML files of the Docs site: +To extract the HTML files of the documentation site: 1. Create the container that holds the HTML files of the documentation website: @@ -158,36 +158,40 @@ To extract the HTML files of the Docs site: ``` 1. Point your web server to serve the contents of `/srv/gitlab/html/`. -1. [Redirect the help links to the new Docs site](#redirect-the-help-links-to-the-new-docs-site). +1. [Redirect the help links to the new documentation site](#redirect-the-help-links-to-the-new-docs-site). ## Redirect the `/help` links to the new Docs site After your local product documentation site is running, [redirect the help links](../user/admin_area/settings/help_page.md#redirect-help-pages) in the GitLab application to your local site, by using the fully qualified domain -name as the docs URL. For example, if you used the +name as the documentation URL. For example, if you used the [Docker method](#self-host-the-product-documentation-with-docker), enter `http://0.0.0.0:4000`. You don't need to append the version. GitLab detects it and appends it to documentation URL requests as needed. For example, if your GitLab version is 14.5: -- The GitLab Docs URL becomes `http://0.0.0.0:4000/14.5/`. +- The GitLab documentation URL becomes `http://0.0.0.0:4000/14.5/`. - The link in GitLab displays as `/help/user/admin_area/settings/help_page#destination-requirements`. - When you select the link, you are redirected to `http://0.0.0.0:4000/14.5/ee/user/admin_area/settings/help_page/#destination-requirements`. -To test the setting, select a **Learn more** link in the GitLab application. +To test the setting, in GitLab, select a **Learn more** link. For example: + +1. On the top bar, in the upper-right corner, select your avatar. +1. Select **Preferences**. +1. In the **Syntax highlighting theme** section, select **Learn more**. ## Upgrade the product documentation to a later version -Upgrading the Docs site to a later version requires downloading the newer Docker image tag. +Upgrading the documentation site to a later version requires downloading the newer Docker image tag. ### Upgrade using Docker To upgrade to a later version [using Docker](#self-host-the-product-documentation-with-docker): -- If you use plain Docker: +- If you use Docker: 1. Stop the running container: @@ -207,7 +211,7 @@ To upgrade to a later version [using Docker](#self-host-the-product-documentatio docker run --detach --name gitlab_docs -it --rm -p 4000:4000 registry.gitlab.com/gitlab-org/gitlab-docs:14.6 ``` -- If you use Docker compose: +- If you use Docker Compose: 1. Change the version in `docker-compose.yaml`, for example 14.6: @@ -231,19 +235,19 @@ To upgrade to a later version [using Docker](#self-host-the-product-documentatio To upgrade to a later version [using GitLab Pages](#self-host-the-product-documentation-with-gitlab-pages): -1. Edit your existing `.gitlab-ci.yml` file, and replace the `image`'s version number: +1. Edit your existing `.gitlab-ci.yml` file, and replace the `image` version number: ```yaml image: registry.gitlab.com/gitlab-org/gitlab-docs:14.5 ``` -1. Commit the changes, push, and GitLab Pages pulls the new Docs site version. +1. Commit the changes, push, and GitLab Pages pulls the new documentation site version. ### Upgrade using your own web-server To upgrade to a later version [using your own web-server](#self-host-the-product-documentation-on-your-own-web-server): -1. Copy the HTML files of the Docs site: +1. Copy the HTML files of the documentation site: ```shell docker create -it --name gitlab_docs registry.gitlab.com/gitlab-org/gitlab-docs:14.6 @@ -261,8 +265,5 @@ To upgrade to a later version [using your own web-server](#self-host-the-product If you self-host the product documentation: -- The version dropdown list displays additional versions that don't exist. Selecting - these versions displays a `404 Not Found` page. -- The search displays results from `docs.gitlab.com` and not the local site. - By default, the landing page redirects to the respective version (for example, `/14.5/`). This causes the landing page to not be displayed. diff --git a/doc/administration/environment_variables.md b/doc/administration/environment_variables.md index d2edc3085f1..a453d703a18 100644 --- a/doc/administration/environment_variables.md +++ b/doc/administration/environment_variables.md @@ -35,8 +35,8 @@ You can use the following environment variables to override certain values: | `GITLAB_ROOT_PASSWORD` | string | Sets the password for the `root` user on installation. | | `GITLAB_SHARED_RUNNERS_REGISTRATION_TOKEN` | string | Sets the initial registration token used for runners. | | `RAILS_ENV` | string | The Rails environment; can be one of `production`, `development`, `staging`, or `test`. | -| `UNSTRUCTURED_RAILS_LOG` | string | Enables the unstructured log in addition to JSON logs (defaults to `true`). | | `GITLAB_RAILS_CACHE_DEFAULT_TTL_SECONDS` | integer | The default TTL used for entries stored in the Rails-cache. Default is `28800`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/95042) in 15.3. | +| `GITLAB_CI_CONFIG_FETCH_TIMEOUT_SECONDS` | integer | Timeout for resolving remote includes in CI config in seconds. Must be between `0` and `60`. Default is `30`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/116383) in 15.11. | ## Adding more variables diff --git a/doc/administration/external_pipeline_validation.md b/doc/administration/external_pipeline_validation.md index 2dec3857f75..9d182a95fc9 100644 --- a/doc/administration/external_pipeline_validation.md +++ b/doc/administration/external_pipeline_validation.md @@ -160,5 +160,4 @@ required number of seconds. } ``` -The `namespace` field is only available in [GitLab Premium](https://about.gitlab.com/pricing/) -and higher. +The `namespace` field is only available in [GitLab Premium and Ultimate](https://about.gitlab.com/pricing/). diff --git a/doc/administration/feature_flags.md b/doc/administration/feature_flags.md index f7237b167e5..26deff0ca84 100644 --- a/doc/administration/feature_flags.md +++ b/doc/administration/feature_flags.md @@ -41,11 +41,15 @@ GitLab, the feature flag status may change. ## Risks when enabling features still in development +Before enabling a disabled feature flag in a production GitLab environment, it is crucial to understand the potential risks involved. + +WARNING: +Data corruption, stability degradation, performance degradation, and security issues may occur if you enable a feature that's disabled by default. + Features that are disabled by default may change or be removed without notice in a future version of GitLab. -Data corruption, stability degradation, performance degradation, or security issues might occur if -you enable a feature that's disabled by default. Problems caused by using a default -disabled feature aren't covered by GitLab Support. +Features behind default-disabled feature flags are not recommended for use in a production environment +and problems caused by using a default disabled features aren't covered by GitLab Support. Security issues found in features that are disabled by default are patched in regular releases and do not follow our regular [maintenance policy](../policy/maintenance.md#security-releases) diff --git a/doc/administration/file_hooks.md b/doc/administration/file_hooks.md index b74ee22d584..dfddf38475c 100644 --- a/doc/administration/file_hooks.md +++ b/doc/administration/file_hooks.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments" type: reference --- diff --git a/doc/administration/geo/disaster_recovery/background_verification.md b/doc/administration/geo/disaster_recovery/background_verification.md index 99ac95cd536..ea4523c66e6 100644 --- a/doc/administration/geo/disaster_recovery/background_verification.md +++ b/doc/administration/geo/disaster_recovery/background_verification.md @@ -6,12 +6,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Automatic background verification **(PREMIUM SELF)** -NOTE: -Automatic background verification of repositories and wikis was added in -GitLab EE 10.6 but is enabled by default only on GitLab EE 11.1. You can -disable or enable this feature manually by following -[these instructions](#disabling-or-enabling-the-automatic-background-verification). - Automatic background verification ensures that the transferred data matches a calculated checksum. If the checksum of the data on the **primary** site matches checksum of the data on the **secondary** site, the data transferred successfully. Following a planned failover, @@ -179,19 +173,5 @@ If the **primary** and **secondary** sites have a checksum verification mismatch ## Current limitations -Automatic background verification doesn't cover attachments, LFS objects, -job artifacts, and user uploads in file storage. You can keep track of the -progress to include them in [Geo: Verify all replicated data](https://gitlab.com/groups/gitlab-org/-/epics/1430). - -For now, you can verify their integrity -manually by following [these instructions](../../raketasks/check.md) on both -sites, and comparing the output between them. - -In GitLab EE 12.1, Geo calculates checksums for attachments, LFS objects, and -archived traces on secondary sites after the transfer, compares it with the -stored checksums, and rejects transfers if mismatched. Geo -currently does not support an automatic way to verify these data if they have -been synced before GitLab EE 12.1. - -Data in object storage is **not verified**, as the object store is responsible -for ensuring the integrity of the data. +For more information on what replication and verification methods are supported, +see the [supported Geo data types](../replication/datatypes.md). diff --git a/doc/administration/geo/disaster_recovery/planned_failover.md b/doc/administration/geo/disaster_recovery/planned_failover.md index e9eb154d398..a52bdc759a2 100644 --- a/doc/administration/geo/disaster_recovery/planned_failover.md +++ b/doc/administration/geo/disaster_recovery/planned_failover.md @@ -38,9 +38,9 @@ performed ahead of the maintenance window; subsequent `rsync`s (including a final transfer inside the maintenance window) then transfers only the *changes* between the **primary** site and the **secondary** sites. -Repository-centric strategies for using `rsync` effectively can be found in the +Git repository-centric strategies for using `rsync` effectively can be found in the [moving repositories](../../operations/moving_repositories.md) documentation; these strategies can -be adapted for use with any other file-based data, such as [GitLab Pages](../../pages/index.md#change-storage-path). +be adapted for use with any other file-based data. ### Container registry diff --git a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md index cef0101dd40..feb5e030b80 100644 --- a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md +++ b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_multi_node.md @@ -6,7 +6,7 @@ type: howto --- WARNING: -This runbook is in [**Alpha**](../../../../policy/alpha-beta-support.md#alpha-features). For complete, production-ready documentation, see the +This runbook is an [Experiment](../../../../policy/alpha-beta-support.md#experiment). For complete, production-ready documentation, see the [disaster recovery documentation](../index.md). # Disaster Recovery (Geo) promotion runbooks **(PREMIUM SELF)** diff --git a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md index 085765ec51e..19150027cca 100644 --- a/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md +++ b/doc/administration/geo/disaster_recovery/runbooks/planned_failover_single_node.md @@ -6,7 +6,7 @@ type: howto --- WARNING: -This runbook is in [**Alpha**](../../../../policy/alpha-beta-support.md#alpha-features). For complete, production-ready documentation, see the +This runbook is an [Experiment](../../../../policy/alpha-beta-support.md#experiment). For complete, production-ready documentation, see the [disaster recovery documentation](../index.md). # Disaster Recovery (Geo) promotion runbooks **(PREMIUM SELF)** diff --git a/doc/administration/geo/index.md b/doc/administration/geo/index.md index 3f98f1e12fe..31de7f5c62f 100644 --- a/doc/administration/geo/index.md +++ b/doc/administration/geo/index.md @@ -9,8 +9,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w Geo is the solution for widely distributed development teams and for providing a warm-standby as part of a disaster recovery strategy. -## Overview - WARNING: Geo undergoes significant changes from release to release. Upgrades are supported and [documented](#upgrading-geo), but you should ensure that you're @@ -123,7 +121,8 @@ The following are required to run Geo: - Note,[PostgreSQL 12 is deprecated](../../update/deprecations.md#postgresql-12-deprecated) and is removed in GitLab 16.0. - Git 2.9 or later - Git-lfs 2.4.2 or later on the user side when using LFS -- All sites must run [the same GitLab and PostgreSQL versions](setup/database.md#postgresql-replication). +- All sites must run the same GitLab version. +- All sites must run [the same PostgreSQL versions](setup/database.md#postgresql-replication). - If using different operating system versions between Geo sites, [check OS locale data compatibility](replication/troubleshooting.md#check-os-locale-data-compatibility) across Geo sites to avoid silent corruption of database indexes. diff --git a/doc/administration/geo/replication/configuration.md b/doc/administration/geo/replication/configuration.md index 912de360e43..5fa6df393b9 100644 --- a/doc/administration/geo/replication/configuration.md +++ b/doc/administration/geo/replication/configuration.md @@ -205,15 +205,16 @@ keys must be manually replicated to the **secondary** site. 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Geo > Sites**. 1. Select **Add site**. - ![Add secondary site](img/adding_a_secondary_v13_3.png) - 1. Fill in **Name** with the `gitlab_rails['geo_node_name']` in - `/etc/gitlab/gitlab.rb`. These values must always match *exactly*, character + ![Add secondary site](img/adding_a_secondary_v15_8.png) + 1. In **Name**, enter the value for `gitlab_rails['geo_node_name']` in + `/etc/gitlab/gitlab.rb`. These values must always match **exactly**, character for character. - 1. Fill in **URL** with the `external_url` in `/etc/gitlab/gitlab.rb`. These + 1. In **External URL**, enter the value for `external_url` in `/etc/gitlab/gitlab.rb`. These values must always match, but it doesn't matter if one ends with a `/` and the other doesn't. - 1. (Optional) Choose which groups or storage shards should be replicated by the - **secondary** site. Leave blank to replicate all. Read more in + 1. Optional. In **Internal URL (optional)**, enter an internal URL for the primary site. + 1. Optional. Select which groups or storage shards should be replicated by the + **secondary** site. Leave blank to replicate all. For more information, see [selective synchronization](#selective-synchronization). 1. Select **Save changes** to add the **secondary** site. 1. SSH into **each Rails, and Sidekiq node on your secondary** site and restart the services: @@ -257,12 +258,12 @@ You can safely skip this step if: #### Custom or self-signed certificate for inbound connections -If your GitLab Geo **primary** site uses a custom or [self-signed certificate to secure inbound HTTPS connections](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates), this certificate can either be single-domain certificate or multi-domain. +If your GitLab Geo **primary** site uses a custom or [self-signed certificate to secure inbound HTTPS connections](https://docs.gitlab.com/omnibus/settings/ssl/index.html#install-custom-public-certificates), this can be either a single-domain or multi-domain certificate. Install the correct certificate based on your certificate type: - **Multi-domain certificate** that includes both primary and secondary site domains: Install the certificate at `/etc/gitlab/ssl` on all **Rails, Sidekiq, and Gitaly** nodes in the **secondary** site. -- **Single-domain certificate** where the certificates are specific to each Geo site domain: Generate a valid certificate for your **secondary** site's domain and install it at `/etc/gitlab/ssl` per [these instructions](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) on all **Rails, Sidekiq, and Gitaly** nodes in the **secondary** site. +- **Single-domain certificate** where the certificates are specific to each Geo site domain: Generate a valid certificate for your **secondary** site's domain and install it at `/etc/gitlab/ssl` following [these instructions](https://docs.gitlab.com/omnibus/settings/ssl/index.html#install-custom-public-certificates) on all **Rails, Sidekiq, and Gitaly** nodes in the **secondary** site. #### Connecting to external services that use custom certificates @@ -303,7 +304,7 @@ If your **primary** site is using a [custom or self-signed certificate for inbou sudo gitlab-ctl reconfigure ``` -### Step 5. Enable Git access over HTTP/HTTPS +### Step 5. Enable Git access over HTTP/HTTPS and SSH Geo synchronizes repositories over HTTP/HTTPS, and therefore requires this clone method to be enabled. This is enabled by default, but if converting an existing site to Geo it should be checked: @@ -313,7 +314,10 @@ On the **primary** site: 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Settings > General**. 1. Expand **Visibility and access controls**. -1. Ensure "Enabled Git access protocols" is set to either "Both SSH and HTTP(S)" or "Only HTTP(S)". +1. If using Git over SSH, then: + 1. Ensure "Enabled Git access protocols" is set to "Both SSH and HTTP(S)". + 1. Follow [Fast lookup of authorized SSH keys in the database](../../operations/fast_ssh_key_lookup.md) on both primary and secondary sites. +1. If not using Git over SSH, then set "Enabled Git access protocols" to "Only HTTP(S)". ### Step 6. Verify proper functioning of the **secondary** site @@ -366,6 +370,9 @@ former is ideal for replicating data belonging to a subset of users, while the latter is more suited to progressively rolling out Geo to a large GitLab instance. +NOTE: +Geo's synchronization logic is outlined in the [documentation](../index.md). Both the solution and the documentation is subject to change from time to time. You must independently determine your legal obligations in regard to privacy and cybersecurity laws, and applicable trade control law on an ongoing basis. + Selective synchronization: 1. Does not restrict permissions from **secondary** sites. diff --git a/doc/administration/geo/replication/container_registry.md b/doc/administration/geo/replication/container_registry.md index 88ca8781dc3..66c67f29c1c 100644 --- a/doc/administration/geo/replication/container_registry.md +++ b/doc/administration/geo/replication/container_registry.md @@ -7,7 +7,7 @@ type: howto # Container Registry for a secondary site **(PREMIUM SELF)** -You can set up a Container Registry on your **secondary** Geo site that mirrors the one on the **primary** Geo site. +You can set up a Container Registry on your **secondary** Geo site that mirrors the one on the **primary** Geo site. NOTE: The Container Registry replication is used only for disaster recovery purposes. We do not recommend @@ -76,7 +76,9 @@ the **primary** site before following the next steps. We need to make Container Registry send notification events to the **primary** site. -1. SSH into your GitLab **primary** server and login as root: +For each application and Sidekiq node on the **primary** site: + +1. SSH into the node and login as the `root` user: ```shell sudo -i @@ -105,17 +107,15 @@ We need to make Container Registry send notification events to the that starts with a letter. You can generate one with `< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c 32 | sed "s/^[0-9]*//"; echo` NOTE: - If you use an external Registry (not the one integrated with GitLab), you must add - these settings to its configuration yourself. In this case, you also have to specify - notification secret in `registry.notification_secret` section of + If you use an external Registry (not the one integrated with GitLab), you also have to specify + the notification secret (`registry['notification_secret']`) in the `/etc/gitlab/gitlab.rb` file. NOTE: - If you use GitLab HA, you also have to specify - the notification secret in `registry.notification_secret` section of + If you use GitLab HA, you also have to specify the notification secret (`registry['notification_secret']`) in `/etc/gitlab/gitlab.rb` file for every web node. -1. Reconfigure the **primary** node for the change to take effect: +1. Reconfigure each node: ```shell gitlab-ctl reconfigure diff --git a/doc/administration/geo/replication/datatypes.md b/doc/administration/geo/replication/datatypes.md index 27e1b990b2b..d6ce5ef5067 100644 --- a/doc/administration/geo/replication/datatypes.md +++ b/doc/administration/geo/replication/datatypes.md @@ -43,12 +43,12 @@ verification methods: | Blobs | CI job artifacts _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | | Blobs | Archived CI build traces _(file system)_ | Geo with API | _Not implemented_ | | Blobs | Archived CI build traces _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | -| Blobs | Container registry _(file system)_ | Geo with API/Docker API | _Not implemented_ | -| Blobs | Container registry _(object storage)_ | Geo with API/Managed/Docker API (*2*) | _Not implemented_ | +| Blobs | Container registry _(file system)_ | Geo with API/Docker API | _SHA256 checksum_ | +| Blobs | Container registry _(object storage)_ | Geo with API/Managed/Docker API (*2*) | _SHA256 checksum_ | | Blobs | Package registry _(file system)_ | Geo with API | SHA256 checksum | | Blobs | Package registry _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | -| Blobs | Infrastructure registry _(file system)_ | Geo with API | SHA256 checksum | -| Blobs | Infrastructure registry _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | +| Blobs | Terraform Module Registry _(file system)_ | Geo with API | SHA256 checksum | +| Blobs | Terraform Module Registry _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | | Blobs | Versioned Terraform State _(file system)_ | Geo with API | SHA256 checksum | | Blobs | Versioned Terraform State _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | | Blobs | External Merge Request Diffs _(file system)_ | Geo with API | SHA256 checksum | @@ -139,7 +139,7 @@ We use PostgreSQL's own replication functionality to replicate data from the **p We use Redis both as a cache store and to hold persistent data for our background jobs system. Because both use-cases have data that are exclusive to the same Geo site, we don't replicate it between sites. -Elasticsearch is an optional database that for advanced searching capabilities. It can improve search +Elasticsearch is an optional database for advanced search. It can improve search in both source-code level, and user generated content in issues, merge requests, and discussions. Elasticsearch is not supported in Geo. @@ -192,17 +192,17 @@ successfully, you must replicate their data using some other means. |:--------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------|:---------------------------------------------------------------------------|:--------------------------------------------------------------------|:----------------------------------------------------------------|:------| |[Application data in PostgreSQL](../../postgresql/index.md) | **Yes** (10.2) | **Yes** (10.2) | N/A | N/A | | |[Project repository](../../../user/project/repository/index.md) | **Yes** (10.2) | **Yes** (10.7) | N/A | N/A | | -|[Project wiki repository](../../../user/project/wiki/index.md) | **Yes** (10.2) | **Yes** (10.7) | N/A | N/A | | +|[Project wiki repository](../../../user/project/wiki/index.md) | **Yes** (10.2)2 | **Yes** (10.7)2 | N/A | N/A | Migrated to [self-service framework](../../../development/geo/framework.md) in 15.11. See GitLab issue [#367925](https://gitlab.com/gitlab-org/gitlab/-/issues/367925) for more details.

Behind feature flag geo_project_wiki_repository_replication, enabled by default in (15.11). | |[Group wiki repository](../../../user/project/wiki/group.md) | [**Yes** (13.10)](https://gitlab.com/gitlab-org/gitlab/-/issues/208147) | No | N/A | N/A | Behind feature flag `geo_group_wiki_repository_replication`, enabled by default. | |[Uploads](../../uploads.md) | **Yes** (10.2) | **Yes** (14.6) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | Replication is behind the feature flag `geo_upload_replication`, enabled by default. Verification was behind the feature flag `geo_upload_verification`, removed in 14.8. | |[LFS objects](../../lfs/index.md) | **Yes** (10.2) | **Yes** (14.6) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | GitLab versions 11.11.x and 12.0.x are affected by [a bug that prevents any new LFS objects from replicating](https://gitlab.com/gitlab-org/gitlab/-/issues/32696).

Replication is behind the feature flag `geo_lfs_object_replication`, enabled by default. Verification was behind the feature flag `geo_lfs_object_verification`, removed in 14.7. | |[Personal snippets](../../../user/snippets.md) | **Yes** (10.2) | **Yes** (10.2) | N/A | N/A | | |[Project snippets](../../../user/snippets.md) | **Yes** (10.2) | **Yes** (10.2) | N/A | N/A | | -|[CI job artifacts](../../../ci/pipelines/job_artifacts.md) | **Yes** (10.4) | **Yes** (14.10) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | Verification is behind the feature flag `geo_job_artifact_replication`, enabled by default in 14.10. | +|[CI job artifacts](../../../ci/jobs/job_artifacts.md) | **Yes** (10.4) | **Yes** (14.10) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | Verification is behind the feature flag `geo_job_artifact_replication`, enabled by default in 14.10. | |[CI Pipeline Artifacts](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/ci/pipeline_artifact.rb) | [**Yes** (13.11)](https://gitlab.com/gitlab-org/gitlab/-/issues/238464) | [**Yes** (13.11)](https://gitlab.com/gitlab-org/gitlab/-/issues/238464) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | Persists additional artifacts after a pipeline completes. | |[CI Secure Files](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/ci/secure_file.rb) | [**Yes** (15.3)](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91430) | [**Yes** (15.3)](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91430) | [**Yes** (15.3)](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91430) | [No](object_storage.md#verification-of-files-in-object-storage) | Verification is behind the feature flag `geo_ci_secure_file_replication`, enabled by default in 15.3. | -|[Container Registry](../../packages/container_registry.md) | **Yes** (12.3)* | No | No | No | See [instructions](container_registry.md) to set up the Container Registry replication. | -|[Infrastructure Registry](../../../user/packages/infrastructure_registry/index.md) | **Yes** (14.0) | **Yes** (14.0) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | Behind feature flag `geo_package_file_replication`, enabled by default. | +|[Container Registry](../../packages/container_registry.md) | **Yes** (12.3)1 | **Yes** (15.10) | **Yes** (12.3)1 | **Yes** (15.10) | See [instructions](container_registry.md) to set up the Container Registry replication. | +|[Terraform Module Registry](../../../user/packages/terraform_module_registry/index.md) | **Yes** (14.0) | **Yes** (14.0) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | Behind feature flag `geo_package_file_replication`, enabled by default. | |[Project designs repository](../../../user/project/issues/design_management.md) | **Yes** (12.7) | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/32467) | N/A | N/A | Designs also require replication of LFS objects and Uploads. | |[Package Registry](../../../user/packages/package_registry/index.md) | **Yes** (13.2) | **Yes** (13.10) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | Behind feature flag `geo_package_file_replication`, enabled by default. | |[Versioned Terraform State](../../terraform_state.md) | **Yes** (13.5) | **Yes** (13.12) | [**Yes** (15.1)](https://gitlab.com/groups/gitlab-org/-/epics/5551) | [No](object_storage.md#verification-of-files-in-object-storage) | Replication is behind the feature flag `geo_terraform_state_version_replication`, enabled by default. Verification was behind the feature flag `geo_terraform_state_version_verification`, which was removed in 14.0. | @@ -217,4 +217,6 @@ successfully, you must replicate their data using some other means. |[Dependency Proxy Images](../../../user/packages/dependency_proxy/index.md) | [**Yes** (15.7)](https://gitlab.com/groups/gitlab-org/-/epics/8833) | [**Yes** (15.7)](https://gitlab.com/groups/gitlab-org/-/epics/8833) | [**Yes** (15.7)](https://gitlab.com/groups/gitlab-org/-/epics/8833) | [No](object_storage.md#verification-of-files-in-object-storage) | | |[Vulnerability Export](../../../user/application_security/vulnerability_report/index.md#export-vulnerability-details) | [Not planned](https://gitlab.com/groups/gitlab-org/-/epics/3111) | No | No | No | Not planned because they are ephemeral and sensitive information. They can be regenerated on demand. | -\* Migrated to [self-service framework](../../../development/geo/framework.md) in 15.5. See GitLab issue [#337436](https://gitlab.com/gitlab-org/gitlab/-/issues/337436) for more details. +1 Migrated to [self-service framework](../../../development/geo/framework.md) in 15.5. See GitLab issue [#337436](https://gitlab.com/gitlab-org/gitlab/-/issues/337436) for more details. + +2 Migrated to [self-service framework](../../../development/geo/framework.md) in 15.11. Behind feature flag `geo_project_wiki_repository_replication`, enabled by default. See GitLab issue [#367925](https://gitlab.com/gitlab-org/gitlab/-/issues/367925) for more details. diff --git a/doc/administration/geo/replication/geo_validation_tests.md b/doc/administration/geo/replication/geo_validation_tests.md index a12dd8d9d68..cad3a396bfc 100644 --- a/doc/administration/geo/replication/geo_validation_tests.md +++ b/doc/administration/geo/replication/geo_validation_tests.md @@ -127,8 +127,7 @@ The following are PostgreSQL upgrade validation tests we performed. - Description: With PostgreSQL 12 available as an opt-in version in GitLab 13.3, we tested upgrading existing Geo installations from PostgreSQL 11 to 12. We also re-tested fresh installations of GitLab with Geo after fixes were made to support PostgreSQL 12. These tests were done using a - [nightly build](https://packages.gitlab.com/gitlab/nightly-builds/packages/ubuntu/bionic/gitlab-ee_13.3.6+rnightly.169516.d5209202-0_amd64.deb) - of GitLab 13.4. + nightly build of GitLab 13.4. - Outcome: Tests were successful for Geo deployments with a single database node on the primary and secondary. We encountered known issues with repmgr and Patroni managed PostgreSQL clusters on the Geo primary. Using PostgreSQL 12 with a database cluster on the primary is not recommended until the issues are resolved. diff --git a/doc/administration/geo/replication/img/adding_a_secondary_v13_3.png b/doc/administration/geo/replication/img/adding_a_secondary_v13_3.png deleted file mode 100644 index e43ace99666..00000000000 Binary files a/doc/administration/geo/replication/img/adding_a_secondary_v13_3.png and /dev/null differ diff --git a/doc/administration/geo/replication/img/adding_a_secondary_v15_8.png b/doc/administration/geo/replication/img/adding_a_secondary_v15_8.png new file mode 100644 index 00000000000..d7c99e6551e Binary files /dev/null and b/doc/administration/geo/replication/img/adding_a_secondary_v15_8.png differ diff --git a/doc/administration/geo/replication/single_sign_on.md b/doc/administration/geo/replication/single_sign_on.md index fc2f23552db..55e77d5657c 100644 --- a/doc/administration/geo/replication/single_sign_on.md +++ b/doc/administration/geo/replication/single_sign_on.md @@ -112,6 +112,15 @@ After configuring your SAML IdP to allow the secondary site's SAML callback URL, If you have configured SAML on the primary site correctly, then it should work on the secondary site without additional configuration. +## OpenID Connect + +If you use an [OpenID Connect (OIDC)](../../auth/oidc.md) OmniAuth provider, +in most cases, it should work without an issue: + +- **OIDC with Unified URL**: If you have configured OIDC on the primary site correctly, then it should work on the secondary site without additional configuration. +- **OIDC with separate URL with proxying disabled**: If you have configured OIDC on the primary site correctly, then it should work on the secondary site without additional configuration. +- **OIDC with separate URL with proxying enabled**: Geo with separate URL with proxying enabled does not support [OpenID Connect](../../auth/oidc.md). For more information, see [issue 396745](https://gitlab.com/gitlab-org/gitlab/-/issues/396745). + ## LDAP If you use LDAP on your **primary** site, you should also set up secondary LDAP servers on each **secondary** site. Otherwise, users cannot perform Git operations over HTTP(s) on the **secondary** site using HTTP basic authentication. However, users can still use Git with SSH and personal access tokens. diff --git a/doc/administration/geo/replication/troubleshooting.md b/doc/administration/geo/replication/troubleshooting.md index 59a67fecfcd..a8736b3ed1d 100644 --- a/doc/administration/geo/replication/troubleshooting.md +++ b/doc/administration/geo/replication/troubleshooting.md @@ -42,7 +42,7 @@ to help identify if something is wrong: ![Geo health check](img/geo_site_health_v14_0.png) -A site shows as "Unhealthy" if the site's status is more than 10 minutes old. In that case, try running the following in the [Rails console](../../operations/rails_console.md) on the affected site: +A site shows as "Unhealthy" if the site's status is more than 10 minutes old. In that case, try running the following in the [Rails console](../../operations/rails_console.md) on the affected secondary site: ```ruby Geo::MetricsUpdateWorker.new.perform @@ -52,7 +52,7 @@ If it raises an error, then the error is probably also preventing the jobs from If it successfully updates the status, then something may be wrong with Sidekiq. Is it running? Do the logs show errors? This job is supposed to be enqueued every minute. It takes an exclusive lease in Redis to ensure that only one of these jobs can run at a time. The primary site updates its status directly in the PostgreSQL database. Secondary sites send an HTTP Post request to the primary site with their status data. -A site also shows as "Unhealthy" if certain health checks fail. You can reveal the failure by running the following in the [Rails console](../../operations/rails_console.md) on the affected site: +A site also shows as "Unhealthy" if certain health checks fail. You can reveal the failure by running the following in the [Rails console](../../operations/rails_console.md) on the affected secondary site: ```ruby Gitlab::Geo::HealthCheck.new.perform_checks @@ -718,6 +718,8 @@ To solve this: ### Very large repositories never successfully synchronize on the **secondary** site +#### GitLab 10.1 and earlier + GitLab places a timeout on all repository clones, including project imports and Geo synchronization operations. If a fresh `git clone` of a repository on the **primary** takes more than the default three hours, you may be affected by this. @@ -740,6 +742,10 @@ add the following line to `/etc/gitlab/gitlab.rb`: This increases the timeout to four hours (14400 seconds). Choose a time long enough to accommodate a full clone of your largest repositories. +#### GitLab 10.2 and later + +Geo [replicates Git repositories over HTTPS](../index.md#how-it-works). GitLab does not place a timeout on these requests. If a Git repository is failing to replicate, with a consistent job execution time, then you should check for timeouts applied by external components such as load balancers. + ### New LFS objects are never replicated If new LFS objects are never replicated to secondary Geo sites, check the version of @@ -959,7 +965,7 @@ This behavior affects only the following data types through GitLab 14.6: | Package Registry | 13.10 | | CI Pipeline Artifacts | 13.11 | | Terraform State Versions | 13.12 | -| Infrastructure Registry | 14.0 | +| Infrastructure Registry (renamed to Terraform Module Registry in GitLab 15.11) | 14.0 | | External MR diffs | 14.6 | | LFS Objects | 14.6 | | Pages Deployments | 14.6 | @@ -1373,6 +1379,23 @@ The bug causes all wildcard domains (`.example.com`) to be ignored except for th You can have only one wildcard domain in the `no_proxy` list. +### Secondary site shows "Unhealthy" in UI after changing the value of `external_url` for the primary site + +If you have updated the value of `external_url` in `/etc/gitlab/gitlab.rb` for the primary site or changed the protocol from `http` to `https`, you may see that secondary sites are shown as `Unhealthy`. You may also find the following error in `geo.log`: + +```plaintext +"class": "Geo::NodeStatusRequestService", +... +"message": "Failed to Net::HTTP::Post to primary url: http://primary-site.gitlab.tld/api/v4/geo/status", + "error": "Failed to open TCP connection to :80 (Connection refused - connect(2) for \"\" port 80)" +``` + +In this case, make sure to update the changed URL on all your sites: + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Admin > Geo > Sites**. +1. Change the URL and save the change. + ## Fixing non-PostgreSQL replication failures If you notice replication failures in `Admin > Geo > Sites` or the [Sync status Rake task](#sync-status-rake-task), you can try to resolve the failures with the following general steps: diff --git a/doc/administration/geo/replication/upgrading_the_geo_sites.md b/doc/administration/geo/replication/upgrading_the_geo_sites.md index 55395a55857..644232a2c9e 100644 --- a/doc/administration/geo/replication/upgrading_the_geo_sites.md +++ b/doc/administration/geo/replication/upgrading_the_geo_sites.md @@ -33,6 +33,7 @@ and all **secondary** sites: 1. SSH into each node of the **primary** site. 1. [Upgrade GitLab on the **primary** site](../../../update/package/index.md#upgrade-using-the-official-repositories). 1. Perform testing on the **primary** site, particularly if you paused replication in step 1 to protect DR. [There are some suggestions for post-upgrade testing](../../../update/plan_your_upgrade.md#pre-upgrade-and-post-upgrade-checks) in the upgrade documentation. +1. Ensure that the secrets in the `/etc/gitlab/gitlab-secrets.json` file of both the primary site and the secondary site are the same. The file must be the same on all of a site’s nodes. 1. SSH into each node of **secondary** sites. 1. [Upgrade GitLab on each **secondary** site](../../../update/package/index.md#upgrade-using-the-official-repositories). 1. If you paused replication in step 1, [resume replication on each **secondary**](../index.md#pausing-and-resuming-replication). diff --git a/doc/administration/geo/secondary_proxy/index.md b/doc/administration/geo/secondary_proxy/index.md index addf894f0a5..35ab1d8252c 100644 --- a/doc/administration/geo/secondary_proxy/index.md +++ b/doc/administration/geo/secondary_proxy/index.md @@ -165,6 +165,7 @@ It does not cover all data types. | LFS objects (using Git) | **{check-circle}** Yes | | Pages | **{dotted-circle}** No 2 | | Advanced search (using the web UI) | **{dotted-circle}** No | +| Container registry | **{dotted-circle}** No | 1. Git reads are served from the local secondary while pushes get proxied to the primary. Selective sync or cases where repositories don't exist locally on the Geo secondary throw a "not found" error. diff --git a/doc/administration/geo/setup/database.md b/doc/administration/geo/setup/database.md index 99f7b32be59..cf0c40dbbc5 100644 --- a/doc/administration/geo/setup/database.md +++ b/doc/administration/geo/setup/database.md @@ -21,7 +21,7 @@ NOTE: The stages of the setup process must be completed in the documented order. If not, [complete all prior stages](../setup/index.md#using-omnibus-gitlab) before proceeding. -Ensure the **secondary** site is running the same version of GitLab Enterprise Edition as the **primary** site. Confirm you have added the [premium or higher licenses](https://about.gitlab.com/pricing/) to your **primary** site. +Ensure the **secondary** site is running the same version of GitLab Enterprise Edition as the **primary** site. Confirm you have added a license for a [Premium or Ultimate subscription](https://about.gitlab.com/pricing/) to your **primary** site. Be sure to read and review all of these steps before you execute them in your testing or production environments. @@ -29,10 +29,10 @@ testing or production environments. ## Single instance database replication A single instance database replication is easier to set up and still provides the same Geo capabilities -as a clusterized alternative. It's useful for setups running on a single machine -or trying to evaluate Geo for a future clusterized installation. +as a clustered alternative. It's useful for setups running on a single machine +or trying to evaluate Geo for a future clustered installation. -A single instance can be expanded to a clusterized version using Patroni, which is recommended for a +A single instance can be expanded to a clustered version using Patroni, which is recommended for a highly available architecture. Follow the instructions below on how to set up PostgreSQL replication as a single instance database. @@ -151,6 +151,13 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o ALTER USER gitlab_replicator WITH REPLICATION ENCRYPTED PASSWORD ''; ``` +1. Edit `/etc/gitlab/gitlab.rb` and set the role to `geo_primary_role` (for more information, see [Geo roles](https://docs.gitlab.com/omnibus/roles/#gitlab-geo-roles)): + + ```ruby + ## Geo Primary role + roles(['geo_primary_role']) + ``` + 1. Configure PostgreSQL to listen on network interfaces: For security reasons, PostgreSQL does not listen on any network interfaces @@ -210,17 +217,6 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o addresses with addresses appropriate to your network configuration: ```ruby - ## - ## Geo Primary role - ## - Configures Postgres settings for replication - ## - Prevents automatic upgrade of Postgres since it requires downtime of - ## streaming replication to Geo secondary sites - ## - Enables standard single-node GitLab services like NGINX, Puma, Redis, - ## or Sidekiq. If you are segregating services, then you will need to - ## explicitly disable unwanted services. - ## - roles(['geo_primary_role']) - ## ## Primary address ## - replace '' with the public or VPC address of your Geo primary node @@ -239,11 +235,13 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o # postgresql['max_replication_slots'] = 1 # Set this to be the number of Geo secondary nodes if you have more than one # postgresql['max_wal_senders'] = 10 # postgresql['wal_keep_segments'] = 10 + ``` - ## - ## Disable automatic database migrations temporarily - ## (until PostgreSQL is restarted and listening on the private address). - ## +1. Disable automatic database migrations temporarily until PostgreSQL is restarted and listening on the private address. + Edit `/etc/gitlab/gitlab.rb` and change the configuration to false: + + ```ruby + ## Disable automatic database migrations gitlab_rails['auto_migrate'] = false ``` @@ -402,6 +400,16 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o Ensure that the contents of `~gitlab-psql/data/server.crt` on the **primary** site match the contents of `~gitlab-psql/.postgresql/root.crt` on the **secondary** site. +1. Edit `/etc/gitlab/gitlab.rb` and set the role to `geo_secondary_role` (for more information, see [Geo roles](https://docs.gitlab.com/omnibus/roles/#gitlab-geo-roles)): + + ```ruby + ## + ## Geo Secondary role + ## - configure dependent flags automatically to enable Geo + ## + roles(['geo_secondary_role']) + ``` + 1. Configure PostgreSQL: This step is similar to how you configured the **primary** instance. @@ -411,12 +419,6 @@ There is an [issue where support is being discussed](https://gitlab.com/gitlab-o addresses with addresses appropriate to your network configuration: ```ruby - ## - ## Geo Secondary role - ## - configure dependent flags automatically to enable Geo - ## - roles(['geo_secondary_role']) - ## ## Secondary address ## - replace '' with the public or VPC address of your Geo secondary site @@ -863,6 +865,7 @@ For each node running a Patroni instance on the secondary site: consul['configuration'] = { retry_join: %w[CONSUL_SECONDARY1_IP CONSUL_SECONDARY2_IP CONSUL_SECONDARY3_IP] } + consul['services'] = %w(postgresql) postgresql['md5_auth_cidr_addresses'] = [ 'PATRONI_SECONDARY1_IP/32', 'PATRONI_SECONDARY2_IP/32', 'PATRONI_SECONDARY3_IP/32', 'PATRONI_SECONDARY_PGBOUNCER/32', @@ -892,7 +895,7 @@ For each node running a Patroni instance on the secondary site: postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' postgresql['listen_address'] = '0.0.0.0' # You can use a public or VPC address here instead - gitlab_rails['dbpassword'] = 'POSTGRESQL_PASSWORD' + gitlab_rails['db_password'] = 'POSTGRESQL_PASSWORD' gitlab_rails['enable'] = true gitlab_rails['auto_migrate'] = false ``` @@ -937,6 +940,8 @@ Omnibus automatically configures a tracking database when `roles(['geo_secondary If you want to run this database in a highly available configuration, don't use the `geo_secondary_role` above. Instead, follow the instructions below. +If you want to run the Geo tracking database on a single node, see [Configure the Geo tracking database on the Geo secondary site](../replication/multiple_servers.md#step-3-configure-the-geo-tracking-database-on-the-geo-secondary-site). + A production-ready and secure setup for the tracking PostgreSQL DB requires at least three Consul nodes: two Patroni nodes, and one PgBouncer node on the secondary site. diff --git a/doc/administration/geo/setup/external_database.md b/doc/administration/geo/setup/external_database.md index 0fefc11f078..65ea2e6e5af 100644 --- a/doc/administration/geo/setup/external_database.md +++ b/doc/administration/geo/setup/external_database.md @@ -211,20 +211,20 @@ the tracking database on port 5432. 1. Set up PostgreSQL according to the [database requirements document](../../../install/requirements.md#database). -1. Set up a `gitlab_geo` user with a password of your choice, create the `gitlabhq_geo_production` database, and make the user an owner of the database. You can see an example of this setup in the [installation from source documentation](../../../install/installation.md#6-database). +1. Set up a `gitlab_geo` user with a password of your choice, create the `gitlabhq_geo_production` database, and make the user an owner of the database. You can see an example of this setup in the [installation from source documentation](../../../install/installation.md#7-database). 1. If you are **not** using a cloud-managed PostgreSQL database, ensure that your secondary site can communicate with your tracking database by manually changing the `pg_hba.conf` that is associated with your tracking database. Remember to restart PostgreSQL afterwards for the changes to take effect: - ```plaintext - ## - ## Geo Tracking Database Role - ## - pg_hba.conf - ## - host all all /32 md5 - host all all /32 md5 - ``` + ```plaintext + ## + ## Geo Tracking Database Role + ## - pg_hba.conf + ## + host all all /32 md5 + host all all /32 md5 + ``` 1. SSH into a GitLab **secondary** server and login as root: diff --git a/doc/administration/geo/setup/index.md b/doc/administration/geo/setup/index.md index c794b8ef219..da7e55509e7 100644 --- a/doc/administration/geo/setup/index.md +++ b/doc/administration/geo/setup/index.md @@ -7,33 +7,33 @@ type: howto # Setting up Geo **(PREMIUM SELF)** -These instructions assume you have a working instance of GitLab. They guide you through: +## Prerequisites -1. Making your existing instance the **primary** site. -1. Adding **secondary** sites. +- Two (or more) independently working GitLab sites: + - One GitLab site serves as the Geo **primary** site. Use the [GitLab reference architectures documentation](../../reference_architectures/index.md) to set this up. You can use different reference architecture sizes for each Geo site. If you already have a working GitLab instance that is in-use, it can be used as a **primary** site. + - The second GitLab site serves as the Geo **secondary** site. Use the [GitLab reference architectures documentation](../../reference_architectures/index.md) to set this up. It's a good idea to sign in and test it. However, be aware that **all of the data on the secondary are lost** as part of the process of replicating from the **primary** site. -You must use a [GitLab Premium](https://about.gitlab.com/pricing/) license or higher, -but you only need one license for all the sites. + NOTE: + Geo supports multiple secondaries. You can follow the same steps and make any changes accordingly. -WARNING: -The steps below should be followed in the order they appear. **Make sure the GitLab version is the same on all sites. Do not create an account or sign in to the new secondary.** +- Ensure the **primary** site has a [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/) subscription to unlock Geo. You only need one license for all the sites. +- Confirm the [requirements for running Geo](../index.md#requirements-for-running-geo) are met by all sites. For example, sites must use the same GitLab version, and sites must be able to communicate with each other over certain ports. ## Using Omnibus GitLab If you installed GitLab using the Omnibus packages (highly recommended): -1. Confirm the [requirements for running Geo](../index.md#requirements-for-running-geo) are met. -1. [Install GitLab Enterprise Edition](https://about.gitlab.com/install/) on the nodes that serve as the **secondary** site. **Do not create an account or sign in** to the new **secondary** site. The **GitLab version must match** across primary and secondary sites. -1. [Add the GitLab License](../../../user/admin_area/license.md) on the **primary** site to unlock Geo. The license must be for [GitLab Premium](https://about.gitlab.com/pricing/) or higher. -1. [Confirm network connectivity](../index.md#firewall-rules) between the **primary** and **secondary** site. 1. [Set up the database replication](database.md) (`primary (read-write) <-> secondary (read-only)` topology). -1. [Configure fast lookup of authorized SSH keys in the database](../../operations/fast_ssh_key_lookup.md). This step is required and needs to be done on **both** the **primary** and **secondary** sites. 1. [Configure GitLab](../replication/configuration.md) to set the **primary** and **secondary** sites. 1. Optional: [Configure Object storage](../../object_storage.md) 1. Optional: [Configure a secondary LDAP server](../../auth/ldap/index.md) for the **secondary** sites. See [notes on LDAP](../index.md#ldap). 1. Optional: [Configure Geo secondary proxying](../secondary_proxy/index.md) to use a single, unified URL for all Geo sites. This step is recommended to accelerate most read requests while transparently proxying writes to the primary Geo site. 1. Follow the [Using a Geo Site](../replication/usage.md) guide. +## Using GitLab Charts + +[Configure the GitLab chart with GitLab Geo](https://docs.gitlab.com/charts/advanced/geo/). + ## Post-installation documentation After installing GitLab on the **secondary** sites and performing the initial configuration, see the [following documentation for post-installation information](../index.md#post-installation-documentation). diff --git a/doc/administration/get_started.md b/doc/administration/get_started.md index d9191440a24..0b5de38a78b 100644 --- a/doc/administration/get_started.md +++ b/doc/administration/get_started.md @@ -1,7 +1,7 @@ --- -info: For assistance with this CSM Onboarding page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. +info: For assistance with this tutorial, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. stage: none -group: unassigned +group: Tutorials --- # Get started administering GitLab **(FREE)** @@ -100,7 +100,6 @@ Unlike other monitoring solutions (for example, Zabbix or New Relic), Prometheus - Prometheus and its exporters are on by default. However, you need to [configure the service](../administration/monitoring/prometheus/index.md#configuring-prometheus). - Learn more about [GitLab architecture](../development/architecture.md). - Find out why [application performance metrics](https://about.gitlab.com/blog/2020/05/07/working-with-performance-metrics/) matter. -- Create a [self-monitoring project](../administration/monitoring/gitlab_self_monitoring_project/index.md) to track the health of your instance. - Integrate Grafana to [build visual dashboards](https://youtu.be/f4R7s0An1qE) based on performance metrics. ### Components of monitoring @@ -151,7 +150,8 @@ Backups of GitLab databases and file systems are taken every 24 hours, and are k - You can use the project export option in: - [The UI](../user/project/settings/import_export.md#export-a-project-and-its-data). - [The API](../api/project_import_export.md#schedule-an-export). -- [Group export](../user/group/settings/import_export.md) does *not* export the projects in it, but does export: +- [Group export by uploading a file export](../user/group/import/index.md#migrate-groups-by-uploading-an-export-file-deprecated) + does **not** export the projects in it, but does export: - Epics - Milestones - Boards diff --git a/doc/administration/git_protocol.md b/doc/administration/git_protocol.md index 349a92de51e..b996b9efae9 100644 --- a/doc/administration/git_protocol.md +++ b/doc/administration/git_protocol.md @@ -17,7 +17,7 @@ More details about the new features and improvements are available in the [Google Open Source Blog](https://opensource.googleblog.com/2018/05/introducing-git-protocol-version-2.html) and the [protocol documentation](https://github.com/git/git/blob/master/Documentation/gitprotocol-v2.txt). -## Requirements +## Prerequisites From the client side, `git` `v2.18.0` or newer must be installed. @@ -111,4 +111,4 @@ URL to use SSH. ### Observe Git protocol version of connections For information on observing the Git protocol versions are being used in a production environment, -see the [relevant documentation](gitaly/monitoring.md#useful-queries). +see the [relevant documentation](gitaly/monitoring.md#queries). diff --git a/doc/administration/gitaly/configure_gitaly.md b/doc/administration/gitaly/configure_gitaly.md index 143f7dca7d3..f3eb2de3f1d 100644 --- a/doc/administration/gitaly/configure_gitaly.md +++ b/doc/administration/gitaly/configure_gitaly.md @@ -6,25 +6,26 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Configure Gitaly **(FREE SELF)** -The Gitaly service itself is configured by using a [TOML configuration file](reference.md). +Configure Gitaly in one of two ways: -To change Gitaly settings: +::Tabs -**For Omnibus GitLab** +:::TabTitle Linux package (Omnibus) 1. Edit `/etc/gitlab/gitlab.rb` and add or change the [Gitaly settings](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/1dd07197c7e5ae23626aad5a4a070a800b670380/files/gitlab-config-template/gitlab.rb.template#L1622-1676). 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -**For installations from source** +:::TabTitle Self-compiled (source) 1. Edit `/home/git/gitaly/config.toml` and add or change the [Gitaly settings](https://gitlab.com/gitlab-org/gitaly/blob/master/config.toml.example). 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). +::EndTabs + The following configuration options are also available: - Enabling [TLS support](#enable-tls-support). -- Configuring the [number of `gitaly-ruby` workers](#configure-number-of-gitaly-ruby-workers). - Limiting [RPC concurrency](#limit-rpc-concurrency). ## About the Gitaly token @@ -138,15 +139,25 @@ Gitaly and GitLab use two shared secrets for authentication: - _Gitaly token_: used to authenticate gRPC requests to Gitaly - _GitLab Shell token_: used for authentication callbacks from GitLab Shell to the GitLab internal API -**For Omnibus GitLab** +Configure authentication in one of two ways: + +::Tabs + +:::TabTitle Linux package (Omnibus) To configure the _Gitaly token_, edit `/etc/gitlab/gitlab.rb`: ```ruby - gitaly['auth_token'] = 'abc123secret' + gitaly['configuration'] = { + # ... + auth: { + # ... + token: 'abc123secret', + }, + } ``` -There are two ways to configure the _GitLab Shell token_. +Configure the _GitLab Shell token_ in one of two ways. Method 1 (recommended): @@ -161,7 +172,7 @@ Edit `/etc/gitlab/gitlab.rb`: gitlab_shell['secret_token'] = 'shellsecret' ``` -**For installations from source** +:::TabTitle Self-compiled (source) 1. Copy `/home/git/gitlab/.gitlab_shell_secret` from the Gitaly client to the same path on the Gitaly servers (and any other Gitaly clients). @@ -183,19 +194,26 @@ Edit `/etc/gitlab/gitlab.rb`: 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). -#### Configure Gitaly server - -**For Omnibus GitLab** +::EndTabs -1. Edit `/etc/gitlab/gitlab.rb`: +#### Configure Gitaly server +Configure Gitaly server in one of two ways: + +::Tabs + +:::TabTitle Linux package (Omnibus) + +1. Edit `/etc/gitlab/gitlab.rb`: + ```ruby # Avoid running unnecessary services on the Gitaly server postgresql['enable'] = false @@ -230,14 +248,21 @@ Updates to example must be made at: # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from Gitaly client to Gitaly server. gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' - # Make Gitaly accept connections on all network interfaces. You must use - # firewalls to restrict access to this address/port. - # Comment out following line if you only want to support TLS connections - gitaly['listen_addr'] = "0.0.0.0:8075" - - # Authentication token to ensure only authorized servers can communicate with - # Gitaly server - gitaly['auth_token'] = 'AUTH_TOKEN' + gitaly['configuration'] = { + # ... + # + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + # Comment out following line if you only want to support TLS connections + listen_addr: '0.0.0.0:8075', + auth: { + # ... + # + # Authentication token to ensure only authorized servers can communicate with + # Gitaly server + token: 'AUTH_TOKEN', + }, + } ``` 1. Append the following to `/etc/gitlab/gitlab.rb` for each respective Gitaly server: @@ -247,24 +272,33 @@ Updates to example must be made at: On `gitaly1.internal`: ```ruby - git_data_dirs({ - 'default' => { - 'path' => '/var/opt/gitlab/git-data' - }, - 'storage1' => { - 'path' => '/mnt/gitlab/git-data' - }, - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'default', + path: '/var/opt/gitlab/git-data', + }, + { + name: 'storage1', + path: '/mnt/gitlab/git-data', + }, + ], + } ``` On `gitaly2.internal`: ```ruby - git_data_dirs({ - 'storage2' => { - 'path' => '/srv/gitlab/git-data' - }, - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'storage2', + path: '/srv/gitlab/git-data', + }, + ], + } ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -272,7 +306,7 @@ Updates to example must be made at: - For GitLab 15.3 and later, run `sudo /opt/gitlab/embedded/bin/gitaly check /var/opt/gitlab/gitaly/config.toml`. - For GitLab 15.2 and earlier, run `sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml`. -**For installations from source** +:::TabTitle Self-compiled (source) 1. Edit `/home/git/gitaly/config.toml`: @@ -323,6 +357,8 @@ Updates to example must be made at: - For GitLab 15.3 and later, run `sudo /opt/gitlab/embedded/bin/gitaly check /var/opt/gitlab/gitaly/config.toml`. - For GitLab 15.2 and earlier, run `sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml`. +::EndTabs + WARNING: If directly copying repository data from a GitLab server to Gitaly, ensure that the metadata file, default path `/var/opt/gitlab/git-data/repositories/.gitaly-metadata`, is not included in the transfer. @@ -359,7 +395,11 @@ You can't define Gitaly servers with some as a local Gitaly server server (with `gitaly_address`) unless you use [mixed configuration](#mixed-configuration). -**For Omnibus GitLab** +Configure Gitaly clients in one of two ways: + +::Tabs + +:::TabTitle Linux package (Omnibus) 1. Edit `/etc/gitlab/gitlab.rb`: @@ -393,7 +433,7 @@ server (with `gitaly_address`) unless you use sudo gitlab-ctl tail gitaly ``` -**For installations from source** +:::TabTitle Self-compiled (source) 1. Edit `/home/git/gitlab/config/gitlab.yml`: @@ -429,6 +469,8 @@ server (with `gitaly_address`) unless you use tail -f /home/git/gitlab/log/gitaly.log ``` +::EndTabs + When you tail the Gitaly logs on your Gitaly server, you should see requests coming in. One sure way to trigger a Gitaly request is to clone a repository from GitLab over HTTP or HTTPS. @@ -461,17 +503,28 @@ example: git_data_dirs({ 'default' => { 'gitaly_address' => 'tcp://gitaly1.internal:8075' }, # Address of the GitLab server that also has Gitaly running on it - 'storage1' => { 'gitaly_address' => 'tcp://gitlab.internal:8075', 'path' => '/mnt/gitlab/git-data' }, + 'storage1' => { 'gitaly_address' => 'tcp://gitlab.internal:8075' }, 'storage2' => { 'gitaly_address' => 'tcp://gitaly2.internal:8075' }, }) -# Make Gitaly accept connections on all network interfaces -gitaly['listen_addr'] = "0.0.0.0:8075" - -# Or for TLS -gitaly['tls_listen_addr'] = "0.0.0.0:9999" -gitaly['certificate_path'] = "/etc/gitlab/ssl/cert.pem" -gitaly['key_path'] = "/etc/gitlab/ssl/key.pem" +gitaly['configuration'] = { + # ... + # + # Make Gitaly accept connections on all network interfaces + listen_addr: '0.0.0.0:8075', + # Or for TLS + tls_listen_addr: '0.0.0.0:9999', + tls: { + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + storage: [ + { + name: 'storage1', + path: '/mnt/gitlab/git-data', + }, + ], +} ``` `path` can be included only for storage shards on the local Gitaly server. @@ -499,9 +552,11 @@ Disabling Gitaly on the GitLab instance makes sense only when you run GitLab in Gitaly runs on a separate machine from the GitLab instance. Disabling Gitaly on all machines in the cluster is not a valid configuration (some machines much act as Gitaly servers). -To disable Gitaly on a GitLab server: +Disable Gitaly on a GitLab server in one of two ways: + +::Tabs -**For Omnibus GitLab** +:::TabTitle Linux package (Omnibus) 1. Edit `/etc/gitlab/gitlab.rb`: @@ -511,7 +566,7 @@ To disable Gitaly on a GitLab server: 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -**For installations from source** +:::TabTitle Self-compiled (source) 1. Edit `/etc/default/gitlab`: @@ -521,10 +576,9 @@ To disable Gitaly on a GitLab server: 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). -## Enable TLS support +::EndTabs -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22602) in GitLab 11.8. -> - [Introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/3160) in GitLab 13.6, outgoing TLS connections to GitLab provide client certificates if configured. +## Enable TLS support Gitaly supports TLS encryption. To communicate with a Gitaly instance that listens for secure connections, use the `tls://` URL scheme in the `gitaly_address` of the corresponding @@ -543,6 +597,8 @@ Additionally, the certificate (or its certificate authority) must be installed o - Gitaly servers. - Gitaly clients that communicate with it. +If you use a load balancer, it must be able to negotiate HTTP/2 using the ALPN TLS extension. + ### Certificate requirements - The certificate must specify the address you use to access the Gitaly server. You must add the hostname or IP address as a Subject Alternative Name to the certificate. @@ -553,9 +609,11 @@ Additionally, the certificate (or its certificate authority) must be installed o ### Configure Gitaly with TLS -To configure Gitaly with TLS: +Configure Gitaly with TLS in one of two ways: + +::Tabs -**For Omnibus GitLab** +:::TabTitle Linux package (Omnibus) 1. Create certificates for Gitaly servers. 1. On the Gitaly clients, copy the certificates (or their certificate authority) into @@ -600,21 +658,26 @@ To configure Gitaly with TLS: ```ruby - gitaly['tls_listen_addr'] = "0.0.0.0:9999" - gitaly['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - gitaly['key_path'] = "/etc/gitlab/ssl/key.pem" + gitaly['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:9999', + tls: { + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). 1. Verify Gitaly traffic is being served over TLS by [observing the types of Gitaly connections](#observe-type-of-gitaly-connections). 1. Optional. Improve security by: - 1. Disabling non-TLS connections by commenting out or deleting `gitaly['listen_addr']` in + 1. Disabling non-TLS connections by commenting out or deleting `gitaly['configuration'][:listen_addr]` in `/etc/gitlab/gitlab.rb`. 1. Saving the file. 1. [Reconfiguring GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -**For installations from source** +:::TabTitle Self-compiled (source) 1. Create certificates for Gitaly servers. 1. On the Gitaly clients, copy the certificates into the system trusted certificates: @@ -690,66 +753,12 @@ To configure Gitaly with TLS: 1. Saving the file. 1. [Restarting GitLab](../restart_gitlab.md#installations-from-source). +::EndTabs + ### Observe type of Gitaly connections For information on observing the type of Gitaly connections being served, see the -[relevant documentation](monitoring.md#useful-queries). - -## `gitaly-ruby` - -Gitaly was developed to replace the Ruby application code in GitLab. - -To save time and avoid the risk of rewriting existing application logic, we chose to copy some -application code from GitLab into Gitaly. - -To be able to run that code, `gitaly-ruby` was created, which is a "sidecar" process for the main -Gitaly Go process. Some examples of things that are implemented in `gitaly-ruby` are: - -- RPCs that deal with wikis. -- RPCs that create commits on behalf of a user, such as merge commits. - -We recommend: - -- At least 300 MB memory per worker. -- No more than one worker per core. - -NOTE: -`gitaly-ruby` is planned to be eventually removed. To track progress, see the -[Remove the Gitaly-Ruby sidecar](https://gitlab.com/groups/gitlab-org/-/epics/2862) epic. - -### Configure number of `gitaly-ruby` workers - -`gitaly-ruby` has much less capacity than Gitaly implemented in Go. If your Gitaly server has to handle lots of -requests, the default setting of having just one active `gitaly-ruby` sidecar might not be enough. - -If you see `ResourceExhausted` errors from Gitaly, it's very likely that you have not enough -`gitaly-ruby` capacity. - -You can increase the number of `gitaly-ruby` processes on your Gitaly server with the following -settings: - -**For Omnibus GitLab** - -1. Edit `/etc/gitlab/gitlab.rb`: - - ```ruby - # Default is 2 workers. The minimum is 2; 1 worker is always reserved as - # a passive stand-by. - gitaly['ruby_num_workers'] = 4 - ``` - -1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). - -**For installations from source** - -1. Edit `/home/git/gitaly/config.toml`: - - ```toml - [gitaly-ruby] - num_workers = 4 - ``` - -1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). +[relevant documentation](monitoring.md#queries). ## Limit RPC concurrency @@ -775,21 +784,23 @@ example: ```ruby # in /etc/gitlab/gitlab.rb - -gitaly['concurrency'] = [ - { - 'rpc' => "/gitaly.SmartHTTPService/PostUploadPackWithSidechannel", - 'max_per_repo' => 20, - 'max_queue_time' => "1s", - 'max_queue_size' => 10 - }, - { - 'rpc' => "/gitaly.SSHService/SSHUploadPackWithSidechannel", - 'max_per_repo' => 20 - 'max_queue_time' => "1s", - 'max_queue_size' => 10 - } -] +gitaly['configuration'] = { + # ... + concurrency: [ + { + rpc: '/gitaly.SmartHTTPService/PostUploadPackWithSidechannel', + max_per_repo: 20, + max_queue_time: '1s', + max_queue_size: 10, + }, + { + rpc: '/gitaly.SSHService/SSHUploadPackWithSidechannel', + max_per_repo: 20, + max_queue_time: '1s', + max_queue_size: 10, + }, + ], +} ``` - `rpc` is the name of the RPC to set a concurrency limit for per repository. @@ -834,7 +845,7 @@ performance. You can use control groups (cgroups) in Linux to impose limits on how much memory and CPU can be consumed by Gitaly processes. See the [`cgroups` Linux man page](https://man7.org/linux/man-pages/man7/cgroups.7.html) for more information. -cgroups can be useful for protecting the system against unexpected resource exhaustion because of over consumption of memory and CPU. +cgroups can help protect the system against unexpected resource exhaustion because of over consumption of memory and CPU. Some Git operations can consume notable resources up to the point of exhaustion in situations such as: @@ -861,43 +872,60 @@ When these limits are reached, performance may be reduced and users may be disco ### Configure repository cgroups (new method) -> This method of configuring repository cgroups was introduced in GitLab 15.1. +> - This method of configuring repository cgroups was introduced in GitLab 15.1. +> - `cpu_quota_us`[introduced](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/5422) in GitLab 15.10. To configure repository cgroups in Gitaly using the new method, use the following settings for the new configuration method -to `gitaly['cgroups']` in `/etc/gitlab/gitlab.rb`: +to `gitaly['configuration'][:cgroups]` in `/etc/gitlab/gitlab.rb`: -- `cgroups_mountpoint` is where the parent cgroup directory is mounted. Defaults to `/sys/fs/cgroup`. -- `cgroups_hierarchy_root` is the parent cgroup under which Gitaly creates groups, and +- `mountpoint` is where the parent cgroup directory is mounted. Defaults to `/sys/fs/cgroup`. +- `hierarchy_root` is the parent cgroup under which Gitaly creates groups, and is expected to be owned by the user and group Gitaly runs as. Omnibus GitLab creates the set of directories `mountpoint//hierarchy_root` when Gitaly starts. -- `cgroups_memory_bytes` is the total memory limit that is imposed collectively on all +- `memory_bytes` is the total memory limit that is imposed collectively on all Git processes that Gitaly spawns. 0 implies no limit. -- `cgroups_cpu_shares` is the CPU limit that is imposed collectively on all Git +- `cpu_shares` is the CPU limit that is imposed collectively on all Git processes that Gitaly spawns. 0 implies no limit. The maximum is 1024 shares, which represents 100% of CPU. -- `cgroups_repositories_count` is the number of cgroups in the cgroups pool. Each time a new Git +- `cpu_quota_us` is the [`cfs_quota_us`](https://docs.kernel.org/scheduler/sched-bwc.html#management) + to throttle the cgroups' processes if they exceed this quota value. We set + `cfs_period_us` to `100ms` so 1 core is `100000`. 0 implies no limit. +- `repositories.count` is the number of cgroups in the cgroups pool. Each time a new Git command is spawned, Gitaly assigns it to one of these cgroups based on the repository the command is for. A circular hashing algorithm assigns Git commands to these cgroups, so a Git command for a repository is always assigned to the same cgroup. -- `cgroups_repositories_memory_bytes` is the total memory limit imposed on all Git processes contained in a repository cgroup. - 0 implies no limit. This value cannot exceed that of the top level `cgroups_memory_bytes`. -- `cgroups_repositories_cpu_shares` is the CPU limit that is imposed on all Git processes contained in a repository cgroup. +- `repositories.memory_bytes` is the total memory limit imposed on all Git processes contained in a repository cgroup. + 0 implies no limit. This value cannot exceed that of the top level `memory_bytes`. +- `repositories.cpu_shares` is the CPU limit that is imposed on all Git processes contained in a repository cgroup. 0 implies no limit. The maximum is 1024 shares, which represents 100% of CPU. - This value cannot exceed that of the top level`cgroups_cpu_shares`. + This value cannot exceed that of the top level`cpu_shares`. +- `repositories.cpu_quota_us` is the [`cfs_quota_us`](https://docs.kernel.org/scheduler/sched-bwc.html#management) + that is imposed on all Git processes contained in a repository cgroup. A Git + process can't use more then the given quota. We set + `cfs_period_us` to `100ms` so 1 core is `100000`. 0 implies no limit. For example: ```ruby # in /etc/gitlab/gitlab.rb -gitaly['cgroups_mountpoint'] = "/sys/fs/cgroup" -gitaly['cgroups_hierarchy_root'] => "gitaly" -gitaly['cgroups_memory_bytes'] = 64424509440, # 60gb -gitaly['cgroups_cpu_shares'] = 1024 -gitaly['cgroups_repositories_count'] => 1000, -gitaly['cgroups_repositories_memory_bytes'] => 32212254720 # 20gb -gitaly['cgroups_repositories_cpu_shares'] => 512 +gitaly['configuration'] = { + # ... + cgroups: { + mountpoint: '/sys/fs/cgroup', + hierarchy_root: 'gitaly', + memory_bytes: 64424509440, # 60gb + cpu_shares: 1024, + cpu_quota_us: 400000 # 4 cores + repositories: { + count: 1000, + memory_bytes: 32212254720, # 20gb + cpu_shares: 512, + cpu_quota_us: 200000 # 2 cores + }, + }, +} ``` ### Configure repository cgroups (legacy method) @@ -953,28 +981,38 @@ This strategy has two main benefits: to 3 child cgroups can concurrently burst up to their max. In general, all 1000 cgroups would use much less than the 20 GB. -## Background Repository Optimization +## Background repository optimization Empty directories and unneeded configuration settings may accumulate in a repository and slow down Git operations. Gitaly can schedule a daily background task with a maximum duration to clean up these items and improve performance. WARNING: -This is an experimental feature and may place significant load on the host while running. +Background repository optimization is an experimental feature and may place significant load on the host while running. Make sure to schedule this during off-peak hours and keep the duration short (for example, 30-60 minutes). -**For Omnibus GitLab** +Configure background repository optimization in one of two ways: + +::Tabs + +:::TabTitle Linux package (Omnibus) Edit `/etc/gitlab/gitlab.rb` and add: ```ruby -gitaly['daily_maintenance_start_hour'] = 4 -gitaly['daily_maintenance_start_minute'] = 30 -gitaly['daily_maintenance_duration'] = '30m' -gitaly['daily_maintenance_storages'] = ["default"] +gitaly['configuration'] = { + # ... + daily_maintenance: { + # ... + start_hour: 4, + start_minute: 30, + duration: '30m', + storages: ['default'], + }, +} ``` -**For installations from source** +:::TabTitle Self-compiled (source) Edit `/home/git/gitaly/config.toml` and add: @@ -986,6 +1024,8 @@ duration = '30m' storages = ["default"] ``` +::EndTabs + ## Rotate Gitaly authentication token Rotating credentials in a production environment often requires downtime, causes outages, or both. @@ -1006,7 +1046,7 @@ server" and "Gitaly client" refers to the same machine. ### Verify authentication monitoring Before rotating a Gitaly authentication token, verify that you can -[monitor the authentication behavior](monitoring.md#useful-queries) of your GitLab installation using +[monitor the authentication behavior](monitoring.md#queries) of your GitLab installation using Prometheus. You can then continue the rest of the procedure. @@ -1018,7 +1058,13 @@ transitioning" mode as follows: ```ruby # in /etc/gitlab/gitlab.rb -gitaly['auth_transitioning'] = true +gitaly['configuration'] = { + # ... + auth: { + # ... + transitioning: true, + }, +} ``` After you have made this change, your [Prometheus query](#verify-authentication-monitoring) @@ -1038,8 +1084,13 @@ To update to a new Gitaly authentication token, on each Gitaly client **and** Gi ```ruby # in /etc/gitlab/gitlab.rb - - gitaly['auth_token'] = '' + gitaly['configuration'] = { + # ... + auth: { + # ... + token: '', + }, + } ``` 1. Restart Gitaly: @@ -1069,7 +1120,13 @@ your Gitaly servers as follows: ```ruby # in /etc/gitlab/gitlab.rb -gitaly['auth_transitioning'] = false +gitaly['configuration'] = { + # ... + auth: { + # ... + transitioning: false, + }, +} ``` WARNING: @@ -1088,17 +1145,11 @@ result as you did at the start. For example: ## Pack-objects cache **(FREE SELF)** -> - [Introduced](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/372) in GitLab 13.11. -> - It's enabled on GitLab.com. -> - It's recommended for production use. - [Gitaly](index.md), the service that provides storage for Git repositories, can be configured to cache a short rolling window of Git fetch responses. This can reduce server load when your server receives lots of CI fetch traffic. -### Overview - The pack-objects cache wraps `git pack-objects`, an internal part of Git that gets invoked indirectly via the PostUploadPack and SSHUploadPack Gitaly RPCs. Gitaly runs PostUploadPack when a @@ -1131,31 +1182,40 @@ automatically keep parts of the pack-objects cache files in RAM, making it faster. Because the pack-objects cache can lead to a significant increase in -disk write IO, it is off by default. +disk write IO, it is off by default. In GitLab 15.11 and later, +the write workload is approximately 50% lower, but the cache is still disabled by default. ### Configure the cache -These are the configuration settings for the pack-objects cache. Each -setting is discussed in greater detail below. +These configuration settings are available for the pack-objects cache. Each setting is discussed in greater detail +below. -|Setting|Default|Description| -|:---|:---|:---| -|`enabled`|`false`|Turns on the cache. When off, Gitaly runs a dedicated `git pack-objects` process for each request. | -|`dir`|`/+gitaly/PackObjectsCache`|Local directory where cache files get stored.| -|`max_age`|`5m` (5 minutes)|Cache entries older than this get evicted and removed from disk.| +| Setting | Default | Description | +|:----------|:---------------------------------------------------|:---------------------------------------------------------------------------------------------------| +| `enabled` | `false` | Turns on the cache. When off, Gitaly runs a dedicated `git pack-objects` process for each request. | +| `dir` | `/+gitaly/PackObjectsCache` | Local directory where cache files get stored. | +| `max_age` | `5m` (5 minutes) | Cache entries older than this get evicted and removed from disk. | +| `min_occurrences` | 1 | Minimum times a key must occur before a cache entry is created. | In `/etc/gitlab/gitlab.rb`, set: ```ruby -gitaly['pack_objects_cache_enabled'] = true -## gitaly['pack_objects_cache_dir'] = '/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache' -## gitaly['pack_objects_cache_max_age'] = '5m' +gitaly['configuration'] = { + # ... + pack_objects_cache: { + # ... + enabled: true, + # dir: '/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache', + # max_age: '5m', + # min_occurrences: 1, + }, +} ``` #### `enabled` defaults to `false` -The cache is disabled by default. This is because in some cases, it -can create an [extreme increase](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/4010#note_534564684) +The cache is disabled by default because in some cases, it can create an +[extreme increase](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/4010#note_534564684) in the number of bytes written to disk. On GitLab.com, we have verified that our repository storage disks can handle this extra workload, but we felt we cannot assume this is true everywhere. @@ -1198,21 +1258,37 @@ The amount of space required depends on: - The size of the `max_age` cache eviction window. If your users pull 100 MB/s and you use a 5 minute window, then on average you have -`5*60*100MB = 30GB` of data in your cache directory. This is an expected average, not +`5*60*100MB = 30GB` of data in your cache directory. This average is an expected average, not a guarantee. Peak size may exceed this average. #### Cache eviction window `max_age` The `max_age` configuration setting lets you control the chance of a cache hit and the average amount of storage used by cache files. -Entries older than `max_age` get evicted from the in-memory metadata -store, and deleted from disk. +Entries older than `max_age` get deleted from the disk. + +Eviction does not interfere with ongoing requests. It is OK for `max_age` to be less than the time it takes to do a +fetch over a slow connection because Unix filesystems do not truly delete a file until all processes that are reading +the deleted file have closed it. + +#### Minimum key occurrences `min_occurrences` + +> [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/2222) in GitLab 15.11. -Eviction does not interfere with ongoing requests, so it is OK -for `max_age` to be less than the time it takes to do a fetch over a -slow connection. This is because Unix filesystems do not truly delete -a file until all processes that are reading the deleted file have -closed it. +The `min_occurrences` setting controls how often an identical request +must occur before we create a new cache entry. The default value is `1`, +meaning that unique requests do not get written into the cache. + +If you: + +- Increase this number, your cache hit rate goes down and the +cache uses less disk space. +- Decrease this number, your cache hit +rate goes up and the cache uses more disk space. + +You should set `min_occurrences` to `1`. On GitLab.com, +going from 0 to 1 saved us 50% cache disk space while barely affecting +the cache hit rate. ### Observe the cache @@ -1315,7 +1391,32 @@ process repositories that do not pass consistency checks. For Omnibus GitLab installations, edit `/etc/gitlab/gitlab.rb` and set the following keys (in this example, to disable the `hasDotgit` consistency check): -- In [GitLab 15.3](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6800) and later: +- In [GitLab 15.10](https://gitlab.com/gitlab-org/gitaly/-/issues/4754) and later: + + ```ruby + ignored_blobs = "/etc/gitlab/instance_wide_ignored_git_blobs.txt" + + gitaly['configuration'] = { + # ... + git: { + # ... + config: [ + # Populate a file with one unabbreviated SHA-1 per line. + # See https://git-scm.com/docs/git-config#Documentation/git-config.txt-fsckskipList + { key: "fsck.skipList", value: ignored_blobs }, + { key: "fetch.fsck.skipList", value: ignored_blobs }, + { key: "receive.fsck.skipList", value: ignored_blobs }, + + { key: "fsck.hasDotgit", value: "ignore" }, + { key: "fetch.fsck.hasDotgit", value: "ignore" }, + { key: "receive.fsck.hasDotgit", value: "ignore" }, + { key: "fsck.missingSpaceBeforeEmail", value: "ignore" }, + ], + }, + } + ``` + +- In [GitLab 15.3](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6800) to GitLab 15.9: ```ruby ignored_blobs = "/etc/gitlab/instance_wide_ignored_git_blobs.txt" @@ -1407,10 +1508,14 @@ By default, Gitaly doesn't sign commits made using GitLab UI. For example, commi - Web IDE. - Merge requests. -You can configure Gitaly to sign commits made using GitLab UI. The commits show as unverified and signed by an unknown user. Support for improvements is -proposed in issue [19185](https://gitlab.com/gitlab-org/gitlab/-/issues/19185). +You can configure Gitaly to sign commits made with the GitLab UI. The commits show as unverified and signed by an unknown +user. Support for improvements is proposed in [issue 19185](https://gitlab.com/gitlab-org/gitlab/-/issues/19185). + +Configure Gitaly to sign commits made with the GitLab UI in one of two ways: + +::Tabs -**For Omnibus GitLab** +:::TabTitle Linux package (Omnibus) 1. [Create a GPG key](../../user/project/repository/gpg_signed_commits/index.md#create-a-gpg-key) and export it. For optimal performance, consider using an EdDSA key. @@ -1423,12 +1528,18 @@ proposed in issue [19185](https://gitlab.com/gitlab-org/gitlab/-/issues/19185). 1. Edit `/etc/gitlab/gitlab.rb` and configure `gitaly['gpg_signing_key_path']`: ```ruby - gitaly['gpg_signing_key_path'] = "/etc/gitlab/gitaly/signing_key.gpg" + gitaly['configuration'] = { + # ... + git: { + # ... + signing_key: '/etc/gitlab/gitaly/signing_key.gpg', + }, + } ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -**For installations from source** +:::TabTitle Self-compiled (source) 1. [Create a GPG key](../../user/project/repository/gpg_signed_commits/index.md#create-a-gpg-key) and export it. For optimal performance, consider using an EdDSA key. @@ -1446,3 +1557,60 @@ proposed in issue [19185](https://gitlab.com/gitlab-org/gitlab/-/issues/19185). ``` 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). + +::EndTabs + +## Generate configuration using an external command + +> [Introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/4828) in GitLab 15.11. + +You can generate parts of the Gitaly configuration using an external command. You might do this: + +- To configure nodes without having to distribute the full configuration to each of them. +- To configure using auto-discovery of the node's settings. For example, using DNS entries. +- To configure secrets at startup of the node, so that don't need to be visible in plain text. + +To generate configuration using an external command, you must provide a script that dumps the +desired configuration of the Gitaly node in JSON format to its standard output. + +For example, the following command configures the HTTP password used to connect to the +GitLab internal API using an AWS secret: + +```ruby +#!/usr/bin/env ruby +require 'json' +JSON.generate({"gitlab": {"http_settings": {"password": `aws get-secret-value --secret-id ...`}}}) +``` + +You must then make the script path known to Gitaly in one of two ways: + +::Tabs + +:::TabTitle Linux package (Omnibus) + +Edit `/etc/gitlab/gitlab.rb` and configure the `config_command`: + +```ruby +gitaly['configuration'] = { + config_command: '/path/to/config_command', +} +``` + +:::TabTitle Self-compiled (source) + +Edit `/home/git/gitaly/config.toml` and configure `config_command`: + +```toml +config_command = "/path/to/config_command" +``` + +::EndTabs + +After configuration, Gitaly executes the command on startup and parses its +standard output as JSON. The resulting configuration is then merged back into +the other Gitaly configuration. + +Gitaly fails to start up if either: + +- The configuration command fails. +- The output produced by the command cannot be parsed as valid JSON. diff --git a/doc/administration/gitaly/gitaly_geo_capabilities.md b/doc/administration/gitaly/gitaly_geo_capabilities.md new file mode 100644 index 00000000000..e4147eec162 --- /dev/null +++ b/doc/administration/gitaly/gitaly_geo_capabilities.md @@ -0,0 +1,41 @@ +--- +stage: Systems +group: Gitaly +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Gitaly and Geo capabilities + +It is common to want the most available, quickly recoverable, highly performant, +and fully resilient solution for your data. However, there are tradeoffs. + +The following tables are intended to guide you to choose the right combination of capabilities based on your requirements. + +## Gitaly capabilities + +| Capability | Availability | Recoverability | Data Resiliency | Performance | Risks/Trade-offs| +|------------|--------------|----------------|-----------------|-------------|-----------------| +|Gitaly Cluster | Very high - tolerant of node failures | RTO for a single node of 10 s with no manual intervention | Data is stored on multiple nodes | Good - While writes may take slightly longer due to voting, read distribution improves read speeds | **Trade-off** - Slight decrease in write speed for redundant, strongly-consistent storage solution. **Risks** - [Does not support snapshot backups](../gitaly/index.md#snapshot-backup-and-recovery-limitations), GitLab backup task can be slow for large data sets | +|Gitaly Shards | Single storage location is a single point of failure | Would need to restore only shards which failed | Single point of failure | Good - can allocate repositories to shards to spread load | **Trade-off** - Need to manually configure repositories into different shards to balance loads / storage space **Risks** - Single point of failure relies on recovery process when single-node failure occurs | +|Gitaly + NFS | Single storage location is a single point of failure | Single node failure requires restoration from backup | Single point of failure | Average - NFS is not ideally suited to large quantities of small reads / writes which can have a detrimental impact on performance | **Trade-off** - Familiar administration though NFS is not ideally suited to Git demands **Risks** - Many instances of NFS compatibility issues which provide very poor customer experiences | + +## Geo capabilities + +If your availability needs to span multiple zones or multiple locations, read about [Geo](../geo/index.md). + +| Capability | Availability | Recoverability | Data Resiliency | Performance | Risks/Trade-offs| +|------------|--------------|----------------|-----------------|-------------|-----------------| +|Geo| Depends on the architecture of the Geo site. It is possible to deploy secondaries in single and multiple node configurations. | Eventually consistent. Recovery point depends on replication lag, which depends on a number of factors such as network speeds. Geo supports failover from a primary to secondary site using manual commands that are scriptable. | Geo replicates 100% of planned data types and verifies 50%. See [limitations table](../geo/replication/datatypes.md#limitations-on-replicationverification) for more detail. | Improves read/clone times for users of a secondary. | Geo is not intended to replace other backup/restore solutions. Because of replication lag and the possibility of replicating bad data from a primary, customers should also take regular backups of their primary site and test the restore process. | + +## Scenarios for failure modes and available mitigation paths + +The following table outlines failure modes and mitigation paths for the product offerings detailed in the tables above. Note - Gitaly Cluster install assumes an odd number replication factor of 3 or greater + +| Gitaly Mode | Loss of Single Gitaly Node | Application / Data Corruption | Regional Outage (Loss of Instance) | Notes | +| ----------- | -------------------------- | ----------------------------- | ---------------------------------- | ----- | +| Single Gitaly Node | Downtime - Must restore from backup | Downtime - Must restore from Backup | Downtime - Must wait for outage to end | | +| Single Gitaly Node + Geo Secondary | Downtime - Must restore from backup, can perform a manual failover to secondary | Downtime - Must restore from Backup, errors could have propagated to secondary | Manual intervention - failover to Geo secondary | | +| Sharded Gitaly Install | Partial Downtime - Only repositories on impacted node affected, must restore from backup | Partial Downtime - Only repositories on impacted node affected, must restore from backup | Downtime - Must wait for outage to end | | +| Sharded Gitaly Install + Geo Secondary | Partial Downtime - Only repositories on impacted node affected, must restore from backup, could perform manual failover to secondary for impacted repositories | Partial Downtime - Only repositories on impacted node affected, must restore from backup, errors could have propagated to secondary | Manual intervention - failover to Geo secondary | | +| Gitaly Cluster Install* | No Downtime - swaps repository primary to another node after 10 seconds | Not applicable; All writes are voted on by multiple Gitaly Cluster nodes | Downtime - Must wait for outage to end | Snapshot backups for Gitaly Cluster nodes not supported at this time | +| Gitaly Cluster Install* + Geo Secondary | No Downtime - swaps repository primary to another node after 10 seconds | Not applicable; All writes are voted on by multiple Gitaly Cluster nodes | Manual intervention - failover to Geo secondary | Snapshot backups for Gitaly Cluster nodes not supported at this time | diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index 405c56284cf..359d4ef90dc 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -69,7 +69,7 @@ the current status of these issues, refer to the referenced issues and epics. | Issue | Summary | How to avoid | |:--------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------| -| Gitaly Cluster + Geo - Issues retrying failed syncs | If Gitaly Cluster is used on a Geo secondary site, repositories that have failed to sync could continue to fail when Geo tries to resync them. Recovering from this state requires assistance from support to run manual steps. | No known solution prior to GitLab 15.0. In GitLab 15.0 to 15.2, enable the [`gitaly_praefect_generated_replica_paths` feature flag](#praefect-generated-replica-paths-gitlab-150-and-later). In GitLab 15.3, the feature flag is enabled by default. | +| Gitaly Cluster + Geo - Issues retrying failed syncs | If Gitaly Cluster is used on a Geo secondary site, repositories that have failed to sync could continue to fail when Geo tries to resync them. Recovering from this state requires assistance from support to run manual steps. | No known solution prior to GitLab 15.0. In GitLab 15.0 to 15.2, enable the [`gitaly_praefect_generated_replica_paths` feature flag](#praefect-generated-replica-paths-gitlab-150-and-later) on your Geo primary site. In GitLab 15.3, the feature flag is enabled by default. | | Praefect unable to insert data into the database due to migrations not being applied after an upgrade | If the database is not kept up to date with completed migrations, then the Praefect node is unable to perform standard operation. | Make sure the Praefect database is up and running with all migrations completed (For example: `/opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml sql-migrate-status` should show a list of all applied migrations). Consider [requesting upgrade assistance](https://about.gitlab.com/support/scheduling-upgrade-assistance/) so your upgrade plan can be reviewed by support. | | Restoring a Gitaly Cluster node from a snapshot in a running cluster | Because the Gitaly Cluster runs with consistent state, introducing a single node that is behind results in the cluster not being able to reconcile the nodes data and other nodes data | Don't restore a single Gitaly Cluster node from a backup snapshot. If you must restore from backup:

1. [Shut down GitLab](../read_only_gitlab.md#shut-down-the-gitlab-ui).
2. Snapshot all Gitaly Cluster nodes at the same time.
3. Take a database dump of the Praefect database. | | Rebuilding or replacing an existing Gitaly Cluster node | There is no way to replace existing nodes in place because the Praefect database is relied on to determine the current state of each Gitaly node. Changing how Gitaly Cluster stores repositories is proposed in issue [4218](https://gitlab.com/gitlab-org/gitaly/-/issues/4218). Turning on [repository verification](praefect.md#repository-verification) is proposed in issue [4429](https://gitlab.com/gitlab-org/gitaly/-/issues/4429) so Praefect can detect that data is missing and needs to replicated to a new Gitaly node. | No known solution at this time. | @@ -78,7 +78,7 @@ the current status of these issues, refer to the referenced issues and epics. Gitaly Cluster does not support snapshot backups. Snapshot backups can cause issues where the Praefect database becomes out of sync with the disk storage. Because of how Praefect rebuilds the replication metadata of Gitaly disk information -during a restore, we recommend using the [official backup and restore Rake tasks](../../raketasks/backup_restore.md). +during a restore, you should use the [official backup and restore Rake tasks](../../raketasks/backup_restore.md). The [incremental backup method](../../raketasks/backup_gitlab.md#incremental-repository-backups) can be used to speed up Gitaly Cluster backups. @@ -100,7 +100,7 @@ These SSDs should have a throughput of at least: - 2,000 IOPS for write operations. These IOPS values are initial recommendations, and may be adjusted to greater or lesser values -depending on the scale of your environment's workload. If you’re running the environment on a +depending on the scale of your environment's workload. If you're running the environment on a cloud provider, refer to their documentation about how to configure IOPS correctly. For repository data, only local storage is supported for Gitaly and Gitaly Cluster for performance and consistency reasons. @@ -363,10 +363,10 @@ For example, `@cluster/repositories/6f/96/54771`. The last component of the replica path, `54771`, is the repository ID. This can be used to identify the repository on the disk. -`/` are the first four hex digits of the SHA256 hash of the string representation of the repository ID. This is used to balance -the repositories evenly into subdirectories to avoid overly large directories that might cause problems on some file -systems. In this case, `54771` hashes to `6f960ab01689464e768366d3315b3d3b2c28f38761a58a70110554eb04d582f7` so the -first four digits are `6f` and `96`. +`/` are the first four hex digits of the SHA256 hash of the string representation of the repository ID. These +digits are used to balance the repositories evenly into subdirectories to avoid overly large directories that might +cause problems on some file systems. In this case, `54771` hashes to +`6f960ab01689464e768366d3315b3d3b2c28f38761a58a70110554eb04d582f7` so the first four digits are `6f` and `96`. #### Identify repositories on disk @@ -387,8 +387,9 @@ Follow the [instructions in hashed storage's documentation](../repository_storag Gitaly Cluster uses the PostgreSQL metadata store with the storage layout to ensure atomicity of repository creation, deletion, and move operations. The disk operations can't be atomically applied across multiple storages. However, PostgreSQL guarantees the atomicity of the metadata operations. Gitaly Cluster models the operations in a manner that the failing operations always leave -the metadata consistent. The disks may contain stale state even after successful operations. This is expected and the leftover state -does not interfere with future operations but may use up disk space unnecessarily until a clean up is performed. +the metadata consistent. The disks may contain stale state even after successful operations. This situation is expected and +the leftover state does not interfere with future operations but may use up disk space unnecessarily until a clean up is +performed. There is on-going work on a [background crawler](https://gitlab.com/gitlab-org/gitaly/-/issues/3719) that cleans up the leftover repositories from the storages. @@ -397,7 +398,7 @@ repositories from the storages. When creating repositories, Praefect: -1. Reserves a repository ID from PostgreSQL. This is atomic and no two creations receive the same ID. +1. Reserves a repository ID from PostgreSQL, which is atomic and no two creations receive the same ID. 1. Creates replicas on the Gitaly storages in the replica path derived from the repository ID. 1. Creates metadata records after the repository is successfully created on disk. @@ -470,9 +471,12 @@ _Up to date_ in this context means that: The primary node is chosen to serve the request if: -- There are no up to date nodes. +- No up-to-date nodes exist. - Any other error occurs during node selection. +If you have a large, heavily modified repository (like a multi-gigabyte monorepo), the primary node can service most or all requests if changes come in faster than Praefect +can replicate to the secondaries. When this occurs, CI/CD jobs and other repository traffic are bottlenecked by the capacity of the primary node. + You can [monitor distribution of reads](monitoring.md#monitor-gitaly-cluster) using Prometheus. #### Strong consistency @@ -677,7 +681,7 @@ Direct Git access is: For the sake of removing complexity, we must remove direct Git access in GitLab. However, we can't remove it as long some GitLab installations require Git repositories on NFS. -There are two facets to our efforts to remove direct Git access in GitLab: +Two facets of our efforts to remove direct Git access in GitLab are: - Reduce the number of inefficient Gitaly queries made by GitLab. - Persuade administrators of fault-tolerant or horizontally-scaled GitLab instances to migrate off diff --git a/doc/administration/gitaly/monitoring.md b/doc/administration/gitaly/monitoring.md index 0fd34d5c89f..00d0499faa2 100644 --- a/doc/administration/gitaly/monitoring.md +++ b/doc/administration/gitaly/monitoring.md @@ -54,6 +54,9 @@ You can observe the status of [control groups (cgroups)](configure_gitaly.md#con - `gitaly_cgroups_cpu_usage`, a gauge that measures CPU usage per cgroup. - `gitaly_cgroup_procs_total`, a gauge that measures the total number of processes Gitaly has spawned under the control of cgroups. +- `gitaly_cgroup_cpu_cfs_periods_total`, a counter that for the value of [`nr_periods`](https://docs.kernel.org/scheduler/sched-bwc.html#statistics). +- `gitaly_cgroup_cpu_cfs_throttled_periods_total`, a counter for the value of [`nr_throttled`](https://docs.kernel.org/scheduler/sched-bwc.html#statistics). +- `gitaly_cgroup_cpu_cfs_throttled_seconds_total`, a counter for the value of [`throttled_time`](https://docs.kernel.org/scheduler/sched-bwc.html#statistics) in seconds. ## `pack-objects` cache @@ -86,9 +89,9 @@ gitaly_streamcache_filestore_removed_total{dir="/var/opt/gitlab/git-data/reposit gitaly_streamcache_index_entries{dir="/var/opt/gitlab/git-data/repositories/+gitaly/PackObjectsCache"} 1 ``` -## Useful queries +## Queries -The following are useful queries for monitoring Gitaly: +The following are some queries for monitoring Gitaly: - Use the following Prometheus query to observe the [type of connections](configure_gitaly.md#enable-tls-support) Gitaly is serving a production @@ -130,8 +133,8 @@ The following are useful queries for monitoring Gitaly: ## Monitor Gitaly Cluster -To monitor Gitaly Cluster (Praefect), you can use these Prometheus metrics. There are two separate metrics -endpoints from which metrics can be scraped: +To monitor Gitaly Cluster (Praefect), you can use these Prometheus metrics. Two separate metrics endpoints are +available from which metrics can be scraped: - The default `/metrics` endpoint. - `/db_metrics`, which contains metrics that require database queries. @@ -153,6 +156,7 @@ The following metrics are available from the `/metrics` endpoint: - `gitaly_praefect_replication_delay_bucket`, a histogram measuring how much time passes between when the replication job is created and when it starts. Available in GitLab 12.10 and later. - `gitaly_praefect_connections_total`, the total number of connections to Praefect. [Introduced](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/4220) in GitLab 14.7. +- `gitaly_praefect_method_types`, a count of accessor and mutator RPCs per node. To monitor [strong consistency](index.md#strong-consistency), you can use the following Prometheus metrics: @@ -179,9 +183,6 @@ To monitor [repository verification](praefect.md#repository-verification), use t - `gitaly_praefect_stale_verification_leases_released_total`, the number of stale verification leases released. -The `/metrics` endpoint also provides all the metrics available under the `/db_metrics` endpoint. Using `/metrics` for `/db_metrics` metrics -is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/390266) in GitLab 15.9 and will be removed in GitLab 16.0. - You can also monitor the [Praefect logs](../logs/index.md#praefect-logs). ### Database metrics `/db_metrics` endpoint diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index 440bd7427ae..51201ec442f 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -142,7 +142,7 @@ with secure tokens as you complete the setup process. 1. `PRAEFECT_EXTERNAL_TOKEN`: repositories hosted on your Praefect cluster can only be accessed by Gitaly clients that carry this token. 1. `PRAEFECT_INTERNAL_TOKEN`: this token is used for replication traffic inside - your Praefect cluster. This is distinct from `PRAEFECT_EXTERNAL_TOKEN` + your Praefect cluster. This token is distinct from `PRAEFECT_EXTERNAL_TOKEN` because Gitaly clients must not be able to access internal nodes of the Praefect cluster directly; that could lead to data loss. 1. `PRAEFECT_SQL_PASSWORD`: this password is used by Praefect to connect to @@ -269,11 +269,16 @@ The database used by Praefect is now configured. You can now configure Praefect to use the database: ```ruby -praefect['database_host'] = POSTGRESQL_HOST -praefect['database_port'] = 5432 -praefect['database_user'] = 'praefect' -praefect['database_password'] = PRAEFECT_SQL_PASSWORD -praefect['database_dbname'] = 'praefect_production' +praefect['configuration'] = { + # ... + database: { + # ... + host: POSTGRESQL_HOST, + port: 5432, + password: PRAEFECT_SQL_PASSWORD, + dbname: 'praefect_production', + } +} ``` If you see Praefect database errors after configuring PostgreSQL, see @@ -285,19 +290,27 @@ Praefect performance can be improved by additionally configuring the `database_d settings: ```ruby -praefect['database_direct_host'] = POSTGRESQL_HOST -praefect['database_direct_port'] = 5432 - -# Use the following to override parameters of direct database connection. -# Comment out where the parameters are the same for both connections. - -praefect['database_direct_user'] = 'praefect' -praefect['database_direct_password'] = PRAEFECT_SQL_PASSWORD -praefect['database_direct_dbname'] = 'praefect_production' -#praefect['database_direct_sslmode'] = '...' -#praefect['database_direct_sslcert'] = '...' -#praefect['database_direct_sslkey'] = '...' -#praefect['database_direct_sslrootcert'] = '...' +praefect['configuration'] = { + # ... + database: { + # ... + session_pooled: { + # ... + host: POSTGRESQL_HOST, + port: 5432 + + # Use the following to override parameters of direct database connection. + # Comment out where the parameters are the same for both connections. + user: 'praefect', + password: PRAEFECT_SQL_PASSWORD, + dbname: 'praefect_production', + # sslmode: '...', + # sslcert: '...', + # sslkey: '...', + # sslrootcert: '...', + } + } +} ``` When configured, this connection is automatically used for the @@ -313,8 +326,8 @@ reads distribution caching is enabled by configuration #### Use PgBouncer -To reduce PostgreSQL resource consumption, we recommend setting up and configuring -[PgBouncer](https://www.pgbouncer.org/) in front of the PostgreSQL instance. However, PgBouncer isn't required because +To reduce PostgreSQL resource consumption, you should set up and configure [PgBouncer](https://www.pgbouncer.org/) in +front of the PostgreSQL instance. However, PgBouncer isn't required because Praefect makes a low number of connections. If you choose to use PgBouncer, you can use the same PgBouncer instance for both the GitLab application database and the Praefect database. @@ -322,15 +335,21 @@ To configure PgBouncer in front of the PostgreSQL instance, you must point Praef parameters on Praefect configuration: ```ruby -praefect['database_host'] = PGBOUNCER_HOST -praefect['database_port'] = 6432 -praefect['database_user'] = 'praefect' -praefect['database_password'] = PRAEFECT_SQL_PASSWORD -praefect['database_dbname'] = 'praefect_production' -#praefect['database_sslmode'] = '...' -#praefect['database_sslcert'] = '...' -#praefect['database_sslkey'] = '...' -#praefect['database_sslrootcert'] = '...' +praefect['configuration'] = { + # ... + database: { + # ... + host: PGBOUNCER_HOST, + port: 6432, + user: 'praefect', + password: PRAEFECT_SQL_PASSWORD, + dbname: 'praefect_production', + # sslmode: '...', + # sslcert: '...', + # sslkey: '...', + # sslrootcert: '...', + } +} ``` Praefect requires an additional connection to the PostgreSQL that supports the @@ -341,12 +360,12 @@ It is not supported in `transaction` pool mode (`pool_mode = transaction`). To configure the additional connection, you must either: - Configure a new PgBouncer database that uses to the same PostgreSQL database endpoint, - but with different pool mode. That is, `pool_mode = session`. + but with different pool mode (`pool_mode = session`). - Connect Praefect directly to PostgreSQL and bypass PgBouncer. #### Configure a new PgBouncer database with `pool_mode = session` -We recommend using PgBouncer with `session` pool mode. You can use the +You should use PgBouncer with `session` pool mode. You can use the [bundled PgBouncer](../postgresql/pgbouncer.md) or use an external PgBouncer and [configure it manually](https://www.pgbouncer.org/config.html). @@ -399,23 +418,30 @@ praefect_production_direct = host=POSTGRESQL_HOST auth_user=pgbouncer dbname=pra Now you can configure Praefect to use PgBouncer for both connections: ```ruby -praefect['database_host'] = PGBOUNCER_HOST -praefect['database_port'] = 6432 -praefect['database_user'] = 'praefect' -# `PRAEFECT_SQL_PASSWORD` is the plain-text password of -# Praefect user. Not to be confused with `PRAEFECT_SQL_PASSWORD_HASH`. -praefect['database_password'] = PRAEFECT_SQL_PASSWORD - -praefect['database_dbname'] = 'praefect_production' -praefect['database_direct_dbname'] = 'praefect_production_direct' - -# There is no need to repeat the following. Parameters of direct -# database connection will fall back to the values above. - -#praefect['database_direct_host'] = PGBOUNCER_HOST -#praefect['database_direct_port'] = 6432 -#praefect['database_direct_user'] = 'praefect' -#praefect['database_direct_password'] = PRAEFECT_SQL_PASSWORD +praefect['configuration'] = { + # ... + database: { + # ... + host: PGBOUNCER_HOST, + port: 6432, + user: 'praefect', + # `PRAEFECT_SQL_PASSWORD` is the plain-text password of + # Praefect user. Not to be confused with `PRAEFECT_SQL_PASSWORD_HASH`. + password: PRAEFECT_SQL_PASSWORD, + dbname: 'praefect_production', + session_pooled: { + # ... + dbname: 'praefect_production_direct', + # There is no need to repeat the following. Parameters of direct + # database connection will fall back to the values above. + # + # host: PGBOUNCER_HOST, + # port: 6432, + # user: 'praefect', + # password: PRAEFECT_SQL_PASSWORD, + }, + }, +} ``` With this configuration, Praefect uses PgBouncer for both connection types. @@ -428,25 +454,34 @@ configuration option is set. For more details, consult the PgBouncer documentati #### Configure Praefect to connect directly to PostgreSQL -As an alternative to configuring PgBouncer with `session` pool mode, Praefect can be configured to use different connection parameters for direct access -to PostgreSQL. This is the connection that supports the `LISTEN` feature. +As an alternative to configuring PgBouncer with `session` pool mode, Praefect can be configured to use different +connection parameters for direct access to PostgreSQL. This connection supports the `LISTEN` feature. An example of Praefect configuration that bypasses PgBouncer and directly connects to PostgreSQL: ```ruby -praefect['database_direct_host'] = POSTGRESQL_HOST -praefect['database_direct_port'] = 5432 - -# Use the following to override parameters of direct database connection. -# Comment out where the parameters are the same for both connections. - -praefect['database_direct_user'] = 'praefect' -praefect['database_direct_password'] = PRAEFECT_SQL_PASSWORD -praefect['database_direct_dbname'] = 'praefect_production' -#praefect['database_direct_sslmode'] = '...' -#praefect['database_direct_sslcert'] = '...' -#praefect['database_direct_sslkey'] = '...' -#praefect['database_direct_sslrootcert'] = '...' +praefect['configuration'] = { + # ... + database: { + # ... + session_pooled: { + # ... + host: POSTGRESQL_HOST, + port: 5432, + + # Use the following to override parameters of direct database connection. + # Comment out where the parameters are the same for both connections. + # + user: 'praefect', + password: PRAEFECT_SQL_PASSWORD, + dbname: 'praefect_production', + # sslmode: '...', + # sslcert: '...', + # sslkey: '...', + # sslrootcert: '...', + }, + }, +} ``` ### Praefect @@ -501,30 +536,42 @@ Updates to example must be made at: `/etc/gitlab/gitlab.rb`: ```ruby - praefect['listen_addr'] = '0.0.0.0:2305' + praefect['configuration'] = { + # ... + listen_addr: '0.0.0.0:2305', + } ``` 1. Configure Prometheus metrics by editing `/etc/gitlab/gitlab.rb`: ```ruby - # Enable Prometheus metrics access to Praefect. You must use firewalls - # to restrict access to this address/port. - # The default metrics endpoint is /metrics - praefect['prometheus_listen_addr'] = '0.0.0.0:9652' - - # Some metrics run queries against the database. Enabling separate database metrics allows - # these metrics to be collected when the metrics are - # scraped on a separate /db_metrics endpoint. - praefect['separate_database_metrics'] = true + praefect['configuration'] = { + # ... + # + # Enable Prometheus metrics access to Praefect. You must use firewalls + # to restrict access to this address/port. + # The default metrics endpoint is /metrics + prometheus_listen_addr: '0.0.0.0:9652', + # Some metrics run queries against the database. Enabling separate database metrics allows + # these metrics to be collected when the metrics are + # scraped on a separate /db_metrics endpoint. + prometheus_exclude_database_from_default_metrics: true, + } ``` -1. Configure a strong `auth_token` for **Praefect** by editing - `/etc/gitlab/gitlab.rb`. This is needed by clients outside the cluster +1. Configure a strong authentication token for **Praefect** by editing + `/etc/gitlab/gitlab.rb`, which is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster: ```ruby - praefect['auth_token'] = 'PRAEFECT_EXTERNAL_TOKEN' + praefect['configuration'] = { + # ... + auth: { + # ... + token: 'PRAEFECT_EXTERNAL_TOKEN', + }, + } ``` 1. Configure **Praefect** to [connect to the PostgreSQL database](#postgresql). We @@ -533,19 +580,32 @@ Updates to example must be made at: If you want to use a TLS client certificate, the options below can be used: ```ruby - # Connect to PostgreSQL using a TLS client certificate - # praefect['database_sslcert'] = '/path/to/client-cert' - # praefect['database_sslkey'] = '/path/to/client-key' - - # Trust a custom certificate authority - # praefect['database_sslrootcert'] = '/path/to/rootcert' + praefect['configuration'] = { + # ... + database: { + # ... + # + # Connect to PostgreSQL using a TLS client certificate + # sslcert: '/path/to/client-cert', + # sslkey: '/path/to/client-key', + # + # Trust a custom certificate authority + # sslrootcert: '/path/to/rootcert', + }, + } ``` By default, Praefect refuses to make an unencrypted connection to PostgreSQL. You can override this by uncommenting the following line: ```ruby - # praefect['database_sslmode'] = 'disable' + praefect['configuration'] = { + # ... + database: { + # ... + # sslmode: 'disable', + }, + } ``` 1. Configure the **Praefect** cluster to connect to each Gitaly node in the @@ -573,29 +633,37 @@ Updates to example must be made at: NOTE: When adding additional Gitaly nodes to a virtual storage, all storage names - within that virtual storage must be unique. Additionally, all Gitaly node + in that virtual storage must be unique. Additionally, all Gitaly node addresses referenced in the Praefect configuration must be unique. ```ruby # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('default') and in git_data_dirs on Gitaly nodes ('gitaly-1') - praefect['virtual_storages'] = { - 'default' => { - 'nodes' => { - 'gitaly-1' => { - 'address' => 'tcp://GITALY_HOST_1:8075', - 'token' => 'PRAEFECT_INTERNAL_TOKEN', - }, - 'gitaly-2' => { - 'address' => 'tcp://GITALY_HOST_2:8075', - 'token' => 'PRAEFECT_INTERNAL_TOKEN' + # server ('default') and in gitaly['configuration'][:storage][INDEX][:name] on Gitaly nodes ('gitaly-1') + praefect['configuration'] = { + # ... + virtual_storage: [ + { + # ... + name: 'default', + node: [ + { + storage: 'gitaly-1', + address: 'tcp://GITALY_HOST_1:8075', + token: 'PRAEFECT_INTERNAL_TOKEN' + }, + { + storage: 'gitaly-2', + address: 'tcp://GITALY_HOST_2:8075', + token: 'PRAEFECT_INTERNAL_TOKEN' + }, + { + storage: 'gitaly-3', + address: 'tcp://GITALY_HOST_3:8075', + token: 'PRAEFECT_INTERNAL_TOKEN' + }, + ], }, - 'gitaly-3' => { - 'address' => 'tcp://GITALY_HOST_3:8075', - 'token' => 'PRAEFECT_INTERNAL_TOKEN' - } - } - } + ], } ``` @@ -662,7 +730,7 @@ for secure connections, you must: Additionally the certificate, or its certificate authority, must be installed on all Gitaly servers and on all Praefect clients that communicate with it following the procedure described in -[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) (and repeated below). +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl/index.html#install-custom-public-certificates) (and repeated below). Note the following: @@ -681,7 +749,14 @@ Note the following: This allows you to do a gradual transition from unencrypted to encrypted traffic, if necessary. - To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. + To disable the unencrypted listener, set: + + ```ruby + praefect['configuration'] = { + # ... + listen_addr: nil, + } + ``` To configure Praefect with TLS: @@ -702,9 +777,15 @@ To configure Praefect with TLS: 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - praefect['tls_listen_addr'] = "0.0.0.0:3305" - praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - praefect['key_path'] = "/etc/gitlab/ssl/key.pem" + praefect['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:3305', + tls: { + # ... + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -790,6 +871,125 @@ To configure Praefect with TLS: 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). +#### Service discovery + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/8971) in GitLab 15.10. + +Prerequisites: + +- A DNS server. + +GitLab uses service discovery to retrieve a list of Praefect hosts. Service +discovery involves periodic checks of a DNS A or AAAA record, with the IPs +retrieved from the record serving as the addresses of the target nodes. +Praefect does not support service discovery by SRV record. + +By default, the minimum time between checks is 5 minutes, regardless of the +records' TTLs. Praefect does not support customizing this interval. When clients +receive an update, they: + +- Establish new connections to the new IP addresses. +- Keep existing connections to intact IP addresses. +- Drop connections to removed IP addresses. + +In-flight requests on to-be-removed connections are still handled until they +finish. Workhorse has a 10-minute timeout, while other clients do not specify a +graceful timeout. + +The DNS server should return all IP addresses instead of load-balancing itself. +Clients can distribute requests to IP addresses in a round-robin fashion. + +Before updating client configuration, ensure that DNS service discovery works +correctly. It should return the list of IP addresses correctly. `dig` is a good +tool to use to verify. + +```console +❯ dig A praefect.service.consul @127.0.0.1 + +; <<>> DiG 9.10.6 <<>> A praefect.service.consul @127.0.0.1 +;; global options: +cmd +;; Got answer: +;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 29210 +;; flags: qr aa rd ra; QUERY: 1, ANSWER: 3, AUTHORITY: 0, ADDITIONAL: 1 + +;; OPT PSEUDOSECTION: +; EDNS: version: 0, flags:; udp: 4096 +;; QUESTION SECTION: +;praefect.service.consul. IN A + +;; ANSWER SECTION: +praefect.service.consul. 0 IN A 10.0.0.3 +praefect.service.consul. 0 IN A 10.0.0.2 +praefect.service.consul. 0 IN A 10.0.0.1 + +;; Query time: 0 msec +;; SERVER: ::1#53(::1) +;; WHEN: Wed Dec 14 12:53:58 +07 2022 +;; MSG SIZE rcvd: 86 +``` + +##### Configure service discovery + +By default, Praefect delegates DNS resolution to the operating system. In such +cases, the Gitaly address can be set in either of these formats: + +- `dns:[host]:[port]` +- `dns:///[host]:[port]` (note the three slashes) + +You can also appoint an authoritative name server by setting it in this format: + +- `dns://[authority_host]:[authority_port]/[host]:[port]` + +::Tabs + +:::TabTitle Linux package (Omnibus) + +1. Edit `/etc/gitlab/gitlab.rb` and add: + + ```ruby + praefect['consul_service_name'] = 'praefect' + ``` + +1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. On the Praefect clients (except Gitaly servers), edit `git_data_dirs` in +`/etc/gitlab/gitlab.rb` as follows. Replace `PRAEFECT_SERVICE_DISCOVERY_ADDRESS` +with Praefect service discovery address, such as `praefect.service.consul`. + + ```ruby + git_data_dirs({ + "default" => { + "gitaly_address" => 'dns:PRAEFECT_SERVICE_DISCOVERY_ADDRESS:2305', + "gitaly_token" => 'PRAEFECT_EXTERNAL_TOKEN' + } + }) + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +:::TabTitle Self-compiled (source) + +1. Install a DNS service discovery service. Register all Praefect nodes with the service. +1. On the Praefect clients (except Gitaly servers), edit `storages` in + `/home/git/gitlab/config/gitlab.yml` as follows: + + ```yaml + gitlab: + repositories: + storages: + default: + gitaly_address: dns:PRAEFECT_SERVICE_DISCOVERY_ADDRESS:2305 + path: /some/local/path + ``` + + NOTE: + `/some/local/path` should be set to a local folder that exists, however no + data is stored in this folder. [Issue 375254](https://gitlab.com/gitlab-org/gitlab/-/issues/375254) + proposes to remove this requirement. + +1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). + +::EndTabs + ### Gitaly NOTE: @@ -813,12 +1013,11 @@ because we rely on Praefect to route operations correctly. Particular attention should be shown to: -- The `gitaly['auth_token']` configured in this section must match the `token` - value under `praefect['virtual_storages']['nodes']` on the Praefect node. This was set - in the [previous section](#praefect). This document uses the placeholder - `PRAEFECT_INTERNAL_TOKEN` throughout. -- The storage names in `git_data_dirs` configured in this section must match the - storage names under `praefect['virtual_storages']` on the Praefect node. This +- The `gitaly['configuration'][:auth][:token]` configured in this section must match the `token` + value under `praefect['configuration'][:virtual_storage][][:node][][:token]` on the Praefect node. This value was + set in the [previous section](#praefect). This document uses the placeholder `PRAEFECT_INTERNAL_TOKEN` throughout. +- The storage names in `gitaly['configuration'][:storage]` configured in this section must match the + storage names under `praefect['configuration'][:virtual_storage]` on the Praefect node. This was set in the [previous section](#praefect). This document uses `gitaly-1`, `gitaly-2`, and `gitaly-3` as Gitaly storage names. @@ -834,7 +1033,7 @@ For more information on Gitaly server configuration, see our 1. Disable all other services by editing `/etc/gitlab/gitlab.rb`: ```ruby - # Disable all other services on the Praefect node + # Disable all other services on the Gitaly node postgresql['enable'] = false redis['enable'] = false nginx['enable'] = false @@ -859,22 +1058,31 @@ For more information on Gitaly server configuration, see our `/etc/gitlab/gitlab.rb`: ```ruby - # Make Gitaly accept connections on all network interfaces. - # Use firewalls to restrict access to this address/port. - gitaly['listen_addr'] = '0.0.0.0:8075' - - # Enable Prometheus metrics access to Gitaly. You must use firewalls - # to restrict access to this address/port. - gitaly['prometheus_listen_addr'] = '0.0.0.0:9236' + gitaly['configuration'] = { + # ... + # + # Make Gitaly accept connections on all network interfaces. + # Use firewalls to restrict access to this address/port. + listen_addr: '0.0.0.0:8075', + # Enable Prometheus metrics access to Gitaly. You must use firewalls + # to restrict access to this address/port. + prometheus_listen_addr: '0.0.0.0:9236', + } ``` 1. Configure a strong `auth_token` for **Gitaly** by editing - `/etc/gitlab/gitlab.rb`. This is needed by clients to communicate with + `/etc/gitlab/gitlab.rb`, which is needed by clients to communicate with this Gitaly nodes. Typically, this token is the same for all Gitaly nodes. ```ruby - gitaly['auth_token'] = 'PRAEFECT_INTERNAL_TOKEN' + gitaly['configuration'] = { + # ... + auth: { + # ... + token: 'PRAEFECT_INTERNAL_TOKEN', + }, + } ``` 1. Configure the GitLab Shell secret token, which is needed for `git push` operations. Either: @@ -903,13 +1111,13 @@ For more information on Gitaly server configuration, see our gitlab_rails['internal_api_url'] = 'http://GITLAB_HOST' ``` -1. Configure the storage location for Git data by setting `git_data_dirs` in +1. Configure the storage location for Git data by setting `gitaly['configuration'][:storage]` in `/etc/gitlab/gitlab.rb`. Each Gitaly node should have a unique storage name (such as `gitaly-1`). - Instead of configuring `git_data_dirs` uniquely for each Gitaly node, it is + Instead of configuring `gitaly['configuration'][:storage]` uniquely for each Gitaly node, it is often easier to have include the configuration for all Gitaly nodes on every - Gitaly node. This is supported because the Praefect `virtual_storages` + Gitaly node. You can do this because the Praefect `virtual_storage` configuration maps each storage name (such as `gitaly-1`) to a specific node, and requests are routed accordingly. This means every Gitaly node in your fleet can share the same configuration. @@ -918,17 +1126,23 @@ For more information on Gitaly server configuration, see our # You can include the data dirs for all nodes in the same config, because # Praefect will only route requests according to the addresses provided in the # prior step. - git_data_dirs({ - "gitaly-1" => { - "path" => "/var/opt/gitlab/git-data" - }, - "gitaly-2" => { - "path" => "/var/opt/gitlab/git-data" - }, - "gitaly-3" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-1', + path: '/var/opt/gitlab/git-data', + }, + { + name: 'gitaly-2', + path: '/var/opt/gitlab/git-data', + }, + { + name: 'gitaly-3', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` 1. Save the changes to `/etc/gitlab/gitlab.rb` and @@ -967,10 +1181,7 @@ scope of the GitLab documentation. NOTE: The load balancer must be configured to accept traffic from the Gitaly nodes in -addition to the GitLab nodes. Some requests handled by -[`gitaly-ruby`](configure_gitaly.md#gitaly-ruby) sidecar processes call into the main Gitaly -process. `gitaly-ruby` uses the Gitaly address set in the GitLab server's -`git_data_dirs` setting to make this connection. +addition to the GitLab nodes. We hope that if you're managing fault-tolerant systems like GitLab, you have a load balancer of choice already. Some examples include [HAProxy](https://www.haproxy.org/) @@ -980,8 +1191,8 @@ Big-IP LTM, and Citrix Net Scaler. This documentation outlines what ports and protocols you need configure. NOTE: -We recommend the equivalent of HAProxy `leastconn` load-balancing strategy because long-running operations (for example, -clones) keep some connections open for extended periods. +You should use the equivalent of HAProxy `leastconn` load-balancing strategy because long-running operations (for +example, clones) keep some connections open for extended periods. | LB Port | Backend Port | Protocol | |:--------|:-------------|:---------| @@ -995,12 +1206,12 @@ To complete this section you need: - [Configured Gitaly nodes](#gitaly) The Praefect cluster needs to be exposed as a storage location to the GitLab -application. This is done by updating the `git_data_dirs`. +application, which is done by updating the `git_data_dirs`. Particular attention should be shown to: - the storage name added to `git_data_dirs` in this section must match the - storage name under `praefect['virtual_storages']` on the Praefect nodes. This + storage name under `praefect['configuration'][:virtual_storage]` on the Praefect nodes. This was set in the [Praefect](#praefect) section of this guide. This document uses `default` as the Praefect storage name. @@ -1219,12 +1430,16 @@ You can configure: The configuration is added to the `/etc/gitlab/gitlab.rb` file: ```ruby - praefect['virtual_storages'] = { - 'default' => { - 'default_replication_factor' => 1, + praefect['configuration'] = { # ... - } - } + virtual_storage: [ + { + # ... + name: 'default', + default_replication_factor: 1, + }, + ], + } ``` - A replication factor for an existing repository using the `set-replication-factor` sub-command. @@ -1313,29 +1528,50 @@ interval is configurable with any valid [Go duration string](https://pkg.go.dev/ To verify the metadata every three days: ```ruby -praefect['background_verification_verification_interval'] = '72h' +praefect['configuration'] = { + # ... + background_verification: { + # ... + verification_interval: '72h', + }, +} ``` Values of 0 and below disable the background verifier. ```ruby -praefect['background_verification_verification_interval'] = '0' +praefect['configuration'] = { + # ... + background_verification: { + # ... + verification_interval: '0', + }, +} ``` #### Enable deletions +> - [Introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/4080) and disabled by default in GitLab 15.0 +> - [Default enabled](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/5321) in GitLab 15.9. + WARNING: -Deletions are disabled by default due to a race condition with repository renames that can cause incorrect -deletions. This is especially prominent in Geo instances as Geo performs more renames than instances without Geo. -You should enable deletions only if the [`gitaly_praefect_generated_replica_paths` feature flag](index.md#praefect-generated-replica-paths-gitlab-150-and-later) is enabled. +Deletions were disabled by default prior to GitLab 15.9 due to a race condition with repository renames +that can cause incorrect deletions, which is especially prominent in Geo instances as Geo performs more renames +than instances without Geo. In GitLab 15.0 to 15.5, you should enable deletions only if the [`gitaly_praefect_generated_replica_paths` feature flag](index.md#praefect-generated-replica-paths-gitlab-150-and-later) is enabled. The feature flag was removed in GitLab 15.6 making deletions always safe to enable. -By default, the worker does not delete invalid metadata records but logs them and outputs Prometheus -metrics for them. +By default, the worker deletes invalid metadata records. It also logs the deleted records and outputs Prometheus +metrics. -You can enable deleting invalid metadata records with: +You can disable deleting invalid metadata records with: ```ruby -praefect['background_verification_delete_invalid_records'] = true +praefect['configuration'] = { + # ... + background_verification: { + # ... + delete_invalid_records: false, + }, +} ``` ### Prioritize verification manually @@ -1370,10 +1606,10 @@ The output includes the number of replicas that were marked unverified. ## Automatic failover and primary election strategies -Praefect regularly checks the health of each Gitaly node. This is used to automatically fail over +Praefect regularly checks the health of each Gitaly node, which is used to automatically fail over to a newly-elected primary Gitaly node if the current primary node is found to be unhealthy. -We recommend using [repository-specific primary nodes](#repository-specific-primary-nodes). This is +You should use [repository-specific primary nodes](#repository-specific-primary-nodes). This is [the only available election strategy](https://gitlab.com/gitlab-org/gitaly/-/issues/3574) from GitLab 14.0. ### Repository-specific primary nodes diff --git a/doc/administration/gitaly/recovery.md b/doc/administration/gitaly/recovery.md index 1207d7af3e7..7d27a633512 100644 --- a/doc/administration/gitaly/recovery.md +++ b/doc/administration/gitaly/recovery.md @@ -74,7 +74,7 @@ Cluster: ### Unavailable repositories > - From GitLab 13.0 through 14.0, repositories became read-only if they were outdated on the primary but fully up to date on a healthy secondary. `dataloss` sub-command displays read-only repositories by default through these versions. -> - Since GitLab 14.1, Praefect contains more responsive failover logic which immediately fails over to one of the fully up to date secondaries rather than placing the repository in read-only mode. Since GitLab 14.1, the `dataloss` sub-command displays repositories which are unavailable due to having no fully up to date copies on healthy Gitaly nodes. +> - From GitLab 14.1, Praefect contains more responsive failover logic which immediately fails over to one of the fully up to date secondaries rather than placing the repository in read-only mode. From GitLab 14.1, the `dataloss` sub-command displays repositories which are unavailable due to having no fully up to date copies on healthy Gitaly nodes. A repository is unavailable if all of its up to date replicas are unavailable. Unavailable repositories are not accessible through Praefect to prevent serving stale data that may break automated tooling. @@ -100,7 +100,7 @@ The following parameters are available: some assigned copies that are not available. NOTE: -`dataloss` is still in [Beta](../../policy/alpha-beta-support.md#beta-features) and the output format is subject to change. +`dataloss` is still in [Beta](../../policy/alpha-beta-support.md#beta) and the output format is subject to change. To check for repositories with outdated primaries or for unavailable repositories, run: @@ -277,15 +277,33 @@ The reconciliation frequency can be changed via the configuration. The value can Examples: ```ruby -praefect['reconciliation_scheduling_interval'] = '5m' # the default value +praefect['configuration'] = { + # ... + reconciliation: { + # ... + scheduling_interval: '5m', # the default value + }, +} ``` ```ruby -praefect['reconciliation_scheduling_interval'] = '30s' # reconcile every 30 seconds +praefect['configuration'] = { + # ... + reconciliation: { + # ... + scheduling_interval: '30s', # reconcile every 30 seconds + }, +} ``` ```ruby -praefect['reconciliation_scheduling_interval'] = '0' # disable the feature +praefect['configuration'] = { + # ... + reconciliation: { + # ... + scheduling_interval: '0', # disable the feature + }, +} ``` ### Manual reconciliation @@ -334,16 +352,21 @@ sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.t sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml remove-repository -virtual-storage -repository -apply ``` -- `-virtual-storage` is the virtual storage the repository is located in. Virtual storages are configured in `/etc/gitlab/gitlab.rb` under `praefect['virtual_storages]` and looks like the following: +- `-virtual-storage` is the virtual storage the repository is located in. Virtual storages are configured in `/etc/gitlab/gitlab.rb` under `praefect['configuration']['virtual_storage]` and looks like the following: ```ruby - praefect['virtual_storages'] = { - 'default' => { - ... - }, - 'storage-1' => { - ... - } + praefect['configuration'] = { + # ... + virtual_storage: [ + { + # ... + name: 'default', + }, + { + # ... + name: 'storage-1', + }, + ], } ``` @@ -415,16 +438,21 @@ The `track-repository` Praefect sub-command adds repositories on disk to the Pra sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml track-repository -virtual-storage -authoritative-storage -repository -replicate-immediately ``` -- `-virtual-storage` is the virtual storage the repository is located in. Virtual storages are configured in `/etc/gitlab/gitlab.rb` under `praefect['virtual_storages]` and looks like the following: +- `-virtual-storage` is the virtual storage the repository is located in. Virtual storages are configured in `/etc/gitlab/gitlab.rb` under `praefect['configuration'][:virtual_storage]` and looks like the following: ```ruby - praefect['virtual_storages'] = { - 'default' => { - ... - }, - 'storage-1' => { - ... - } + praefect['configuration'] = { + # ... + virtual_storage: [ + { + # ... + name: 'default', + }, + { + # ... + name: 'storage-1', + }, + ], } ``` @@ -473,7 +501,7 @@ The command validates that all entries: - Are formatted correctly and contain required fields. - Correspond to a valid Git repository on disk. -- Are not currently tracked in the Praefect tracking database. +- Are not tracked in the Praefect tracking database. If any entry fails these checks, the command aborts prior to attempting to track a repository. diff --git a/doc/administration/gitaly/reference.md b/doc/administration/gitaly/reference.md index 1516b82a906..81b3faf859e 100644 --- a/doc/administration/gitaly/reference.md +++ b/doc/administration/gitaly/reference.md @@ -8,11 +8,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w Gitaly is configured via a [TOML](https://github.com/toml-lang/toml) configuration file. Unlike installations from source, in Omnibus GitLab, you -would not edit this file directly. +would not edit this file directly. For Omnibus GitLab installations, the default file location is `/var/opt/gitlab/gitaly/config.toml`. -The configuration file is passed as an argument to the `gitaly` -executable. This is usually done by either Omnibus GitLab or your -[init](https://en.wikipedia.org/wiki/Init) script. +The configuration file is passed as an argument to the `gitaly` executable, which is usually done by either Omnibus +GitLab or your [init](https://en.wikipedia.org/wiki/Init) script. An [example configuration file](https://gitlab.com/gitlab-org/gitaly/blob/master/config.toml.example) can be found in the Gitaly project. @@ -42,7 +41,7 @@ prometheus_listen_addr = "localhost:9236" ### Authentication Gitaly can be configured to reject requests that do not contain a -specific bearer token in their headers. This is a security measure to +specific bearer token in their headers, which is a security measure to be used when serving requests over TCP: ```toml @@ -70,7 +69,7 @@ Remember to disable `transitioning` when you are done changing your token settings. All authentication attempts are counted in Prometheus under -the [`gitaly_authentications_total` metric](monitoring.md#useful-queries). +the [`gitaly_authentications_total` metric](monitoring.md#queries). ### TLS @@ -156,36 +155,6 @@ Prometheus query to see the hit rate: sum(rate(gitaly_catfile_cache_total{type="hit"}[5m])) / sum(rate(gitaly_catfile_cache_total{type=~"(hit)|(miss)"}[5m])) ``` -### `gitaly-ruby` - -A Gitaly process uses one or more `gitaly-ruby` helper processes to -execute RPCs implemented in Ruby instead of Go. The `[gitaly-ruby]` -section of the configuration file contains settings for these helper processes. - -These processes are known to occasionally suffer from memory leaks. -Gitaly restarts its `gitaly-ruby` helpers when their memory exceeds the -`max_rss` limit. - -| Name | Type | Required | Description | -| ---- | ---- | -------- | ----------- | -| `dir` | string | yes | Path to where `gitaly-ruby` is installed (needed to boot the process).| -| `max_rss` | integer | no | Resident set size limit that triggers a `gitaly-ruby` restart, in bytes. Default is `200000000` (200 MB). | -| `graceful_restart_timeout` | string | no | Grace period before a `gitaly-ruby` process is forcibly terminated after exceeding `max_rss`. Default is `10m` (10 minutes).| -| `restart_delay` | string | no |Time that `gitaly-ruby` memory must remain high before a restart. Default is `5m` (5 minutes).| -| `num_workers` | integer | no |Number of `gitaly-ruby` worker processes. Try increasing this number in case of `ResourceExhausted` errors. Default is `2`, minimum is `2`.| -| `linguist_languages_path` | string | no | Override for dynamic `languages.json` discovery. Defaults to an empty string (use of dynamic discovery).| - -Example: - -```toml -[gitaly-ruby] -dir = "/home/git/gitaly/ruby" -max_rss = 200000000 -graceful_restart_timeout = "10m" -restart_delay = "5m" -num_workers = 2 -``` - ### GitLab Shell For historical reasons @@ -233,7 +202,6 @@ The following values configure logging in Gitaly under the `[logging]` section. | `level` | string | no | Log level: `debug`, `info`, `warn`, `error`, `fatal`, or `panic`. Default: `info`. | | `sentry_dsn` | string | no | Sentry DSN (Data Source Name) for exception monitoring. | | `sentry_environment` | string | no | [Sentry Environment](https://docs.sentry.io/product/sentry-basics/environments/) for exception monitoring. | -| `ruby_sentry_dsn` | string | no | Sentry DSN (Data Source Name) for `gitaly-ruby` exception monitoring. | While the main Gitaly application logs go to `stdout`, there are some extra log files that go to a configured directory, like the GitLab Shell logs. diff --git a/doc/administration/gitaly/troubleshooting.md b/doc/administration/gitaly/troubleshooting.md index 5f05b6322b0..dbbd348556c 100644 --- a/doc/administration/gitaly/troubleshooting.md +++ b/doc/administration/gitaly/troubleshooting.md @@ -66,7 +66,7 @@ for details. ### Client side gRPC logs Gitaly uses the [gRPC](https://grpc.io/) RPC framework. The Ruby gRPC -client has its own log file which may contain useful information when +client has its own log file which may contain helpful information when you are seeing Gitaly errors. You can control the log level of the gRPC client with the `GRPC_LOG_LEVEL` environment variable. The default level is `WARN`. @@ -85,7 +85,7 @@ check for an SSL or TLS problem: ``` Check whether `Verify return code` field indicates a -[known Omnibus GitLab configuration problem](https://docs.gitlab.com/omnibus/settings/ssl.html). +[known Omnibus GitLab configuration problem](https://docs.gitlab.com/omnibus/settings/ssl/index.html). If `openssl` succeeds but `gitlab-rake gitlab:gitaly:check` fails, check [certificate requirements](configure_gitaly.md#certificate-requirements) for Gitaly. @@ -123,27 +123,6 @@ sudo cat /proc/$PID/environ | tr '\0' '\n' | grep ^CORRELATION_ID= This method isn't reliable for `git cat-file` processes, because Gitaly internally pools and re-uses those across RPCs. -### Observing `gitaly-ruby` traffic - -[`gitaly-ruby`](configure_gitaly.md#gitaly-ruby) is an internal implementation detail of Gitaly, -so, there's not that much visibility into what goes on inside -`gitaly-ruby` processes. - -If you have Prometheus set up to scrape your Gitaly process, you can see -request rates and error codes for individual RPCs in `gitaly-ruby` by -querying `grpc_client_handled_total`. - -All gRPC calls made by `gitaly-ruby` itself are internal calls from the main Gitaly process to one of its `gitaly-ruby` -sidecars. - -Assuming your `grpc_client_handled_total` counter only observes Gitaly, -the following query shows you RPCs are (most likely) internally -implemented as calls to `gitaly-ruby`: - -```prometheus -sum(rate(grpc_client_handled_total[5m])) by (grpc_method) > 0 -``` - ### Repository changes fail with a `401 Unauthorized` error If you run Gitaly on its own server and notice these conditions: @@ -345,7 +324,7 @@ You might see the following in Gitaly and Praefect logs: } ``` -This is a gRPC call +This information in the logs is a gRPC call [error response code](https://grpc.github.io/grpc/core/md_doc_statuscodes.html). If this error occurs, even though @@ -358,7 +337,7 @@ server to keep them synchronized. ### Gitaly not listening on new address after reconfiguring -When updating the `gitaly['listen_addr']` or `gitaly['prometheus_listen_addr']` values, Gitaly may +When updating the `gitaly['configuration'][:listen_addr]` or `gitaly['configuration'][:prometheus_listen_addr]` values, Gitaly may continue to listen on the old address after a `sudo gitlab-ctl reconfigure`. When this occurs, run `sudo gitlab-ctl restart` to resolve the issue. This should no longer be @@ -401,6 +380,12 @@ push, which causes a significant delay. If Git pushes are too slow when Dynatrace is enabled, disable Dynatrace. +### `gitaly check` fails with `401` status code + +`gitaly check` can fail with `401` status code if Gitaly can't access the internal GitLab API. + +One way to resolve this is to make sure the entry is correct for the GitLab internal API URL configured in `gitlab.rb` with `gitlab_rails['internal_api_url']`. + ## Gitaly fails to fork processes stored on `noexec` file systems Because of changes [introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/5999) in GitLab 14.10, applying the `noexec` option to a mount @@ -492,7 +477,7 @@ in sync so the token check succeeds. This check helps identify the root cause of `permission denied` [errors being logged by Praefect](#permission-denied-errors-appearing-in-gitaly-or-praefect-logs-when-accessing-repositories). -For offline environments where access to public [`pool.ntp.org`](https://pool.ntp.org) servers is not possible, the Praefect `check` sub-command fails this +For offline environments where access to public `pool.ntp.org` servers is not possible, the Praefect `check` sub-command fails this check with an error message similar to: ```plaintext @@ -514,9 +499,9 @@ Here are common errors and potential causes: - 500 response code - `ActionView::Template::Error (7:permission denied)` - - `praefect['auth_token']` and `gitlab_rails['gitaly_token']` do not match on the GitLab server. + - `praefect['configuration'][:auth][:token]` and `gitlab_rails['gitaly_token']` do not match on the GitLab server. - `Unable to save project. Error: 7:permission denied` - - Secret token in `praefect['storage_nodes']` on GitLab server does not match the + - Secret token in `praefect['configuration'][:virtual_storage]` on GitLab server does not match the value in `gitaly['auth_token']` on one or more Gitaly servers. - 503 response code - `GRPC::Unavailable (14:failed to connect to all addresses)` @@ -530,7 +515,7 @@ Here are common errors and potential causes: Some common reasons for the Praefect database to experience elevated CPU usage include: - Prometheus metrics scrapes [running an expensive query](https://gitlab.com/gitlab-org/gitaly/-/issues/3796). If you have GitLab 14.2 - or above, set `praefect['separate_database_metrics'] = true` in `gitlab.rb`. + or above, set `praefect['configuration'][:prometheus_exclude_database_from_default_metrics] = true` in `gitlab.rb`. - [Read distribution caching](praefect.md#reads-distribution-caching) is disabled, increasing the number of queries made to the database when user traffic is high. Ensure read distribution caching is enabled. @@ -544,9 +529,8 @@ To determine the primary node of a repository: - With legacy election strategies in GitLab 13.12 and earlier, the primary was the same for all repositories in a virtual storage. To determine the current primary Gitaly node for a specific virtual storage: - - Use the `Shard Primary Election` [Grafana chart](praefect.md#grafana) on the + - (Recommended) Use the `Shard Primary Election` [Grafana chart](praefect.md#grafana) on the [`Gitlab Omnibus - Praefect` dashboard](https://gitlab.com/gitlab-org/grafana-dashboards/-/blob/master/omnibus/praefect.json). - This is recommended. - If you do not have Grafana set up, use the following command on each host of each Praefect node: @@ -650,7 +634,7 @@ If the supplied value for `-virtual-storage` is incorrect, the command returns t get metadata: rpc error: code = NotFound desc = repository not found ``` -The documented examples specify `-virtual-storage default`. Check the Praefect server setting `praefect['virtual_storages']` in `/etc/gitlab/gitlab.rb`. +The documented examples specify `-virtual-storage default`. Check the Praefect server setting `praefect['configuration'][:virtual_storage]` in `/etc/gitlab/gitlab.rb`. ### Check that repositories are in sync @@ -669,7 +653,7 @@ However, the Praefect database tables are not created on initial reconfigure and errors that relations do not exist if either: - The `gitlab-ctl reconfigure` command isn't executed. -- There are errors during the execution. +- Errors occur during the execution. For example: @@ -693,7 +677,7 @@ praefect sql-migrate: OK (applied 21 migrations) This indicates that the virtual storage name used in the [Praefect configuration](praefect.md#praefect) does not match the storage name used in -[`git_data_dirs` setting](praefect.md#gitaly) for GitLab. +[`gitaly['configuration'][:storage][][:name]` setting](praefect.md#gitaly) for GitLab. Resolve this by matching the virtual storage names used in Praefect and GitLab configuration. @@ -715,9 +699,13 @@ Possible solutions: - Provision larger VMs to gain access to larger network traffic allowances. - Use your cloud service's monitoring and logging to check that the Praefect nodes are not exhausting their traffic allowances. +### `gitlab-ctl reconfigure` fails with error: `STDOUT: praefect: configuration error: error reading config file: toml: cannot store TOML string into a Go int` + +This error occurs when `praefect['database_port']` or `praefect['database_direct_port']` are configured as a string instead of an integer. + ## Profiling Gitaly -Gitaly exposes several of the Golang built-in performance profiling tools on the Prometheus listen port. For example, if Prometheus is listening +Gitaly exposes several of the Go built-in performance profiling tools on the Prometheus listen port. For example, if Prometheus is listening on port `9236` of the GitLab server: - Get a list of running `goroutines` and their backtraces: diff --git a/doc/administration/housekeeping.md b/doc/administration/housekeeping.md index d58989db70c..3a2b3657145 100644 --- a/doc/administration/housekeeping.md +++ b/doc/administration/housekeeping.md @@ -166,6 +166,10 @@ You can change this default in Gitaly configuration. The following snippet enables daily background repository maintenance starting at 23:00 for 1 hour for the `default` storage: +::Tabs + +:::TabTitle Self-compiled (source) + ```toml [daily_maintenance] start_hour = 23 @@ -182,6 +186,33 @@ maintenance: disabled = true ``` +:::TabTitle Linux package (Omnibus) + +```ruby +gitaly['configuration'] = { + daily_maintenance: { + disabled: false, + start_hour: 23, + start_minute: 00, + duration: '1h', + storages: ['default'], + }, +} +``` + +Use the following snippet to completely disable background repository +maintenance: + +```ruby +gitaly['configuration'] = { + daily_maintenance: { + disabled: true, + }, +} +``` + +::EndTabs + ## Object pool repositories Object pool repositories are used by GitLab to deduplicate objects across forks diff --git a/doc/administration/incoming_email.md b/doc/administration/incoming_email.md index ea051e2067d..3b4eaeb161c 100644 --- a/doc/administration/incoming_email.md +++ b/doc/administration/incoming_email.md @@ -68,8 +68,7 @@ this method only supports replies, and not the other features of [incoming email ## Accepted headers -> - Accepting `Received` headers [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81489) in GitLab 14.9 [with a flag](feature_flags.md) named `use_received_header_for_incoming_emails`. Enabled by default. -> - Accepting `Received` headers: [feature flag](feature_flags.md) named `use_received_header_for_incoming_emails` [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/362596) in GitLab 14.1. +> Accepting `Received` headers [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81489) in GitLab 14.9. Email is processed correctly when a configured email address is present in one of the following headers (sorted in the order they are checked): diff --git a/doc/administration/index.md b/doc/administration/index.md index 9f4477bcbf7..ded6eabdba7 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -7,235 +7,14 @@ description: 'Learn how to install, configure, update, and maintain your GitLab # Administer GitLab **(FREE SELF)** -If you use GitLab.com, only GitLab team members have access to administration tools and settings. -If you use a self-managed GitLab instance, learn how to administer it. +Learn how to administer a self-managed GitLab instance. -Only administrator users can access GitLab administration tools and settings. +- [Get started](../administration/get_started.md) +- [All feature flags](../user/feature_flags.md) +- [Configure your installation](../administration/configure.md) +- [Configure GitLab Dedicated](../administration/dedicated/index.md) +- [Maintain your installation](../administration/operations/index.md) +- [Secure your installation](../security/index.md) +- [Administer users](../user/profile/account/create_accounts.md) -## Available distributions - -GitLab has two product distributions available through [different subscriptions](https://about.gitlab.com/pricing/): - -- The open source [GitLab Community Edition (CE)](https://gitlab.com/gitlab-org/gitlab-foss). -- The open core [GitLab Enterprise Edition (EE)](https://gitlab.com/gitlab-org/gitlab). - -You can [install either GitLab CE or GitLab EE](https://about.gitlab.com/install/ce-or-ee/). -However, the features you have access to depend on your chosen subscription. - -GitLab Community Edition installations have access only to Free features. - -## Installing and maintaining GitLab - -Learn how to install, configure, update, and maintain your GitLab instance. - -### Installing GitLab - -- [Install](../install/index.md): Requirements, directory structures, and installation methods. -- [Reference architectures](reference_architectures/index.md): Add additional resources to support more users. -- [Installing GitLab on Amazon Web Services (AWS)](../install/aws/index.md): Set up GitLab on Amazon AWS. -- [Geo](geo/index.md): Replicate your GitLab instance to other geographic locations as a read-only fully operational version. -- [Disaster Recovery](geo/disaster_recovery/index.md): Quickly fail-over to a different site with minimal effort in a disaster situation. -- [Add License](../user/admin_area/license.md): Add a license at install time to unlock features that are in paid tiers of GitLab. - -### Configuring GitLab - -- [Adjust your instance's time zone](timezone.md): Customize the default time zone of GitLab. -- [System hooks](system_hooks.md): Notifications when users, projects and keys are changed. -- [Security](../security/index.md): Learn what you can do to further secure your GitLab instance. -- [Usage statistics, version check, and Service Ping](../user/admin_area/settings/usage_statistics.md): Enable or disable information about your instance to be sent to GitLab, Inc. -- [Global user settings](user_settings.md): Configure instance-wide user permissions. -- [Polling](polling.md): Configure how often the GitLab UI polls for updates. -- [GitLab Pages configuration](pages/index.md): Enable and configure GitLab Pages. -- [GitLab Pages configuration for GitLab source installations](pages/source.md): - Enable and configure GitLab Pages on [source installations](../install/installation.md#installation-from-source). -- [Uploads administration](uploads.md): Configure GitLab uploads storage. -- [Environment variables](environment_variables.md): Supported environment - variables that can be used to override their default values to configure - GitLab. -- [File hooks](file_hooks.md): With custom file hooks, GitLab administrators can - introduce custom integrations without modifying GitLab source code. -- [Enforcing Terms of Service](../user/admin_area/settings/terms.md) -- [Customer experience improvement and third-party offers](../user/admin_area/settings/third_party_offers.md) -- [Compliance](compliance.md): A collection of features from across the - application that you may configure to help ensure that your GitLab instance - and DevOps workflow meet compliance standards. -- [Diff limits](../user/admin_area/diff_limits.md): Configure the diff rendering - size limits of branch comparison pages. -- [Merge request diffs storage](merge_request_diffs.md): Configure merge - requests diffs external storage. -- [Broadcast Messages](../user/admin_area/broadcast_messages.md): Send messages - to GitLab users through the UI. -- [Elasticsearch](../integration/advanced_search/elasticsearch.md): Enable Elasticsearch to - empower Advanced Search. Use when you deal with a huge amount of data. -- [External Classification Policy Authorization](../user/admin_area/settings/external_authorization.md) -- [Add a license](../user/admin_area/license.md): Add a license to unlock - features that are in paid tiers of GitLab. -- [Admin Area](../user/admin_area/index.md): for self-managed instance-wide - configuration and maintenance. -- [S/MIME Signing](smime_signing_email.md): how to sign all outgoing notification - emails with S/MIME. -- [Enabling and disabling features flags](feature_flags.md): how to enable and - disable GitLab features deployed behind feature flags. -- [Application settings cache expiry interval](application_settings_cache.md) -- [Database Load Balancing](postgresql/database_load_balancing.md): Distribute database queries among multiple database servers. -- [Omnibus support for log forwarding](https://docs.gitlab.com/omnibus/settings/logs.html#udp-log-shipping-gitlab-enterprise-edition-only). - -#### Customizing GitLab appearance - -- [Header logo](../user/admin_area/appearance.md#top-bar): Change the logo on all pages and email headers. -- [Favicon](../user/admin_area/appearance.md#favicon): Change the default favicon to your own logo. -- [Branded login page](../user/admin_area/appearance.md#sign-in--sign-up-pages): Customize the login page with your own logo, title, and description. -- ["New Project" page](../user/admin_area/appearance.md#new-project-pages): Customize the text to be displayed on the page that opens whenever your users create a new project. -- [Additional custom email text](../user/admin_area/settings/email.md#custom-additional-text): Add additional custom text to emails sent from GitLab. - -### Maintaining GitLab - -- [Rake tasks](../raketasks/index.md): Perform various tasks for maintenance, backups, automatic webhooks setup, and more. - - [Backup and restore](../raketasks/backup_restore.md): Backup and restore your GitLab instance. -- [Operations](operations/index.md): Keeping GitLab up and running (clean up Redis sessions, moving repositories, Sidekiq MemoryKiller, Puma). -- [Restart GitLab](restart_gitlab.md): Learn how to restart GitLab and its components. -- [Invalidate Markdown cache](invalidate_markdown_cache.md): Invalidate any cached Markdown. -- [Instance review](instance_review.md): Request a free review of your GitLab instance. - -#### Updating GitLab - -- [GitLab versions and maintenance policy](../policy/maintenance.md): Understand GitLab versions and releases (Major, Minor, Patch, Security), as well as update recommendations. -- [GitLab in maintenance mode](maintenance_mode/index.md): Put GitLab in maintenance mode. -- [Update GitLab](../update/index.md): Update guides to upgrade your installation to a new version. -- [Upgrading without downtime](../update/index.md#upgrading-without-downtime): Upgrade to a newer major, minor, or patch version of GitLab without taking your GitLab instance offline. - -### Upgrading or downgrading GitLab - -- [Upgrade from GitLab CE to GitLab EE](../update/index.md#upgrading-between-editions): Learn how to upgrade GitLab Community Edition to GitLab Enterprise Editions. -- [Downgrade from GitLab EE to GitLab CE](../downgrade_ee_to_ce/index.md): Learn how to downgrade GitLab Enterprise Editions to Community Edition. - -### GitLab platform integrations - -- [Mattermost](../integration/mattermost/index.md): Integrate with [Mattermost](https://mattermost.com), an open source, private cloud workplace for web messaging. -- [PlantUML](integration/plantuml.md): Create diagrams in AsciiDoc and Markdown documents - created in snippets, wikis, and repositories. -- [Web terminals](integration/terminal.md): Provide terminal access to your applications deployed to Kubernetes from GitLab CI/CD [environments](../ci/environments/index.md#web-terminals-deprecated). - -## User settings and permissions - -- [Creating users](../user/profile/account/create_accounts.md): Create users manually or through authentication integrations. -- [Libravatar](libravatar.md): Use Libravatar instead of Gravatar for user avatars. -- [Sign-up restrictions](../user/admin_area/settings/sign_up_restrictions.md): block email addresses of specific domains, or allow only specific domains. -- [Admin mode](../user/admin_area/settings/sign_in_restrictions.md#admin-mode): require that administrators authenticate separately to use administrative access, like `sudo`. -- [Access restrictions](../user/admin_area/settings/visibility_and_access_controls.md#configure-enabled-git-access-protocols): Define which Git access protocols can be used to talk to GitLab (SSH, HTTP, HTTPS). -- [Authentication and Authorization](auth/index.md): Configure external authentication with LDAP, SAML, CAS, and additional providers. - - [Sync LDAP](auth/ldap/index.md) - - [Kerberos authentication](../integration/kerberos.md) - - See also other [authentication](../topics/authentication/index.md#gitlab-administrators) topics (for example, enforcing 2FA). -- [Email users](../user/admin_area/email_from_gitlab.md): Email GitLab users from GitLab. -- [User Cohorts](../user/admin_area/user_cohorts.md): Display the monthly cohorts of new users and their activities over time. -- [Audit events](audit_events.md): View the changes made on the GitLab server for: - - Groups and projects. - - Instances. -- [Auditor users](auditor_users.md): Users with read-only access to all projects, groups, and other resources on the GitLab instance. -- [Incoming email](incoming_email.md): Configure incoming emails to allow - users to [reply by email](reply_by_email.md), create [issues by email](../user/project/issues/create_issues.md#by-sending-an-email) and - [merge requests by email](../user/project/merge_requests/creating_merge_requests.md#by-sending-an-email), and to enable [Service Desk](../user/project/service_desk.md). - - [Postfix for incoming email](reply_by_email_postfix_setup.md): Set up a - basic Postfix mail server with IMAP authentication on Ubuntu for incoming - emails. -- [Abuse reports](../user/admin_area/review_abuse_reports.md): View and resolve abuse reports from your users. -- [Credentials Inventory](../user/admin_area/credentials_inventory.md): With Credentials inventory, GitLab administrators can keep track of the credentials used by their users in their GitLab self-managed instance. - -## Project settings - -- [Issue closing pattern](issue_closing_pattern.md): Customize how to close an issue from commit messages. -- [Gitaly](gitaly/index.md): Configuring Gitaly, the Git repository storage service for GitLab. -- [Default labels](../user/admin_area/labels.md): Create labels that are automatically added to every new project. -- [Restrict the use of public or internal projects](../user/public_access.md#restrict-use-of-public-or-internal-projects): Restrict the use of visibility levels for users when they create a project or a snippet. -- [Custom project templates](../user/admin_area/custom_project_templates.md): Configure a set of projects to be used as custom templates when creating a new project. - -## Package Registry administration - -- [Container Registry](packages/container_registry.md): Configure GitLab to act as a registry for containers. -- [Package Registry](packages/index.md): Enable GitLab to act as a registry for packages. -- [Dependency Proxy](packages/dependency_proxy.md): Configure the Dependency Proxy, a local proxy for frequently used upstream images/packages. - -### Repository settings - -- [Repository checks](repository_checks.md): Check your repository for data corruption. -- [Repository storage paths](repository_storage_paths.md): Manage the paths used to store repositories. -- [Repository storage types](repository_storage_types.md): Information about the different repository storage types. -- [Repository storage Rake tasks](raketasks/storage.md): A collection of Rake tasks to list and migrate existing projects and attachments associated with it from Legacy storage to Hashed storage. -- [Limit repository size](../user/admin_area/settings/account_and_limit_settings.md): Set a hard limit for your repositories' size. -- [Static objects external storage](static_objects_external_storage.md): Set external storage for static objects in a repository. - -## CI/CD settings - -- [Enable or disable GitLab CI/CD](cicd.md#disable-gitlab-cicd-in-new-projects): Set new projects in your instance to have GitLab CI/CD enabled or disabled by default. -- [GitLab CI/CD administration settings](../user/admin_area/settings/continuous_integration.md): Enable or disable Auto DevOps site-wide and define the artifacts' max size and expiration time. -- [External Pipeline Validation](external_pipeline_validation.md): Enable, disable, and configure external pipeline validation. -- [Job artifacts](job_artifacts.md): Enable, disable, and configure job artifacts (a set of files and directories which are outputted by a job when it completes successfully). -- [Job logs](job_logs.md): Information about the job logs. -- [Register runners](../ci/runners/runners_scope.md): Learn how to register and configure runners. -- [Shared runners quota of CI/CD minutes](../ci/pipelines/cicd_minutes.md): Limit the usage of CI/CD minutes for shared runners. -- [Enable or disable Auto DevOps](../topics/autodevops/index.md#enable-or-disable-auto-devops): Enable or disable Auto DevOps for your instance. - -## Snippet settings - -- [Setting snippet content size limit](snippets/index.md): Set a maximum content size limit for snippets. - -## Wiki settings - -- [Setting wiki page content size limit](wikis/index.md): Set a maximum content size limit for wiki pages. - -## Git configuration options - -- [Git server hooks](server_hooks.md): Git server hooks (on the file system) for when webhooks aren't enough. Previously called server hooks. -- [Git LFS configuration](lfs/index.md): Learn how to configure LFS for GitLab. -- [Housekeeping](housekeeping.md): Keep your Git repositories tidy and fast. -- [Configuring Git Protocol v2](git_protocol.md): Git protocol version 2 support. - -## Monitoring GitLab - -- [Monitoring GitLab](monitoring/index.md): - - [Monitoring uptime](../user/admin_area/monitoring/health_check.md): Check the server status using the health check endpoint. - - [IP allowlist](monitoring/ip_allowlist.md): Monitor endpoints that provide health check information when probed. - - [Monitoring GitHub imports](monitoring/github_imports.md): The GitLab GitHub Importer displays Prometheus metrics to monitor the health and progress of the importer. - -### Performance Monitoring - -- [GitLab Performance Monitoring](monitoring/performance/index.md): - - [Enable Performance Monitoring](monitoring/performance/gitlab_configuration.md): Enable GitLab Performance Monitoring. - - [GitLab performance monitoring with Prometheus](monitoring/prometheus/index.md): Configure GitLab and Prometheus for measuring performance metrics. - - [GitLab performance monitoring with Grafana](monitoring/performance/grafana_configuration.md): Configure GitLab to visualize time series metrics through graphs and dashboards. - - [Performance Bar](monitoring/performance/performance_bar.md): Get performance information for the current page. - -## Troubleshooting - -- [Log system](logs/index.md): Where to look for logs. -- [Sidekiq Troubleshooting](sidekiq/sidekiq_troubleshooting.md): Debug when Sidekiq appears hung and is not processing jobs. -- [Navigating GitLab via Rails console](operations/rails_console.md) -- [GitLab application limits](instance_limits.md) -- [Responding to security incidents](../security/responding_to_security_incidents.md) - -### Support Team Documentation - -The GitLab Support Team has collected a lot of information about troubleshooting GitLab. -The following documents are used by the Support Team or by customers -with direct guidance from a Support Team member. GitLab administrators may find the -information useful for troubleshooting. However, if you are experiencing trouble with your -GitLab instance, you should check your [support options](https://about.gitlab.com/support/) -before referring to these documents. - -WARNING: -The commands in the following documentation might result in data loss or -other damage to a GitLab instance. They should be used only by experienced administrators -who are aware of the risks. - -- [Diagnostics tools](troubleshooting/diagnostics_tools.md) -- [Linux commands](troubleshooting/linux_cheat_sheet.md) -- [Troubleshooting Kubernetes](https://docs.gitlab.com/charts/troubleshooting/kubernetes_cheat_sheet.html) -- [Troubleshooting PostgreSQL](troubleshooting/postgresql.md) -- [Guide to test environments](troubleshooting/test_environments.md) (for Support Engineers) -- [Troubleshooting SSL](troubleshooting/ssl.md) -- Related links: - - [GitLab Developer Documentation](../development/index.md) - - [Repairing and recovering broken Git repositories](https://git.seveas.net/repairing-and-recovering-broken-git-repositories.html) - - [Testing with OpenSSL](https://www.feistyduck.com/library/openssl-cookbook/online/testing-with-openssl/index.html) - - [`strace` zine](https://wizardzines.com/zines/strace/) +Only GitLab team members have access to administration tools and settings on GitLab.com. diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index b1d97fd16a9..03c7c51251b 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -126,7 +126,7 @@ Read more about [import/export rate limits](../user/admin_area/settings/import_e Limit the maximum daily member invitations allowed per group hierarchy. -- GitLab.com: Free members may invite 20 members per day. +- GitLab.com: Free members may invite 20 members per day, Premium trial and Ultimate trial members may invite 50 members per day. - Self-managed: Invites are not limited. ### Webhook rate limit @@ -157,16 +157,17 @@ Set the limit to `0` to disable it. ### Search rate limit > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80631) in GitLab 14.9. -> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104208) to include issue, merge request, and epic searches to the rate limit in GitLab 15.9. +> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104208) in GitLab 15.9 to include issue, merge request, and epic searches in the rate limit. +> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118525) in GitLab 16.0 to apply rate limits to [search scopes](../user/search/index.md#global-search-scopes) for authenticated requests. This setting limits search requests as follows: | Limit | Default (requests per minute) | |-------------------------|-------------------------------| -| Authenticated user | 30 | -| Unauthenticated user | 10 | +| Authenticated user | 300 | +| Unauthenticated user | 100 | -Depending on the number of enabled [scopes](../user/search/index.md#global-search-scopes), a global search request can consume two to seven requests per minute. You may want to disable one or more scopes to use fewer requests. Search requests that exceed the search rate limit per minute return the following error: +Search requests that exceed the search rate limit per minute return the following error: ```plaintext This endpoint has been requested too many times. Try again later. @@ -290,7 +291,7 @@ Plan.default.actual_limits.update!(group_hooks: 100) Set the limit to `0` to disable it. -The default maximum number of webhooks is `100` per project, `50` per group. +The default maximum number of webhooks is `100` per project and `50` per group. Webhooks in a child group do not count towards the webhook limit of their parent group. For GitLab.com, see the [webhook limits for GitLab.com](../user/gitlab_com/index.md#webhooks). @@ -430,38 +431,6 @@ Plan.default.actual_limits.update!(ci_active_jobs: 500) Set the limit to `0` to disable it. -### Number of pipelines running concurrently - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32823) in GitLab 12.5. - -The total number of pipelines running concurrently can be limited per project. -When enabled, the limit is checked each time a new pipeline is created. -Without a concurrent pipelines limit, a sudden flood of triggered pipelines could -overwhelm the instance resources. - -If a new pipeline would cause the total number of pipelines to exceed the limit, -the pipeline fails with a `The pipeline activity limit was exceeded.` error. - -On [GitLab Premium](https://about.gitlab.com/pricing/) self-managed or -higher installations, this limit is defined under a `default` plan that affects all -projects. This limit is disabled (`0`) by default. GitLab SaaS subscribers have different -limits [defined per plan](../user/gitlab_com/index.md#gitlab-cicd), and they affect -all projects under that plan. - -To set this limit for a self-managed installation, enable the **Maximum number of active pipelines per project** -[setting in the Admin Area](../user/admin_area/settings/continuous_integration.md#set-cicd-limits). - -Alternatively, you can run the following in the [GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session): - -```ruby -# If limits don't exist for the default plan, you can create one with: -# Plan.default.create_limits! - -Plan.default.actual_limits.update!(ci_active_pipelines: 100) -``` - -Set the limit to `0` to disable it. - ### Maximum time jobs can run > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16777) in GitLab 12.3. @@ -509,10 +478,10 @@ checked each time a new subscription is created. If a new subscription would cause the total number of subscription to exceed the limit, the subscription is considered invalid. -- GitLab SaaS subscribers have different limits [defined per plan](../user/gitlab_com/index.md#gitlab-cicd), +- On GitLab SaaS: Limits are [defined per plan](../user/gitlab_com/index.md#gitlab-cicd), and they affect all projects under that plan. -- On [GitLab Premium](https://about.gitlab.com/pricing/) self-managed - or higher installations, this limit is defined under a `default` plan that +- On self-managed: On [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/), + this limit is defined under a `default` plan that affects all projects. By default, there is a limit of `2` subscriptions. To set this limit for a self-managed installation, run the following in the @@ -632,6 +601,42 @@ To update this limit to a new value on a self-managed installation, run the foll Plan.default.actual_limits.update!(ci_instance_level_variables: 30) ``` +### Number of group level variables + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362227) in GitLab 15.7. + +The total number of group level CI/CD variables is limited at the instance level. +This limit is checked each time a new group level variable is created. If a new variable +would cause the total number of variables to exceed the limit, the new variable is not created. + +On self-managed instances this limit is defined for the `default` plan. By default, +this limit is set to `30000`. + +To update this limit to a new value on a self-managed installation, run the following in the +[GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session): + +```ruby +Plan.default.actual_limits.update!(group_ci_variables: 40000) +``` + +### Number of project level variables + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362227) in GitLab 15.7. + +The total number of project level CI/CD variables is limited at the instance level. +This limit is checked each time a new project level variable is created. If a new variable +would cause the total number of variables to exceed the limit, the new variable is not created. + +On self-managed instances this limit is defined for the `default` plan. By default, +this limit is set to `8000`. + +To update this limit to a new value on a self-managed installation, run the following in the +[GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session): + +```ruby +Plan.default.actual_limits.update!(project_ci_variables: 10000) +``` + ### Maximum file size per type of artifact > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37226) in GitLab 13.3. @@ -723,22 +728,23 @@ GitLab checks these limits against runners that have been active in the last 3 m A runner's registration fails if it exceeds the limit for the scope determined by the runner registration token. If the limit value is set to zero, the limit is disabled. -- GitLab SaaS subscribers have different limits defined per plan, affecting all projects using that plan. -- Self-managed GitLab Premium and Ultimate limits are defined by a default plan that affects all projects: +GitLab SaaS subscribers have different limits defined per plan, affecting all projects using that plan. - | Runner scope | Default value | - |---------------------------------------------|---------------| - | `ci_registered_group_runners` | 1000 | - | `ci_registered_project_runners` | 1000 | +Self-managed GitLab Premium and Ultimate limits are defined by a default plan that affects all projects: - To update these limits, run the following in the - [GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session): +| Runner scope | Default value | +|---------------------------------------------|---------------| +| `ci_registered_group_runners` | 1000 | +| `ci_registered_project_runners` | 1000 | + +To update these limits, run the following in the +[GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session): - ```ruby - # Use ci_registered_group_runners or ci_registered_project_runners - # depending on desired scope - Plan.default.actual_limits.update!(ci_registered_project_runners: 100) - ``` +```ruby +# Use ci_registered_group_runners or ci_registered_project_runners +# depending on desired scope +Plan.default.actual_limits.update!(ci_registered_project_runners: 100) +``` ### Maximum file size for job logs @@ -789,7 +795,7 @@ You can change these limits in the [GitLab Rails console](operations/rails_conso The `max_yaml_size_bytes` value is not directly tied to the size of the YAML file, but rather the memory allocated for the relevant objects. -- To update the maximum YAML depth, update `max_yaml_depth` with the new value in megabytes: +- To update the maximum YAML depth, update `max_yaml_depth` with the new value in number of lines: ```ruby ApplicationSetting.update(max_yaml_depth: 125) @@ -922,7 +928,7 @@ Reports that go over the 20 MB limit aren't loaded. Affected reports: - [CI/CD parameter `artifacts:expose_as`](../ci/yaml/index.md#artifactsexpose_as) - [Unit test reports](../ci/testing/unit_test_reports.md) -## Advanced Search limits +## Advanced search limits ### Maximum file size indexed @@ -945,7 +951,7 @@ is pre-allocated during indexing. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201826) in GitLab 12.8. -You can set a limit on the content of text fields indexed for Advanced Search. +You can set a limit on the content of text fields indexed for advanced search. Setting a maximum helps to reduce the load of the indexing processes. If any text field exceeds this limit, then the text is truncated to this number of characters. The rest of the text is not indexed, and not searchable. diff --git a/doc/administration/integration/kroki.md b/doc/administration/integration/kroki.md index 081c4d8a08c..f90458200b3 100644 --- a/doc/administration/integration/kroki.md +++ b/doc/administration/integration/kroki.md @@ -9,12 +9,12 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/241744) in GitLab 13.7. > - Support for reStructuredText and Textile documents [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/324766) in GitLab 13.12. -When [Kroki](https://kroki.io) integration is enabled and configured in -GitLab, you can use it to create diagrams in AsciiDoc, Markdown, reStructuredText, and Textile documents. +With the [Kroki](https://kroki.io) integration, +you can create diagrams in AsciiDoc, Markdown, reStructuredText, and Textile. -## Kroki Server +## Kroki server -When Kroki is enabled, GitLab sends diagrams to an instance of Kroki to display them as images. +When you enable Kroki, GitLab sends diagrams to an instance of Kroki to display them as images. You can use the free public cloud instance `https://kroki.io` or you can [install Kroki](https://docs.kroki.io/kroki/setup/install/) on your own infrastructure. After you've installed Kroki, make sure to update the server URL to point to your instance. @@ -29,11 +29,16 @@ docker run -d --name kroki -p 8080:8000 yuzutech/kroki The **Kroki URL** is the hostname of the server running the container. -The [`yuzutech/kroki`](https://hub.docker.com/r/yuzutech/kroki) image contains the following diagrams libraries out-of-the-box: +The [`yuzutech/kroki`](https://hub.docker.com/r/yuzutech/kroki) Docker image contains several diagram +libraries out of the box. For a complete list, see the +[`asciidoctor-kroki` README](https://github.com/ggrossetie/asciidoctor-kroki/blob/master/README.md#supported-diagram-types). +Supported libraries include: - [Bytefield](https://bytefield-svg.deepsymmetry.org/) +- [D2](https://d2lang.com/tour/intro/) +- [DBML](https://www.dbml.org/home/) - [Ditaa](https://ditaa.sourceforge.net) - [Erd](https://github.com/BurntSushi/erd) - [GraphViz](https://www.graphviz.org/) diff --git a/doc/administration/integration/plantuml.md b/doc/administration/integration/plantuml.md index 33434e15c4e..fcfae6cbe70 100644 --- a/doc/administration/integration/plantuml.md +++ b/doc/administration/integration/plantuml.md @@ -7,9 +7,8 @@ type: reference, howto # PlantUML **(FREE)** -When the [PlantUML](https://plantuml.com) integration is enabled and configured in -GitLab, you can create diagrams in snippets, wikis, and repositories. This integration -is enabled on GitLab.com for all SaaS users and does not require any additional configuration. +With the [PlantUML](https://plantuml.com) integration, you can create diagrams in snippets, wikis, and repositories. +This integration is enabled on GitLab.com for all SaaS users and does not require any additional configuration. To set up the integration on a self-managed instance, you must: @@ -117,7 +116,7 @@ services: image: 'gitlab/gitlab-ee:12.2.5-ee.0' environment: GITLAB_OMNIBUS_CONFIG: | - nginx['custom_gitlab_server_config'] = "location /-/plantuml/ { \n proxy_cache off; \n proxy_pass http://plantuml:8080/; \n}\n" + nginx['custom_gitlab_server_config'] = "location /-/plantuml/ { \n rewrite ^/-/plantuml/(.*) /$1 break;\n proxy_cache off; \n proxy_pass http://plantuml:8080/; \n}\n" plantuml: image: 'plantuml/plantuml-server:tomcat' @@ -148,7 +147,7 @@ using Tomcat: ``` The Tomcat service should restart. After the restart is complete, the -PlantUML service is ready and listening for requests on port 8080: +PlantUML integration is ready and listening for requests on port 8080: `http://localhost:8080/plantuml` To change these defaults, edit the `/etc/tomcat8/server.xml` file. @@ -169,7 +168,7 @@ following: - `http://plantuml:8080/` - `http://localhost:8080/plantuml/` -If you're running [GitLab with TLS](https://docs.gitlab.com/omnibus/settings/ssl.html) +If you're running [GitLab with TLS](https://docs.gitlab.com/omnibus/settings/ssl/index.html) you must configure this redirection, because PlantUML uses the insecure HTTP protocol. Newer browsers such as [Google Chrome 86+](https://www.chromestatus.com/feature/4926989725073408) don't load insecure HTTP resources on pages served over HTTPS. @@ -180,10 +179,10 @@ To enable this redirection: ```ruby # Docker deployment - nginx['custom_gitlab_server_config'] = "location /-/plantuml/ { \n proxy_cache off; \n proxy_pass http://plantuml:8080/; \n}\n" + nginx['custom_gitlab_server_config'] = "location /-/plantuml/ { \n rewrite ^/-/plantuml/(.*) /$1 break;\n proxy_cache off; \n proxy_pass http://plantuml:8080/; \n}\n" # Built from source - nginx['custom_gitlab_server_config'] = "location /-/plantuml { \n rewrite ^/-/(plantuml.*) /$1 break;\n proxy_cache off; \n proxy_pass http://localhost:8080/plantuml; \n}\n" + nginx['custom_gitlab_server_config'] = "location /-/plantuml { \n rewrite ^/-/plantuml/(.*) /$1 break;\n proxy_cache off; \n proxy_pass http://localhost:8080/plantuml; \n}\n" ``` 1. To activate the changes, run the following command: diff --git a/doc/administration/integration/terminal.md b/doc/administration/integration/terminal.md index a1522ccac31..df9bb6b6e17 100644 --- a/doc/administration/integration/terminal.md +++ b/doc/administration/integration/terminal.md @@ -1,10 +1,10 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Web terminals (DEPRECATED) **(FREE)** +# Web terminals (deprecated) **(FREE)** > - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. > - [Disabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/353410) in GitLab 15.0. diff --git a/doc/administration/job_artifacts.md b/doc/administration/job_artifacts.md index 4f2eb20877e..099e7ec9e6f 100644 --- a/doc/administration/job_artifacts.md +++ b/doc/administration/job_artifacts.md @@ -1,13 +1,13 @@ --- stage: Verify -group: Pipeline Insights +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Jobs artifacts administration **(FREE SELF)** This is the administration documentation. To learn how to use job artifacts in your GitLab CI/CD pipeline, -see the [job artifacts configuration documentation](../ci/pipelines/job_artifacts.md). +see the [job artifacts configuration documentation](../ci/jobs/job_artifacts.md). An artifact is a list of files and directories attached to a job after it finishes. This feature is enabled by default in all GitLab installations. @@ -174,7 +174,7 @@ In a multi-server setup you must use one of the options to [eliminate local disk usage for job logs](job_logs.md#prevent-local-disk-usage), or job logs could be lost. In GitLab 13.2 and later, you should use the -[consolidated object storage settings](object_storage.md#consolidated-object-storage-configuration). +[consolidated object storage settings](object_storage.md#configure-a-single-storage-connection-for-all-object-types-consolidated-form). ### Migrating to object storage @@ -233,7 +233,7 @@ processing is done in a background worker and requires **no downtime**. ::EndTabs - 1. Verify that all packages migrated to object storage with the following + 1. Verify that all artifacts migrated to object storage with the following SQL query. The number of `objectstg` should be the same as `total`: ```shell @@ -276,7 +276,7 @@ to clean up orphaned artifacts. ### Migrating from object storage to local storage To migrate back to local storage, you must -[selectively disable the artifacts storage](object_storage.md#selectively-disabling-object-storage). +[selectively disable the artifacts storage](object_storage.md#disable-object-storage-for-specific-features). ## Expiring artifacts @@ -419,6 +419,8 @@ reasons are: to remove these. This script should always find work to do, as it also removes empty directories (see above). - [Artifact housekeeping was changed significantly](#artifacts-housekeeping-disabled-in-gitlab-146-to-152), and you might need to enable a feature flag to used the updated system. +- The [keep latest artifacts from most recent success jobs](../ci/jobs/job_artifacts.md#keep-artifacts-from-most-recent-successful-jobs) + feature is enabled. In these and other cases, identify the projects most responsible for disk space usage, figure out what types of artifacts are using the most @@ -461,7 +463,7 @@ To check if the feature flags are enabled: ``` These changes include switching artifacts from `unlocked` to `locked` if -they [should be retained](../ci/pipelines/job_artifacts.md#keep-artifacts-from-most-recent-successful-jobs). +they [should be retained](../ci/jobs/job_artifacts.md#keep-artifacts-from-most-recent-successful-jobs). Artifacts created before this feature was introduced have a status of `unknown`. After they expire, these artifacts are not processed by the new housekeeping jobs. @@ -673,7 +675,7 @@ If you need to manually remove job artifacts associated with multiple jobs while NOTE: This step also erases artifacts that users have chosen to - ["keep"](../ci/pipelines/job_artifacts.md#download-job-artifacts). + ["keep"](../ci/jobs/job_artifacts.md#download-job-artifacts). ```ruby builds_to_clear = builds_with_artifacts.where("finished_at < ?", 1.week.ago) @@ -779,7 +781,7 @@ In both cases, you might need to add `region` to the job artifact [object storag ### Job artifact upload fails with `500 Internal Server Error (Missing file)` -Bucket names that include folder paths are not supported with [consolidated object storage](object_storage.md#consolidated-object-storage-configuration). +Bucket names that include folder paths are not supported with [consolidated object storage](object_storage.md#configure-a-single-storage-connection-for-all-object-types-consolidated-form). For example, `bucket/path`. If a bucket name has a path in it, you might receive an error similar to: ```plaintext @@ -807,3 +809,18 @@ To work around this issue, you can try: - For older kernels, using the `nolease` mount option to disable file leasing. For more information, [see the investigation details](https://gitlab.com/gitlab-org/gitlab/-/issues/389995). + +### Usage quota shows incorrect artifact storage usage + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/238536) in GitLab 14.10. + +Sometimes the [artifacts storage usage](../user/usage_quotas.md) displays an incorrect +value for the total storage space used by artifacts. To recalculate the artifact +usage statistics for all projects in the instance, you can run this background script: + +```shell +bin/rake 'gitlab:refresh_project_statistics_build_artifacts_size[file.csv]' +``` + +The artifact usage value can fluctuate to `0` while the script is running. After +recalculation, usage should display as expected again. diff --git a/doc/administration/lfs/index.md b/doc/administration/lfs/index.md index 3638729a6b6..ee5cd04f3d6 100644 --- a/doc/administration/lfs/index.md +++ b/doc/administration/lfs/index.md @@ -2,7 +2,6 @@ stage: Create group: Source Code info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments" -disqus_identifier: 'https://docs.gitlab.com/ee/workflow/lfs/lfs_administration.html' --- # GitLab Git Large File Storage (LFS) Administration **(FREE SELF)** @@ -157,14 +156,14 @@ You can store LFS objects in remote object storage. This allows you to reduce reads and writes to the local disk, and free up disk space significantly. In GitLab 13.2 and later, you should use the -[consolidated object storage settings](../object_storage.md#consolidated-object-storage-configuration). +[consolidated object storage settings](../object_storage.md#configure-a-single-storage-connection-for-all-object-types-consolidated-form). ### Migrating to object storage You can migrate the LFS objects from local storage to object storage. The processing is done in the background and requires **no downtime**. -1. [Configure the object storage](../object_storage.md#consolidated-object-storage-configuration). +1. [Configure the object storage](../object_storage.md#configure-a-single-storage-connection-for-all-object-types-consolidated-form). 1. Migrate the LFS objects: ::Tabs @@ -272,7 +271,7 @@ To migrate back to local storage: ``` 1. Edit `/etc/gitlab/gitlab.rb` and - [disable object storage](../object_storage.md#selectively-disabling-object-storage) + [disable object storage](../object_storage.md#disable-object-storage-for-specific-features) for LFS objects: ```ruby @@ -421,6 +420,18 @@ To check an installed Git LFS client's version, run this command: git lfs version ``` +### `Connection refused` errors + +If you push or mirror LFS objects and receive errors like the following: + +- `dial tcp :443: connect: connection refused` +- `Connection refused - connect(2) for \"\" port 443` + +a firewall or proxy rule may be terminating the connection. + +If connection checks with standard Unix tools or manual Git pushes are successful, +the rule may be related to the size of the request. + ## Error viewing a PDF file When LFS has been configured with object storage and `proxy_download` set to diff --git a/doc/administration/load_balancer.md b/doc/administration/load_balancer.md index 298d22f1da5..e43fe851aa2 100644 --- a/doc/administration/load_balancer.md +++ b/doc/administration/load_balancer.md @@ -31,18 +31,18 @@ Configure your load balancers to pass connections on port 443 as 'TCP' rather than 'HTTP(S)' protocol. This passes the connection to the application nodes NGINX service untouched. NGINX has the SSL certificate and listen on port 443. -See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html) for details on managing SSL certificates and configuring NGINX. ### Load Balancers terminate SSL without backend SSL Configure your load balancers to use the `HTTP(S)` protocol rather than `TCP`. -The load balancers is be responsible for managing SSL certificates and +The load balancers are responsible for managing SSL certificates and terminating SSL. Because communication between the load balancers and GitLab isn't secure, there is some additional configuration needed. See the -[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) +[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) for details. ### Load Balancers terminate SSL with backend SSL @@ -55,7 +55,7 @@ Traffic is secure between the load balancers and NGINX in this scenario. There is no need to add configuration for proxied SSL because the connection is secure all the way. However, configuration must be added to GitLab to configure SSL certificates. See -the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html) for details on managing SSL certificates and configuring NGINX. ## Ports @@ -131,5 +131,5 @@ The default ciphers for a GitLab version can be viewed in the [`files/gitlab-cookbooks/gitlab/attributes/default.rb`](https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/master/files/gitlab-cookbooks/gitlab/attributes/default.rb) file and selecting the Git tag that correlates with your target GitLab version (for example `15.0.5+ee.0`). If required by your load balancer, you can then define -[custom SSL ciphers](https://docs.gitlab.com/omnibus/settings/ssl.html#use-custom-ssl-ciphers) +[custom SSL ciphers](https://docs.gitlab.com/omnibus/settings/ssl/index.html#use-custom-ssl-ciphers) for NGINX. diff --git a/doc/administration/logs/index.md b/doc/administration/logs/index.md index eab4c9b7d83..7fab97f76da 100644 --- a/doc/administration/logs/index.md +++ b/doc/administration/logs/index.md @@ -96,6 +96,7 @@ except those captured by `runit`. | [LogRotate logs](#logrotate-logs) | **{dotted-circle}** No | **{check-circle}** Yes | | [Mailroom](#mail_room_jsonlog-default) | **{check-circle}** Yes | **{check-circle}** Yes | | [NGINX](#nginx-logs) | **{check-circle}** Yes | **{check-circle}** Yes | +| [PgBouncer logs](#pgbouncer-logs) | **{dotted-circle}** No | **{check-circle}** Yes | | [PostgreSQL logs](#postgresql-logs) | **{dotted-circle}** No | **{check-circle}** Yes | | [Praefect logs](#praefect-logs) | **{dotted-circle}** Yes | **{check-circle}** Yes | | [Prometheus logs](#prometheus-logs) | **{dotted-circle}** No | **{check-circle}** Yes | @@ -180,7 +181,7 @@ In addition, the log contains the originating IP address, (`remote_ip`), the user's ID (`user_id`), and username (`username`). Some endpoints (such as `/search`) may make requests to Elasticsearch if using -[Advanced Search](../../user/search/advanced_search.md). These +[advanced search](../../user/search/advanced_search.md). These additionally log `elasticsearch_calls` and `elasticsearch_call_duration_s`, which correspond to: @@ -349,15 +350,17 @@ the `view_duration_s` is calculated by [`duration_s - db_duration_s`](https://gi Therefore, `view_duration_s` can be affected by multiple different factors, like read-write process on Redis or external HTTP, not only the serialization process. -## `application.log` +## `application.log` (deprecated) + +> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/111046) in GitLab 15.10. Depending on your installation method, this file is located at: - Omnibus GitLab: `/var/log/gitlab/gitlab-rails/application.log` - Installations from source: `/home/git/gitlab/log/application.log` -It helps you discover events happening in your instance such as user creation -and project deletion. For example: +It contains a less structured version of the logs in +[`application_json.log`](#application_jsonlog), like this example: ```plaintext October 06, 2014 11:56: User "Administrator" (admin@example.com) was created @@ -376,7 +379,8 @@ Depending on your installation method, this file is located at: - Omnibus GitLab: `/var/log/gitlab/gitlab-rails/application_json.log` - Installations from source: `/home/git/gitlab/log/application_json.log` -It contains the JSON version of the logs in `application.log`, like this example: +It helps you discover events happening in your instance such as user creation +and project deletion. For example: ```json { @@ -426,7 +430,7 @@ like this example: } ``` -## `kubernetes.log` (DEPRECATED) +## `kubernetes.log` (deprecated) > [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. @@ -686,14 +690,6 @@ Unix timestamp format, and `gzip`-compressed (like `@1584057562.s`). This file is at `/var/log/gitlab/gitlab-rails/grpc.log` for Omnibus GitLab packages. Native [gRPC](https://grpc.io/) logging used by Gitaly. -### `gitaly_ruby_json.log` - -> [Introduced](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/2678) in GitLab 13.6. - -This file is at `/var/log/gitlab/gitaly/gitaly_ruby_json.log` and is -produced by [`gitaly-ruby`](../gitaly/reference.md#gitaly-ruby). It contains an -access log of gRPC calls made by Gitaly to `gitaly-ruby`. - ### `gitaly_hooks.log` This file is at `/var/log/gitlab/gitaly/gitaly_hooks.log` and is @@ -733,7 +729,7 @@ Depending on your installation method, this file is located at: - Omnibus GitLab: `/var/log/gitlab/gitlab-rails/importer.log` - Installations from source: `/home/git/gitlab/log/importer.log` -It logs the progress of the import process. +This file logs the progress of [project imports and migrations](../../user/project/import/index.md). ## `exporter.log` @@ -771,6 +767,24 @@ are recorded in this file. For example: {"severity":"INFO","time":"2020-11-24T02:31:29.329Z","correlation_id":null,"key":"cd_auto_rollback","action":"remove"} ``` +## `ci_resource_groups_json.log` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/384180) in GitLab 15.9. + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/ci_resource_group_json.log` +- Installations from source: `/home/git/gitlab/log/ci_resource_group_json.log` + +It contains information about [resource group](../../ci/resource_groups/index.md) acquisition. For example: + +```json +{"severity":"INFO","time":"2023-02-10T23:02:06.095Z","correlation_id":"01GRYS10C2DZQ9J1G12ZVAD4YD","resource_group_id":1,"processable_id":288,"message":"attempted to assign resource to processable","success":true} +{"severity":"INFO","time":"2023-02-10T23:02:08.945Z","correlation_id":"01GRYS138MYEG32C0QEWMC4BDM","resource_group_id":1,"processable_id":288,"message":"attempted to release resource from processable","success":true} +``` + +The examples show the `resource_group_id`, `processable_id`, `message`, and `success` fields for each entry. + ## `auth.log` > Introduced in GitLab 12.0. @@ -787,6 +801,28 @@ This log records: - In GitLab versions [12.3](https://gitlab.com/gitlab-org/gitlab/-/issues/29239) and later, user ID and username, if available. +## `auth_json.log` + +Depending on your installation method, this file is located at: + +- Omnibus GitLab: `/var/log/gitlab/gitlab-rails/auth_json.log` +- Installations from source: `/home/git/gitlab/log/auth_json.log` + +This file contains the JSON version of the logs in `auth.log`, for example: + +```json +{ + "severity":"ERROR", + "time":"2023-04-19T22:14:25.893Z", + "correlation_id":"01GYDSAKAN2SPZPAMJNRWW5H8S", + "message":"Rack_Attack", + "env":"blocklist", + "remote_ip":"x.x.x.x", + "request_method":"GET", + "path":"/group/project.git/info/refs?service=git-upload-pack" +} +``` + ## `graphql_json.log` > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/59587) in GitLab 12.0. @@ -811,6 +847,8 @@ Depending on your installation method, this file is located at: - Omnibus GitLab: `/var/log/gitlab/gitlab-rails/migrations.log` - Installations from source: `/home/git/gitlab/log/migrations.log` +This file logs the progress of [database migrations](../raketasks/maintenance.md#display-status-of-database-migrations). + ## `mail_room_json.log` (default) > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/19186) in GitLab 12.6. @@ -1016,9 +1054,13 @@ For Omnibus GitLab installations, NGINX logs are in: Below is the default GitLab NGINX access log format: ```plaintext -$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" +'$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"' ``` +The `$request` and `$http_referer` are +[filtered](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/support/nginx/gitlab) +for sensitive query string parameters such as secret tokens. + ## Pages logs For Omnibus GitLab installations, Pages logs are in `/var/log/gitlab/gitlab-pages/current`. @@ -1059,6 +1101,10 @@ For Omnibus GitLab installations, Mattermost logs are in these locations: For Omnibus GitLab installations, Workhorse logs are in `/var/log/gitlab/gitlab-workhorse/current`. +## PgBouncer logs + +For Omnibus GitLab installations, PgBouncer logs are in `/var/log/gitlab/pgbouncer/current`. + ## PostgreSQL logs For Omnibus GitLab installations, PostgreSQL logs are in `/var/log/gitlab/postgresql/current`. diff --git a/doc/administration/logs/log_parsing.md b/doc/administration/logs/log_parsing.md index 84c517e6120..d9520ea9bc0 100644 --- a/doc/administration/logs/log_parsing.md +++ b/doc/administration/logs/log_parsing.md @@ -158,7 +158,7 @@ CT: 297 ROUTE: /api/:version/projects/:id/repository/tags DURS: 731.39, CT: 190 ROUTE: /api/:version/projects/:id/repository/commits DURS: 1079.02, 979.68, 958.21 ``` -### Print top API user agents +#### Print top API user agents ```shell jq --raw-output '[.route, .ua] | @tsv' api_json.log | sort | uniq -c | sort -n @@ -180,9 +180,20 @@ if the output contains many: You can also [use `fast-stats top`](#parsing-gitlab-logs-with-jq) to extract performance statistics. +### Parsing `gitlab-rails/importer.log` + +To troubleshoot [project imports](../../administration/raketasks/project_import_export.md) or +[migrations](../../user/project/import/index.md), run this command: + +```shell +jq 'select(.project_path == "/").error_messages' importer.log +``` + +For common issues, see [troubleshooting](../../administration/raketasks/project_import_export.md#troubleshooting). + ### Parsing `gitlab-workhorse/current` -### Print top Workhorse user agents +#### Print top Workhorse user agents ```shell jq --raw-output '[.uri, .user_agent] | @tsv' current | sort | uniq -c | sort -n @@ -212,9 +223,12 @@ repeatedly reports that some items never reach 100%, the following command helps to focus on the most common errors. ```shell -jq --raw-output 'select(.severity == "ERROR") | [.project_path, .message] | @tsv' geo.log | sort | uniq -c | sort | tail +jq --raw-output 'select(.severity == "ERROR") | [.project_path, .class, .message, .error] | @tsv' geo.log | sort | uniq -c | sort | tail ``` +Refer to our [Geo troubleshooting page](../geo/replication/troubleshooting.md) +for advice about specific error messages. + ### Parsing `gitaly/current` Use the following examples to [troubleshoot Gitaly](../gitaly/troubleshooting.md). diff --git a/doc/administration/maintenance_mode/index.md b/doc/administration/maintenance_mode/index.md index c30465b4df9..8347015a8a3 100644 --- a/doc/administration/maintenance_mode/index.md +++ b/doc/administration/maintenance_mode/index.md @@ -8,17 +8,17 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2149) in GitLab 13.9. -Maintenance Mode allows administrators to reduce write operations to a minimum while maintenance tasks are performed. The main goal is to block all external actions that change the internal state, including the PostgreSQL database, but especially files, Git repositories, and Container repositories. +Maintenance Mode allows administrators to reduce write operations to a minimum while maintenance tasks are performed. The main goal is to block all external actions that change the internal state. The internal state includes the PostgreSQL database, but especially files, Git repositories, and Container repositories. -When Maintenance Mode is enabled, in-progress actions finish relatively quickly since no new actions are coming in, and internal state changes are minimal. -In that state, various maintenance tasks are easier, and services can be stopped completely or be -further degraded for a much shorter period of time than might otherwise be needed. For example, stopping cron jobs and draining queues should be fairly quick. +When Maintenance Mode is enabled, in-progress actions finish relatively quickly because no new actions are coming in, and internal state changes are minimal. +In that state, various maintenance tasks are easier. Services can be stopped completely or +further degraded for a shorter period of time than might otherwise be needed. For example, stopping cron jobs and draining queues should be fairly quick. Maintenance Mode allows most external actions that do not change internal state. On a high-level, HTTP `POST`, `PUT`, `PATCH`, and `DELETE` requests are blocked and a detailed overview of [how special cases are handled](#rest-api) is available. ## Enable Maintenance Mode -There are three ways to enable Maintenance Mode as an administrator: +Enable Maintenance Mode as an administrator in one of these ways: - **Web UI**: 1. On the top bar, select **Main menu > Admin**. @@ -42,7 +42,7 @@ There are three ways to enable Maintenance Mode as an administrator: ## Disable Maintenance Mode -There are three ways to disable Maintenance Mode: +Disable Maintenance Mode in one of three ways: - **Web UI**: 1. On the top bar, select **Main menu > Admin**. @@ -73,7 +73,7 @@ An error is displayed when a user tries to perform a write operation that isn't ![Maintenance Mode banner and error message](img/maintenance_mode_error_message.png) NOTE: -In some cases, the visual feedback from an action could be misleading, for example when starring a project, the **Star** button changes to show the **Unstar** action, however, this is only the frontend update, and it doesn't take into account the failed status of the POST request. These visual bugs are to be fixed [in follow-up iterations](https://gitlab.com/gitlab-org/gitlab/-/issues/295197). +In some cases, the visual feedback from an action could be misleading. For example, when starring a project, the **Star** button changes to show the **Unstar** action. However, this is only the frontend update, and it doesn't take into account the failed status of the POST request. These visual bugs are to be fixed [in follow-up iterations](https://gitlab.com/gitlab-org/gitlab/-/issues/295197). ### Administrator functions @@ -84,7 +84,7 @@ them to disable Maintenance Mode after it's been enabled. All users can sign in and out of the GitLab instance but no new users can be created. -If there are [LDAP syncs](../auth/ldap/index.md) scheduled for that time, they fail since user creation is disabled. Similarly, [user creations based on SAML](../../integration/saml.md#configure-saml-support-in-gitlab) fail. +If there are [LDAP syncs](../auth/ldap/index.md) scheduled for that time, they fail because user creation is disabled. Similarly, [user creations based on SAML](../../integration/saml.md#configure-saml-support-in-gitlab) fail. ### Git actions @@ -135,22 +135,22 @@ For most JSON requests, `POST`, `PUT`, `PATCH`, and `DELETE` are blocked, and th even if they finish running on the GitLab Runner. - Jobs in the `running` state for longer than the project's time limit do not time out. - Pipelines cannot be started, retried or canceled. No new jobs can be created either. -- The status of the runners in `/admin/runners` won't be updated. -- `gitlab-runner verify` will return the error `ERROR: Verifying runner... is removed`. +- The status of the runners in `/admin/runners` isn't updated. +- `gitlab-runner verify` returns the error `ERROR: Verifying runner... is removed`. After Maintenance Mode is disabled, new jobs are picked up again. Jobs that were in the `running` state before enabling Maintenance Mode resume and their logs start updating again. NOTE: -It is recommended that you restart previously `running` pipelines after Maintenance Mode +You should restart previously `running` pipelines after Maintenance Mode is turned off. ### Deployments Deployments don't go through because pipelines are unfinished. -It is recommended to disable auto deploys during Maintenance Mode, and enable them when it is disabled. +You should disable auto deploys during Maintenance Mode, and enable them when it is disabled. #### Terraform integration @@ -169,7 +169,7 @@ Package Registry allows you to install but not publish packages. Background jobs (cron jobs, Sidekiq) continue running as is, because background jobs are not automatically disabled. [During a planned Geo failover](../geo/disaster_recovery/planned_failover.md#prevent-updates-to-the-primary-site), -it is recommended that you disable all cron jobs except for those related to Geo. +you should disable all cron jobs except for those related to Geo. To monitor queues and disable jobs: @@ -206,7 +206,7 @@ SAST and Secret Detection cannot be initiated because they depend on passing CI ## An example use case: a planned failover -In the use case of [a planned failover](../geo/disaster_recovery/planned_failover.md), a few writes in the primary database are acceptable, since they are replicated quickly and are not significant in number. +In the use case of [a planned failover](../geo/disaster_recovery/planned_failover.md), a few writes in the primary database are acceptable, because they are replicated quickly and are not significant in number. For the same reason we don't automatically block background jobs when Maintenance Mode is enabled. diff --git a/doc/administration/merge_request_diffs.md b/doc/administration/merge_request_diffs.md index 85677512860..3250a2339e2 100644 --- a/doc/administration/merge_request_diffs.md +++ b/doc/administration/merge_request_diffs.md @@ -105,7 +105,7 @@ be configured already. ### Object Storage Settings In GitLab 13.2 and later, you should use the -[consolidated object storage settings](object_storage.md#consolidated-object-storage-configuration). +[consolidated object storage settings](object_storage.md#configure-a-single-storage-connection-for-all-object-types-consolidated-form). This section describes the earlier configuration format. For source installations, these settings are nested under `external_diffs:` and @@ -121,7 +121,7 @@ then `object_store:`. On Omnibus installations, they are prefixed by #### S3 compatible connection settings -See [the available connection settings for different providers](object_storage.md#connection-settings). +See [the available connection settings for different providers](object_storage.md#configure-the-connection-settings). **In Omnibus installations:** diff --git a/doc/administration/monitoring/github_imports.md b/doc/administration/monitoring/github_imports.md index 097109585af..74e584db3a0 100644 --- a/doc/administration/monitoring/github_imports.md +++ b/doc/administration/monitoring/github_imports.md @@ -1,6 +1,6 @@ --- -stage: Monitor -group: Respond +stage: Manage +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/administration/monitoring/gitlab_self_monitoring_project/img/self_monitoring_overview_dashboard.png b/doc/administration/monitoring/gitlab_self_monitoring_project/img/self_monitoring_overview_dashboard.png deleted file mode 100644 index 1d61823ce41..00000000000 Binary files a/doc/administration/monitoring/gitlab_self_monitoring_project/img/self_monitoring_overview_dashboard.png and /dev/null differ diff --git a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md index 566bc070347..dbdcdf22007 100644 --- a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md +++ b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md @@ -2,120 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../index.md' --- -# Self-monitoring project (DEPRECATED) **(FREE SELF)** +# Self-monitoring project (removed) **(FREE SELF)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32351) in GitLab 12.7 [with a flag](../../feature_flags.md) named `self_monitoring_project`. Disabled by default. -> - Generally available in GitLab 12.8. [Feature flag `self_monitoring_project`](https://gitlab.com/gitlab-org/gitlab/-/issues/198511) removed. -> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/348909) in GitLab 14.9. Planned for removal in GitLab 16.0. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/348909) -in GitLab 14.9, and is planned for removal in GitLab 16.0. - -GitLab provides administrators insights into the health of their GitLab instance. - -To provide a native experience (similar interacting with an application deployed using GitLab), a -project called **Monitoring** is created: - -- With [internal visibility](../../../user/public_access.md#internal-projects-and-groups). -- Under a group called **GitLab Instance**. - -The project is created specifically for visualizing and configuring the monitoring of your GitLab -instance. - -When the project and group are created, all administrators are given the [Maintainer role](../../../user/permissions.md). -As an administrator, you can add new members to the group to give them the Maintainer role for the project. - -This project can be used to: - -- Self-monitor your GitLab instance. The metrics dashboard of the project shows some basic resource - usage charts, such as CPU and memory usage of each server in - [Omnibus GitLab](https://docs.gitlab.com/omnibus/) installations. -- Also configure your own [custom metrics](../../../operations/metrics/index.md#adding-custom-metrics) - using metrics exposed by the [GitLab exporter](../prometheus/gitlab_metrics.md#metrics-available). - -## Create the self-monitoring project - -1. On the top bar, select **Main menu > Admin**. -1. On the left sidebar, select **Settings > Metrics and profiling** and expand **Self-monitoring**. -1. Toggle **Self-monitoring** on. -1. After your GitLab instance creates the project, GitLab displays a link to the - project in the text above the **Self-monitoring** toggle. You can also find it - from the top bar by selecting **Main menu > Projects**. - -## Delete the self-monitoring project - -WARNING: -Deleting the self-monitoring project removes any changes made to the project. If -you create the project again, it's created in its default state. - -1. On the top bar, select **Main menu > Admin**. -1. On the left sidebar, go to **Settings > Metrics and profiling** and expand **Self-monitoring**. -1. Toggle **Self-monitoring** off. -1. In the confirmation dialog that opens, select **Delete self-monitoring project**. - It can take a few seconds for it to be deleted. -1. After the project is deleted, GitLab displays a message confirming your action. - -## Dashboards available in Omnibus GitLab - -Omnibus GitLab provides a dashboard that displays CPU and memory usage -of each GitLab server. To select the servers to be displayed in the -panels, provide a regular expression in the **Instance label regex** field. -The dashboard uses metrics available in -[Omnibus GitLab](https://docs.gitlab.com/omnibus/) installations. - -![GitLab self-monitoring overview dashboard](img/self_monitoring_overview_dashboard.png) - -You can also -[create your own dashboards](../../../operations/metrics/dashboards/index.md). - -## Connect to Prometheus - -The project is automatically configured to connect to the -[internal Prometheus](../prometheus/index.md) instance if the Prometheus instance is present. -This should be the case if GitLab was installed using Omnibus GitLab and you haven't disabled it. - -If that's not the case, or if you have an external Prometheus instance or a customized setup, -you [configure it manually](../../../user/project/integrations/prometheus.md#manual-configuration-of-prometheus). - -## Take action on Prometheus alerts **(ULTIMATE)** - -You can [add a Prometheus integration](../../../operations/incident_management/integrations.md) -to GitLab to receive notifications of any alerts. - -When the integration is set up, you can -[take action on incoming alerts](../../../operations/metrics/alerts.md#trigger-actions-from-alerts). - -## Add custom metrics to the self-monitoring project - -You can add custom metrics in the self-monitoring project by: - -1. [Duplicating](../../../operations/metrics/dashboards/index.md#duplicate-a-gitlab-defined-dashboard) the overview dashboard. -1. [Editing](../../../operations/metrics/index.md) the newly created dashboard file and configuring it with [dashboard YAML properties](../../../operations/metrics/dashboards/yaml.md). - -## Troubleshooting - -### Error message in logs: `Could not create instance administrators group. Errors: ["You don't have permission to create groups."]` - -A [bug](https://gitlab.com/gitlab-org/gitlab/-/issues/208676) causes project creation to fail with -the following error in the log file when the first administrator user is an -[external user](../../../user/admin_area/external_users.md): - -```plaintext -Could not create instance administrators group. Errors: ["You don't have permission to create groups."] -``` - -Run the following in a Rails console to check if the first administrator user is an external user: - -```ruby -User.admins.active.first.external? -``` - -If this returns true, the first administrator user is an external user. - -If you face this issue, you can temporarily -[make the administrator user a non-external user](../../../user/admin_area/external_users.md) -and then try to create the project. -After the project is created, the administrator user can be changed back to an external user. +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/348909) +in GitLab 14.9 and [removed](https://gitlab.com/groups/gitlab-org/-/epics/10030) in 16.0. diff --git a/doc/administration/monitoring/index.md b/doc/administration/monitoring/index.md index e57156e6513..1b23d6b7f49 100644 --- a/doc/administration/monitoring/index.md +++ b/doc/administration/monitoring/index.md @@ -8,9 +8,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w Explore our features to monitor your GitLab instance: -- [GitLab self-monitoring](gitlab_self_monitoring_project/index.md): The - GitLab instance administration project helps to monitor the GitLab instance and - take action on alerts. - [Performance monitoring](performance/index.md): GitLab Performance Monitoring makes it possible to measure a wide variety of statistics of your instance. - [Prometheus](prometheus/index.md): Prometheus is a powerful time-series monitoring diff --git a/doc/administration/monitoring/performance/grafana_configuration.md b/doc/administration/monitoring/performance/grafana_configuration.md index 3dec34ebace..1113dcfef32 100644 --- a/doc/administration/monitoring/performance/grafana_configuration.md +++ b/doc/administration/monitoring/performance/grafana_configuration.md @@ -6,10 +6,41 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Grafana Configuration **(FREE SELF)** +> [Deprecated](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/7772) in GitLab 16.0. + +WARNING: +Bundled Grafana was deprecated GitLab 16.0 and is no longer supported. It will be removed in GitLab 16.3. +For more information, see [deprecation notes](#deprecation-of-bundled-grafana). + [Grafana](https://grafana.com/) is a tool that enables you to visualize time series metrics through graphs and dashboards. GitLab writes performance data to Prometheus, and Grafana allows you to query the data to display useful graphs. +## Deprecation of bundled Grafana + +Bundled Grafana was an optional Omnibus GitLab service that provided a user interface to GitLab metrics. + +The version of Grafana that is bundled with Omnibus GitLab is no longer supported. If you're using the bundled Grafana, you +should switch to a newer version from [Grafana Labs](https://grafana.com/grafana/). + +### Switch to new Grafana instance + +To switch away from bundled Grafana to a newer version of Grafana from Grafana Labs: + +1. Set up a version of Grafana from Grafana Labs. +1. [Export the existing dashboards](https://grafana.com/docs/grafana/latest/dashboards/manage-dashboards/#export-a-dashboard) from bundled Grafana. +1. [Import the existing dashboards](https://grafana.com/docs/grafana/latest/dashboards/manage-dashboards/#import-a-dashboard) in the new Grafana instance. +1. [Configure GitLab](#integration-with-gitlab-ui) to use the new Grafana instance. + +### Temporary workaround + +In GitLab versions 16.0 to 16.2, you can still force Omnibus GitLab to enable and configure Grafana by setting the following: + +- `grafana['enable'] = true`. +- `grafana['enable_deprecated_service'] = true`. + +You see a deprecation message when reconfiguring GitLab. + ## Installation Omnibus GitLab can [help you install Grafana (recommended)](https://docs.gitlab.com/omnibus/settings/grafana.html) diff --git a/doc/administration/monitoring/performance/index.md b/doc/administration/monitoring/performance/index.md index 89f1270532a..cbdd68804e8 100644 --- a/doc/administration/monitoring/performance/index.md +++ b/doc/administration/monitoring/performance/index.md @@ -1,6 +1,6 @@ --- -stage: Monitor -group: Respond +stage: Data Stores +group: Application Performance info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index ed55e7d7ff3..c0a175c76cd 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -1,6 +1,6 @@ --- -stage: Monitor -group: Respond +stage: Shared responsibility based on functional area +group: Shared responsibility based on functional area info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -30,16 +30,16 @@ For enabling and viewing metrics from Sidekiq nodes, see [Sidekiq metrics](#side ## Metrics available +> `caller_id` [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/392622) from `redis_hit_miss_operations_total` and `redis_cache_generation_duration_seconds` in GitLab 15.11. + The following metrics are available: | Metric | Type | Since | Description | Labels | | :--------------------------------------------------------------- | :---------- | ------: | :-------------------------------------------------------------------------------------------------------------------- | :-------------------------------------------------------- | -| `gitlab_banzai_cached_render_real_duration_seconds` | Histogram | 9.4 | Duration of rendering Markdown into HTML when cached output exists | `controller`, `action` | -| `gitlab_banzai_cacheless_render_real_duration_seconds` | Histogram | 9.4 | Duration of rendering Markdown into HTML when cached output does not exist | `controller`, `action` | | `gitlab_cache_misses_total` | Counter | 10.2 | Cache read miss | `controller`, `action` | -| `gitlab_cache_operation_duration_seconds` | Histogram | 10.2 | Cache access time | | -| `gitlab_cache_operations_total` | Counter | 12.2 | Cache operations by controller or action | `controller`, `action`, `operation` | -| `gitlab_cache_read_multikey_count` | Histogram | 15.7 | Count of keys in multi-key cache read operations | `controller`, `action` | +| `gitlab_cache_operation_duration_seconds` | Histogram | 10.2 | Cache access time | `operation`, `store` | +| `gitlab_cache_operations_total` | Counter | 12.2 | Cache operations by controller or action | `controller`, `action`, `operation`, `store` | +| `gitlab_cache_read_multikey_count` | Histogram | 15.7 | Count of keys in multi-key cache read operations | `controller`, `action`, `store` | | `gitlab_ci_pipeline_builder_scoped_variables_duration` | Histogram | 14.5 | Time in seconds it takes to create the scoped variables for a CI/CD job | `gitlab_ci_pipeline_creation_duration_seconds` | Histogram | 13.0 | Time in seconds it takes to create a CI/CD pipeline | `gitlab` | | `gitlab_ci_pipeline_size_builds` | Histogram | 13.1 | Total number of builds within a pipeline grouped by a pipeline source | `source` | @@ -129,10 +129,12 @@ The following metrics are available: | `action_cable_single_client_transmissions_total` | Counter | 13.10 | The number of ActionCable messages transmitted to any client in any channel | `server_mode` | | `action_cable_subscription_confirmations_total` | Counter | 13.10 | The number of ActionCable subscriptions from clients confirmed | `server_mode` | | `action_cable_subscription_rejections_total` | Counter | 13.10 | The number of ActionCable subscriptions from clients rejected | `server_mode` | -| `action_cable_transmitted_bytes` | Histogram | 14.1 | Message size, in bytes, transmitted over action cable | `operation`, `channel` | +| `action_cable_transmitted_bytes_total` | Counter | 16.0 | Total number of bytes transmitted over ActionCable | `operation`, `channel` | | `gitlab_issuable_fast_count_by_state_total` | Counter | 13.5 | Total number of row count operations on issue/merge request list pages | | | `gitlab_issuable_fast_count_by_state_failures_total` | Counter | 13.5 | Number of soft-failed row count operations on issue/merge request list pages | | | `gitlab_ci_trace_finalize_duration_seconds` | Histogram | 13.6 | Duration of build trace chunks migration to object storage | | +| `gitlab_vulnerability_report_branch_comparison_real_duration_seconds` | Histogram | 15.11 | Execution duration of vulnerability report present on default branch sql query | | +| `gitlab_vulnerability_report_branch_comparison_cpu_duration_seconds` | Histogram | 15.11 | Execution duration of vulnerability report present on default branch sql query | | | `gitlab_external_http_total` | Counter | 13.8 | Total number of HTTP calls to external systems | `controller`, `action` | | `gitlab_external_http_duration_seconds` | Counter | 13.8 | Duration in seconds spent on each HTTP call to external systems | | | `gitlab_external_http_exception_total` | Counter | 13.8 | Total number of exceptions raised when making external HTTP calls | | @@ -151,8 +153,8 @@ The following metrics are available: | `gitlab_ci_build_trace_errors_total` | Counter | 14.4 | Total amount of different error types on a build trace | `error_reason` | | `gitlab_presentable_object_cacheless_render_real_duration_seconds` | Histogram | 15.3 | Duration of real time spent caching and representing specific web request objects | `controller`, `action` | | `cached_object_operations_total` | Counter | 15.3 | Total number of objects cached for specific web requests | `controller`, `action` | -| `redis_hit_miss_operations_total` | Counter | 15.6 | Total number of Redis cache hits and misses | `cache_hit`, `caller_id`, `cache_identifier`, `feature_category`, `backing_resource` | -| `redis_cache_generation_duration_seconds` | Histogram | 15.6 | Time to generate Redis cache | `cache_hit`, `caller_id`, `cache_identifier`, `feature_category`, `backing_resource` | +| `redis_hit_miss_operations_total` | Counter | 15.6 | Total number of Redis cache hits and misses | `cache_hit`, `cache_identifier`, `feature_category`, `backing_resource` | +| `redis_cache_generation_duration_seconds` | Histogram | 15.6 | Time to generate Redis cache | `cache_hit`, `cache_identifier`, `feature_category`, `backing_resource` | | `gitlab_diffs_reorder_real_duration_seconds` | Histogram | 15.8 | Duration in seconds spend on reordering of diff files on diffs batch request | `controller`, `action` | | `gitlab_diffs_collection_real_duration_seconds` | Histogram | 15.8 | Duration in seconds spent on querying merge request diff files on diffs batch request | `controller`, `action` | | `gitlab_diffs_comparison_real_duration_seconds` | Histogram | 15.8 | Duration in seconds spent on getting comparison data on diffs batch request | `controller`, `action` | @@ -163,12 +165,15 @@ The following metrics are available: | `gitlab_diffs_render_real_duration_seconds` | Histogram | 15.8 | Duration in seconds spent on serializing and rendering diffs on diffs batch request | `controller`, `action` | | `gitlab_memwd_violations_total` | Counter | 15.9 | Total number of times a Ruby process violated a memory threshold | | | `gitlab_memwd_violations_handled_total` | Counter | 15.9 | Total number of times Ruby process memory violations were handled | | +| `gitlab_sli_rails_request_apdex_total` | Counter | 14.4 | Total number of request Apdex measurements. For more information, see [Rails request SLIs](../../../development/application_slis/rails_request.md) | `endpoint_id`, `feature_category`, `request_urgency` | +| `gitlab_sli_rails_request_apdex_success_total` | Counter | 14.4 | Total number of successful requests that met the target duration for their urgency. Divide by `gitlab_sli_rails_requests_apdex_total` to get a success ratio | `endpoint_id`, `feature_category`, `request_urgency` | +| `gitlab_sli_rails_request_error_total` | Counter | 15.7 | Total number of request error measurements. For more information, see [Rails request SLIs](../../../development/application_slis/rails_request.md) | `endpoint_id`, `feature_category`, `request_urgency`, `error` | ## Metrics controlled by a feature flag The following metrics can be controlled by feature flags: -| Metric | Feature Flag | +| Metric | Feature flag | |:---------------------------------------------------------------|:-------------------------------------------------------------------| | `gitlab_view_rendering_duration_seconds` | `prometheus_metrics_view_instrumentation` | @@ -206,16 +211,16 @@ configuration option in `gitlab.yml`. These metrics are served from the | `geo_repositories` | Gauge | 10.2 | Total number of repositories available on primary | `url` | | `geo_repositories_synced` | Gauge | 10.2 | Number of repositories synced on secondary | `url` | | `geo_repositories_failed` | Gauge | 10.2 | Number of repositories failed to sync on secondary | `url` | -| `geo_lfs_objects` | Gauge | 10.2 | Number of LFS objects on primary | `url` | +| `geo_lfs_objects` | Gauge | 10.2 | Number of LFS objects on primary | `url` | | `geo_lfs_objects_checksummed` | Gauge | 14.6 | Number of LFS objects checksummed successfully on primary | `url` | | `geo_lfs_objects_checksum_failed` | Gauge | 14.6 | Number of LFS objects failed to calculate the checksum on primary | `url` | -| `geo_lfs_objects_checksum_total` | Gauge | 14.6 | Number of LFS objects tried to checksum on primary | `url` | +| `geo_lfs_objects_checksum_total` | Gauge | 14.6 | Number of LFS objects that need to be checksummed on primary | `url` | | `geo_lfs_objects_synced` | Gauge | 10.2 | Number of syncable LFS objects synced on secondary | `url` | | `geo_lfs_objects_failed` | Gauge | 10.2 | Number of syncable LFS objects failed to sync on secondary | `url` | | `geo_lfs_objects_registry` | Gauge | 14.6 | Number of LFS objects in the registry | `url` | -| `geo_lfs_objects_verified` | Gauge | 14.6 | Number of LFS objects verified on secondary | `url` | -| `geo_lfs_objects_verification_failed` | Gauge | 14.6 | Number of LFS objects' verifications failed on secondary | `url` | -| `geo_lfs_objects_verification_total` | Gauge | 14.6 | Number of LFS objects' verifications tried on secondary | `url` |LFS objects failed to sync on secondary | `url` | +| `geo_lfs_objects_verified` | Gauge | 14.6 | Number of LFS objects successfully verified on secondary | `url` | +| `geo_lfs_objects_verification_failed` | Gauge | 14.6 | Number of LFS objects that failed verifications on secondary | `url` | +| `geo_lfs_objects_verification_total` | Gauge | 14.6 | Number of LFS objects to attempt to verify on secondary | `url` | | `geo_attachments` | Gauge | 10.2 | Total number of file attachments available on primary | `url` | | `geo_attachments_synced` | Gauge | 10.2 | Number of attachments synced on secondary | `url` | | `geo_attachments_failed` | Gauge | 10.2 | Number of attachments failed to sync on secondary | `url` | @@ -230,11 +235,11 @@ configuration option in `gitlab.yml`. These metrics are served from the | `geo_repositories_checksum_failed` | Gauge | 10.7 | Number of repositories failed to calculate the checksum on primary | `url` | | `geo_wikis_checksummed` | Gauge | 10.7 | Number of wikis checksummed on primary | `url` | | `geo_wikis_checksum_failed` | Gauge | 10.7 | Number of wikis failed to calculate the checksum on primary | `url` | -| `geo_repositories_verified` | Gauge | 10.7 | Number of repositories verified on secondary | `url` | -| `geo_repositories_verification_failed` | Gauge | 10.7 | Number of repositories failed to verify on secondary | `url` | +| `geo_repositories_verified` | Gauge | 10.7 | Number of repositories successfully verified on secondary | `url` | +| `geo_repositories_verification_failed` | Gauge | 10.7 | Number of repositories that failed verification on secondary | `url` | | `geo_repositories_checksum_mismatch` | Gauge | 10.7 | Number of repositories that checksum mismatch on secondary | `url` | -| `geo_wikis_verified` | Gauge | 10.7 | Number of wikis verified on secondary | `url` | -| `geo_wikis_verification_failed` | Gauge | 10.7 | Number of wikis failed to verify on secondary | `url` | +| `geo_wikis_verified` | Gauge | 10.7 | Number of wikis successfully verified on secondary | `url` | +| `geo_wikis_verification_failed` | Gauge | 10.7 | Number of wikis that failed verification on secondary | `url` | | `geo_wikis_checksum_mismatch` | Gauge | 10.7 | Number of wikis that checksum mismatch on secondary | `url` | | `geo_repositories_checked` | Gauge | 11.1 | Number of repositories that have been checked via `git fsck` | `url` | | `geo_repositories_checked_failed` | Gauge | 11.1 | Number of repositories that have a failure from `git fsck` | `url` | @@ -249,25 +254,25 @@ configuration option in `gitlab.yml`. These metrics are served from the | `geo_terraform_state_versions` | Gauge | 13.5 | Number of terraform state versions on primary | `url` | | `geo_terraform_state_versions_checksummed` | Gauge | 13.5 | Number of terraform state versions checksummed successfully on primary | `url` | | `geo_terraform_state_versions_checksum_failed` | Gauge | 13.5 | Number of terraform state versions failed to calculate the checksum on primary | `url` | -| `geo_terraform_state_versions_checksum_total` | Gauge | 13.12 | Number of terraform state versions tried to checksum on primary | `url` | +| `geo_terraform_state_versions_checksum_total` | Gauge | 13.12 | Number of terraform state versions that need to be checksummed on primary | `url` | | `geo_terraform_state_versions_synced` | Gauge | 13.5 | Number of syncable terraform state versions synced on secondary | `url` | | `geo_terraform_state_versions_failed` | Gauge | 13.5 | Number of syncable terraform state versions failed to sync on secondary | `url` | | `geo_terraform_state_versions_registry` | Gauge | 13.5 | Number of terraform state versions in the registry | `url` | -| `geo_terraform_state_versions_verified` | Gauge | 13.12 | Number of terraform state versions verified on secondary | `url` | -| `geo_terraform_state_versions_verification_failed` | Gauge | 13.12 | Number of terraform state versions verifications failed on secondary | `url` | -| `geo_terraform_state_versions_verification_total` | Gauge | 13.12 | Number of terraform state versions verifications tried on secondary | `url` | +| `geo_terraform_state_versions_verified` | Gauge | 13.12 | Number of terraform state versions successfully verified on secondary | `url` | +| `geo_terraform_state_versions_verification_failed` | Gauge | 13.12 | Number of terraform state versions that failed verification on secondary | `url` | +| `geo_terraform_state_versions_verification_total` | Gauge | 13.12 | Number of terraform state versions to attempt to verify on secondary | `url` | | `global_search_bulk_cron_queue_size` | Gauge | 12.10 | Number of database records waiting to be synchronized to Elasticsearch | | | `global_search_awaiting_indexing_queue_size` | Gauge | 13.2 | Number of database updates waiting to be synchronized to Elasticsearch while indexing is paused | | | `geo_merge_request_diffs` | Gauge | 13.4 | Number of merge request diffs on primary | `url` | -| `geo_merge_request_diffs_checksum_total` | Gauge | 13.12 | Number of merge request diffs tried to checksum on primary | `url` | -| `geo_merge_request_diffs_checksummed` | Gauge | 13.4 | Number of merge request diffs successfully checksummed on primary | `url` | +| `geo_merge_request_diffs_checksum_total` | Gauge | 13.12 | Number of merge request diffs to checksum on primary | `url` | +| `geo_merge_request_diffs_checksummed` | Gauge | 13.4 | Number of merge request diffs that successfully calculated the checksum on primary | `url` | | `geo_merge_request_diffs_checksum_failed` | Gauge | 13.4 | Number of merge request diffs failed to calculate the checksum on primary | `url` | | `geo_merge_request_diffs_synced` | Gauge | 13.4 | Number of syncable merge request diffs synced on secondary | `url` | | `geo_merge_request_diffs_failed` | Gauge | 13.4 | Number of syncable merge request diffs failed to sync on secondary | `url` | | `geo_merge_request_diffs_registry` | Gauge | 13.4 | Number of merge request diffs in the registry | `url` | -| `geo_merge_request_diffs_verification_total` | Gauge | 13.12 | Number of merge request diffs verifications tried on secondary | `url` | -| `geo_merge_request_diffs_verified` | Gauge | 13.12 | Number of merge request diffs verified on secondary | `url` | -| `geo_merge_request_diffs_verification_failed` | Gauge | 13.12 | Number of merge request diffs verifications failed on secondary | `url` | +| `geo_merge_request_diffs_verification_total` | Gauge | 13.12 | Number of merge request diffs to attempt to verify on secondary | `url` | +| `geo_merge_request_diffs_verified` | Gauge | 13.12 | Number of merge request diffs successfully verified on secondary | `url` | +| `geo_merge_request_diffs_verification_failed` | Gauge | 13.12 | Number of merge request diffs that failed verification on secondary | `url` | | `geo_snippet_repositories` | Gauge | 13.4 | Number of snippets on primary | `url` | | `geo_snippet_repositories_checksummed` | Gauge | 13.4 | Number of snippets checksummed on primary | `url` | | `geo_snippet_repositories_checksum_failed` | Gauge | 13.4 | Number of snippets failed to calculate the checksum on primary | `url` | @@ -281,25 +286,25 @@ configuration option in `gitlab.yml`. These metrics are served from the | `geo_group_wiki_repositories_failed` | Gauge | 13.10 | Number of syncable group wikis failed on secondary | `url` | | `geo_group_wiki_repositories_registry` | Gauge | 13.10 | Number of syncable group wikis in the registry | `url` | | `geo_pages_deployments` | Gauge | 14.3 | Number of pages deployments on primary | `url` | -| `geo_pages_deployments_checksum_total` | Gauge | 14.6 | Number of pages deployments tried to checksum on primary | `url` | -| `geo_pages_deployments_checksummed` | Gauge | 14.6 | Number of pages deployments successfully checksummed on primary | `url` | +| `geo_pages_deployments_checksum_total` | Gauge | 14.6 | Number of pages deployments to checksum on primary | `url` | +| `geo_pages_deployments_checksummed` | Gauge | 14.6 | Number of pages deployments that successfully calculated the checksum on primary | `url` | | `geo_pages_deployments_checksum_failed` | Gauge | 14.6 | Number of pages deployments failed to calculate the checksum on primary | `url` | | `geo_pages_deployments_synced` | Gauge | 14.3 | Number of syncable pages deployments synced on secondary | `url` | | `geo_pages_deployments_failed` | Gauge | 14.3 | Number of syncable pages deployments failed to sync on secondary | `url` | | `geo_pages_deployments_registry` | Gauge | 14.3 | Number of pages deployments in the registry | `url` | -| `geo_pages_deployments_verification_total` | Gauge | 14.6 | Number of pages deployments verifications tried on secondary | `url` | -| `geo_pages_deployments_verified` | Gauge | 14.6 | Number of pages deployments verified on secondary | `url` | +| `geo_pages_deployments_verification_total` | Gauge | 14.6 | Number of pages deployments to attempt to verify on secondary | `url` | +| `geo_pages_deployments_verified` | Gauge | 14.6 | Number of pages deployments successfully verified on secondary | `url` | | `geo_pages_deployments_verification_failed` | Gauge | 14.6 | Number of pages deployments verifications failed on secondary | `url` | | `geo_job_artifacts` | Gauge | 14.8 | Number of job artifacts on primary | `url` | -| `geo_job_artifacts_checksum_total` | Gauge | 14.8 | Number of job artifacts tried to checksum on primary | `url` | -| `geo_job_artifacts_checksummed` | Gauge | 14.8 | Number of job artifacts successfully checksummed on primary | `url` | +| `geo_job_artifacts_checksum_total` | Gauge | 14.8 | Number of job artifacts to checksum on primary | `url` | +| `geo_job_artifacts_checksummed` | Gauge | 14.8 | Number of job artifacts that successfully calculated the checksum on primary | `url` | | `geo_job_artifacts_checksum_failed` | Gauge | 14.8 | Number of job artifacts failed to calculate the checksum on primary | `url` | | `geo_job_artifacts_synced` | Gauge | 14.8 | Number of syncable job artifacts synced on secondary | `url` | | `geo_job_artifacts_failed` | Gauge | 14.8 | Number of syncable job artifacts failed to sync on secondary | `url` | | `geo_job_artifacts_registry` | Gauge | 14.8 | Number of job artifacts in the registry | `url` | -| `geo_job_artifacts_verification_total` | Gauge | 14.8 | Number of job artifacts verifications tried on secondary | `url` | -| `geo_job_artifacts_verified` | Gauge | 14.8 | Number of job artifacts verified on secondary | `url` | -| `geo_job_artifacts_verification_failed` | Gauge | 14.8 | Number of job artifacts verifications failed on secondary | `url` | +| `geo_job_artifacts_verification_total` | Gauge | 14.8 | Number of job artifacts to attempt to verify on secondary | `url` | +| `geo_job_artifacts_verified` | Gauge | 14.8 | Number of job artifacts successfully verified on secondary | `url` | +| `geo_job_artifacts_verification_failed` | Gauge | 14.8 | Number of job artifacts that failed verification on secondary | `url` | | `limited_capacity_worker_running_jobs` | Gauge | 13.5 | Number of running jobs | `worker` | | `limited_capacity_worker_max_running_jobs` | Gauge | 13.5 | Maximum number of running jobs | `worker` | | `limited_capacity_worker_remaining_work_count` | Gauge | 13.5 | Number of jobs waiting to be enqueued | `worker` | @@ -310,51 +315,66 @@ configuration option in `gitlab.yml`. These metrics are served from the | `geo_uploads_synced` | Gauge | 14.1 | Number of uploads synced on secondary | `url` | | `geo_uploads_failed` | Gauge | 14.1 | Number of syncable uploads failed to sync on secondary | `url` | | `geo_uploads_registry` | Gauge | 14.1 | Number of uploads in the registry | `url` | -| `geo_uploads_checksum_total` | Gauge | 14.6 | Number of uploads tried to checksum on primary | `url` | -| `geo_uploads_checksummed` | Gauge | 14.6 | Number of uploads successfully checksummed on primary | `url` | +| `geo_uploads_checksum_total` | Gauge | 14.6 | Number of uploads to checksum on primary | `url` | +| `geo_uploads_checksummed` | Gauge | 14.6 | Number of uploads that successfully calculated the checksum on primary | `url` | | `geo_uploads_checksum_failed` | Gauge | 14.6 | Number of uploads failed to calculate the checksum on primary | `url` | -| `geo_uploads_verification_total` | Gauge | 14.6 | Number of uploads verifications tried on secondary | `url` | -| `geo_uploads_verified` | Gauge | 14.6 | Number of uploads verified on secondary | `url` | -| `geo_uploads_verification_failed` | Gauge | 14.6 | Number of uploads verifications failed on secondary | `url` | +| `geo_uploads_verification_total` | Gauge | 14.6 | Number of uploads to attempt to verify on secondary | `url` | +| `geo_uploads_verified` | Gauge | 14.6 | Number of uploads successfully verified on secondary | `url` | +| `geo_uploads_verification_failed` | Gauge | 14.6 | Number of uploads that failed verification on secondary | `url` | | `geo_container_repositories` | Gauge | 15.4 | Number of container repositories on primary | `url` | | `geo_container_repositories_synced` | Gauge | 15.4 | Number of container repositories synced on secondary | `url` | | `geo_container_repositories_failed` | Gauge | 15.4 | Number of syncable container repositories failed to sync on secondary | `url` | | `geo_container_repositories_registry` | Gauge | 15.4 | Number of container repositories in the registry | `url` | -| `gitlab_sli:rails_request_apdex:total` | Counter | 14.4 | The number of request-apdex measurements, [more information the development documentation](../../../development/application_slis/rails_request_apdex.md) | `endpoint_id`, `feature_category`, `request_urgency` | -| `gitlab_sli:rails_request_apdex:success_total` | Counter | 14.4 | The number of successful requests that met the target duration for their urgency. Divide by `gitlab_sli:rails_requests_apdex:total` to get a success ratio | `endpoint_id`, `feature_category`, `request_urgency` | +| `geo_container_repositories_checksum_total` | Gauge | 15.10 | Number of container repositories checksummed successfully on primary | `url` | +| `geo_container_repositories_checksummed` | Gauge | 15.10 | Number of container repositories tried to checksum on primary | `url` | +| `geo_container_repositories_checksum_failed` | Gauge | 15.10 | Number of container repositories failed to calculate the checksum on primary | `url` | +| `geo_container_repositories_verification_total` | Gauge | 15.10 | Number of container repositories' verifications tried on secondary | `url` | +| `geo_container_repositories_verified` | Gauge | 15.10 | Number of container repositories verified on secondary | `url` | +| `geo_container_repositories_verification_failed` | Gauge | 15.10 | Number of container repositories' failed verifications on secondary | `url` | | `geo_ci_secure_files` | Gauge | 15.3 | Number of secure files on primary | `url` | -| `geo_ci_secure_files_checksum_total` | Gauge | 15.3 | Number of secure files tried to checksum on primary | `url` | -| `geo_ci_secure_files_checksummed` | Gauge | 15.3 | Number of secure files successfully checksummed on primary | `url` | +| `geo_ci_secure_files_checksum_total` | Gauge | 15.3 | Number of secure files to checksum on primary | `url` | +| `geo_ci_secure_files_checksummed` | Gauge | 15.3 | Number of secure files that successfully calculated the checksum on primary | `url` | | `geo_ci_secure_files_checksum_failed` | Gauge | 15.3 | Number of secure files failed to calculate the checksum on primary | `url` | | `geo_ci_secure_files_synced` | Gauge | 15.3 | Number of syncable secure files synced on secondary | `url` | | `geo_ci_secure_files_failed` | Gauge | 15.3 | Number of syncable secure files failed to sync on secondary | `url` | | `geo_ci_secure_files_registry` | Gauge | 15.3 | Number of secure files in the registry | `url` | -| `geo_ci_secure_files_verification_total` | Gauge | 15.3 | Number of secure files verifications tried on secondary | `url` | -| `geo_ci_secure_files_verified` | Gauge | 15.3 | Number of secure files verified on secondary | `url` | -| `geo_ci_secure_files_verification_failed` | Gauge | 15.3 | Number of secure files verifications failed on secondary | `url` | +| `geo_ci_secure_files_verification_total` | Gauge | 15.3 | Number of secure files to attempt to verify on secondary | `url` | +| `geo_ci_secure_files_verified` | Gauge | 15.3 | Number of secure files successfully verified on secondary | `url` | +| `geo_ci_secure_files_verification_failed` | Gauge | 15.3 | Number of secure files that failed verification on secondary | `url` | | `geo_dependency_proxy_blob` | Gauge | 15.6 | Number of dependency proxy blobs on primary | | -| `geo_dependency_proxy_blob_checksum_total` | Gauge | 15.6 | Number of dependency proxy blobs tried to checksum on primary | | -| `geo_dependency_proxy_blob_checksummed` | Gauge | 15.6 | Number of dependency proxy blobs successfully checksummed on primary | | +| `geo_dependency_proxy_blob_checksum_total` | Gauge | 15.6 | Number of dependency proxy blobs to checksum on primary | | +| `geo_dependency_proxy_blob_checksummed` | Gauge | 15.6 | Number of dependency proxy blobs that successfully calculated the checksum on primary | | | `geo_dependency_proxy_blob_checksum_failed` | Gauge | 15.6 | Number of dependency proxy blobs failed to calculate the checksum on primary | | | `geo_dependency_proxy_blob_synced` | Gauge | 15.6 | Number of dependency proxy blobs synced on secondary | | | `geo_dependency_proxy_blob_failed` | Gauge | 15.6 | Number of dependency proxy blobs failed to sync on secondary | | | `geo_dependency_proxy_blob_registry` | Gauge | 15.6 | Number of dependency proxy blobs in the registry | | -| `geo_dependency_proxy_blob_verification_total` | Gauge | 15.6 | Number of dependency proxy blobs verifications tried on secondary | | -| `geo_dependency_proxy_blob_verified` | Gauge | 15.6 | Number of dependency proxy blobs verified on secondary | | -| `geo_dependency_proxy_blob_verification_failed` | Gauge | 15.6 | Number of dependency proxy blobs verifications failed on secondary | | +| `geo_dependency_proxy_blob_verification_total` | Gauge | 15.6 | Number of dependency proxy blobs to attempt to verify on secondary | | +| `geo_dependency_proxy_blob_verified` | Gauge | 15.6 | Number of dependency proxy blobs successfully verified on secondary | | +| `geo_dependency_proxy_blob_verification_failed` | Gauge | 15.6 | Number of dependency proxy blobs that failed verification on secondary | | | `geo_dependency_proxy_manifests` | Gauge | 15.6 | Number of dependency proxy manifests on primary | `url` | -| `geo_dependency_proxy_manifests_checksum_total` | Gauge | 15.6 | Number of dependency proxy manifests tried to checksum on primary | `url` | -| `geo_dependency_proxy_manifests_checksummed` | Gauge | 15.6 | Number of dependency proxy manifests successfully checksummed on primary | `url` | +| `geo_dependency_proxy_manifests_checksum_total` | Gauge | 15.6 | Number of dependency proxy manifests to checksum on primary | `url` | +| `geo_dependency_proxy_manifests_checksummed` | Gauge | 15.6 | Number of dependency proxy manifests that successfully calculated the checksum on primary | `url` | | `geo_dependency_proxy_manifests_checksum_failed` | Gauge | 15.6 | Number of dependency proxy manifests failed to calculate the checksum on primary | `url` | | `geo_dependency_proxy_manifests_synced` | Gauge | 15.6 | Number of syncable dependency proxy manifests synced on secondary | `url` | | `geo_dependency_proxy_manifests_failed` | Gauge | 15.6 | Number of syncable dependency proxy manifests failed to sync on secondary | `url` | | `geo_dependency_proxy_manifests_registry` | Gauge | 15.6 | Number of dependency proxy manifests in the registry | `url` | -| `geo_dependency_proxy_manifests_verification_total` | Gauge | 15.6 | Number of dependency proxy manifests verifications tried on secondary | `url` | -| `geo_dependency_proxy_manifests_verified` | Gauge | 15.6 | Number of dependency proxy manifests verified on secondary | `url` | -| `geo_dependency_proxy_manifests_verification_failed` | Gauge | 15.6 | Number of dependency proxy manifests verifications failed on secondary | `url` | +| `geo_dependency_proxy_manifests_verification_total` | Gauge | 15.6 | Number of dependency proxy manifests to attempt to verify on secondary | `url` | +| `geo_dependency_proxy_manifests_verified` | Gauge | 15.6 | Number of dependency proxy manifests successfully verified on secondary | `url` | +| `geo_dependency_proxy_manifests_verification_failed` | Gauge | 15.6 | Number of dependency proxy manifests that failed verification on secondary | `url` | +| `geo_project_wiki_repositories` | Gauge | 15.10 | Number of Project Wiki Repositories on primary | `url` | +| `geo_project_wiki_repositories_checksum_total` | Gauge | 15.10 | Number of Project Wiki Repositories to checksum on primary | `url` | +| `geo_project_wiki_repositories_checksummed` | Gauge | 15.10 | Number of Project Wiki Repositories that successfully calculated the checksum on primary | `url` | +| `geo_project_wiki_repositories_checksum_failed` | Gauge | 15.10 | Number of Project Wiki Repositories that failed to calculate the checksum on primary | `url` | +| `geo_project_wiki_repositories_synced` | Gauge | 15.10 | Number of syncable Project Wiki Repositories synced on secondary | `url` | +| `geo_project_wiki_repositories_failed` | Gauge | 15.10 | Number of syncable Project Wiki Repositories failed to sync on secondary | `url` | +| `geo_project_wiki_repositories_registry` | Gauge | 15.10 | Number of Project Wiki Repositories in the registry | `url` | +| `geo_project_wiki_repositories_verification_total` | Gauge | 15.10 | Number of Project Wiki Repositories to attempt to verify on secondary | `url` | +| `geo_project_wiki_repositories_verified` | Gauge | 15.10 | Number of Project Wiki Repositories successfully verified on secondary | `url` | +| `geo_project_wiki_repositories_verification_failed` | Gauge | 15.10 | Number of Project Wiki Repositories that failed verification on secondary | `url` | | `gitlab_memwd_violations_total` | Counter | 15.9 | Total number of times a Sidekiq process violated a memory threshold | | | `gitlab_memwd_violations_handled_total` | Counter | 15.9 | Total number of times Sidekiq process memory violations were handled | | | `sidekiq_watchdog_running_jobs_total` | Counter | 15.9 | Current running jobs when RSS limit was reached | `worker_class` | +| `gitlab_maintenance_mode` | Gauge | 15.11 | Is GitLab Maintenance Mode enabled? | | ## Database load balancing metrics **(PREMIUM SELF)** @@ -445,6 +465,7 @@ instance. For example, `cache` or `shared_state`. | `gitlab_redis_client_exceptions_total` | Counter | 13.2 | Number of Redis client exceptions, broken down by exception class | | `gitlab_redis_client_requests_total` | Counter | 13.2 | Number of Redis client requests | | `gitlab_redis_client_requests_duration_seconds` | Histogram | 13.2 | Redis request latency, excluding blocking commands | +| `gitlab_redis_client_redirections_total` | Counter | 15.10 | Number of Redis Cluster MOVED/ASK redirections, broken down by redirection type | ## Metrics shared directory diff --git a/doc/administration/monitoring/prometheus/index.md b/doc/administration/monitoring/prometheus/index.md index 56583deca89..013c4515268 100644 --- a/doc/administration/monitoring/prometheus/index.md +++ b/doc/administration/monitoring/prometheus/index.md @@ -22,7 +22,7 @@ Prometheus services are on by default. Prometheus and its exporters don't authenticate users, and are available to anyone who can access them. -## Overview +## How Prometheus works Prometheus works by periodically connecting to data sources and collecting their performance metrics through the [various exporters](#bundled-software-metrics). To view @@ -201,7 +201,10 @@ To use an external Prometheus server: postgres_exporter['listen_address'] = '0.0.0.0:9187' # Gitaly nodes - gitaly['prometheus_listen_addr'] = "0.0.0.0:9236" + gitaly['configuration'] = { + # ... + prometheus_listen_addr: '0.0.0.0:9236', + } ``` 1. Install and set up a dedicated Prometheus instance, if necessary, using the [official installation instructions](https://prometheus.io/docs/prometheus/latest/installation/). diff --git a/doc/administration/monitoring/prometheus/node_exporter.md b/doc/administration/monitoring/prometheus/node_exporter.md index c2f84773b3a..337ab9becaa 100644 --- a/doc/administration/monitoring/prometheus/node_exporter.md +++ b/doc/administration/monitoring/prometheus/node_exporter.md @@ -1,6 +1,6 @@ --- -stage: Monitor -group: Respond +stage: Shared responsibility based on functional area +group: Shared responsibility based on functional area info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/administration/monitoring/prometheus/pgbouncer_exporter.md b/doc/administration/monitoring/prometheus/pgbouncer_exporter.md index 56da3155fd9..f8d48832288 100644 --- a/doc/administration/monitoring/prometheus/pgbouncer_exporter.md +++ b/doc/administration/monitoring/prometheus/pgbouncer_exporter.md @@ -1,6 +1,6 @@ --- -stage: Monitor -group: Respond +stage: Data Stores +group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/administration/monitoring/prometheus/postgres_exporter.md b/doc/administration/monitoring/prometheus/postgres_exporter.md index 0458a4a5bae..1cae28a069f 100644 --- a/doc/administration/monitoring/prometheus/postgres_exporter.md +++ b/doc/administration/monitoring/prometheus/postgres_exporter.md @@ -1,6 +1,6 @@ --- -stage: Monitor -group: Respond +stage: Data Stores +group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/administration/monitoring/prometheus/redis_exporter.md b/doc/administration/monitoring/prometheus/redis_exporter.md index b50a2a4aebf..b17bf9787a7 100644 --- a/doc/administration/monitoring/prometheus/redis_exporter.md +++ b/doc/administration/monitoring/prometheus/redis_exporter.md @@ -1,6 +1,6 @@ --- -stage: Monitor -group: Respond +stage: Shared responsibility based on functional area +group: Shared responsibility based on functional area info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/administration/monitoring/prometheus/registry_exporter.md b/doc/administration/monitoring/prometheus/registry_exporter.md index b79e97bd937..3f4d6cfb14d 100644 --- a/doc/administration/monitoring/prometheus/registry_exporter.md +++ b/doc/administration/monitoring/prometheus/registry_exporter.md @@ -1,6 +1,6 @@ --- -stage: Monitor -group: Respond +stage: Package +group: Package Registry info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/administration/nfs.md b/doc/administration/nfs.md index 7349e8ad9ba..3e3712c9645 100644 --- a/doc/administration/nfs.md +++ b/doc/administration/nfs.md @@ -12,7 +12,7 @@ recommended for performance reasons. For data objects such as LFS, Uploads, and Artifacts, an [Object Storage service](object_storage.md) is recommended over NFS where possible, due to better performance. -When eliminating the usage of NFS, there are [additional steps you need to take](object_storage.md#other-alternatives-to-file-system-storage) +When eliminating the usage of NFS, there are [additional steps you need to take](object_storage.md#alternatives-to-file-system-storage) in addition to moving to Object Storage. File system performance can impact overall GitLab performance, especially for diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index 4b3e251d407..1e6605e41a8 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -11,101 +11,428 @@ It's recommended over NFS and in general it's better in larger setups as object storage is typically much more performant, reliable, and scalable. -## Options +To configure the object storage, you have two options: -GitLab is tightly integrated with `Fog`, so you can refer to its -[documentation](https://fog.io/about/provider_documentation.html) to check -which storage services can be integrated with GitLab. +- Recommended. [Configure a single storage connection for all object types](#configure-a-single-storage-connection-for-all-object-types-consolidated-form): + A single credential is shared by all supported object types. This is called the consolidated form. +- [Configure each object type to define its own storage connection](#configure-each-object-type-to-define-its-own-storage-connection-storage-specific-form): + Every object defines its own object storage connection and configuration. This is called the storage-specific form. + + If you already use the storage-specific form, see how to + [transition to the consolidated form](#transition-to-consolidated-form). + +If you store data locally, see how to +[migrate to object storage](#migrate-to-object-storage). + +## Supported object storage providers + +GitLab is tightly integrated with the Fog library, so you can see which +[providers](https://fog.io/about/provider_documentation.html) can be used +with GitLab. Specifically, GitLab has been tested by vendors and customers on a number of object storage providers: -- [Amazon S3](https://aws.amazon.com/s3/) +- [Amazon S3](https://aws.amazon.com/s3/) ([Object Lock](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html) + is not supported, see [issue #335775](https://gitlab.com/gitlab-org/gitlab/-/issues/335775) + for more information) - [Google Cloud Storage](https://cloud.google.com/storage) -- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces) (S3 compatible) - [Oracle Cloud Infrastructure](https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) - [OpenStack Swift (S3 compatible mode)](https://docs.openstack.org/swift/latest/s3_compat.html) - [Azure Blob storage](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) +- [MinIO](https://min.io/) (S3 compatible) - On-premises hardware and appliances from various storage vendors, whose list is not officially established. -- MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. -### Known compatibility issues +## Configure a single storage connection for all object types (consolidated form) -- Dell EMC ECS: Prior to GitLab 13.3, there is a - [known bug in GitLab Workhorse that prevents HTTP Range Requests from working with CI job artifacts](https://gitlab.com/gitlab-org/gitlab/-/issues/223806). - Be sure to upgrade to GitLab 13.3.0 or above if you use S3 storage with this hardware. +> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4368) in GitLab 13.2. -- Ceph S3 prior to [Kraken 11.0.2](https://ceph.com/releases/kraken-11-0-2-released/) does not support the [Upload Copy Part API](https://gitlab.com/gitlab-org/gitlab/-/issues/300604). You may need to [disable multi-threaded copying](#multi-threaded-copying). +Most types of objects, such as CI artifacts, LFS files, and upload attachments +can be saved in object storage by specifying a single credential for object +storage with multiple buckets. -- Amazon S3 [Object Lock](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html) - is not supported. Follow [issue #335775](https://gitlab.com/gitlab-org/gitlab/-/issues/335775) - for progress on enabling this option. +Configuring the object storage using the consolidated form has a number of advantages: -## Configuration guides +- It can simplify your GitLab configuration since the connection details are shared + across object types. +- It enables the use of [encrypted S3 buckets](#encrypted-s3-buckets). +- It [uploads files to S3 with proper `Content-MD5` headers](https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/222). -There are two ways of specifying object storage configuration in GitLab: +When the consolidated form is used, +[direct upload](../development/uploads/index.md#direct-upload) is enabled +automatically. Thus, only the following providers can be used: -- [Consolidated form](#consolidated-object-storage-configuration): A single credential is - shared by all supported object types. -- [Storage-specific form](#storage-specific-configuration): Every object defines its - own object storage [connection and configuration](#connection-settings). +- [Amazon S3-compatible providers](#amazon-s3) +- [Google Cloud Storage](#google-cloud-storage-gcs) +- [Azure Blob storage](#azure-blob-storage) -For more information on the differences and to transition from one form to another, see -[Transition to consolidated form](#transition-to-consolidated-form). +The consolidated form configuration can't be used for backups or +Mattermost. Backups can be configured with +[server side encryption](../raketasks/backup_gitlab.md#s3-encrypted-buckets) +separately. See the +[table for a complete list](#configure-each-object-type-to-define-its-own-storage-connection-storage-specific-form) +of supported object storage types. -If you are currently storing data locally, see -[Migrate to object storage](#migrate-to-object-storage) for migration details. +Enabling the consolidated form enables object storage for all object +types. If not all buckets are specified, you may see an error like: -### Consolidated object storage configuration +```plaintext +Object storage for must have a bucket specified +``` -> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4368) in GitLab 13.2. +If you want to use local storage for specific object types, you can +[disable object storage for specific features](#disable-object-storage-for-specific-features). -Using the consolidated object storage configuration has a number of advantages: +### Configure the common parameters -- It can simplify your GitLab configuration since the connection details are shared - across object types. -- It enables the use of [encrypted S3 buckets](#encrypted-s3-buckets). -- It [uploads files to S3 with proper `Content-MD5` headers](https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/222). +In the consolidated form, the `object_store` section defines a +common set of parameters. -Because [direct upload mode](../development/uploads/index.md#direct-upload) -must be enabled, only the following providers can be used: +| Setting | Description | +|-------------------|-----------------------------------| +| `enabled` | Enable or disable object storage. | +| `proxy_download` | Set to `true` to [enable proxying all files served](#proxy-download). Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data. | +| `connection` | Various [connection options](#configure-the-connection-settings) described below. | +| `storage_options` | Options to use when saving new objects, such as [server side encryption](#server-side-encryption-headers). Introduced in GitLab 13.3. | +| `objects` | [Object-specific configuration](#configure-the-parameters-of-each-object). | -- [Amazon S3-compatible providers](#s3-compatible-connection-settings) -- [Google Cloud Storage](#google-cloud-storage-gcs) -- [Azure Blob storage](#azure-blob-storage) +For an example, see how to [use the consolidated form and Amazon S3](#full-example-using-the-consolidated-form-and-amazon-s3). + +### Configure the parameters of each object -When consolidated object storage is used, direct upload is enabled -automatically. For storage-specific -configuration, [direct upload may become the default](https://gitlab.com/gitlab-org/gitlab/-/issues/27331) +Each object type must at least define the bucket name where it will be stored. + +The following table lists the valid `objects` that can be used: + +| Type | Description | +|--------------------|----------------------------------------------------------------------------| +| `artifacts` | [CI artifacts](job_artifacts.md) | +| `external_diffs` | [Merge request diffs](merge_request_diffs.md) | +| `uploads` | [User uploads](uploads.md) | +| `lfs` | [Git Large File Storage objects](lfs/index.md) | +| `packages` | [Project packages (for example, PyPI, Maven, or NuGet)](packages/index.md) | +| `dependency_proxy` | [Dependency Proxy](packages/dependency_proxy.md) | +| `terraform_state` | [Terraform state files](terraform_state.md) | +| `pages` | [Pages](pages/index.md) | + +Within each object type, three parameters can be defined: + +| Setting | Required? | Description | +|------------------|------------------------|-------------------------------------| +| `bucket` | **{check-circle}** Yes\* | Bucket name for the object type. Not required if `enabled` is set to `false`. | +| `enabled` | **{dotted-circle}** No | Overrides the [common parameter](#configure-the-common-parameters). | +| `proxy_download` | **{dotted-circle}** No | Overrides the [common parameter](#configure-the-common-parameters). | + +For an example, see how to [use the consolidated form and Amazon S3](#full-example-using-the-consolidated-form-and-amazon-s3). + +#### Disable object storage for specific features + +As seen above, object storage can be disabled for specific types by +setting the `enabled` flag to `false`. For example, to disable object +storage for CI artifacts: + +```ruby +gitlab_rails['object_store']['objects']['artifacts']['enabled'] = false +``` + +A bucket is not needed if the feature is disabled entirely. For example, +no bucket is needed if CI artifacts are disabled with this setting: + +```ruby +gitlab_rails['artifacts_enabled'] = false +``` + +## Configure each object type to define its own storage connection (storage-specific form) + +With the storage-specific form, every object defines its own object +storage connection and configuration. If you're using GitLab 13.2 and later, +you should [transition to the consolidated form](#transition-to-consolidated-form). + +The use of [encrypted S3 buckets](#encrypted-s3-buckets) with non-consolidated form is not supported. +You may get [ETag mismatch errors](#etag-mismatch) if you use it. + +NOTE: +For the storage-specific form, +[direct upload may become the default](https://gitlab.com/gitlab-org/gitlab/-/issues/27331) because it does not require a shared folder. -Consolidated object storage configuration can't be used for backups or -Mattermost. See the [full table for a complete list](#storage-specific-configuration). -However, backups can be configured with [server side encryption](../raketasks/backup_gitlab.md#s3-encrypted-buckets) separately. +For configuring object storage in GitLab 13.1 and earlier, or for storage types not +supported by consolidated form, refer to the following guides: + +| Object storage type | Supported by consolidated form? | +|---------------------|------------------------------------------| +| [Backups](../raketasks/backup_gitlab.md#upload-backups-to-a-remote-cloud-storage) | **{dotted-circle}** No | +| [Container Registry](packages/container_registry.md#use-object-storage) (optional feature) | **{dotted-circle}** No | +| [Mattermost](https://docs.mattermost.com/configure/file-storage-configuration-settings.html)| **{dotted-circle}** No | +| [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | **{dotted-circle}** No | +| [Job artifacts](job_artifacts.md#using-object-storage) including archived job logs | **{check-circle}** Yes | +| [LFS objects](lfs/index.md#storing-lfs-objects-in-remote-object-storage) | **{check-circle}** Yes | +| [Uploads](uploads.md#using-object-storage) | **{check-circle}** Yes | +| [Merge request diffs](merge_request_diffs.md#using-object-storage) | **{check-circle}** Yes | +| [Packages](packages/index.md#use-object-storage) (optional feature) | **{check-circle}** Yes | +| [Dependency Proxy](packages/dependency_proxy.md#using-object-storage) (optional feature) | **{check-circle}** Yes | +| [Terraform state files](terraform_state.md#using-object-storage) | **{check-circle}** Yes | +| [Pages content](pages/index.md#using-object-storage) | **{check-circle}** Yes | + +## Configure the connection settings + +Both consolidated and storage-specific form must configure a connection. The following sections describe parameters that can be used +in the `connection` setting. + +### Amazon S3 + +The connection settings match those provided by [fog-aws](https://github.com/fog/fog-aws): + +| Setting | Description | Default | +|---------------------------------------------|------------------------------------|---------| +| `provider` | Always `AWS` for compatible hosts. | `AWS` | +| `aws_access_key_id` | AWS credentials, or compatible. | | +| `aws_secret_access_key` | AWS credentials, or compatible. | | +| `aws_signature_version` | AWS signature version to use. `2` or `4` are valid options. Digital Ocean Spaces and other providers may need `2`. | `4` | +| `enable_signature_v4_streaming` | Set to `true` to enable HTTP chunked transfers with [AWS v4 signatures](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be `false`. | `true` | +| `region` | AWS region. | | +| `host` | DEPRECATED: Use `endpoint` instead. S3 compatible host for when not using AWS. For example, `localhost` or `storage.example.com`. HTTPS and port 443 is assumed. | `s3.amazonaws.com` | +| `endpoint` | Can be used when configuring an S3 compatible service such as [MinIO](https://min.io), by entering a URL such as `http://127.0.0.1:9000`. This takes precedence over `host`. Always use `endpoint` for consolidated form. | (optional) | +| `path_style` | Set to `true` to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Set to `true` for using [MinIO](https://min.io). Leave as `false` for AWS S3. | `false`. | +| `use_iam_profile` | Set to `true` to use IAM profile instead of access keys. | `false` | +| `aws_credentials_refresh_threshold_seconds` | Sets the [automatic refresh threshold](https://github.com/fog/fog-aws#controlling-credential-refresh-time-with-iam-authentication) in seconds when using temporary credentials in IAM. | `15` | + +#### Use Amazon instance profiles + +Instead of supplying AWS access and secret keys in object storage +configuration, you can configure GitLab to use Amazon Identity Access and Management (IAM) roles to set up an +[Amazon instance profile](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). +When this is used, GitLab fetches temporary credentials each time an +S3 bucket is accessed, so no hard-coded values are needed in the +configuration. + +Prerequisites: + +- GitLab must be able to connect to the + [instance metadata endpoint](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html). +- If GitLab is [configured to use an internet proxy](https://docs.gitlab.com/omnibus/settings/environment-variables.html), the endpoint IP + address must be added to the `no_proxy` list. + +To set up an instance profile: -Enabling consolidated object storage enables object storage for all object -types. If not all buckets are specified, `sudo gitlab-ctl reconfigure` may fail with the error like: +1. Create an IAM role with the necessary permissions. The + following example is a role for an S3 bucket named `test-bucket`: + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject" + ], + "Resource": "arn:aws:s3:::test-bucket/*" + } + ] + } + ``` + +1. [Attach this role](https://aws.amazon.com/premiumsupport/knowledge-center/attach-replace-ec2-instance-profile/) + to the EC2 instance hosting your GitLab instance. +1. Set the `use_iam_profile` GitLab configuration option to `true`. + +#### Encrypted S3 buckets + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/466) in GitLab 13.1 for instance profiles only and [S3 default encryption](https://docs.aws.amazon.com/AmazonS3/latest/dev/bucket-encryption.html). +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34460) in GitLab 13.2 for static credentials when the [consolidated form](#configure-a-single-storage-connection-for-all-object-types-consolidated-form) and [S3 default encryption](https://docs.aws.amazon.com/AmazonS3/latest/dev/bucket-encryption.html) is used. + +When configured either with an instance profile or with the consolidated +form, GitLab Workhorse properly uploads files to S3 +buckets that have [SSE-S3 or SSE-KMS encryption enabled by default](https://docs.aws.amazon.com/kms/latest/developerguide/services-s3.html). +AWS KMS keys and SSE-C encryption are +[not supported since this requires sending the encryption keys in every request](https://gitlab.com/gitlab-org/gitlab/-/issues/226006). + +#### Server-side encryption headers + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38240) in GitLab 13.3. + +Setting a default encryption on an S3 bucket is the easiest way to +enable encryption, but you may want to +[set a bucket policy to ensure only encrypted objects are uploaded](https://repost.aws/knowledge-center/s3-bucket-store-kms-encrypted-objects). +To do this, you must configure GitLab to send the proper encryption headers +in the `storage_options` configuration section: + +| Setting | Description | +|-------------------------------------|------------------------------------------| +| `server_side_encryption` | Encryption mode (`AES256` or `aws:kms`). | +| `server_side_encryption_kms_key_id` | Amazon Resource Name. Only needed when `aws:kms` is used in `server_side_encryption`. See the [Amazon documentation on using KMS encryption](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html). | + +As with the case for default encryption, these options only work when +the Workhorse S3 client is enabled. One of the following two conditions +must be fulfilled: + +- `use_iam_profile` is `true` in the connection settings. +- Consolidated form is in use. + +[ETag mismatch errors](#etag-mismatch) occur if server side +encryption headers are used without enabling the Workhorse S3 client. + +### Oracle Cloud S3 + +Oracle Cloud S3 must be sure to use the following settings: + +| Setting | Value | +|---------------------------------|---------| +| `enable_signature_v4_streaming` | `false` | +| `path_style` | `true` | + +If `enable_signature_v4_streaming` is set to `true`, you may see the +following error in `production.log`: ```plaintext -Object storage for must have a bucket specified +STREAMING-AWS4-HMAC-SHA256-PAYLOAD is not supported ``` -If you want to use local storage for specific object types, you can -[selectively disable object storages](#selectively-disabling-object-storage). +### Google Cloud Storage (GCS) + +Here are the valid connection parameters for GCS: -Most types of objects, such as CI artifacts, LFS files, and upload -attachments can be saved in object storage by specifying a single -credential for object storage with multiple buckets. +| Setting | Description | Example | +|------------------------------|-------------------|---------| +| `provider` | Provider name. | `Google` | +| `google_project` | GCP project name. | `gcp-project-12345` | +| `google_json_key_location` | JSON key path. | `/path/to/gcp-project-12345-abcde.json` | +| `google_json_key_string` | JSON key string. | `{ "type": "service_account", "project_id": "example-project-382839", ... }` | +| `google_application_default` | Set to `true` to use [Google Cloud Application Default Credentials](https://cloud.google.com/docs/authentication#adc) to locate service account credentials. | | -When the consolidated form is: +GitLab reads the value of `google_json_key_location`, then `google_json_key_string`, and finally, `google_application_default`. +It uses the first of these settings that has a value. -- Used with an S3-compatible object storage, Workhorse uses its internal S3 client to - upload files. -- Not used with an S3-compatible object storage, Workhorse falls back to using - pre-signed URLs. +The service account must have permission to access the bucket. For more information, +see the [Cloud Storage authentication documentation](https://cloud.google.com/storage/docs/authentication). + +NOTE: +Bucket encryption with the [Cloud Key Management Service (KMS)](https://cloud.google.com/kms/docs) is not supported and results in [ETag mismatch errors](#etag-mismatch). + +#### GCS example + +For Omnibus installations, this is an example of the `connection` setting +in the consolidated form: + +```ruby +gitlab_rails['object_store']['connection'] = { + 'provider' => 'Google', + 'google_project' => '', + 'google_json_key_location' => '' +} +``` + +#### GCS example with ADC + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/275979) in GitLab 13.6. + +Google Cloud Application Default Credentials (ADC) are typically +used with GitLab to use the default service account. This eliminates the +need to supply credentials for the instance. For example, in the consolidated form: + +```ruby +gitlab_rails['object_store']['connection'] = { + 'provider' => 'Google', + 'google_project' => '', + 'google_application_default' => true +} +``` + +If you use ADC, be sure that: + +- The service account that you use has the +[`iam.serviceAccounts.signBlob` permission](https://cloud.google.com/iam/docs/reference/credentials/rest/v1/projects.serviceAccounts/signBlob). + Typically this is done by granting the `Service Account Token Creator` role to the service account. +- Your virtual machines have the [correct access scopes to access Google Cloud APIs](https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances#changeserviceaccountandscopes). If the machines do not have the right scope, the error logs may show: -See the section on [ETag mismatch errors](#etag-mismatch) for more details. + ```markdown + Google::Apis::ClientError (insufficientPermissions: Request had insufficient authentication scopes.) + ``` -#### Use AWS S3 +### Azure Blob storage + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/25877) in GitLab 13.4. + +Although Azure uses the word `container` to denote a collection of +blobs, GitLab standardizes on the term `bucket`. Be sure to configure +Azure container names in the `bucket` settings. + +Azure Blob storage can only be used with the [consolidated form](#configure-a-single-storage-connection-for-all-object-types-consolidated-form) +because a single set of credentials are used to access multiple +containers. The [storage-specific form](#configure-each-object-type-to-define-its-own-storage-connection-storage-specific-form) +is not supported. For more details, see [how to transition to consolidated form](#transition-to-consolidated-form). + +The following are the valid connection parameters for Azure. For more information, see the +[Azure Blob Storage documentation](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction). + +| Setting | Description | Example | +|------------------------------|----------------|-----------| +| `provider` | Provider name. | `AzureRM` | +| `azure_storage_account_name` | Name of the Azure Blob Storage account used to access the storage. | `azuretest` | +| `azure_storage_access_key` | Storage account access key used to access the container. This is typically a secret, 512-bit encryption key encoded in base64. | `czV2OHkvQj9FKEgrTWJRZVRoV21ZcTN0Nnc5eiRDJkYpSkBOY1JmVWpYbjJy\nNHU3eCFBJUQqRy1LYVBkU2dWaw==\n` | +| `azure_storage_domain` | Domain name used to contact the Azure Blob Storage API (optional). Defaults to `blob.core.windows.net`. Set this if you are using Azure China, Azure Germany, Azure US Government, or some other custom Azure domain. | `blob.core.windows.net` | + +- For Omnibus installations, this is an example of the `connection` setting + in the consolidated form: + + ```ruby + gitlab_rails['object_store']['connection'] = { + 'provider' => 'AzureRM', + 'azure_storage_account_name' => '', + 'azure_storage_access_key' => '', + 'azure_storage_domain' => '' + } + ``` + +- For source installations, Workhorse also needs to be configured with Azure + credentials. This isn't needed in Omnibus installs, because the Workhorse + settings are populated from the previous settings. + + 1. Edit `/home/git/gitlab-workhorse/config.toml` and add or amend the following lines: + + ```toml + [object_storage] + provider = "AzureRM" + + [object_storage.azurerm] + azure_storage_account_name = "" + azure_storage_access_key = "" + ``` + + If you are using a custom Azure storage domain, + `azure_storage_domain` does **not** have to be set in the Workhorse + configuration. This information is exchanged in an API call between + GitLab Rails and Workhorse. + +### Storj Gateway (SJ) + +NOTE: +The Storj Gateway [does not support](https://github.com/storj/gateway-st/blob/4b74c3b92c63b5de7409378b0d1ebd029db9337d/docs/s3-compatibility.md) multi-threaded copying (see `UploadPartCopy` in the table). +While an implementation [is planned](https://github.com/storj/roadmap/issues/40), you must [disable multi-threaded copying](#multi-threaded-copying) until completion. + +The [Storj Network](https://www.storj.io/) provides an S3-compatible API gateway. Use the following configuration example: + +```ruby +gitlab_rails['object_store']['connection'] = { + 'provider' => 'AWS', + 'endpoint' => 'https://gateway.storjshare.io', + 'path_style' => true, + 'region' => 'eu1', + 'aws_access_key_id' => 'ACCESS_KEY', + 'aws_secret_access_key' => 'SECRET_KEY', + 'aws_signature_version' => 2, + 'enable_signature_v4_streaming' => false +} +``` + +The signature version must be `2`. Using v4 results in a HTTP 411 Length Required error. +For more information, see [issue #4419](https://gitlab.com/gitlab-org/gitlab/-/issues/4419). + +## Full example using the consolidated form and Amazon S3 The following example uses AWS S3 to enable object storage for all supported services: @@ -116,42 +443,42 @@ The following example uses AWS S3 to enable object storage for all supported ser 1. Edit `/etc/gitlab/gitlab.rb` and add the following lines, substituting the values you want: - ```ruby - # Consolidated object storage configuration - gitlab_rails['object_store']['enabled'] = true - gitlab_rails['object_store']['proxy_download'] = true - gitlab_rails['object_store']['connection'] = { - 'provider' => 'AWS', - 'region' => 'eu-central-1', - 'aws_access_key_id' => '', - 'aws_secret_access_key' => '' - } - # OPTIONAL: The following lines are only needed if server side encryption is required - gitlab_rails['object_store']['storage_options'] = { - 'server_side_encryption' => '', - 'server_side_encryption_kms_key_id' => '' - } - gitlab_rails['object_store']['objects']['artifacts']['bucket'] = 'gitlab-artifacts' - gitlab_rails['object_store']['objects']['external_diffs']['bucket'] = 'gitlab-mr-diffs' - gitlab_rails['object_store']['objects']['lfs']['bucket'] = 'gitlab-lfs' - gitlab_rails['object_store']['objects']['uploads']['bucket'] = 'gitlab-uploads' - gitlab_rails['object_store']['objects']['packages']['bucket'] = 'gitlab-packages' - gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = 'gitlab-dependency-proxy' - gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = 'gitlab-terraform-state' - gitlab_rails['object_store']['objects']['ci_secure_files']['bucket'] = 'gitlab-ci-secure-files' - gitlab_rails['object_store']['objects']['pages']['bucket'] = 'gitlab-pages' - ``` - - If you’re using [AWS IAM profiles](#using-amazon-instance-profiles), omit - the AWS access key and secret access key/value pairs. For example: - - ```ruby - gitlab_rails['object_store']['connection'] = { - 'provider' => 'AWS', - 'region' => 'eu-central-1', - 'use_iam_profile' => true - } - ``` + ```ruby + # Consolidated object storage configuration + gitlab_rails['object_store']['enabled'] = true + gitlab_rails['object_store']['proxy_download'] = true + gitlab_rails['object_store']['connection'] = { + 'provider' => 'AWS', + 'region' => 'eu-central-1', + 'aws_access_key_id' => '', + 'aws_secret_access_key' => '' + } + # OPTIONAL: The following lines are only needed if server side encryption is required + gitlab_rails['object_store']['storage_options'] = { + 'server_side_encryption' => '', + 'server_side_encryption_kms_key_id' => '' + } + gitlab_rails['object_store']['objects']['artifacts']['bucket'] = 'gitlab-artifacts' + gitlab_rails['object_store']['objects']['external_diffs']['bucket'] = 'gitlab-mr-diffs' + gitlab_rails['object_store']['objects']['lfs']['bucket'] = 'gitlab-lfs' + gitlab_rails['object_store']['objects']['uploads']['bucket'] = 'gitlab-uploads' + gitlab_rails['object_store']['objects']['packages']['bucket'] = 'gitlab-packages' + gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = 'gitlab-dependency-proxy' + gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = 'gitlab-terraform-state' + gitlab_rails['object_store']['objects']['ci_secure_files']['bucket'] = 'gitlab-ci-secure-files' + gitlab_rails['object_store']['objects']['pages']['bucket'] = 'gitlab-pages' + ``` + + If you're using [AWS IAM profiles](#use-amazon-instance-profiles), omit + the AWS access key and secret access key/value pairs. For example: + + ```ruby + gitlab_rails['object_store']['connection'] = { + 'provider' => 'AWS', + 'region' => 'eu-central-1', + 'use_iam_profile' => true + } + ``` 1. Save the file and reconfigure GitLab: @@ -171,7 +498,7 @@ The following example uses AWS S3 to enable object storage for all supported ser aws_secret_access_key: ``` - If you’re using [AWS IAM profiles](#using-amazon-instance-profiles), omit + If you're using [AWS IAM profiles](#use-amazon-instance-profiles), omit the AWS access key and secret access key/value pairs. For example: ```yaml @@ -293,7 +620,7 @@ The following example uses AWS S3 to enable object storage for all supported ser gitlab_rails['object_store']['objects']['pages']['bucket'] = 'gitlab-pages' ``` - If you’re using [AWS IAM profiles](#using-amazon-instance-profiles), omit + If you're using [AWS IAM profiles](#use-amazon-instance-profiles), omit the AWS access key and secret access key/value pairs. For example: ```ruby @@ -337,363 +664,59 @@ The following example uses AWS S3 to enable object storage for all supported ser uploads: bucket: gitlab-uploads packages: - bucket: gitlab-packages - dependency_proxy: - bucket: gitlab-dependency-proxy - terraform_state: - bucket: gitlab-terraform-state - ci_secure_files: - bucket: gitlab-ci-secure-files - pages: - bucket: gitlab-pages - ``` - - If you’re using [AWS IAM profiles](#using-amazon-instance-profiles), omit - the AWS access key and secret access key/value pairs. For example: - - ```yaml - connection: - provider: AWS - region: eu-central-1 - use_iam_profile: true - ``` - -1. Edit `/home/git/gitlab-workhorse/config.toml` and add or amend the following lines: - - ```toml - [object_storage] - provider = "AWS" - - [object_storage.s3] - aws_access_key_id = "" - aws_secret_access_key = "" - ``` - - If you’re using [AWS IAM profiles](#using-amazon-instance-profiles), omit - the AWS access key and secret access key/value pairs. For example: - - ```yaml - [object_storage.s3] - use_iam_profile = true - ``` - -1. Save the file and restart GitLab: - - ```shell - # For systems running systemd - sudo systemctl restart gitlab.target - - # For systems running SysV init - sudo service gitlab restart - ``` - -::EndTabs - -#### Common parameters - -In the consolidated configuration, the `object_store` section defines a -common set of parameters. Here we use the YAML from the source -installation because it's easier to see the inheritance: - -```yaml - object_store: - enabled: true - proxy_download: true - connection: - provider: AWS - aws_access_key_id: - aws_secret_access_key: - objects: - ... -``` - -The Omnibus configuration maps directly to this: - -```ruby -gitlab_rails['object_store']['enabled'] = true -gitlab_rails['object_store']['proxy_download'] = true -gitlab_rails['object_store']['connection'] = { - 'provider' => 'AWS', - 'aws_access_key_id' => ' '' -} -``` - -| Setting | Description | -|-------------------|-----------------------------------| -| `enabled` | Enable or disable object storage. | -| `proxy_download` | Set to `true` to [enable proxying all files served](#proxy-download). Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data. | -| `connection` | Various [connection options](#connection-settings) described below. | -| `storage_options` | Options to use when saving new objects, such as [server side encryption](#server-side-encryption-headers). Introduced in GitLab 13.3. | -| `objects` | [Object-specific configuration](#object-specific-configuration). | - -### Connection settings - -Both consolidated configuration form and storage-specific configuration form must configure a connection. The following sections describe parameters that can be used -in the `connection` setting. - -#### S3-compatible connection settings - -The connection settings match those provided by [fog-aws](https://github.com/fog/fog-aws): - -| Setting | Description | Default | -|---------------------------------------------|------------------------------------|---------| -| `provider` | Always `AWS` for compatible hosts. | `AWS` | -| `aws_access_key_id` | AWS credentials, or compatible. | | -| `aws_secret_access_key` | AWS credentials, or compatible. | | -| `aws_signature_version` | AWS signature version to use. `2` or `4` are valid options. Digital Ocean Spaces and other providers may need `2`. | `4` | -| `enable_signature_v4_streaming` | Set to `true` to enable HTTP chunked transfers with [AWS v4 signatures](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be `false`. | `true` | -| `region` | AWS region. | | -| `host` | DEPRECATED: Use `endpoint` instead. S3 compatible host for when not using AWS. For example, `localhost` or `storage.example.com`. HTTPS and port 443 is assumed. | `s3.amazonaws.com` | -| `endpoint` | Can be used when configuring an S3 compatible service such as [MinIO](https://min.io), by entering a URL such as `http://127.0.0.1:9000`. This takes precedence over `host`. Always use `endpoint` for consolidated form. | (optional) | -| `path_style` | Set to `true` to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Set to `true` for using [MinIO](https://min.io). Leave as `false` for AWS S3. | `false`. | -| `use_iam_profile` | Set to `true` to use IAM profile instead of access keys. | `false` | -| `aws_credentials_refresh_threshold_seconds` | Sets the [automatic refresh threshold](https://github.com/fog/fog-aws#controlling-credential-refresh-time-with-iam-authentication) when using temporary credentials in IAM. | `15` | - -#### Oracle Cloud S3 connection settings - -Oracle Cloud S3 must be sure to use the following settings: - -| Setting | Value | -|---------------------------------|---------| -| `enable_signature_v4_streaming` | `false` | -| `path_style` | `true` | - -If `enable_signature_v4_streaming` is set to `true`, you may see the -following error in `production.log`: - -```plaintext -STREAMING-AWS4-HMAC-SHA256-PAYLOAD is not supported -``` - -#### Google Cloud Storage (GCS) - -Here are the valid connection parameters for GCS: - -| Setting | Description | Example | -|------------------------------|-------------------|---------| -| `provider` | Provider name. | `Google` | -| `google_project` | GCP project name. | `gcp-project-12345` | -| `google_json_key_location` | JSON key path. | `/path/to/gcp-project-12345-abcde.json` | -| `google_json_key_string` | JSON key string. | `{ "type": "service_account", "project_id": "example-project-382839", ... }` | -| `google_application_default` | Set to `true` to use [Google Cloud Application Default Credentials](https://cloud.google.com/docs/authentication#adc) to locate service account credentials. | | - -GitLab reads the value of `google_json_key_location`, then `google_json_key_string`, and finally, `google_application_default`. -It uses the first of these settings that has a value. - -The service account must have permission to access the bucket. For more information, -see the [Cloud Storage authentication documentation](https://cloud.google.com/storage/docs/authentication). - -NOTE: -Bucket encryption with the [Cloud Key Management Service (KMS)](https://cloud.google.com/kms/docs) is not supported and results in [ETag mismatch errors](#etag-mismatch). - -##### Google example (consolidated form) - -For Omnibus installations, this is an example of the `connection` setting: - -```ruby -gitlab_rails['object_store']['connection'] = { - 'provider' => 'Google', - 'google_project' => '', - 'google_json_key_location' => '' -} -``` - -##### Google example with ADC (consolidated form) - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/275979) in GitLab 13.6. - -Google Cloud Application Default Credentials (ADC) are typically -used with GitLab to use the default service account. This eliminates the -need to supply credentials for the instance. For example: - -```ruby -gitlab_rails['object_store']['connection'] = { - 'provider' => 'Google', - 'google_project' => '', - 'google_application_default' => true -} -``` - -If you use ADC, be sure that: - -- The service account that you use has the -[`iam.serviceAccounts.signBlob` permission](https://cloud.google.com/iam/docs/reference/credentials/rest/v1/projects.serviceAccounts/signBlob). - Typically this is done by granting the `Service Account Token Creator` role to the service account. -- Your virtual machines have the [correct access scopes to access Google Cloud APIs](https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances#changeserviceaccountandscopes). If the machines do not have the right scope, the error logs may show: - - ```markdown - Google::Apis::ClientError (insufficientPermissions: Request had insufficient authentication scopes.) - ``` - -#### Azure Blob storage - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/25877) in GitLab 13.4. - -Although Azure uses the word `container` to denote a collection of -blobs, GitLab standardizes on the term `bucket`. Be sure to configure -Azure container names in the `bucket` settings. - -Azure Blob storage can only be used with the [consolidated form](#consolidated-object-storage-configuration) -because a single set of credentials are used to access multiple -containers. The [storage-specific form](#storage-specific-configuration) -is not supported. For more details, see [how to transition to consolidated form](#transition-to-consolidated-form). - -The following are the valid connection parameters for Azure. For more information, see the -[Azure Blob Storage documentation](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction). - -| Setting | Description | Example | -|------------------------------|----------------|-----------| -| `provider` | Provider name. | `AzureRM` | -| `azure_storage_account_name` | Name of the Azure Blob Storage account used to access the storage. | `azuretest` | -| `azure_storage_access_key` | Storage account access key used to access the container. This is typically a secret, 512-bit encryption key encoded in base64. | `czV2OHkvQj9FKEgrTWJRZVRoV21ZcTN0Nnc5eiRDJkYpSkBOY1JmVWpYbjJy\nNHU3eCFBJUQqRy1LYVBkU2dWaw==\n` | -| `azure_storage_domain` | Domain name used to contact the Azure Blob Storage API (optional). Defaults to `blob.core.windows.net`. Set this if you are using Azure China, Azure Germany, Azure US Government, or some other custom Azure domain. | `blob.core.windows.net` | - -##### Azure example (consolidated form) - -For Omnibus installations, this is an example of the `connection` setting: - -```ruby -gitlab_rails['object_store']['connection'] = { - 'provider' => 'AzureRM', - 'azure_storage_account_name' => '', - 'azure_storage_access_key' => '', - 'azure_storage_domain' => '' -} -``` + bucket: gitlab-packages + dependency_proxy: + bucket: gitlab-dependency-proxy + terraform_state: + bucket: gitlab-terraform-state + ci_secure_files: + bucket: gitlab-ci-secure-files + pages: + bucket: gitlab-pages + ``` -###### Azure Workhorse settings (source installs only) + If you're using [AWS IAM profiles](#use-amazon-instance-profiles), omit + the AWS access key and secret access key/value pairs. For example: -For source installations, Workhorse also needs to be configured with Azure -credentials. This isn't needed in Omnibus installs, because the Workhorse -settings are populated from the previous settings. + ```yaml + connection: + provider: AWS + region: eu-central-1 + use_iam_profile: true + ``` 1. Edit `/home/git/gitlab-workhorse/config.toml` and add or amend the following lines: ```toml [object_storage] - provider = "AzureRM" + provider = "AWS" - [object_storage.azurerm] - azure_storage_account_name = "" - azure_storage_access_key = "" + [object_storage.s3] + aws_access_key_id = "" + aws_secret_access_key = "" ``` -If you are using a custom Azure storage domain, -`azure_storage_domain` does **not** have to be set in the Workhorse -configuration. This information is exchanged in an API call between -GitLab Rails and Workhorse. - -#### Storj Gateway Configuration (SJ) - -NOTE: -The Storj Gateway [does not support](https://github.com/storj/gateway-st/blob/4b74c3b92c63b5de7409378b0d1ebd029db9337d/docs/s3-compatibility.md) multi-threaded copying (see `UploadPartCopy` in the table). -While an implementation [is planned](https://github.com/storj/roadmap/issues/40), you must [disable multi-threaded copying](#multi-threaded-copying) until completion. - -The [Storj Network](https://www.storj.io/) provides an S3-compatible API gateway. Use the following configuration example: - -```ruby -gitlab_rails['object_store']['connection'] = { - 'provider' => 'AWS', - 'endpoint' => 'https://gateway.storjshare.io', - 'path_style' => true, - 'region' => 'eu1', - 'aws_access_key_id' => 'ACCESS_KEY', - 'aws_secret_access_key' => 'SECRET_KEY', - 'aws_signature_version' => 2, - 'enable_signature_v4_streaming' => false -} -``` - -The signature version must be `2`. Using v4 results in a HTTP 411 Length Required error. -For more information, see [issue #4419](https://gitlab.com/gitlab-org/gitlab/-/issues/4419). - -### Object-specific configuration - -The following YAML shows how the `object_store` section defines -object-specific configuration block and how the `enabled` and -`proxy_download` flags can be overridden. The `bucket` is the only -required parameter within each type: - -```yaml - object_store: - connection: - ... - objects: - artifacts: - bucket: artifacts - proxy_download: false - external_diffs: - bucket: external-diffs - lfs: - bucket: lfs-objects - uploads: - bucket: uploads - packages: - bucket: packages - dependency_proxy: - enabled: false - bucket: dependency_proxy - terraform_state: - bucket: terraform - pages: - bucket: pages -``` - -This maps to this Omnibus GitLab configuration: - -```ruby -gitlab_rails['object_store']['objects']['artifacts']['bucket'] = 'artifacts' -gitlab_rails['object_store']['objects']['artifacts']['proxy_download'] = false -gitlab_rails['object_store']['objects']['external_diffs']['bucket'] = 'external-diffs' -gitlab_rails['object_store']['objects']['lfs']['bucket'] = 'lfs-objects' -gitlab_rails['object_store']['objects']['uploads']['bucket'] = 'uploads' -gitlab_rails['object_store']['objects']['packages']['bucket'] = 'packages' -gitlab_rails['object_store']['objects']['dependency_proxy']['enabled'] = false -gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = 'dependency-proxy' -gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = 'terraform-state' -gitlab_rails['object_store']['objects']['pages']['bucket'] = 'pages' -``` - -This is the list of valid `objects` that can be used: - -| Type | Description | -|--------------------|----------------------------------------------------------------------------| -| `artifacts` | [CI artifacts](job_artifacts.md) | -| `external_diffs` | [Merge request diffs](merge_request_diffs.md) | -| `uploads` | [User uploads](uploads.md) | -| `lfs` | [Git Large File Storage objects](lfs/index.md) | -| `packages` | [Project packages (for example, PyPI, Maven, or NuGet)](packages/index.md) | -| `dependency_proxy` | [Dependency Proxy](packages/dependency_proxy.md) | -| `terraform_state` | [Terraform state files](terraform_state.md) | -| `pages` | [Pages](pages/index.md) | - -Within each object type, three parameters can be defined: - -| Setting | Required? | Description | -|------------------|------------------------|-------------------------------------| -| `bucket` | **{check-circle}** Yes | Bucket name for the object storage. | -| `enabled` | **{dotted-circle}** No | Overrides the common parameter. | -| `proxy_download` | **{dotted-circle}** No | Overrides the common parameter. | + If you're using [AWS IAM profiles](#use-amazon-instance-profiles), omit + the AWS access key and secret access key/value pairs. For example: -#### Selectively disabling object storage + ```yaml + [object_storage.s3] + use_iam_profile = true + ``` -As seen above, object storage can be disabled for specific types by -setting the `enabled` flag to `false`. For example, to disable object -storage for CI artifacts: +1. Save the file and restart GitLab: -```ruby -gitlab_rails['object_store']['objects']['artifacts']['enabled'] = false -``` + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target -A bucket is not needed if the feature is disabled entirely. For example, -no bucket is needed if CI artifacts are disabled with this setting: + # For systems running SysV init + sudo service gitlab restart + ``` -```ruby -gitlab_rails['artifacts_enabled'] = false -``` +::EndTabs -### Migrate to object storage +## Migrate to object storage To migrate existing local data to object storage see the following guides: @@ -706,7 +729,7 @@ To migrate existing local data to object storage see the following guides: - [Terraform state files](terraform_state.md#migrate-to-object-storage) - [Pages content](pages/index.md#migrate-pages-deployments-to-object-storage) -### Transition to consolidated form +## Transition to consolidated form Prior to GitLab 13.2: @@ -737,36 +760,55 @@ additional complexity and unnecessary redundancy. Since both GitLab Rails and Workhorse components need access to object storage, the consolidated form avoids excessive duplication of credentials. -The consolidated object storage configuration is used _only_ if all lines from +The consolidated form is used _only_ if all lines from the original form is omitted. To move to the consolidated form, remove the original configuration (for example, `artifacts_object_store_enabled`, or `uploads_object_store_connection`) -### Storage-specific configuration +## Migrate objects to a different object storage provider -For configuring object storage in GitLab 13.1 and earlier, or for storage types not -supported by consolidated configuration form, refer to the following guides: +You may need to migrate GitLab data in object storage to a different object storage provider. The following steps show you how do this using [Rclone](https://rclone.org/). -| Object storage type | Supported by consolidated configuration? | -|---------------------|------------------------------------------| -| [Backups](../raketasks/backup_gitlab.md#upload-backups-to-a-remote-cloud-storage) | **{dotted-circle}** No | -| [Job artifacts](job_artifacts.md#using-object-storage) including archived job logs | **{check-circle}** Yes | -| [LFS objects](lfs/index.md#storing-lfs-objects-in-remote-object-storage) | **{check-circle}** Yes | -| [Uploads](uploads.md#using-object-storage) | **{check-circle}** Yes | -| [Container Registry](packages/container_registry.md#use-object-storage) (optional feature) | **{dotted-circle}** No | -| [Merge request diffs](merge_request_diffs.md#using-object-storage) | **{check-circle}** Yes | -| [Mattermost](https://docs.mattermost.com/configure/file-storage-configuration-settings.html)| **{dotted-circle}** No | -| [Packages](packages/index.md#use-object-storage) (optional feature) | **{check-circle}** Yes | -| [Dependency Proxy](packages/dependency_proxy.md#using-object-storage) (optional feature) | **{check-circle}** Yes | -| [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | **{dotted-circle}** No | -| [Terraform state files](terraform_state.md#using-object-storage) | **{check-circle}** Yes | -| [Pages content](pages/index.md#using-object-storage) | **{check-circle}** Yes | +The steps assume you are moving the `uploads` bucket, but the same process works for other buckets. + +Prerequisites: + +- Choose the computer to run Rclone on. Depending on how much data you are migrating, Rclone may have to run for a long time so you should avoid using a laptop or desktop computer that can go into power saving. You can use your GitLab server to run Rclone. + +1. [Install](https://rclone.org/downloads/) Rclone. +1. Configure Rclone by running the following: + + ```shell + rclone config + ``` + + The configuration process is interactive. Add at least two "remotes": one for the object storage provider your data is currently on (`old`), and one for the provider you are moving to (`new`). + +1. Verify that you can read the old data. The following example refers to the `uploads` bucket , but your bucket may have a different name: + + ```shell + rclone ls old:uploads | head + ``` + + This should print a partial list of the objects currently stored in your `uploads` bucket. If you get an error, or if + the list is empty, go back and update your Rclone configuration using `rclone config`. + +1. Perform an initial copy. You do not need to take your GitLab server offline for this step. + + ```shell + rclone sync -P old:uploads new:uploads + ``` -WARNING: -The use of [encrypted S3 buckets](#encrypted-s3-buckets) with non-consolidated configuration is not supported. -You may start getting [ETag mismatch errors](#etag-mismatch) if you use it. +1. After the first sync completes, use the web UI or command-line interface of your new object storage provider to + verify that there are objects in the new bucket. If there are none, or if you encounter an error while running `rclone + sync`, check your Rclone configuration and try again. + +After you have done at least one successful Rclone copy from the old location to the new location, schedule maintenance and take your GitLab server offline. During your maintenance window you must do two things: + +1. Perform a final `rclone sync` run, knowing that your users cannot add new objects so you do not leave any behind in the old bucket. +1. Update the object storage configuration of your GitLab server to use the new provider for `uploads`. -### Other alternatives to file system storage +## Alternatives to file system storage If you're working to [scale out](reference_architectures/index.md) your GitLab implementation, or add fault tolerance and redundancy, you may be @@ -779,11 +821,11 @@ See the following additional guides: 1. [Prevent local disk usage for job logs](job_logs.md#prevent-local-disk-usage). 1. [Disable Pages local storage](pages/index.md#disable-pages-local-storage). -## Warnings, limitations, and known issues +## Troubleshooting ### Objects are not included in GitLab backups -As noted in [our backup documentation](../raketasks/backup_restore.md), +As noted in [the backup documentation](../raketasks/backup_restore.md), objects are not included in GitLab backups. You can enable backups with your object storage provider instead. @@ -882,14 +924,14 @@ If you are seeing this ETag mismatch error with Amazon Web Services S3, it's likely this is due to [encryption settings on your bucket](https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html). To fix this issue, you have two options: -- [Use the consolidated object configuration](#consolidated-object-storage-configuration). -- [Use Amazon instance profiles](#using-amazon-instance-profiles). +- [Use the consolidated form](#configure-a-single-storage-connection-for-all-object-types-consolidated-form). +- [Use Amazon instance profiles](#use-amazon-instance-profiles). The first option is recommended for MinIO. Otherwise, the [workaround for MinIO](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/1564#note_244497658) is to use the `--compat` parameter on the server. -Without consolidated object store configuration or instance profiles enabled, +Without the consolidated form or instance profiles enabled, GitLab Workhorse uploads files to S3 using pre-signed URLs that do not have a `Content-MD5` HTTP header computed for them. To ensure data is not corrupted, Workhorse checks that the MD5 hash of the data sent @@ -897,90 +939,16 @@ equals the ETag header returned from the S3 server. When encryption is enabled, this is not the case, which causes Workhorse to report an `ETag mismatch` error during an upload. -With the consolidated object configuration and instance profile, Workhorse has -S3 credentials so that it can compute the `Content-MD5` header. This -eliminates the need to compare ETag headers returned from the S3 server. - -Encrypting buckets with the GCS [Cloud Key Management Service (KMS)](https://cloud.google.com/kms/docs) is not supported and results in ETag mismatch errors. - -### Using Amazon instance profiles - -Instead of supplying AWS access and secret keys in object storage -configuration, GitLab can be configured to use IAM roles to set up an -[Amazon instance profile](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html). -When this is used, GitLab fetches temporary credentials each time an -S3 bucket is accessed, so no hard-coded values are needed in the -configuration. - -To use an Amazon instance profile, GitLab must be able to connect to the -[instance metadata endpoint](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html). -If GitLab is [configured to use an Internet proxy](https://docs.gitlab.com/omnibus/settings/environment-variables.html), the endpoint IP -address must be added to the `no_proxy` list. - -#### Encrypted S3 buckets - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/466) in GitLab 13.1 for instance profiles only and [S3 default encryption](https://docs.aws.amazon.com/AmazonS3/latest/dev/bucket-encryption.html). -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34460) in GitLab 13.2 for static credentials when [consolidated object storage configuration](#consolidated-object-storage-configuration) and [S3 default encryption](https://docs.aws.amazon.com/AmazonS3/latest/dev/bucket-encryption.html) are used. - -When configured either with an instance profile or with the consolidated -object configuration, GitLab Workhorse properly uploads files to S3 -buckets that have [SSE-S3 or SSE-KMS encryption enabled by default](https://docs.aws.amazon.com/kms/latest/developerguide/services-s3.html). -Customer master keys (CMKs) and SSE-C encryption are -[not supported since this requires sending the encryption keys in every request](https://gitlab.com/gitlab-org/gitlab/-/issues/226006). - -##### Server-side encryption headers - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38240) in GitLab 13.3. - -Setting a default encryption on an S3 bucket is the easiest way to -enable encryption, but you may want to -[set a bucket policy to ensure only encrypted objects are uploaded](https://repost.aws/knowledge-center/s3-bucket-store-kms-encrypted-objects). -To do this, you must configure GitLab to send the proper encryption headers -in the `storage_options` configuration section: - -| Setting | Description | -|-------------------------------------|------------------------------------------| -| `server_side_encryption` | Encryption mode (`AES256` or `aws:kms`). | -| `server_side_encryption_kms_key_id` | Amazon Resource Name. Only needed when `aws:kms` is used in `server_side_encryption`. See the [Amazon documentation on using KMS encryption](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html). | - -As with the case for default encryption, these options only work when -the Workhorse S3 client is enabled. One of the following two conditions -must be fulfilled: - -- `use_iam_profile` is `true` in the connection settings. -- Consolidated object storage settings are in use. - -[ETag mismatch errors](#etag-mismatch) occur if server side -encryption headers are used without enabling the Workhorse S3 client. - -#### IAM Permissions - -To set up an instance profile: - -1. Create an Amazon Identity Access and Management (IAM) role with the necessary permissions. The - following example is a role for an S3 bucket named `test-bucket`: +When the consolidated form is: - ```json - { - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "VisualEditor0", - "Effect": "Allow", - "Action": [ - "s3:PutObject", - "s3:GetObject", - "s3:DeleteObject" - ], - "Resource": "arn:aws:s3:::test-bucket/*" - } - ] - } - ``` +- Used with an S3-compatible object storage or an istance profile, Workhorse + uses its internal S3 client which has S3 credentials so that it can compute + the `Content-MD5` header. This eliminates the need to compare ETag headers + returned from the S3 server. +- Not used with an S3-compatible object storage, Workhorse falls back to using + pre-signed URLs. -1. [Attach this role](https://aws.amazon.com/premiumsupport/knowledge-center/attach-replace-ec2-instance-profile/) - to the EC2 instance hosting your GitLab instance. -1. Configure GitLab to use it via the `use_iam_profile` configuration option. +Encrypting buckets with the GCS [Cloud Key Management Service (KMS)](https://cloud.google.com/kms/docs) is not supported and results in ETag mismatch errors. ### Multi-threaded copying @@ -996,46 +964,3 @@ to run the following command: ```ruby Feature.disable(:s3_multithreaded_uploads) ``` - -## Migrate objects to a different object storage provider - -You may need to migrate GitLab data in object storage to a different object storage provider. The following steps show you how do this using [Rclone](https://rclone.org/). - -The steps assume you are moving the `uploads` bucket, but the same process works for other buckets. - -Prerequisites: - -- Choose the computer to run Rclone on. Depending on how much data you are migrating, Rclone may have to run for a long time so you should avoid using a laptop or desktop computer that can go into power saving. You can use your GitLab server to run Rclone. - -1. [Install](https://rclone.org/downloads/) Rclone. -1. Configure Rclone by running the following: - - ```shell - rclone config - ``` - - The configuration process is interactive. Add at least two "remotes": one for the object storage provider your data is currently on (`old`), and one for the provider you are moving to (`new`). - -1. Verify that you can read the old data. The following example refers to the `uploads` bucket , but your bucket may have a different name: - - ```shell - rclone ls old:uploads | head - ``` - - This should print a partial list of the objects currently stored in your `uploads` bucket. If you get an error, or if - the list is empty, go back and update your Rclone configuration using `rclone config`. - -1. Perform an initial copy. You do not need to take your GitLab server offline for this step. - - ```shell - rclone sync -P old:uploads new:uploads - ``` - -1. After the first sync completes, use the web UI or command-line interface of your new object storage provider to - verify that there are objects in the new bucket. If there are none, or if you encounter an error while running `rclone - sync`, check your Rclone configuration and try again. - -After you have done at least one successful Rclone copy from the old location to the new location, schedule maintenance and take your GitLab server offline. During your maintenance window you must do two things: - -1. Perform a final `rclone sync` run, knowing that your users cannot add new objects so you do not leave any behind in the old bucket. -1. Update the object storage configuration of your GitLab server to use the new provider for `uploads`. diff --git a/doc/administration/operations/gitlab_sshd.md b/doc/administration/operations/gitlab_sshd.md index 7b61631fe3a..249d6232616 100644 --- a/doc/administration/operations/gitlab_sshd.md +++ b/doc/administration/operations/gitlab_sshd.md @@ -6,13 +6,13 @@ info: To determine the technical writer assigned to the Stage/Group associated w # `gitlab-sshd` **(FREE SELF)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/299109) in GitLab 14.5 as an **Alpha** release for self-managed customers. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/299109) in GitLab 14.5 as an Experiment for self-managed customers. > - Ready for production use with [Cloud Native GitLab in GitLab 15.1](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/2540) and [Omnibus GitLab in GitLab 15.9](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5937). `gitlab-sshd` is [a standalone SSH server](https://gitlab.com/gitlab-org/gitlab-shell/-/tree/main/internal/sshd) written in Go. It is provided as a part of the `gitlab-shell` package. It has a lower memory use as a OpenSSH alternative, and supports -[group access restriction by IP address](../../user/group/index.md) for applications +[group access restriction by IP address](../../user/group/access_and_permissions.md#restrict-group-access-by-ip-address) for applications running behind the proxy. `gitlab-sshd` is a lightweight alternative to OpenSSH for providing @@ -27,8 +27,9 @@ If you are considering switching from OpenSSH to `gitlab-sshd`, consider these c - `gitlab-sshd` supports the PROXY protocol. It can run behind proxy servers that rely on it, such as HAProxy. The PROXY protocol is not enabled by default, but [it can be enabled](#proxy-protocol-support). -- `gitlab-sshd` **does not** support SSH certificates. For more details, read - [issue #495](https://gitlab.com/gitlab-org/gitlab-shell/-/issues/495). +- `gitlab-sshd` **does not** support SSH certificates. For more details, see the + [confidential issue](../../user/project/issues/confidential_issues.md) + `https://gitlab.com/gitlab-org/gitlab-shell/-/issues/495`. ## Enable `gitlab-sshd` @@ -110,11 +111,11 @@ To enable the PROXY protocol: 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_sshd['proxy_protocol'] = true - # # Proxy protocol policy ("use", "require", "reject", "ignore"), "use" is the default value - gitlab_sshd['proxy_policy'] = "use" - ``` + ```ruby + gitlab_sshd['proxy_protocol'] = true + # # Proxy protocol policy ("use", "require", "reject", "ignore"), "use" is the default value + gitlab_sshd['proxy_policy'] = "use" + ``` 1. Save the file and reconfigure GitLab: diff --git a/doc/administration/operations/index.md b/doc/administration/operations/index.md index f6ab46b9fbf..867ef3236ee 100644 --- a/doc/administration/operations/index.md +++ b/doc/administration/operations/index.md @@ -4,29 +4,21 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Performing operations in GitLab **(FREE SELF)** +# Maintain your GitLab installation **(FREE SELF)** -Keep your GitLab instance up and running smoothly. +Keep your GitLab instance up and running. -- [Rake tasks](../../raketasks/index.md): Tasks for common administration and operational processes such as - [cleaning up unneeded items from GitLab instance](../../raketasks/cleanup.md), integrity checks, - and more. -- [Moving repositories](moving_repositories.md): Moving all repositories managed - by GitLab to another file system or another server. -- [Sidekiq MemoryKiller](../sidekiq/sidekiq_memory_killer.md): Configure Sidekiq MemoryKiller - to restart Sidekiq. -- [Multiple Sidekiq processes](../sidekiq/extra_sidekiq_processes.md): Configure multiple Sidekiq processes to ensure certain queues always have dedicated workers, no matter the number of jobs that must be processed. **(FREE SELF)** -- [Puma](puma.md): Understand Puma and puma-worker-killer. -- [`gitlab-sshd`](gitlab_sshd.md): Use GitLab SSH daemon instead of OpenSSH. -- Speed up SSH operations by - [Authorizing SSH users via a fast, indexed lookup to the GitLab database](fast_ssh_key_lookup.md), and/or - by [doing away with user SSH keys stored on GitLab entirely in favor of SSH certificates](ssh_certificates.md). -- [File System Performance Benchmarking](filesystem_benchmarking.md): File system - performance can have a big impact on GitLab performance, especially for actions - that read or write Git repositories. This information helps benchmark - file system performance against known good and bad real-world systems. -- [The Rails Console](rails_console.md): Provides a way to interact with your GitLab instance from the command line. - Used for troubleshooting a problem or retrieving some data that can only be done through direct access to GitLab. -- [ChatOps Scripts](https://gitlab.com/gitlab-com/chatops): The GitLab.com Infrastructure team uses this repository to house - common ChatOps scripts they use to troubleshoot and maintain the production instance of GitLab.com. - These scripts can be used by administrators of GitLab instances of all sizes. +- [Housekeeping](../../administration/housekeeping.md) +- [Activate GitLab EE with license](../../user/admin_area/license_file.md) +- [Fast SSH key lookup](../../administration/operations/fast_ssh_key_lookup.md) +- [File system benchmarking](../../administration/operations/filesystem_benchmarking.md) +- [`gitlab-sshd`](../../administration/operations/gitlab_sshd.md) +- [Rails console](../../administration/operations/rails_console.md) +- [Use SSH certificates](../../administration/operations/ssh_certificates.md) +- [Enable encrypted configuration](../../administration/encrypted_configuration.md) +- [Rake tasks](../../raketasks/index.md) +- [Backup and restore](../../raketasks/backup_restore.md) +- [Inactive project deletion](../../administration/inactive_project_deletion.md) +- [Move repositories](../../administration/operations/moving_repositories.md) +- [Read-only state](../../administration/read_only_gitlab.md) +- [Restart GitLab](../../administration/restart_gitlab.md) diff --git a/doc/administration/operations/moving_repositories.md b/doc/administration/operations/moving_repositories.md index aa0477be788..e9d829f3f08 100644 --- a/doc/administration/operations/moving_repositories.md +++ b/doc/administration/operations/moving_repositories.md @@ -139,14 +139,14 @@ To move all snippets by using the API: To move all groups by using the API: 1. [Schedule repository storage moves for all groups on a storage shard](../../api/group_repository_storage_moves.md#schedule-repository-storage-moves-for-all-groups-on-a-storage-shard). - For example: - - ```shell - curl --request POST --header "PRIVATE-TOKEN: " \ - --header "Content-Type: application/json" \ - --data '{"source_storage_name":"","destination_storage_name":""}' \ - "https://gitlab.example.com/api/v4/group_repository_storage_moves" - ``` + For example: + + ```shell + curl --request POST --header "PRIVATE-TOKEN: " \ + --header "Content-Type: application/json" \ + --data '{"source_storage_name":"","destination_storage_name":""}' \ + "https://gitlab.example.com/api/v4/group_repository_storage_moves" + ``` 1. [Query the most recent repository moves](../../api/group_repository_storage_moves.md#retrieve-all-group-repository-storage-moves). The response indicates either: diff --git a/doc/administration/operations/puma.md b/doc/administration/operations/puma.md index f2f9f1cdcda..efc55a5fbc3 100644 --- a/doc/administration/operations/puma.md +++ b/doc/administration/operations/puma.md @@ -99,7 +99,7 @@ To change the worker timeout to 600 seconds: ## Disable Puma clustered mode in memory-constrained environments WARNING: -This is an experimental [Alpha feature](../../policy/alpha-beta-support.md#alpha-features) and subject to change without notice. The feature +This feature is an [Experiment](../../policy/alpha-beta-support.md#experiment) and subject to change without notice. The feature is not ready for production use. If you want to use this feature, you should test outside of production first. See the [known issues](#puma-single-mode-known-issues) for additional details. @@ -182,7 +182,7 @@ steps below: NOTE: If using a self-signed certificate from a custom Certificate Authority (CA), - follow [the documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) + follow [the documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html#install-custom-public-certificates) to make them trusted by other GitLab components. 1. Edit `/etc/gitlab/gitlab.rb`: diff --git a/doc/administration/operations/rails_console.md b/doc/administration/operations/rails_console.md index 652a4fa5497..ac550d30566 100644 --- a/doc/administration/operations/rails_console.md +++ b/doc/administration/operations/rails_console.md @@ -569,7 +569,7 @@ def disable_two_factor! end def two_factor_enabled? - two_factor_otp_enabled? || two_factor_u2f_enabled? + two_factor_otp_enabled? || two_factor_webauthn_enabled? end ``` diff --git a/doc/administration/package_information/defaults.md b/doc/administration/package_information/defaults.md index 3b52b6bca82..96b56388ea9 100644 --- a/doc/administration/package_information/defaults.md +++ b/doc/administration/package_information/defaults.md @@ -30,7 +30,7 @@ by default: | PgBouncer exporter | No | Port | X | 9188 | | GitLab Exporter | Yes | Port | X | 9168 | | Sidekiq exporter | Yes | Port | X | 8082 | -| Sidekiq health check | No | Port | X | 8092[^Sidekiq-health] | +| Sidekiq health check | Yes | Port | X | 8092[^Sidekiq-health] | | Web exporter | No | Port | X | 8083 | | Geo PostgreSQL | No | Socket | Port (5431) | X | | Redis Sentinel | No | Port | X | 26379 | @@ -49,8 +49,10 @@ by default: | PgBouncer | No | Port | X | 6432 | | Consul | No | Port | X | 8300, 8301(UDP), 8500, 8600[^Consul-notes] | | Patroni | No | Port | X | 8008 | -| GitLab KAS | Yes | Port | X | 8150 | -| Gitaly | No | Port | X | 8075 | +| GitLab KAS | Yes | Port | X | 8150 | +| Gitaly | Yes | Socket | Port (8075) | 8075 or 9999 (TLS) | +| Gitaly exporter | Yes | Port | X | 9236 | +| Praefect | No | Port | X | 2305 or 3305 (TLS) | Legend: diff --git a/doc/administration/package_information/supported_os.md b/doc/administration/package_information/supported_os.md index 9d1c8dcde5a..a1f589015cb 100644 --- a/doc/administration/package_information/supported_os.md +++ b/doc/administration/package_information/supported_os.md @@ -15,13 +15,17 @@ operating systems supported by GitLab are listed in the The following lists the currently supported OSs and their possible EOL dates. +NOTE: +`amd64` and `x86_64` refer to the same 64-bit architecture. +The names `arm64` and `aarch64` are also interchangeable and refer to the same +architecture. + | OS Version | First supported GitLab version | Arch | Install Documentation | OS EOL | Details | | ------------------------------------------------------------ | ------------------------------ | --------------- | :----------------------------------------------------------: | ---------- | ------------------------------------------------------------ | | AlmaLinux 8 | GitLab CE / GitLab EE 14.5.0 | x86_64, aarch64 | [AlmaLinux Install Documentation](https://about.gitlab.com/install/#almalinux-8) | 2029 | | | CentOS 7 | GitLab CE / GitLab EE 7.10.0 | x86_64 | [CentOS Install Documentation](https://about.gitlab.com/install/#centos-7) | June 2024 | | | Debian 10 | GitLab CE / GitLab EE 12.2.0 | amd64, arm64 | [Debian Install Documentation](https://about.gitlab.com/install/#debian) | 2024 | | | Debian 11 | GitLab CE / GitLab EE 14.6.0 | amd64, arm64 | [Debian Install Documentation](https://about.gitlab.com/install/#debian) | 2026 | | -| OpenSUSE 15.3 | GitLab CE / GitLab EE 14.5.0 | x86_64, aarch64 | [OpenSUSE Install Documentation](https://about.gitlab.com/install/#opensuse-leap) | Dec 2022 | | | OpenSUSE 15.4 | GitLab CE / GitLab EE 15.7.0 | x86_64, aarch64 | [OpenSUSE Install Documentation](https://about.gitlab.com/install/#opensuse-leap) | Nov 2023 | | | RHEL 8 | GitLab CE / GitLab EE 12.8.1 | x86_64, arm64 | [Use CentOS Install Documentation](https://about.gitlab.com/install/#centos-7) | May 2029 | [RHEL Details](https://access.redhat.com/support/policy/updates/errata/#Life_Cycle_Dates) | | SLES 12 | GitLab EE 9.0.0 | x86_64 | [Use OpenSUSE Install Documentation](https://about.gitlab.com/install/#opensuse-leap) | Oct 2027 | | @@ -31,7 +35,8 @@ The following lists the currently supported OSs and their possible EOL dates. | Ubuntu 18.04 | GitLab CE / GitLab EE 10.7.0 | amd64 | [Ubuntu Install Documentation](https://about.gitlab.com/install/#ubuntu) | April 2023 | | | Ubuntu 20.04 | GitLab CE / GitLab EE 13.2.0 | amd64, arm64 | [Ubuntu Install Documentation](https://about.gitlab.com/install/#ubuntu) | April 2025 | | | Ubuntu 22.04 | GitLab CE / GitLab EE 15.5.0 | amd64, arm64 | [Ubuntu Install Documentation](https://about.gitlab.com/install/#ubuntu) | April 2027 | | -| Amazon Linux 2 | GitLab CE / GitLab EE 14.9.0 | amd64, arm64 | [Amazon Linux 2 Install Documentation](https://about.gitlab.com/install/#amazonlinux-2) | June 2023 | | +| Amazon Linux 2 | GitLab CE / GitLab EE 14.9.0 | amd64, arm64 | [Amazon Linux 2 Install Documentation](https://about.gitlab.com/install/#amazonlinux-2) | June 2025 | | +| Amazon Linux 2022 | GitLab CE / GitLab EE 15.9.0 | amd64, arm64 | [Amazon Linux 2022 Install Documentation](https://about.gitlab.com/install/#amazonlinux-2022) | October 2027 | | | Raspberry Pi OS (Buster) (formerly known as Raspbian Buster) | GitLab CE 12.2.0 | armhf | [Raspberry Pi Install Documentation](https://about.gitlab.com/install/#raspberry-pi-os) | 2024 | [Raspberry Pi Details](https://www.raspberrypi.com/news/new-old-functionality-with-raspberry-pi-os-legacy/) | | Raspberry Pi OS (Bullseye) | GitLab CE 15.5.0 | armhf | [Raspberry Pi Install Documentation](https://about.gitlab.com/install/#raspberry-pi-os) | 2026 | [Raspberry Pi Details](https://www.raspberrypi.com/news/raspberry-pi-os-debian-bullseye/) | @@ -55,6 +60,14 @@ although [new versions have been released](https://about.gitlab.com/releases/cat of the [Linux package install guide](https://about.gitlab.com/install/#content). Future GitLab upgrades are fetched according to your upgraded OS. +## Update both GitLab and the operating system + +To upgrade both the operating system (OS) and GitLab: + +1. Upgrade the OS. +1. Check if it's necessary to [update the GitLab package sources](#update-gitlab-package-sources-after-upgrading-the-os). +1. [Upgrade GitLab](../../update/index.md). + ## Packages for ARM64 > [Introduced](https://gitlab.com/gitlab-org/gitlab-omnibus-builder/-/issues/27) in GitLab 13.4. @@ -94,6 +107,7 @@ release for them can be found below: | Ubuntu 16.04 | [April 2021](https://ubuntu.com/info/release-end-of-life) | [GitLab CE](https://packages.gitlab.com/app/gitlab/gitlab-ce/search?q=gitlab-ce_13.12&dist=ubuntu%2Fxenial) / [GitLab EE](https://packages.gitlab.com/app/gitlab/gitlab-ee/search?q=gitlab-ee_13.12&dist=ubuntu%2Fxenial) 13.12 | | OpenSUSE 15.2 | [December 2021](https://en.opensuse.org/Lifetime#Discontinued_distributions) | [GitLab CE](https://packages.gitlab.com/app/gitlab/gitlab-ce/search?q=gitlab-ce-14.7&dist=opensuse%2F15.2) / [GitLab EE](https://packages.gitlab.com/app/gitlab/gitlab-ee/search?q=gitlab-ee-14.7&dist=opensuse%2F15.2) 14.7 | | Debian 9 "Stretch" | [June 2022](https://lists.debian.org/debian-lts-announce/2022/07/msg00002.html) | [GitLab CE](https://packages.gitlab.com/app/gitlab/gitlab-ce/search?q=gitlab-ce_15.2&dist=debian%2Fstretch) / [GitLab EE](https://packages.gitlab.com/app/gitlab/gitlab-ee/search?q=gitlab-ee_15.2&dist=debian%2Fstretch) 15.2 | +| OpenSUSE 15.3 | [December 2022](https://en.opensuse.org/Lifetime#Discontinued_distributions) | [GitLab CE](https://packages.gitlab.com/app/gitlab/gitlab-ce/search?q=gitlab-ce-15.10&dist=opensuse%2F15.3) / [GitLab EE](https://packages.gitlab.com/app/gitlab/gitlab-ee/search?q=gitlab-ee-15.10&dist=opensuse%2F15.3) 15.10 | NOTE: An exception to this deprecation policy is when we are unable to provide diff --git a/doc/administration/packages/container_registry.md b/doc/administration/packages/container_registry.md index 34acf57ce70..fd3cbb2ad05 100644 --- a/doc/administration/packages/container_registry.md +++ b/doc/administration/packages/container_registry.md @@ -21,10 +21,8 @@ Registry, see the [user documentation](../../user/packages/container_registry/in If you installed GitLab by using the Omnibus installation package, the Container Registry may or may not be available by default. -The Container Registry is automatically enabled and available on your GitLab domain, port 5050 if: - -- You're using the built-in [Let's Encrypt integration](https://docs.gitlab.com/omnibus/settings/ssl.html#enable-the-lets-encrypt-integration), and -- You're using GitLab 12.5 or later. +The Container Registry is automatically enabled and available on your GitLab domain, port 5050 if +you're using the built-in [Let's Encrypt integration](https://docs.gitlab.com/omnibus/settings/ssl/index.html#enable-the-lets-encrypt-integration). Otherwise, the Container Registry is not enabled. To enable it: @@ -96,7 +94,7 @@ If `auth` is not set up, users can pull Docker images without authentication. ## Container Registry domain configuration -There are two ways you can configure the Registry's external domain. Either: +You can configure the Registry's external domain in either of these ways: - [Use the existing GitLab domain](#configure-container-registry-under-an-existing-gitlab-domain). The Registry listens on a port and reuses the TLS certificate from GitLab. @@ -110,13 +108,15 @@ for the first time. ### Configure Container Registry under an existing GitLab domain -If the Registry is configured to use the existing GitLab domain, you can -expose the Registry on a port. This way you can reuse the existing GitLab TLS +If the Container Registry is configured to use the existing GitLab domain, you can +expose the Container Registry on a port. This way you can reuse the existing GitLab TLS certificate. -If the GitLab domain is `https://gitlab.example.com` and the port to the outside world is `5050`, here is what you need to set -in `gitlab.rb` or `gitlab.yml` if you are using Omnibus GitLab or installed -GitLab from source respectively. +If the GitLab domain is `https://gitlab.example.com` and the port to the outside world is `5050`, +to configure the Container Registry: + +- Edit `gitlab.rb` if you are using Omnibus GitLab. +- Edit `gitlab.yml` if you installed GitLab from source. Ensure you choose a port different than the one that Registry listens to (`5000` by default), otherwise conflicts occur. @@ -202,7 +202,7 @@ domain. For example, `*.gitlab.example.com`, is a wildcard that matches `registr and is distinct from `*.example.com`. As well as manually generated SSL certificates (explained here), certificates automatically -generated by Let's Encrypt are also [supported in Omnibus installs](https://docs.gitlab.com/omnibus/settings/ssl.html). +generated by Let's Encrypt are also [supported in Omnibus installs](https://docs.gitlab.com/omnibus/settings/ssl/index.html). Let's assume that you want the container Registry to be accessible at `https://registry.gitlab.example.com`. @@ -261,7 +261,7 @@ docker login registry.gitlab.example.com ## Disable Container Registry site-wide When you disable the Registry by following these steps, you do not -remove any existing Docker images. This is handled by the +remove any existing Docker images. Docker image removal is handled by the Registry application itself. **Omnibus GitLab** @@ -557,7 +557,7 @@ you can pull from the Container Registry, but you cannot push. 1. To perform the final data sync, [put the Container Registry in `read-only` mode](#performing-garbage-collection-without-downtime) and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -1. Sync any changes since the initial data load to your S3 bucket and delete files that exist in the destination bucket but not in the source: +1. Sync any changes dating from after the initial data load to your S3 bucket, and delete files that exist in the destination bucket but not in the source: ```shell sudo aws --endpoint-url https://your-object-storage-backend.com s3 sync registry s3://mybucket --delete --dryrun @@ -590,15 +590,15 @@ you can pull from the Container Registry, but you cannot push. #### Moving to Azure Object Storage -> The default configuration for the storage driver will be [changed](https://gitlab.com/gitlab-org/container-registry/-/issues/854) in GitLab 16.0. +> The default configuration for the storage driver is scheduled to be [changed](https://gitlab.com/gitlab-org/container-registry/-/issues/854) in GitLab 16.0. WARNING: -The default configuration for the storage driver will be [changed](https://gitlab.com/gitlab-org/container-registry/-/issues/854) in GitLab 16.0. The storage driver will use `/` as the default root directory. You can add `trimlegacyrootprefix: false` to your current configuration now to avoid any disruptions. For more information, see the [Container Registry configuration](https://gitlab.com/gitlab-org/container-registry/-/tree/master/docs-gitlab#azure-storage-driver) documentation. +The default configuration for the storage driver is scheduled to be [changed](https://gitlab.com/gitlab-org/container-registry/-/issues/854) in GitLab 16.0. The storage driver will use `/` as the default root directory. You can add `trimlegacyrootprefix: false` to your current configuration now to avoid any disruptions. For more information, see the [Container Registry configuration](https://gitlab.com/gitlab-org/container-registry/-/tree/master/docs-gitlab#azure-storage-driver) documentation. When moving from an existing file system or another object storage provider to Azure Object Storage, you must configure the registry to use the standard root directory. -This configuration is done by setting [`trimlegacyrootprefix: true]`](https://gitlab.com/gitlab-org/container-registry/-/blob/a3f64464c3ec1c5a599c0a2daa99ebcbc0100b9a/docs-gitlab/README.md#azure-storage-driver) in the Azure storage driver section of the registry configuration. +Configure it by setting [`trimlegacyrootprefix: true]`](https://gitlab.com/gitlab-org/container-registry/-/blob/a3f64464c3ec1c5a599c0a2daa99ebcbc0100b9a/docs-gitlab/README.md#azure-storage-driver) in the Azure storage driver section of the registry configuration. Without this configuration, the Azure storage driver uses `//` instead of `/` as the first section of the root path, rendering the migrated images inaccessible. **Omnibus GitLab installations** @@ -607,10 +607,10 @@ Without this configuration, the Azure storage driver uses `//` instead of `/` as registry['storage'] = { 'azure' => { 'accountname' => 'accountname', - 'accesskey' => 'base64encodedaccountkey', + 'accountkey' => 'base64encodedaccountkey', 'container' => 'containername', 'rootdirectory' => '/azure/virtual/container', - 'trimlegacyrootprefix' => 'true' + 'trimlegacyrootprefix' => true } } ``` @@ -627,6 +627,8 @@ storage: trimlegacyrootprefix: true ``` +By default, Azure Storage Driver uses the `core.windows.net` realm. You can set another value for `realm` in the `azure` section (for example, `core.usgovcloudapi.net` for Azure Government Cloud). For more information, see the [Docker documentation](https://docs.docker.com/registry/storage-drivers/azure/). + ### Disable redirect for storage driver By default, users accessing a registry configured with a remote backend are redirected to the default backend for the storage driver. For example, registries can be configured using the `s3` storage driver, which redirects requests to a remote S3 bucket to alleviate load on the GitLab server. @@ -637,20 +639,20 @@ However, this behavior is undesirable for registries used by internal hosts that 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - registry['storage'] = { - 's3' => { - 'accesskey' => 's3-access-key', - 'secretkey' => 's3-secret-key-for-access-key', - 'bucket' => 'your-s3-bucket', - 'region' => 'your-s3-region', - 'regionendpoint' => 'your-s3-regionendpoint' - }, - 'redirect' => { - 'disable' => true - } - } - ``` + ```ruby + registry['storage'] = { + 's3' => { + 'accesskey' => 's3-access-key', + 'secretkey' => 's3-secret-key-for-access-key', + 'bucket' => 'your-s3-bucket', + 'region' => 'your-s3-region', + 'regionendpoint' => 'your-s3-regionendpoint' + }, + 'redirect' => { + 'disable' => true + } + } + ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -658,21 +660,21 @@ However, this behavior is undesirable for registries used by internal hosts that 1. Add the `redirect` flag to your registry configuration YAML file: - ```yaml - storage: - s3: - accesskey: 'AKIAKIAKI' - secretkey: 'secret123' - bucket: 'gitlab-registry-bucket-AKIAKIAKI' - region: 'your-s3-region' - regionendpoint: 'your-s3-regionendpoint' - redirect: - disable: true - cache: - blobdescriptor: inmemory - delete: - enabled: true - ``` + ```yaml + storage: + s3: + accesskey: 'AKIAKIAKI' + secretkey: 'secret123' + bucket: 'gitlab-registry-bucket-AKIAKIAKI' + region: 'your-s3-region' + regionendpoint: 'your-s3-regionendpoint' + redirect: + disable: true + cache: + blobdescriptor: inmemory + delete: + enabled: true + ``` 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. @@ -680,7 +682,7 @@ However, this behavior is undesirable for registries used by internal hosts that You can use server-side encryption with AWS KMS for S3 buckets that have [SSE-S3 or SSE-KMS encryption enabled by default](https://docs.aws.amazon.com/kms/latest/developerguide/services-s3.html). -Customer master keys (CMKs) and SSE-C encryption aren't supported since this requires sending the +Customer master keys (CMKs) and SSE-C encryption aren't supported because this requires sending the encryption keys in every request. For SSE-S3, you must enable the `encrypt` option in the registry settings. How you do this depends @@ -690,18 +692,18 @@ For Omnibus GitLab installations: 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - registry['storage'] = { - 's3' => { - 'accesskey' => 's3-access-key', - 'secretkey' => 's3-secret-key-for-access-key', - 'bucket' => 'your-s3-bucket', - 'region' => 'your-s3-region', - 'regionendpoint' => 'your-s3-regionendpoint', - 'encrypt' => true - } - } - ``` + ```ruby + registry['storage'] = { + 's3' => { + 'accesskey' => 's3-access-key', + 'secretkey' => 's3-secret-key-for-access-key', + 'bucket' => 'your-s3-bucket', + 'region' => 'your-s3-region', + 'regionendpoint' => 'your-s3-regionendpoint', + 'encrypt' => true + } + } + ``` 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -710,23 +712,23 @@ For installations from source: 1. Edit your registry configuration YAML file: - ```yaml - storage: - s3: - accesskey: 'AKIAKIAKI' - secretkey: 'secret123' - bucket: 'gitlab-registry-bucket-AKIAKIAKI' - region: 'your-s3-region' - regionendpoint: 'your-s3-regionendpoint' - encrypt: true - ``` + ```yaml + storage: + s3: + accesskey: 'AKIAKIAKI' + secretkey: 'secret123' + bucket: 'gitlab-registry-bucket-AKIAKIAKI' + region: 'your-s3-region' + regionendpoint: 'your-s3-regionendpoint' + encrypt: true + ``` 1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. ### Storage limitations -Currently, there is no storage limitation, which means a user can upload an +There is no storage limitation, which means a user can upload an infinite amount of Docker images with arbitrary sizes. This setting should be configurable in future releases. @@ -765,7 +767,11 @@ project, you can [disable it from your project's settings](../../user/project/se ## Use an external container registry with GitLab as an auth endpoint -> Support for external container registries in GitLab is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/376217) in GitLab 15.8 and will be removed in GitLab 16.0. +WARNING: +Using external container registries in GitLab is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/376217) +in GitLab 15.8 and the end of support is scheduled for GitLab 16.0. +If you need to use external container registries instead of the GitLab Container Registry, +tell us about your use cases in [feedback issue 958](https://gitlab.com/gitlab-org/container-registry/-/issues/958). If you use an external container registry, some features associated with the container registry may be unavailable or have [inherent risks](../../user/packages/container_registry/reduce_container_registry_storage.md#use-with-external-container-registries). @@ -871,7 +877,7 @@ You can use GitLab as an auth endpoint with an external container registry. ## Configure Container Registry notifications You can configure the Container Registry to send webhook notifications in -response to events happening within the registry. +response to events happening in the registry. Read more about the Container Registry notifications configuration options in the [Docker Registry notifications documentation](https://docs.docker.com/registry/notifications/). @@ -924,13 +930,17 @@ notifications: WARNING: If you're using a distributed architecture and Sidekiq is running on a different node, the cleanup -policies don't work. To fix this, you must configure the `gitlab.rb` file on the Sidekiq nodes to -point to the correct registry URL and copy the `registry.key` file to each Sidekiq node. For more -information, see the [Sidekiq configuration](../sidekiq/index.md) +policies don't work. To fix this: + +1. Configure the `gitlab.rb` file on the Sidekiq nodes to + point to the correct registry URL. +1. Copy the `registry.key` file to each Sidekiq node. + +For more information, see the [Sidekiq configuration](../sidekiq/index.md) page. To reduce the amount of [Container Registry disk space used by a given project](#registry-disk-space-usage-by-project), -administrators can clean up image tags +administrators can setup cleanup policies and [run garbage collection](#container-registry-garbage-collection). ### Registry Disk Space Usage by Project @@ -993,7 +1003,7 @@ You can also [run cleanup on a schedule](../../user/packages/container_registry/ ## Container Registry garbage collection NOTE: -Retention policies within your object storage provider, such as Amazon S3 Lifecycle, may prevent +Retention policies in your object storage provider, such as Amazon S3 Lifecycle, may prevent objects from being properly deleted. Container Registry can use considerable amounts of disk space. To clear up @@ -1037,9 +1047,9 @@ docker build -t my.registry.com/my.group/my.project:latest . docker push my.registry.com/my.group/my.project:latest ``` -Now, the `:latest` tag points to manifest of `sha256:222222`. However, due to -the architecture of registry, this data is still accessible when pulling the -image `my.registry.com/my.group/my.project@sha256:111111`, even though it is +Now, the `:latest` tag points to manifest of `sha256:222222`. +Due to the architecture of registry, this data is still accessible when pulling the +image `my.registry.com/my.group/my.project@sha256:111111`, though it is no longer directly accessible via the `:latest` tag. ### Recycling unused tags diff --git a/doc/administration/packages/dependency_proxy.md b/doc/administration/packages/dependency_proxy.md index 78efd75bbe3..a29d6d93051 100644 --- a/doc/administration/packages/dependency_proxy.md +++ b/doc/administration/packages/dependency_proxy.md @@ -122,7 +122,7 @@ To change the local storage path: Instead of relying on the local storage, you can use an object storage to store the blobs of the Dependency Proxy. In GitLab 13.2 and later, you should use the -[consolidated object storage settings](../object_storage.md#consolidated-object-storage-configuration). +[consolidated object storage settings](../object_storage.md#configure-a-single-storage-connection-for-all-object-types-consolidated-form). This section describes the earlier configuration format. [Migration steps still apply](#migrate-local-dependency-proxy-blobs-and-manifests-to-object-storage). [Read more about using object storage with GitLab](../object_storage.md). diff --git a/doc/administration/packages/index.md b/doc/administration/packages/index.md index 6e53d77109f..f0f238aa5ba 100644 --- a/doc/administration/packages/index.md +++ b/doc/administration/packages/index.md @@ -124,7 +124,7 @@ Instead of relying on the local storage, you can use an object storage to store packages. For more information, see how to use the -[consolidated object storage settings](../object_storage.md#consolidated-object-storage-configuration). +[consolidated object storage settings](../object_storage.md#configure-a-single-storage-connection-for-all-object-types-consolidated-form). ### Migrate local packages to object storage diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index ed08b10fe97..1626a4fd41a 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -1,8 +1,7 @@ --- -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -description: 'Learn how to administer GitLab Pages.' --- # GitLab Pages administration **(FREE SELF)** @@ -15,7 +14,7 @@ This guide is for Omnibus GitLab installations. If you have installed GitLab from source, see [GitLab Pages administration for source installations](source.md). -## Overview +## The GitLab Pages daemon GitLab Pages makes use of the [GitLab Pages daemon](https://gitlab.com/gitlab-org/gitlab-pages), a basic HTTP server written in Go that can listen on an external IP address and provide support for @@ -247,20 +246,20 @@ control over how the Pages daemon runs and serves content in your environment. | `enable` | Enable or disable GitLab Pages on the current system. | | `external_http` | Configure Pages to bind to one or more secondary IP addresses, serving HTTP requests. Multiple addresses can be given as an array, along with exact ports, for example `['1.2.3.4', '1.2.3.5:8063']`. Sets value for `listen_http`. | | `external_https` | Configure Pages to bind to one or more secondary IP addresses, serving HTTPS requests. Multiple addresses can be given as an array, along with exact ports, for example `['1.2.3.4', '1.2.3.5:8063']`. Sets value for `listen_https`. | -| `server_shutdown_timeout` | GitLab Pages server shutdown timeout in seconds (default: 30 s). | -| `gitlab_client_http_timeout` | GitLab API HTTP client connection timeout in seconds (default: 10 s). | -| `gitlab_client_jwt_expiry` | JWT Token expiry time in seconds (default: 30 s). | -| `gitlab_cache_expiry` | The maximum time a domain's configuration is stored in the cache (default: 600 s). | -| `gitlab_cache_refresh` | The interval at which a domain's configuration is set to be due to refresh (default: 60 s). | -| `gitlab_cache_cleanup` | The interval at which expired items are removed from the cache (default: 60 s). | -| `gitlab_retrieval_timeout` | The maximum time to wait for a response from the GitLab API per request (default: 30 s). | -| `gitlab_retrieval_interval` | The interval to wait before retrying to resolve a domain's configuration via the GitLab API (default: 1 s). | +| `server_shutdown_timeout` | GitLab Pages server shutdown timeout in seconds (default: `30s`). | +| `gitlab_client_http_timeout` | GitLab API HTTP client connection timeout in seconds (default: `10s`). | +| `gitlab_client_jwt_expiry` | JWT Token expiry time in seconds (default: `30s`). | +| `gitlab_cache_expiry` | The maximum time a domain's configuration is stored in the cache (default: `600s`). | +| `gitlab_cache_refresh` | The interval at which a domain's configuration is set to be due to refresh (default: `60s`). | +| `gitlab_cache_cleanup` | The interval at which expired items are removed from the cache (default: `60s`). | +| `gitlab_retrieval_timeout` | The maximum time to wait for a response from the GitLab API per request (default: `30s`). | +| `gitlab_retrieval_interval` | The interval to wait before retrying to resolve a domain's configuration via the GitLab API (default: `1s`). | | `gitlab_retrieval_retries` | The maximum number of times to retry to resolve a domain's configuration via the API (default: 3). | | `domain_config_source` | This parameter was removed in 14.0, on earlier versions it can be used to enable and test API domain configuration source | | `gitlab_id` | The OAuth application public ID. Leave blank to automatically fill when Pages authenticates with GitLab. | | `gitlab_secret` | The OAuth application secret. Leave blank to automatically fill when Pages authenticates with GitLab. | | `auth_scope` | The OAuth application scope to use for authentication. Must match GitLab Pages OAuth application settings. Leave blank to use `api` scope by default. | -| `auth_cookie_session_timeout` | Authentication cookie session timeout in seconds (default: 600 s). A value of `0` means the cookie is deleted after the browser session ends. | +| `auth_cookie_session_timeout` | Authentication cookie session timeout in seconds (default: `10m`). A value of `0` means the cookie is deleted after the browser session ends. | | `gitlab_server` | Server to use for authentication when access control is enabled; defaults to GitLab `external_url`. | | `headers` | Specify any additional http headers that should be sent to the client with each response. Multiple headers can be given as an array, header and value as one string, for example `['my-header: myvalue', 'my-other-header: my-other-value']` | | `enable_disk` | Allows the GitLab Pages daemon to serve content from disk. Shall be disabled if shared disk storage isn't available. | @@ -300,6 +299,10 @@ control over how the Pages daemon runs and serves content in your environment. | `rate_limit_source_ip_burst` | Rate limit per source IP maximum burst allowed per second. | | `rate_limit_domain` | Rate limit per domain in number of requests per second. Set to `0` to disable this feature. | | `rate_limit_domain_burst` | Rate limit per domain maximum burst allowed per second. | +| `rate_limit_tls_source_ip` | Rate limit per source IP in number of TLS connections per second. Set to `0` to disable this feature. | +| `rate_limit_tls_source_ip_burst` | Rate limit per source IP maximum TLS connections burst allowed per second. | +| `rate_limit_tls_domain` | Rate limit per domain in number of TLS connections per second. Set to `0` to disable this feature. | +| `rate_limit_tls_domain_burst` | Rate limit per domain maximum TLS connections burst allowed per second. | | `server_read_timeout` | Maximum duration to read the request headers and body. For no timeout, set to `0` or a negative value. Default: `5s` | | `server_read_header_timeout` | Maximum duration to read the request headers. For no timeout, set to `0` or a negative value. Default: `1s` | | `server_write_timeout` | Maximum duration to write all files in the response. Larger files require more time. For no timeout, set to `0` or a negative value. Default: `0` | @@ -514,7 +517,7 @@ internet connectivity is gated by a proxy. To use a proxy for GitLab Pages: ### Using a custom Certificate Authority (CA) When using certificates issued by a custom CA, [Access Control](../../user/project/pages/pages_access_control.md) and -the [online view of HTML job artifacts](../../ci/pipelines/job_artifacts.md#download-job-artifacts) +the [online view of HTML job artifacts](../../ci/jobs/job_artifacts.md#download-job-artifacts) fails to work if the custom CA is not recognized. This usually results in this error: @@ -523,7 +526,7 @@ This usually results in this error: For installation from source, this can be fixed by installing the custom Certificate Authority (CA) in the system certificate store. -For Omnibus, this is fixed by [installing a custom CA in Omnibus GitLab](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). +For Omnibus, this is fixed by [installing a custom CA in Omnibus GitLab](https://docs.gitlab.com/omnibus/settings/ssl/index.html#install-custom-public-certificates). ### ZIP serving and cache configuration @@ -540,22 +543,22 @@ archive. You can modify the cache behavior by changing the following configurati | Setting | Description | | ------- | ----------- | -| `zip_cache_expiration` | The cache expiration interval of ZIP archives. Must be greater than zero to avoid serving stale content. Default is 60 s. | -| `zip_cache_cleanup` | The interval at which archives are cleaned from memory if they have already expired. Default is 30 s. | -| `zip_cache_refresh` | The time interval in which an archive is extended in memory if accessed before `zip_cache_expiration`. This works together with `zip_cache_expiration` to determine if an archive is extended in memory. See the [example below](#zip-cache-refresh-example) for important details. Default is 30 s. | +| `zip_cache_expiration` | The cache expiration interval of ZIP archives. Must be greater than zero to avoid serving stale content. Default is `60s`. | +| `zip_cache_cleanup` | The interval at which archives are cleaned from memory if they have already expired. Default is `30s`. | +| `zip_cache_refresh` | The time interval in which an archive is extended in memory if accessed before `zip_cache_expiration`. This works together with `zip_cache_expiration` to determine if an archive is extended in memory. See the [example below](#zip-cache-refresh-example) for important details. Default is `30s`. | | `zip_open_timeout` | The maximum time allowed to open a ZIP archive. Increase this time for big archives or slow network connections, as doing so may affect the latency of serving Pages. Default is 30 s. | -| `zip_http_client_timeout` | The maximum time for the ZIP HTTP client. Default is 30 m. | +| `zip_http_client_timeout` | The maximum time for the ZIP HTTP client. Default is `30m`. | #### ZIP cache refresh example Archives are refreshed in the cache (extending the time they are held in memory) if they're accessed before `zip_cache_expiration`, and the time left before expiring is less than or equal to -`zip_cache_refresh`. For example, if `archive.zip` is accessed at time 0 s, it expires in 60 s (the -default for `zip_cache_expiration`). In the example below, if the archive is opened again after 15 s -it is **not** refreshed because the time left for expiry (45 s) is greater than `zip_cache_refresh` -(default 30 s). However, if the archive is accessed again after 45 s (from the first time it was +`zip_cache_refresh`. For example, if `archive.zip` is accessed at time `0s`, it expires in `60s` (the +default for `zip_cache_expiration`). In the example below, if the archive is opened again after `15s` +it is **not** refreshed because the time left for expiry (`45s`) is greater than `zip_cache_refresh` +(default `30s`). However, if the archive is accessed again after `45s` (from the first time it was opened) it's refreshed. This extends the time the archive remains in memory from -`45s + zip_cache_expiration (60s)`, for a total of 105 s. +`45s + zip_cache_expiration (60s)`, for a total of `105s`. After an archive reaches `zip_cache_expiration`, it's marked as expired and removed on the next `zip_cache_cleanup` interval. @@ -631,34 +634,6 @@ are stored. 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -Alternatively, if you have existing Pages deployed you can follow -the below steps to do a no downtime transfer to a new storage location. - -1. Pause Pages deployments by setting the following in `/etc/gitlab/gitlab.rb`: - - ```ruby - sidekiq['queue_selector'] = true - sidekiq['queue_groups'] = [ - "feature_category!=pages" - ] - ``` - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -1. `rsync` contents from the current storage location to the new storage location: `sudo rsync -avzh --progress /var/opt/gitlab/gitlab-rails/shared/pages/ /mnt/storage/pages` -1. Set the new storage location in `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_rails['pages_path'] = "/mnt/storage/pages" - ``` - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -1. Verify Pages are still being served up as expected. -1. Resume Pages deployments by removing from `/etc/gitlab/gitlab.rb` the `sidekiq` setting set above. -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -1. Trigger a new Pages deployment and verify it's working as expected. -1. Remove the old Pages storage location: `sudo rm -rf /var/opt/gitlab/gitlab-rails/shared/pages` -1. Verify Pages are still being served up as expected. - ## Configure listener for reverse proxy requests Follow the steps below to configure the proxy listener of GitLab Pages. @@ -683,23 +658,23 @@ Follow the steps below to configure the proxy listener of GitLab Pages. ## Set global maximum size of each GitLab Pages site **(FREE SELF)** -Prerequisites: +Prerequisite: -- Only GitLab administrators can edit this setting. +- You must have administrator access to the instance. To set the global maximum pages size for a project: 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Settings > Preferences**. 1. Expand **Pages**. -1. Enter a value under **Maximum size of pages**. +1. In **Maximum size of pages**, enter a value. The default is `100`. 1. Select **Save changes**. ## Set maximum size of each GitLab Pages site in a group **(PREMIUM SELF)** -Prerequisites: +Prerequisite: -- You must have at least the Maintainer role for the group. +- You must have administrator access to the instance. To set the maximum size of each GitLab Pages site in a group, overriding the inherited setting: @@ -711,9 +686,9 @@ To set the maximum size of each GitLab Pages site in a group, overriding the inh ## Set maximum size of GitLab Pages site in a project **(PREMIUM SELF)** -Prerequisites: +Prerequisite: -- You must have at least the Maintainer role for the project. +- You must have administrator access to the instance. To set the maximum size of GitLab Pages site in a project, overriding the inherited setting: @@ -729,7 +704,7 @@ To set the maximum size of GitLab Pages site in a project, overriding the inheri Prerequisite: -- You must be an administrator of a self-managed GitLab instance. +- You must have administrator access to the instance. To set the maximum number of GitLab Pages custom domains for a project: @@ -738,6 +713,15 @@ To set the maximum number of GitLab Pages custom domains for a project: 1. Enter a value for **Maximum number of custom domains per project**. Use `0` for unlimited domains. 1. Select **Save changes**. +## Set maximum number of files per GitLab Pages website + +The total number of file entries (including directories and symlinks) is limited to `200,000` per GitLab Pages website. + +You can update the limit in your self-managed instance using the +[GitLab Rails console](../operations/rails_console.md#starting-a-rails-console-session). + +For more information, see [GitLab application limits](../instance_limits.md#number-of-files-per-gitlab-pages-website). + ## Running GitLab Pages on a separate server You can run the GitLab Pages daemon on a separate server to decrease the load on @@ -846,7 +830,7 @@ From [GitLab 13.3 to GitLab 13.12](#domain-source-configuration-before-140) GitL Starting from [GitLab 14.0](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5993) GitLab Pages uses API by default and fails to start if it can't connect to it. -For common issues, see the [troubleshooting section](#failed-to-connect-to-the-internal-gitlab-api). +For common issues, see [troubleshooting](troubleshooting.md#failed-to-connect-to-the-internal-gitlab-api). For more details see this [blog post](https://about.gitlab.com/blog/2020/08/03/how-gitlab-pages-uses-the-gitlab-api-to-serve-content/). @@ -891,7 +875,7 @@ Incorrect configuration of these values may result in intermittent or persistent errors, or the Pages Daemon serving old content. NOTE: -Expiry, interval and timeout flags use [Golang's duration formatting](https://pkg.go.dev/time#ParseDuration). +Expiry, interval and timeout flags use [Go duration formatting](https://pkg.go.dev/time#ParseDuration). A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as `300ms`, `1.5h` or `2h45m`. Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. @@ -923,7 +907,7 @@ configuration is tried to be resolved automatically before reporting an error. > [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5577) in GitLab 13.6. -[Read more about using object storage with GitLab](../object_storage.md). +For more information, see [object storage](../object_storage.md). ### Object storage settings @@ -945,10 +929,10 @@ If you want to stop using and disconnect the NFS server, you need to #### S3-compatible connection settings In GitLab 13.2 and later, you should use the -[consolidated object storage settings](../object_storage.md#consolidated-object-storage-configuration). +[consolidated object storage settings](../object_storage.md#configure-a-single-storage-connection-for-all-object-types-consolidated-form). This section describes the earlier configuration format. -See [the available connection settings for different providers](../object_storage.md#connection-settings). +See [the available connection settings for different providers](../object_storage.md#configure-the-connection-settings). In Omnibus installations: @@ -1128,8 +1112,8 @@ In GitLab 14.0 a number of breaking changes were introduced which may require so The steps below describe the best way to migrate without causing any downtime for your GitLab instance. A GitLab instance running on a single server typically upgrades to 14.0 smoothly, and there should be minimal issues after the upgrade is complete. -Regardless, we recommend everyone follow the migration steps to ensure a successful upgrade. -If at any point you run into issues, consult the [troubleshooting section](#troubleshooting). +Regardless, you should follow the migration steps to ensure a successful upgrade. +For common issues, see [troubleshooting](troubleshooting.md). If your current GitLab version is lower than 13.12, then you must first update to 13.12. Updating directly to 14.0 is [not supported](../../update/index.md#upgrade-paths) @@ -1158,14 +1142,14 @@ than GitLab to prevent XSS attacks. You can enforce rate limits to help minimize the risk of a Denial of Service (DoS) attack. GitLab Pages uses a [token bucket algorithm](https://en.wikipedia.org/wiki/Token_bucket) to enforce rate limiting. By default, -requests that exceed the specified limits are reported but not rejected. +requests or TLS connections that exceed the specified limits are reported but not rejected. GitLab Pages supports the following types of rate limiting: -- Per `source_ip`. It limits how many requests are allowed from the single client IP address. -- Per `domain`. It limits how many requests are allowed per domain hosted on GitLab Pages. It can be a custom domain like `example.com`, or group domain like `group.gitlab.io`. +- Per `source_ip`. It limits how many requests or TLS connections are allowed from the single client IP address. +- Per `domain`. It limits how many requests or TLS connections are allowed per domain hosted on GitLab Pages. It can be a custom domain like `example.com`, or group domain like `group.gitlab.io`. -Rate limits are enforced using the following: +HTTP request-based rate limits are enforced using the following: - `rate_limit_source_ip`: Set the maximum threshold in number of requests per client IP per second. Set to 0 to disable this feature. - `rate_limit_source_ip_burst`: Sets the maximum threshold of number of requests allowed in an initial outburst of requests per client IP. @@ -1173,7 +1157,15 @@ Rate limits are enforced using the following: - `rate_limit_domain`: Set the maximum threshold in number of requests per hosted pages domain per second. Set to 0 to disable this feature. - `rate_limit_domain_burst`: Sets the maximum threshold of number of requests allowed in an initial outburst of requests per hosted pages domain. -#### Enable source-IP rate limits +TLS connection-based rate limits are enforced using the following: + +- `rate_limit_tls_source_ip`: Set the maximum threshold in number of TLS connections per client IP per second. Set to 0 to disable this feature. +- `rate_limit_tls_source_ip_burst`: Sets the maximum threshold of number of TLS connections allowed in an initial outburst of TLS connections per client IP. + For example, when you load a web page from different web browsers at the same time. +- `rate_limit_tls_domain`: Set the maximum threshold in number of TLS connections per hosted pages domain per second. Set to 0 to disable this feature. +- `rate_limit_tls_domain_burst`: Sets the maximum threshold of number of TLS connections allowed in an initial outburst of TLS connections per hosted pages domain. + +#### Enable HTTP requests rate limits by source-IP > [Introduced](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/631) in GitLab 14.5. @@ -1184,16 +1176,9 @@ Rate limits are enforced using the following: gitlab_pages['rate_limit_source_ip_burst'] = 600 ``` -1. To reject requests that exceed the specified limits, enable the `FF_ENFORCE_IP_RATE_LIMITS` feature flag in - `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_pages['env'] = {'FF_ENFORCE_IP_RATE_LIMITS' => 'true'} - ``` - 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -#### Enable domain rate limits +#### Enable HTTP requests rate limits by domain > [Introduced](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/630) in GitLab 14.7. @@ -1204,321 +1189,34 @@ Rate limits are enforced using the following: gitlab_pages['rate_limit_domain_burst'] = 5000 ``` -1. To reject requests that exceed the specified limits, enable the `FF_ENFORCE_DOMAIN_RATE_LIMITS` feature flag in - `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_pages['env'] = {'FF_ENFORCE_DOMAIN_RATE_LIMITS' => 'true'} - ``` - 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). - - -## Troubleshooting - -### How to see GitLab Pages logs - -You can see Pages daemon logs by running: - -```shell -sudo gitlab-ctl tail gitlab-pages -``` - -You can also find the log file in `/var/log/gitlab/gitlab-pages/current`. - -### `unsupported protocol scheme \"\""` - -If you see the following error: - -```plaintext -{"error":"failed to connect to internal Pages API: Get \"/api/v4/internal/pages/status\": unsupported protocol scheme \"\"","level":"warning","msg":"attempted to connect to the API","time":"2021-06-23T20:03:30Z"} -``` - -It means you didn't set the HTTP(S) protocol scheme in the Pages server settings. -To fix it: - -1. Edit `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_pages['gitlab_server'] = "https://" - gitlab_pages['internal_gitlab_server'] = "https://" - ``` - -1. Reconfigure GitLab: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -### 502 error when connecting to GitLab Pages proxy when server does not listen over IPv6 - -In some cases, NGINX might default to using IPv6 to connect to the GitLab Pages -service even when the server does not listen over IPv6. You can identify when -this is happening if you see something similar to the log entry below in the -`gitlab_pages_error.log`: - -```plaintext -2020/02/24 16:32:05 [error] 112654#0: *4982804 connect() failed (111: Connection refused) while connecting to upstream, client: 123.123.123.123, server: ~^(?.*)\.pages\.example\.com$, request: "GET /-/group/project/-/jobs/1234/artifacts/artifact.txt HTTP/1.1", upstream: "http://[::1]:8090//-/group/project/-/jobs/1234/artifacts/artifact.txt", host: "group.example.com" -``` - -To resolve this, set an explicit IP and port for the GitLab Pages `listen_proxy` setting -to define the explicit address that the GitLab Pages daemon should listen on: - -```ruby -gitlab_pages['listen_proxy'] = '127.0.0.1:8090' -``` - -### Intermittent 502 errors or after a few days - -If you run Pages on a system that uses `systemd` and -[`tmpfiles.d`](https://www.freedesktop.org/software/systemd/man/tmpfiles.d.html), -you may encounter intermittent 502 errors trying to serve Pages with an error similar to: - -```plaintext -dial tcp: lookup gitlab.example.com on [::1]:53: dial udp [::1]:53: connect: no route to host" -``` - -GitLab Pages creates a [bind mount](https://man7.org/linux/man-pages/man8/mount.8.html) -inside `/tmp/gitlab-pages-*` that includes files like `/etc/hosts`. -However, `systemd` may clean the `/tmp/` directory on a regular basis so the DNS -configuration may be lost. - -To stop `systemd` from cleaning the Pages related content: - -1. Tell `tmpfiles.d` to not remove the Pages `/tmp` directory: - - ```shell - echo 'x /tmp/gitlab-pages-*' >> /etc/tmpfiles.d/gitlab-pages-jail.conf - ``` - -1. Restart GitLab Pages: - - ```shell - sudo gitlab-ctl restart gitlab-pages - ``` - -### Unable to access GitLab Pages - -If you can't access your GitLab Pages (such as receiving `502 Bad Gateway` errors, or a login loop) -and in your Pages log shows this error: - -```plaintext -"error":"retrieval context done: context deadline exceeded","host":"root.docs-cit.otenet.gr","level":"error","msg":"could not fetch domain information from a source" -``` - -1. Add the following to `/etc/gitlab/gitlab.rb`: - - ```ruby - gitlab_pages['internal_gitlab_server'] = 'http://localhost:8080' - ``` - -1. Restart GitLab Pages: - - ```shell - sudo gitlab-ctl restart gitlab-pages - ``` - -### Failed to connect to the internal GitLab API - -If you see the following error: +#### Enable TLS connections rate limits by source-IP -```plaintext -ERRO[0010] Failed to connect to the internal GitLab API after 0.50s error="failed to connect to internal Pages API: HTTP status: 401" -``` - -If you are [Running GitLab Pages on a separate server](#running-gitlab-pages-on-a-separate-server) -you must copy the `/etc/gitlab/gitlab-secrets.json` file -from the **GitLab server** to the **Pages server** after upgrading to GitLab 13.3, -as described in that section. - -Other reasons may include network connectivity issues between your -**GitLab server** and your **Pages server** such as firewall configurations or closed ports. -For example, if there is a connection timeout: - -```plaintext -error="failed to connect to internal Pages API: Get \"https://gitlab.example.com:3000/api/v4/internal/pages/status\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)" -``` - -### Pages cannot communicate with an instance of the GitLab API - -If you use the default value for `domain_config_source=auto` and run multiple instances of GitLab -Pages, you may see intermittent 502 error responses while serving Pages content. You may also see -the following warning in the Pages logs: - -```plaintext -WARN[0010] Pages cannot communicate with an instance of the GitLab API. Please sync your gitlab-secrets.json file https://gitlab.com/gitlab-org/gitlab-pages/-/issues/535#workaround. error="pages endpoint unauthorized" -``` - -This can happen if your `gitlab-secrets.json` file is out of date between GitLab Rails and GitLab -Pages. Follow steps 8-10 of [Running GitLab Pages on a separate server](#running-gitlab-pages-on-a-separate-server), -in all of your GitLab Pages instances. - -### Intermittent 502 errors when using an AWS Network Load Balancer and GitLab Pages - -Connections will time out when using a Network Load Balancer with client IP preservation enabled and [the request is looped back to the source server](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-troubleshooting.html#loopback-timeout). -This can happen to GitLab instances with multiple servers -running both the core GitLab application and GitLab Pages. This can also happen when a single -container is running both the core GitLab application and GitLab Pages. - -AWS [recommends using an IP target type](https://aws.amazon.com/premiumsupport/knowledge-center/target-connection-fails-load-balancer/) -to resolve this issue. - -Turning off [client IP preservation](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-target-groups.html#client-ip-preservation) -may resolve this issue when the core GitLab application and GitLab Pages run on the same host or -container. - -### 500 error with `securecookie: failed to generate random iv` and `Failed to save the session` - -This problem most likely results from an [out-dated operating system](../package_information/supported_os.md#os-versions-that-are-no-longer-supported). -The [Pages daemon uses the `securecookie` library](https://gitlab.com/search?group_id=9970&project_id=734943&repository_ref=master&scope=blobs&search=securecookie&snippets=false) to get random strings via [`crypto/rand` in Go](https://pkg.go.dev/crypto/rand#pkg-variables). -This requires the `getrandom` system call or `/dev/urandom` to be available on the host OS. -Upgrading to an [officially supported operating system](https://about.gitlab.com/install/) is recommended. - -### The requested scope is invalid, malformed, or unknown - -This problem comes from the permissions of the GitLab Pages OAuth application. To fix it: - -1. On the top bar, select **Main menu > Admin**. -1. On the left sidebar, select **Applications > GitLab Pages**. -1. Edit the application. -1. Under **Scopes**, ensure that the `api` scope is selected. -1. Save your changes. - -When running a [separate Pages server](#running-gitlab-pages-on-a-separate-server), -this setting needs to be configured on the main GitLab server. - -### Workaround in case no wildcard DNS entry can be set - -If the wildcard DNS [prerequisite](#prerequisites) can't be met, you can still use GitLab Pages in a limited fashion: - -1. [Move](../../user/project/settings/index.md#transfer-a-project-to-another-namespace) - all projects you need to use Pages with into a single group namespace, for example `pages`. -1. Configure a [DNS entry](#dns-configuration) without the `*.`-wildcard, for example `pages.example.io`. -1. Configure `pages_external_url http://example.io/` in your `gitlab.rb` file. - Omit the group namespace here, because it automatically is prepended by GitLab. - -### Pages daemon fails with permission denied errors - -If `/tmp` is mounted with `noexec`, the Pages daemon fails to start with an error like: - -```plaintext -{"error":"fork/exec /gitlab-pages: permission denied","level":"fatal","msg":"could not create pages daemon","time":"2021-02-02T21:54:34Z"} -``` +> [Introduced](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/632) in GitLab 14.9. -In this case, change `TMPDIR` to a location that is not mounted with `noexec`. Add the following to -`/etc/gitlab/gitlab.rb`: - -```ruby -gitlab_pages['env'] = {'TMPDIR' => ''} -``` - -Once added, reconfigure with `sudo gitlab-ctl reconfigure` and restart GitLab with -`sudo gitlab-ctl restart`. - -### `The redirect URI included is not valid.` when using Pages Access Control - -You may see this error if `pages_external_url` was updated at some point of time. Verify the following: - -1. The **Callback URL**/Redirect URI in the GitLab Pages [System OAuth application](../../integration/oauth_provider.md#create-an-instance-wide-application) -is using the protocol (HTTP or HTTPS) that `pages_external_url` is configured to use. -1. The domain and path components of `Redirect URI` are valid: they should look like `projects./auth`. - -### 500 error `cannot serve from disk` - -If you get a 500 response from Pages and encounter an error similar to: - -```plaintext -ERRO[0145] cannot serve from disk error="gitlab: disk access is disabled via enable-disk=false" project_id=27 source_path="file:///shared/pages/@hashed/67/06/670671cd97404156226e507973f2ab8330d3022ca96e0c93bdbdb320c41adcaf/pages_deployments/14/artifacts.zip" source_type=zip -``` - -It means that GitLab Rails is telling GitLab Pages to serve content from a location on disk, -however, GitLab Pages was configured to disable disk access. - -To enable disk access: - -1. Enable disk access for GitLab Pages in `/etc/gitlab/gitlab.rb`: +1. Set rate limits in `/etc/gitlab/gitlab.rb`: ```ruby - gitlab_pages['enable_disk'] = true + gitlab_pages['rate_limit_tls_source_ip'] = 20.0 + gitlab_pages['rate_limit_tls_source_ip_burst'] = 600 ``` 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -### `httprange: new resource 403` - -If you see an error similar to: - -```plaintext -{"error":"httprange: new resource 403: \"403 Forbidden\"","host":"root.pages.example.com","level":"error","msg":"vfs.Root","path":"/pages1/","time":"2021-06-10T08:45:19Z"} -``` - -And you run pages on the separate server syncing files via NFS, it may mean that -the shared pages directory is mounted on a different path on the main GitLab server and the -GitLab Pages server. +#### Enable TLS connections rate limits by domain -In that case, it's highly recommended you to configure -[object storage and migrate any existing pages data to it](#using-object-storage). +> [Introduced](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/632) in GitLab 14.9. -Alternatively, you can mount the GitLab Pages shared directory to the same path on -both servers. - -### GitLab Pages doesn't work after upgrading to GitLab 14.0 or above - -GitLab 14.0 introduces a number of changes to GitLab Pages which may require manual intervention. - -1. Firstly [follow the migration guide](#prepare-gitlab-pages-for-140). -1. Try to upgrade to GitLab 14.3 or above. Some of the issues were fixed in GitLab 14.1, 14.2 and 14.3. -1. If it doesn't work, see [GitLab Pages logs](#how-to-see-gitlab-pages-logs), and if you see any errors there then search them on this page. - -WARNING: -In GitLab 14.0-14.2 you can temporarily enable legacy storage and configuration mechanisms. - -To do that: - -1. Describe the issue you're seeing in the [migration feedback issue](https://gitlab.com/gitlab-org/gitlab/-/issues/331699). - -1. Edit `/etc/gitlab/gitlab.rb`: +1. Set rate limits in `/etc/gitlab/gitlab.rb`: ```ruby - gitlab_pages['use_legacy_storage'] = true + gitlab_pages['rate_limit_tls_domain'] = 1000 + gitlab_pages['rate_limit_tls_domain_burst'] = 5000 ``` 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -### GitLab Pages deploy job fails with error "is not a recognized provider" - -If the **pages** job succeeds but the **deploy** job gives the error "is not a recognized provider": - -![Pages Deploy Failure](img/pages_deploy_failure_v14_8.png) - -The error message `is not a recognized provider` could be coming from the `fog` gem that GitLab uses to connect to cloud providers for object storage. - -To fix that: - -1. Check your `gitlab.rb` file. If you have `gitlab_rails['pages_object_store_enabled']` enabled, but no bucket details have been configured, either: - - - Configure object storage for your Pages deployments, following the [S3-compatible connection settings](#s3-compatible-connection-settings) guide. - - Store your deployments locally, by commenting out that line. - -1. Save the changes you made to your `gitlab.rb` file, then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). - -### 404 error `The page you're looking for could not be found` - -If you get a `404 Page Not Found` response from GitLab Pages: - -1. Check `.gitlab-ci.yml` contains the job `pages:`. -1. Check the current project's pipeline to confirm the job `pages:deploy` is being run. +## Related topics -Without the `pages:deploy` job, the updates to your GitLab Pages site are never published. +- [Troubleshooting GitLab Pages administration](troubleshooting.md) diff --git a/doc/administration/pages/source.md b/doc/administration/pages/source.md index db76d15ec58..b36b87f3b1d 100644 --- a/doc/administration/pages/source.md +++ b/doc/administration/pages/source.md @@ -1,6 +1,6 @@ --- -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -10,8 +10,8 @@ NOTE: Before attempting to enable GitLab Pages, first make sure you have [installed GitLab](../../install/installation.md) successfully. -This is the documentation for configuring a GitLab Pages when you have installed -GitLab from source and not using the Omnibus packages. +This document explains how to configure GitLab Pages when you have installed +GitLab from source and not the Omnibus packages. You are encouraged to read the [Omnibus documentation](index.md) as it provides invaluable information about the configuration of GitLab Pages. @@ -20,12 +20,11 @@ We also highly recommend that you use the Omnibus GitLab packages. We optimize them specifically for GitLab, and we take care of upgrading GitLab Pages to the latest supported version. -## Overview +## How GitLab Pages works -GitLab Pages makes use of the [GitLab Pages daemon](https://gitlab.com/gitlab-org/gitlab-pages), a simple HTTP server -written in Go that can listen on an external IP address and provide support for -custom domains and custom certificates. It supports dynamic certificates through -SNI and exposes pages using HTTP2 by default. +GitLab Pages makes use of the [GitLab Pages daemon](https://gitlab.com/gitlab-org/gitlab-pages), a lightweight HTTP server that listens on an external IP address and provides support for +custom domains and certificates. It supports dynamic certificates through +`SNI` and exposes pages using HTTP2 by default. You are encouraged to read its [README](https://gitlab.com/gitlab-org/gitlab-pages/blob/master/README.md) to fully understand how it works. @@ -57,7 +56,7 @@ Before proceeding with the Pages configuration, make sure that: assume that to be `example.io`. - You have configured a **wildcard DNS record** for that domain. - You have installed the `zip` and `unzip` packages in the same server that - GitLab is installed since they are needed to compress and decompress the + GitLab is installed because they are needed to compress and decompress the Pages artifacts. - Optional. You have a **wildcard certificate** for the Pages domain if you decide to serve Pages (`*.example.io`) under HTTPS. @@ -86,7 +85,7 @@ see the [security section](#security). Depending on your needs, you can set up GitLab Pages in 4 different ways. The following options are listed from the easiest setup to the most advanced one. The absolute minimum requirement is to set up the wildcard DNS -since that is needed in all configurations. +because that is needed in all configurations. ### Wildcard domains @@ -96,7 +95,7 @@ since that is needed in all configurations. URL scheme: `http://.example.io/` -This is the minimum setup that you can use Pages with. It is the base for all +This setup is the minimum you can use Pages with. It is the base for all other setups as described below. NGINX proxies all requests to the daemon. The Pages daemon doesn't listen to the outside world. @@ -150,8 +149,8 @@ The Pages daemon doesn't listen to the outside world. You may use an `http` address, when running GitLab Pages and GitLab on the same host. If you use `https` and use a self-signed certificate, be sure to - make your custom CA available to GitLab Pages, for example by setting the - `SSL_CERT_DIR` environment variable. + make your custom CA available to GitLab Pages. For example, you can do this + by setting the `SSL_CERT_DIR` environment variable. 1. Add the secret API key: @@ -264,8 +263,8 @@ that without TLS certificates. URL scheme: `http://.example.io/` and `http://custom-domain.com` -In that case, the pages daemon is running, NGINX still proxies requests to -the daemon but the daemon is also able to receive requests from the outside +In that case, the pages daemon is running. NGINX still proxies requests to +the daemon, but the daemon is also able to receive requests from the outside world. Custom domains are supported, but no TLS. 1. Install the Pages daemon: @@ -296,10 +295,9 @@ world. Custom domains are supported, but no TLS. external_http: 192.0.2.2:80 ``` -1. Edit `/etc/default/gitlab` and set `gitlab_pages_enabled` to `true` in - order to enable the pages daemon. In `gitlab_pages_options` the - `-pages-domain` and `-listen-http` must match the `host` and `external_http` - settings that you set above respectively: +1. To enable the daemon, edit `/etc/default/gitlab` and set `gitlab_pages_enabled` to `true`. + In `gitlab_pages_options`, the value for `-pages-domain` must match the `host` and `-listen-http` must match + the `external_http`: ```ini gitlab_pages_enabled=true @@ -329,8 +327,8 @@ world. Custom domains are supported, but no TLS. URL scheme: `https://.example.io/` and `https://custom-domain.com` -In that case, the pages daemon is running, NGINX still proxies requests to -the daemon but the daemon is also able to receive requests from the outside +In that case, the pages daemon is running. NGINX still proxies requests to +the daemon, but the daemon is also able to receive requests from the outside world. Custom domains and TLS are supported. 1. Install the Pages daemon: @@ -416,8 +414,8 @@ server_name ~^.*\.pages\.example\.io$; ## Access control -GitLab Pages access control can be configured per-project, and allows access to a Pages -site to be controlled based on a user's membership to that project. +GitLab Pages access control can be configured per project. Access to a Pages +site can be controlled based on a user's membership to that project. Access control works by registering the Pages daemon as an OAuth application with GitLab. Whenever a request to access a private Pages site is made by an diff --git a/doc/administration/pages/troubleshooting.md b/doc/administration/pages/troubleshooting.md new file mode 100644 index 00000000000..6f00ae074f5 --- /dev/null +++ b/doc/administration/pages/troubleshooting.md @@ -0,0 +1,304 @@ +--- +stage: Plan +group: Knowledge +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Troubleshooting GitLab Pages administration **(FREE SELF)** + +This page contains a list of issues you might encounter when administering GitLab Pages. + +## How to see GitLab Pages logs + +You can see Pages daemon logs by running: + +```shell +sudo gitlab-ctl tail gitlab-pages +``` + +You can also find the log file in `/var/log/gitlab/gitlab-pages/current`. + +## `unsupported protocol scheme \"\""` + +If you see the following error: + +```plaintext +{"error":"failed to connect to internal Pages API: Get \"/api/v4/internal/pages/status\": unsupported protocol scheme \"\"","level":"warning","msg":"attempted to connect to the API","time":"2021-06-23T20:03:30Z"} +``` + +It means you didn't set the HTTP(S) protocol scheme in the Pages server settings. +To fix it: + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_pages['gitlab_server'] = "https://" + gitlab_pages['internal_gitlab_server'] = "https://" # optional, gitlab_pages['gitlab_server'] is used as default + ``` + +1. Reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +## 502 error when connecting to GitLab Pages proxy when server does not listen over IPv6 + +In some cases, NGINX might default to using IPv6 to connect to the GitLab Pages +service even when the server does not listen over IPv6. You can identify when +this is happening if you see something similar to the log entry below in the +`gitlab_pages_error.log`: + +```plaintext +2020/02/24 16:32:05 [error] 112654#0: *4982804 connect() failed (111: Connection refused) while connecting to upstream, client: 123.123.123.123, server: ~^(?.*)\.pages\.example\.com$, request: "GET /-/group/project/-/jobs/1234/artifacts/artifact.txt HTTP/1.1", upstream: "http://[::1]:8090//-/group/project/-/jobs/1234/artifacts/artifact.txt", host: "group.example.com" +``` + +To resolve this, set an explicit IP and port for the GitLab Pages `listen_proxy` setting +to define the explicit address that the GitLab Pages daemon should listen on: + +```ruby +gitlab_pages['listen_proxy'] = '127.0.0.1:8090' +``` + +## Intermittent 502 errors or after a few days + +If you run Pages on a system that uses `systemd` and +[`tmpfiles.d`](https://www.freedesktop.org/software/systemd/man/tmpfiles.d.html), +you may encounter intermittent 502 errors trying to serve Pages with an error similar to: + +```plaintext +dial tcp: lookup gitlab.example.com on [::1]:53: dial udp [::1]:53: connect: no route to host" +``` + +GitLab Pages creates a [bind mount](https://man7.org/linux/man-pages/man8/mount.8.html) +inside `/tmp/gitlab-pages-*` that includes files like `/etc/hosts`. +However, `systemd` may clean the `/tmp/` directory on a regular basis so the DNS +configuration may be lost. + +To stop `systemd` from cleaning the Pages related content: + +1. Tell `tmpfiles.d` to not remove the Pages `/tmp` directory: + + ```shell + echo 'x /tmp/gitlab-pages-*' >> /etc/tmpfiles.d/gitlab-pages-jail.conf + ``` + +1. Restart GitLab Pages: + + ```shell + sudo gitlab-ctl restart gitlab-pages + ``` + +## Unable to access GitLab Pages + +If you can't access your GitLab Pages (such as receiving `502 Bad Gateway` errors, or a login loop) +and in your Pages log shows this error: + +```plaintext +"error":"retrieval context done: context deadline exceeded","host":"root.docs-cit.otenet.gr","level":"error","msg":"could not fetch domain information from a source" +``` + +1. Add the following to `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_pages['internal_gitlab_server'] = 'http://localhost:8080' + ``` + +1. Restart GitLab Pages: + + ```shell + sudo gitlab-ctl restart gitlab-pages + ``` + +## Failed to connect to the internal GitLab API + +If you see the following error: + +```plaintext +ERRO[0010] Failed to connect to the internal GitLab API after 0.50s error="failed to connect to internal Pages API: HTTP status: 401" +``` + +If you are [Running GitLab Pages on a separate server](index.md#running-gitlab-pages-on-a-separate-server) +you must copy the `/etc/gitlab/gitlab-secrets.json` file +from the **GitLab server** to the **Pages server** after upgrading to GitLab 13.3, +as described in that section. + +Other reasons may include network connectivity issues between your +**GitLab server** and your **Pages server** such as firewall configurations or closed ports. +For example, if there is a connection timeout: + +```plaintext +error="failed to connect to internal Pages API: Get \"https://gitlab.example.com:3000/api/v4/internal/pages/status\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)" +``` + +## Pages cannot communicate with an instance of the GitLab API + +If you use the default value for `domain_config_source=auto` and run multiple instances of GitLab +Pages, you may see intermittent 502 error responses while serving Pages content. You may also see +the following warning in the Pages logs: + +```plaintext +WARN[0010] Pages cannot communicate with an instance of the GitLab API. Please sync your gitlab-secrets.json file https://gitlab.com/gitlab-org/gitlab-pages/-/issues/535#workaround. error="pages endpoint unauthorized" +``` + +This can happen if your `gitlab-secrets.json` file is out of date between GitLab Rails and GitLab +Pages. Follow steps 8-10 of [Running GitLab Pages on a separate server](index.md#running-gitlab-pages-on-a-separate-server), +in all of your GitLab Pages instances. + +## Intermittent 502 errors when using an AWS Network Load Balancer and GitLab Pages + +Connections will time out when using a Network Load Balancer with client IP preservation enabled and [the request is looped back to the source server](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-troubleshooting.html#loopback-timeout). +This can happen to GitLab instances with multiple servers +running both the core GitLab application and GitLab Pages. This can also happen when a single +container is running both the core GitLab application and GitLab Pages. + +AWS [recommends using an IP target type](https://aws.amazon.com/premiumsupport/knowledge-center/target-connection-fails-load-balancer/) +to resolve this issue. + +Turning off [client IP preservation](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-target-groups.html#client-ip-preservation) +may resolve this issue when the core GitLab application and GitLab Pages run on the same host or +container. + +## 500 error with `securecookie: failed to generate random iv` and `Failed to save the session` + +This problem most likely results from an [out-dated operating system](../package_information/supported_os.md#os-versions-that-are-no-longer-supported). +The [Pages daemon uses the `securecookie` library](https://gitlab.com/search?group_id=9970&project_id=734943&repository_ref=master&scope=blobs&search=securecookie&snippets=false) to get random strings via [`crypto/rand` in Go](https://pkg.go.dev/crypto/rand#pkg-variables). +This requires the `getrandom` system call or `/dev/urandom` to be available on the host OS. +Upgrading to an [officially supported operating system](https://about.gitlab.com/install/) is recommended. + +## The requested scope is invalid, malformed, or unknown + +This problem comes from the permissions of the GitLab Pages OAuth application. To fix it: + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Applications > GitLab Pages**. +1. Edit the application. +1. Under **Scopes**, ensure that the `api` scope is selected. +1. Save your changes. + +When running a [separate Pages server](index.md#running-gitlab-pages-on-a-separate-server), +this setting needs to be configured on the main GitLab server. + +## Workaround in case no wildcard DNS entry can be set + +If the wildcard DNS [prerequisite](index.md#prerequisites) can't be met, you can still use GitLab Pages in a limited fashion: + +1. [Move](../../user/project/settings/index.md#transfer-a-project-to-another-namespace) + all projects you need to use Pages with into a single group namespace, for example `pages`. +1. Configure a [DNS entry](index.md#dns-configuration) without the `*.`-wildcard, for example `pages.example.io`. +1. Configure `pages_external_url http://example.io/` in your `gitlab.rb` file. + Omit the group namespace here, because it automatically is prepended by GitLab. + +## Pages daemon fails with permission denied errors + +If `/tmp` is mounted with `noexec`, the Pages daemon fails to start with an error like: + +```plaintext +{"error":"fork/exec /gitlab-pages: permission denied","level":"fatal","msg":"could not create pages daemon","time":"2021-02-02T21:54:34Z"} +``` + +In this case, change `TMPDIR` to a location that is not mounted with `noexec`. Add the following to +`/etc/gitlab/gitlab.rb`: + +```ruby +gitlab_pages['env'] = {'TMPDIR' => ''} +``` + +Once added, reconfigure with `sudo gitlab-ctl reconfigure` and restart GitLab with +`sudo gitlab-ctl restart`. + +## `The redirect URI included is not valid.` when using Pages Access Control + +You may see this error if `pages_external_url` was updated at some point of time. Verify the following: + +1. The **Callback URL**/Redirect URI in the GitLab Pages [System OAuth application](../../integration/oauth_provider.md#create-an-instance-wide-application) +is using the protocol (HTTP or HTTPS) that `pages_external_url` is configured to use. +1. The domain and path components of `Redirect URI` are valid: they should look like `projects./auth`. + +## 500 error `cannot serve from disk` + +If you get a 500 response from Pages and encounter an error similar to: + +```plaintext +ERRO[0145] cannot serve from disk error="gitlab: disk access is disabled via enable-disk=false" project_id=27 source_path="file:///shared/pages/@hashed/67/06/670671cd97404156226e507973f2ab8330d3022ca96e0c93bdbdb320c41adcaf/pages_deployments/14/artifacts.zip" source_type=zip +``` + +It means that GitLab Rails is telling GitLab Pages to serve content from a location on disk, +however, GitLab Pages was configured to disable disk access. + +To enable disk access: + +1. Enable disk access for GitLab Pages in `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_pages['enable_disk'] = true + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +## `httprange: new resource 403` + +If you see an error similar to: + +```plaintext +{"error":"httprange: new resource 403: \"403 Forbidden\"","host":"root.pages.example.com","level":"error","msg":"vfs.Root","path":"/pages1/","time":"2021-06-10T08:45:19Z"} +``` + +And you run pages on the separate server syncing files via NFS, it may mean that +the shared pages directory is mounted on a different path on the main GitLab server and the +GitLab Pages server. + +In that case, it's highly recommended you to configure +[object storage and migrate any existing pages data to it](index.md#using-object-storage). + +Alternatively, you can mount the GitLab Pages shared directory to the same path on +both servers. + +## GitLab Pages doesn't work after upgrading to GitLab 14.0 or above + +GitLab 14.0 introduces a number of changes to GitLab Pages which may require manual intervention. + +1. Firstly [follow the migration guide](index.md#prepare-gitlab-pages-for-140). +1. Try to upgrade to GitLab 14.3 or above. Some of the issues were fixed in GitLab 14.1, 14.2 and 14.3. +1. If it doesn't work, see [GitLab Pages logs](#how-to-see-gitlab-pages-logs), and if you see any errors there then search them on this page. + +WARNING: +In GitLab 14.0-14.2 you can temporarily enable legacy storage and configuration mechanisms. + +To do that: + +1. Describe the issue you're seeing in the [migration feedback issue](https://gitlab.com/gitlab-org/gitlab/-/issues/331699). + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_pages['use_legacy_storage'] = true + ``` + +1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +## GitLab Pages deploy job fails with error "is not a recognized provider" + +If the **pages** job succeeds but the **deploy** job gives the error "is not a recognized provider": + +![Pages Deploy Failure](img/pages_deploy_failure_v14_8.png) + +The error message `is not a recognized provider` could be coming from the `fog` gem that GitLab uses to connect to cloud providers for object storage. + +To fix that: + +1. Check your `gitlab.rb` file. If you have `gitlab_rails['pages_object_store_enabled']` enabled, but no bucket details have been configured, either: + + - Configure object storage for your Pages deployments, following the [S3-compatible connection settings](index.md#s3-compatible-connection-settings) guide. + - Store your deployments locally, by commenting out that line. + +1. Save the changes you made to your `gitlab.rb` file, then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +## 404 error `The page you're looking for could not be found` + +If you get a `404 Page Not Found` response from GitLab Pages: + +1. Check `.gitlab-ci.yml` contains the job `pages:`. +1. Check the current project's pipeline to confirm the job `pages:deploy` is being run. + +Without the `pages:deploy` job, the updates to your GitLab Pages site are never published. diff --git a/doc/administration/postgresql/database_load_balancing.md b/doc/administration/postgresql/database_load_balancing.md index 15129770888..d5cf93a135a 100644 --- a/doc/administration/postgresql/database_load_balancing.md +++ b/doc/administration/postgresql/database_load_balancing.md @@ -235,3 +235,8 @@ operation retries up to 3 times using an exponential back-off. When using load balancing, you should be able to safely restart a database server without it immediately leading to errors being presented to the users. + +### Development guide + +For detailed development guide on database load balancing, +see [the development documentation](../../development/database/load_balancing.md). diff --git a/doc/administration/postgresql/external.md b/doc/administration/postgresql/external.md index 8605ee94255..8f664f9809e 100644 --- a/doc/administration/postgresql/external.md +++ b/doc/administration/postgresql/external.md @@ -17,7 +17,7 @@ If you use a cloud-managed service, or provide your own PostgreSQL instance: 1. Set up PostgreSQL according to the [database requirements document](../../install/requirements.md#database). -1. Set up a `gitlab` user with a password of your choice, create the `gitlabhq_production` database, and make the user an owner of the database. You can see an example of this setup in the [installation from source documentation](../../install/installation.md#6-database). +1. Set up a `gitlab` user with a password of your choice, create the `gitlabhq_production` database, and make the user an owner of the database. You can see an example of this setup in the [installation from source documentation](../../install/installation.md#7-database). 1. If you are using a cloud-managed service, you may need to grant additional roles to your `gitlab` user: - Amazon RDS requires the [`rds_superuser`](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.html#Appendix.PostgreSQL.CommonDBATasks.Roles) role. @@ -30,18 +30,18 @@ If you use a cloud-managed service, or provide your own PostgreSQL instance: 1. Configure the GitLab application servers with the appropriate connection details for your external PostgreSQL service in your `/etc/gitlab/gitlab.rb` file: - ```ruby - # Disable the bundled Omnibus provided PostgreSQL - postgresql['enable'] = false + ```ruby + # Disable the bundled Omnibus provided PostgreSQL + postgresql['enable'] = false - # PostgreSQL connection details - gitlab_rails['db_adapter'] = 'postgresql' - gitlab_rails['db_encoding'] = 'unicode' - gitlab_rails['db_host'] = '10.1.0.5' # IP/hostname of database server - gitlab_rails['db_password'] = 'DB password' - ``` + # PostgreSQL connection details + gitlab_rails['db_adapter'] = 'postgresql' + gitlab_rails['db_encoding'] = 'unicode' + gitlab_rails['db_host'] = '10.1.0.5' # IP/hostname of database server + gitlab_rails['db_password'] = 'DB password' + ``` - For more information on GitLab multi-node setups, refer to the [reference architectures](../reference_architectures/index.md). + For more information on GitLab multi-node setups, refer to the [reference architectures](../reference_architectures/index.md). 1. Reconfigure for the changes to take effect: diff --git a/doc/administration/postgresql/multiple_databases.md b/doc/administration/postgresql/multiple_databases.md index 836736fb73f..857fd4fc9c5 100644 --- a/doc/administration/postgresql/multiple_databases.md +++ b/doc/administration/postgresql/multiple_databases.md @@ -1,6 +1,6 @@ --- stage: Data Stores -group: Pods +group: Tenant Scale info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -15,23 +15,83 @@ By default, GitLab uses a single application database, referred to as the `main` To scale GitLab, you can configure GitLab to use multiple application databases. -Due to [known issues](#known-issues), configuring GitLab with multiple databases is in [**Alpha**](../../policy/alpha-beta-support.md#alpha-features). +Due to [known issues](#known-issues), configuring GitLab with multiple databases is an [Experiment](../../policy/alpha-beta-support.md#experiment). + +After you have set up multiple databases, GitLab uses a second application database for +[CI/CD features](../../ci/index.md), referred to as the `ci` database. + +All tables have exactly the same structure in both the `main`, and `ci` +databases. Some examples: + +- When multiple databases are configured, the `ci_pipelines` table exists in + both the `main` and `ci` databases, but GitLab reads and writes only to the + `ci_pipelines` table in the `ci` database. +- Similarly, the `projects` table exists in + both the `main` and `ci` databases, but GitLab reads and writes only to the + `projects` table in the `main` database. +- For some tables (such as `loose_foreign_keys_deleted_records`) GitLab reads and writes to both the `main` and `ci` databases. See the + [development documentation](../../development/database/multiple_databases.md#gitlab-schema) ## Known issues -- Migrating data from the `main` database to the `ci` database is not supported or documented yet. - Once data is migrated to the `ci` database, you cannot migrate it back. -## Set up multiple databases +## Migrate existing installations -Use the following content to set up multiple databases with a new GitLab installation. +To migrate existing data from the `main` database to the `ci` database, you can +copy the database across. -There is no documentation for existing GitLab installations yet. +### Existing source installation -After you have set up multiple databases, GitLab uses a second application database for -[CI/CD features](../../ci/index.md), referred to as the `ci` database. For -example, GitLab reads and writes to the `ci_pipelines` table in the `ci` -database. +1. Stop GitLab, except for PostgreSQL: + + ```shell + sudo service gitlab stop + sudo service postgresql start + ``` + +1. Dump the `main` database: + + ```shell + sudo -u git pg_dump -f gitlabhq_production.sql gitlabhq_production + ``` + +1. Create the `ci` database, and copy the data from the previous dump: + + ```shell + sudo -u postgres psql -d template1 -c "CREATE DATABASE gitlabhq_production_ci OWNER git;" + sudo -u git psql -f gitlabhq_production.sql gitlabhq_production_ci + ``` + +1. Configure GitLab to [use multiple databases](#set-up-multiple-databases). + +### Existing Omnibus installation + +1. Stop GitLab, except for PostgreSQL: + + ```shell + sudo gitlab-ctl stop + sudo gitlab-ctl start postgresql + ``` + +1. Dump the `main` database: + + ```shell + sudo -u gitlab-psql /opt/gitlab/embedded/bin/pg_dump -h /var/opt/gitlab/postgresql -f gitlabhq_production.sql gitlabhq_production + ``` + +1. Create the `ci` database, and copy the data from the previous dump: + + ```shell + sudo -u gitlab-psql /opt/gitlab/embedded/bin/psql -h /var/opt/gitlab/postgresql -d template1 -c "CREATE DATABASE gitlabhq_production_ci OWNER gitlab;" + sudo -u gitlab-psql /opt/gitlab/embedded/bin/psql -h /var/opt/gitlab/postgresql -f gitlabhq_production.sql gitlabhq_production_ci + ``` + +1. Configure GitLab to [use multiple databases](#set-up-multiple-databases). + +## Set up multiple databases + +To configure GitLab to use multiple application databases, follow the instructions below for your installation type. WARNING: You must stop GitLab before setting up multiple databases. This prevents @@ -40,6 +100,9 @@ the other way around. ### Installations from source +1. For existing installations, + [migrate the data](#migrate-existing-installations) first. + 1. [Back up GitLab](../../raketasks/backup_restore.md) in case of unforeseen issues. @@ -70,7 +133,7 @@ the other way around. 1. Update the service files to set the `GITLAB_ALLOW_SEPARATE_CI_DATABASE` environment variable to `true`. -1. Create the `gitlabhq_production_ci` database: +1. For new installations only. Create the `gitlabhq_production_ci` database: ```shell sudo -u postgres psql -d template1 -c "CREATE DATABASE gitlabhq_production OWNER git;" @@ -91,6 +154,9 @@ the other way around. ### Omnibus GitLab installations +1. For existing installations, + [migrate the data](#migrate-existing-installations) first. + 1. [Back up GitLab](../../raketasks/backup_restore.md) in case of unforeseen issues. @@ -116,7 +182,8 @@ the other way around. sudo gitlab-ctl reconfigure ``` -1. Optional. Reconfiguring GitLab should create the `gitlabhq_production_ci`. If it did not, manually create the `gitlabhq_production_ci`: +1. Optional, for new installations only. Reconfiguring GitLab should create the + `gitlabhq_production_ci` database if it does not exist. If the database is not created automatically, create it manually: ```shell sudo gitlab-ctl start postgresql diff --git a/doc/administration/postgresql/pgbouncer.md b/doc/administration/postgresql/pgbouncer.md index 5dd0aad7162..59aac9141a4 100644 --- a/doc/administration/postgresql/pgbouncer.md +++ b/doc/administration/postgresql/pgbouncer.md @@ -174,10 +174,12 @@ ote_pid | tls ## Procedure for bypassing PgBouncer +### Omnibus installations + Some database changes have to be done directly, and not through PgBouncer. -Read more about the affected tasks: [database restores](../../raketasks/backup_restore.md#back-up-and-restore-for-installations-using-pgbouncer) -and [GitLab upgrades](../../update/zero_downtime.md#postgresql). +The main affected tasks are [database restores](../../raketasks/backup_restore.md#back-up-and-restore-for-installations-using-pgbouncer) +and [GitLab upgrades with database migrations](../../update/zero_downtime.md#postgresql). 1. To find the primary node, run the following on a database node: @@ -195,7 +197,7 @@ and [GitLab upgrades](../../update/zero_downtime.md#postgresql). sudo gitlab-ctl reconfigure ``` -Once you've performed the tasks or procedure, switch back to using PgBouncer: +After you've performed the tasks or procedure, switch back to using PgBouncer: 1. Change back `/etc/gitlab/gitlab.rb` to point to PgBouncer. 1. Run reconfigure: @@ -204,6 +206,19 @@ Once you've performed the tasks or procedure, switch back to using PgBouncer: sudo gitlab-ctl reconfigure ``` +### Helm chart installations + +High-availability deployments also need to bypass PgBouncer for the same reasons as Omnibus-based ones. +For this type of installation: + +- Database backup and restore tasks are performed by the toolbox container. +- Migration tasks are performed by the migrations container. + +You should override the PostgreSQL port on each subchart, so these tasks can execute and connect to PostgreSQL directly: + +- [Toolbox](https://gitlab.com/gitlab-org/charts/gitlab/-/blob/master/charts/gitlab/charts/toolbox/values.yaml#L40) +- [Migrations](https://gitlab.com/gitlab-org/charts/gitlab/-/blob/master/charts/gitlab/charts/migrations/values.yaml#L46) + ## Fine tuning PgBouncer's default settings suit the majority of installations. diff --git a/doc/administration/postgresql/replication_and_failover.md b/doc/administration/postgresql/replication_and_failover.md index 46890b0b2ca..8ac3ac40a75 100644 --- a/doc/administration/postgresql/replication_and_failover.md +++ b/doc/administration/postgresql/replication_and_failover.md @@ -1255,8 +1255,6 @@ To do the switch on **all** PgBouncer nodes: ``` 1. Run `gitlab-ctl reconfigure`. -1. You must also run `rm /var/opt/gitlab/consul/config.d/watcher_postgresql.json`. - This is a [known issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/7293). #### Clean up diff --git a/doc/administration/raketasks/github_import.md b/doc/administration/raketasks/github_import.md index d089682f78e..1babafc902e 100644 --- a/doc/administration/raketasks/github_import.md +++ b/doc/administration/raketasks/github_import.md @@ -6,7 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # GitHub import Rake task **(FREE SELF)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/390690) in GitLab 15.9, Rake task no longer automatically creates namespaces or groups that don't exist. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/390690) in GitLab 15.9, Rake task no longer automatically creates namespaces or groups that don't exist. +> - Requirement for Maintainer role instead of Developer role introduced in GitLab 16.0 and backported to GitLab 15.11.1 and GitLab 15.10.5. To retrieve and import GitHub repositories, you need a [GitHub personal access token](https://github.com/settings/tokens). A username should be passed as the second argument to the Rake task, @@ -19,10 +20,9 @@ before/after the brackets. Also, some shells (for example, Zsh) can interpret th Prerequisite: -- At least the Maintainer role on the destination group to import to. Using the Developer role for this purpose was - [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387891) in GitLab 15.8 and will be removed in GitLab 16.0. +- At least the Maintainer role on the destination group to import to. -## Caveats +## Rate limit If the GitHub [rate limit](https://docs.github.com/en/rest/rate-limit) is reached while importing, the importing process waits (`sleep()`) until it can continue importing. diff --git a/doc/administration/raketasks/maintenance.md b/doc/administration/raketasks/maintenance.md index 5c258d73fdb..06f7203f695 100644 --- a/doc/administration/raketasks/maintenance.md +++ b/doc/administration/raketasks/maintenance.md @@ -11,7 +11,7 @@ GitLab provides Rake tasks for general maintenance. ## Gather GitLab and system information This command gathers information about your GitLab installation and the system it runs on. -These may be useful when asking for help or reporting issues. +These may be useful when asking for help or reporting issues. In a multi-node environment, run this command on nodes running GitLab Rails to avoid PostgreSQL socket errors. **Omnibus Installation** @@ -117,7 +117,7 @@ If you're running Geo, see also the [Geo Health check Rake task](../geo/replicat You may also have a look at our troubleshooting guides for: -- [GitLab](../index.md#troubleshooting) +- [GitLab](../troubleshooting/index.md) - [Omnibus GitLab](https://docs.gitlab.com/omnibus/index.html#troubleshooting) Additionally you should also [verify database values can be decrypted using the current secrets](check.md#verify-database-values-can-be-decrypted-using-the-current-secrets). @@ -386,7 +386,7 @@ You can also [enable reindexing as a regular cron job](https://docs.gitlab.com/o Sometimes you may need to re-import the common metrics that power the Metrics dashboards. -This could be as a result of [updating existing metrics](../../development/prometheus_metrics.md#update-existing-metrics), or as a [troubleshooting measure](../../operations/metrics/dashboards/index.md#troubleshooting). +This could be as a result of [updating existing metrics](../../development/prometheus_metrics.md#update-existing-metrics). To re-import the metrics you can run: diff --git a/doc/administration/raketasks/project_import_export.md b/doc/administration/raketasks/project_import_export.md index 4694af18af2..17a0eb46a30 100644 --- a/doc/administration/raketasks/project_import_export.md +++ b/doc/administration/raketasks/project_import_export.md @@ -4,61 +4,42 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Project import/export administration **(FREE SELF)** +# Project import and export Rake tasks **(FREE SELF)** -GitLab provides Rake tasks relating to project import and export. For more information, see: +GitLab provides Rake tasks for [project import and export](../../user/project/settings/import_export.md). -- [Project import/export documentation](../../user/project/settings/import_export.md). -- [Project import/export API](../../api/project_import_export.md). -- [Developer documentation: project import/export](../../development/import_export.md) +You can only import from a [compatible](../../user/project/settings/import_export.md#compatibility) GitLab instance. -## Project import status +## Import large projects -You can query an import through the [Project import/export API](../../api/project_import_export.md#import-status). -As described in the API documentation, the query may return an import error or exceptions. +> The [Rake task](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/tasks/gitlab/import_export/import.rake) was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20724) in GitLab 12.6, replacing a GitLab.com Ruby script. -## Import/export Rake tasks +This script was introduced in GitLab 12.6 for importing large GitLab project exports. -The GitLab import/export version can be checked by using the following command: +As part of this script, we also disable direct upload. This avoids uploading a huge archive to GCS, which can cause idle transaction timeouts. -```shell -# Omnibus installations -sudo gitlab-rake gitlab:import_export:version +We can run this script from the terminal: -# Installations from source -bundle exec rake gitlab:import_export:version RAILS_ENV=production -``` +Parameters: -The current list of DB tables to export can be listed by using the following command: +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `username` | string | yes | User name | +| `namespace_path` | string | yes | Namespace path | +| `project_path` | string | yes | Project path | +| `archive_path` | string | yes | Path to the exported project tarball you want to import | ```shell -# Omnibus installations -sudo gitlab-rake gitlab:import_export:data - -# Installations from source -bundle exec rake gitlab:import_export:data RAILS_ENV=production +bundle exec rake "gitlab:import_export:import[root, group/subgroup, testingprojectimport, /path/to/file.tar.gz]" ``` -Note the following: - -- Importing is only possible if the version of the import and export GitLab instances are - [compatible](../../user/project/settings/import_export.md#compatibility). -- The project import option must be enabled: - - 1. On the top bar, select **Main menu > Admin**. - 1. On the left sidebar, select **Settings > General**. - 1. Expand **Visibility and access controls**. - 1. Under **Import sources**, check the "Project export enabled" option. - 1. Select **Save changes**. - -- The exports are stored in a temporary directory and are deleted every - 24 hours by a specific worker. - -### Import large projects using a Rake task +If you're running Omnibus, run the following Rake task: -If you have a larger project, consider using a Rake task as described in our [developer documentation](../../development/import_project.md#importing-via-a-rake-task). +```shell +gitlab-rake "gitlab:import_export:import[root, group/subgroup, testingprojectimport, /path/to/file.tar.gz]" +``` -### Export using a Rake task +## Export large projects > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25598) in GitLab 12.9. @@ -88,3 +69,86 @@ IMPORT_DEBUG=true gitlab-rake "gitlab:import_export:import[root, group/subgroup, # Export EXPORT_DEBUG=true gitlab-rake "gitlab:import_export:export[root, group/subgroup, projectnametoexport, /tmp/export_file.tar.gz]" ``` + +Check the common errors listed below, what they mean, and how to fix them. + +### `Exception: undefined method 'name' for nil:NilClass` + +The `username` is not valid. + +### `Exception: undefined method 'full_path' for nil:NilClass` + +The `namespace_path` does not exist. +For example, one of the groups or subgroups is mistyped or missing, +or you've specified the project name in the path. + +The task only creates the project. +If you want to import it to a new group or subgroup, create it first. + +### `Exception: No such file or directory @ rb_sysopen - (filename)` + +The specified project export file in `archive_path` is missing. + +### `Exception: Permission denied @ rb_sysopen - (filename)` + +The specified project export file cannot be accessed by the `git` user. + +To fix the issue: + +1. Set the file owner to `git:git`. +1. Change the file permissions to `0400`. +1. Move the file to a public folder (for example `/tmp/`). + +### `Name can contain only letters, digits, emojis ...` + +```plaintext +Name can contain only letters, digits, emojis, '_', '.', '+', dashes, or spaces. It must start with a letter, +digit, emoji, or '_', and Path can contain only letters, digits, '_', '-', or '.'. It cannot start +with '-', end in '.git', or end in '.atom'. +``` + +The project name specified in `project_path` is not valid for one of the specified reasons. + +Only put the project name in `project_path`. For example, if you provide a path of subgroups +it fails with this error as `/` is not a valid character in a project name. + +### `Name has already been taken and Path has already been taken` + +A project with that name already exists. + +### `Exception: Error importing repository into (namespace) - No space left on device` + +The disk has insufficient space to complete the import. + +During import, the tarball is cached in your configured `shared_path` directory. Verify the +disk has enough free space to accommodate both the cached tarball and the unpacked +project files on disk. + +### Import is successful, but with a `Total number of not imported relations: XX` message, and issues are not created during the import + +If you receive a `Total number of not imported relations: XX` message, and issues +aren't created during the import, check [exceptions_json.log](../logs/index.md#exceptions_jsonlog). +You might see an error like `N is out of range for ActiveModel::Type::Integer with limit 4 bytes`, +where `N` is the integer exceeding the 4-byte integer limit. If that's the case, you +are likely hitting the issue with rebalancing of `relative_position` field of the issues. + +```ruby +# Check the current maximum value of relative_position +Issue.where(project_id: Project.find(ID).root_namespace.all_projects).maximum(:relative_position) + +# Run the rebalancing process and check if the maximum value of relative_position has changed +Issues::RelativePositionRebalancingService.new(Project.find(ID).root_namespace.all_projects).execute +Issue.where(project_id: Project.find(ID).root_namespace.all_projects).maximum(:relative_position) +``` + +Repeat the import attempt and check if the issues are imported successfully. + +### Gitaly calls error when importing + +If you're attempting to import a large project into a development environment, Gitaly might throw an error about too many calls or invocations. For example: + +```plaintext +Error importing repository into qa-perf-testing/gitlabhq - GitalyClient#call called 31 times from single request. Potential n+1? +``` + +This error is due to a [n+1 calls limit for development setups](../../development/gitaly.md#toomanyinvocationserror-errors). To resolve this error, set `GITALY_DISABLE_REQUEST_LIMITS=1` as an environment variable. Then restart your development environment and import again. diff --git a/doc/administration/raketasks/service_desk_email.md b/doc/administration/raketasks/service_desk_email.md index 10de379b1cd..1cbdec35171 100644 --- a/doc/administration/raketasks/service_desk_email.md +++ b/doc/administration/raketasks/service_desk_email.md @@ -12,7 +12,7 @@ The following are Service Desk email-related Rake tasks. ## Secrets -GitLab can use [Service Desk email](../../user/project/service_desk.md#configuring-a-custom-mailbox) secrets read from an encrypted file instead of storing them in plaintext in the file system. The following Rake tasks are provided for updating the contents of the encrypted file. +GitLab can use [Service Desk email](../../user/project/service_desk.md#configure-a-custom-mailbox) secrets read from an encrypted file instead of storing them in plaintext in the file system. The following Rake tasks are provided for updating the contents of the encrypted file. ### Show secret diff --git a/doc/administration/raketasks/uploads/sanitize.md b/doc/administration/raketasks/uploads/sanitize.md index 831abee9739..567a20a37f3 100644 --- a/doc/administration/raketasks/uploads/sanitize.md +++ b/doc/administration/raketasks/uploads/sanitize.md @@ -11,7 +11,7 @@ In GitLab 11.9 and later, EXIF data is automatically stripped from JPG or TIFF i EXIF data may contain sensitive information (for example, GPS location), so you can remove EXIF data from existing images that were uploaded to an earlier version of GitLab. -## Requirements +## Prerequisite To run this Rake task, you need `exiftool` installed on your system. If you installed GitLab: diff --git a/doc/administration/read_only_gitlab.md b/doc/administration/read_only_gitlab.md index 3718741e2e9..3842cf0846b 100644 --- a/doc/administration/read_only_gitlab.md +++ b/doc/administration/read_only_gitlab.md @@ -70,28 +70,28 @@ the database is read-only: in case things don't go as expected. 1. Enter PostgreSQL on the console as an administrator user: - ```shell - sudo \ - -u gitlab-psql /opt/gitlab/embedded/bin/psql \ - -h /var/opt/gitlab/postgresql gitlabhq_production - ``` + ```shell + sudo \ + -u gitlab-psql /opt/gitlab/embedded/bin/psql \ + -h /var/opt/gitlab/postgresql gitlabhq_production + ``` 1. Create the `gitlab_read_only` user. The password is set to `mypassword`, change that to your liking: - ```sql - -- NOTE: Use the password defined earlier - CREATE USER gitlab_read_only WITH password 'mypassword'; - GRANT CONNECT ON DATABASE gitlabhq_production to gitlab_read_only; - GRANT USAGE ON SCHEMA public TO gitlab_read_only; - GRANT SELECT ON ALL TABLES IN SCHEMA public TO gitlab_read_only; - GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO gitlab_read_only; - - -- Tables created by "gitlab" should be made read-only for "gitlab_read_only" - -- automatically. - ALTER DEFAULT PRIVILEGES FOR USER gitlab IN SCHEMA public GRANT SELECT ON TABLES TO gitlab_read_only; - ALTER DEFAULT PRIVILEGES FOR USER gitlab IN SCHEMA public GRANT SELECT ON SEQUENCES TO gitlab_read_only; - ``` + ```sql + -- NOTE: Use the password defined earlier + CREATE USER gitlab_read_only WITH password 'mypassword'; + GRANT CONNECT ON DATABASE gitlabhq_production to gitlab_read_only; + GRANT USAGE ON SCHEMA public TO gitlab_read_only; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO gitlab_read_only; + GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO gitlab_read_only; + + -- Tables created by "gitlab" should be made read-only for "gitlab_read_only" + -- automatically. + ALTER DEFAULT PRIVILEGES FOR USER gitlab IN SCHEMA public GRANT SELECT ON TABLES TO gitlab_read_only; + ALTER DEFAULT PRIVILEGES FOR USER gitlab IN SCHEMA public GRANT SELECT ON SEQUENCES TO gitlab_read_only; + ``` 1. Get the hashed password of the `gitlab_read_only` user and copy the result: @@ -101,10 +101,10 @@ the database is read-only: 1. Edit `/etc/gitlab/gitlab.rb` and add the password from the previous step: - ```ruby - postgresql['sql_user_password'] = 'a2e20f823772650f039284619ab6f239' - postgresql['sql_user'] = "gitlab_read_only" - ``` + ```ruby + postgresql['sql_user_password'] = 'a2e20f823772650f039284619ab6f239' + postgresql['sql_user'] = "gitlab_read_only" + ``` 1. Reconfigure GitLab and restart PostgreSQL: diff --git a/doc/administration/redis/replication_and_failover.md b/doc/administration/redis/replication_and_failover.md index 2e56884e309..a37794ec44b 100644 --- a/doc/administration/redis/replication_and_failover.md +++ b/doc/administration/redis/replication_and_failover.md @@ -766,6 +766,49 @@ redis['master'] = false You can find the relevant attributes defined in [`gitlab_rails.rb`](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/libraries/gitlab_rails.rb). +### Control startup behavior + +> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/6646) in GitLab 15.10. + +To prevent the bundled Redis service from starting at boot or restarting after changing its configuration: + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + redis['start_down'] = true + ``` + +1. Reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +If you need to test a new replica node, you may set `start_down` to +`true` and manually start the node. After the new replica node is confirmed +working in the Redis cluster, set `start_down` to `false` and reconfigure GitLab +to ensure the node starts and restarts as expected during operation. + +### Control replica configuration + +> [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/6646) in GitLab 15.10. + +To prevent the `replicaof` line from rendering in the Redis configuration file: + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + redis['set_replicaof'] = false + ``` + +1. Reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +This setting can be used to prevent replication of a Redis node independently of other Redis settings. + ## Troubleshooting See the [Redis troubleshooting guide](troubleshooting.md). diff --git a/doc/administration/redis/replication_and_failover_external.md b/doc/administration/redis/replication_and_failover_external.md index 23c9ce33c2d..14378b478da 100644 --- a/doc/administration/redis/replication_and_failover_external.md +++ b/doc/administration/redis/replication_and_failover_external.md @@ -32,15 +32,15 @@ Note the Redis node's IP address or hostname, port, and password (if required). 1. Configure the GitLab application servers with the appropriate connection details for your external Redis service in your `/etc/gitlab/gitlab.rb` file: - ```ruby - redis['enable'] = false + ```ruby + redis['enable'] = false - gitlab_rails['redis_host'] = 'redis.example.com' - gitlab_rails['redis_port'] = 6379 + gitlab_rails['redis_host'] = 'redis.example.com' + gitlab_rails['redis_port'] = 6379 - # Required if Redis authentication is configured on the Redis node - gitlab_rails['redis_password'] = 'Redis Password' - ``` + # Required if Redis authentication is configured on the Redis node + gitlab_rails['redis_password'] = 'Redis Password' + ``` 1. Reconfigure for the changes to take effect: @@ -87,7 +87,7 @@ valuable information for the general setup. Assuming that the Redis primary instance IP is `10.0.0.1`: -1. [Install Redis](../../install/installation.md#7-redis). +1. [Install Redis](../../install/installation.md#8-redis). 1. Edit `/etc/redis/redis.conf`: ```conf @@ -113,7 +113,7 @@ Assuming that the Redis primary instance IP is `10.0.0.1`: Assuming that the Redis replica instance IP is `10.0.0.2`: -1. [Install Redis](../../install/installation.md#7-redis). +1. [Install Redis](../../install/installation.md#8-redis). 1. Edit `/etc/redis/redis.conf`: ```conf @@ -250,8 +250,8 @@ unauthorized access from other machines, and block traffic from the outside ([Internet](https://gitlab.com/gitlab-org/gitlab-foss/uploads/c4cc8cd353604bd80315f9384035ff9e/The_Internet_IT_Crowd.png)). For this example, **Sentinel 1** is configured in the same machine as the -**Redis Primary**, **Sentinel 2** and **Sentinel 3** in the same machines as the -**Replica 1** and **Replica 2** respectively. +**Redis Primary**, **Sentinel 2** in the same machine as **Replica 1**, and +**Sentinel 3** in the same machine as **Replica 2**. Here is a list and description of each **machine** and the assigned **IP**: diff --git a/doc/administration/redis/troubleshooting.md b/doc/administration/redis/troubleshooting.md index 29407f65efd..947db2c1d4e 100644 --- a/doc/administration/redis/troubleshooting.md +++ b/doc/administration/redis/troubleshooting.md @@ -113,15 +113,19 @@ To make sure your configuration is correct: redis.info ``` - Keep this screen open and try to simulate a failover below. + Keep this screen open, and proceed to trigger a failover as described below. -1. To simulate a failover on primary Redis, SSH into the Redis server and run: +1. To trigger a failover on the primary Redis, SSH into the Redis server and run: ```shell # port must match your primary redis port, and the sleep time must be a few seconds bigger than defined one redis-cli -h localhost -p 6379 DEBUG sleep 20 ``` + WARNING: + This action affects services, and takes the instance down for up to 20 seconds. If successful, + it should recover after that. + 1. Then back in the Rails console from the first step, run: ```ruby diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index 529621813aa..ba4c5fd9046 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -31,8 +31,8 @@ full list of reference architectures, see | Gitaly5 6 | 3 | 16 vCPU, 60 GB memory | `n1-standard-16` | `m5.4xlarge` | | Praefect5 | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Praefect PostgreSQL1 | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | -| GitLab Rails | 3 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | +| Sidekiq7 | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | +| GitLab Rails7 | 3 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | | Monitoring node | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | | Object storage4 | - | - | - | - | @@ -53,6 +53,8 @@ full list of reference architectures, see 6. Gitaly has been designed and tested with repositories of varying sizes that follow best practices. However, large repositories or monorepos that don't follow these practices can significantly impact Gitaly requirements. Refer to [Large repositories](index.md#large-repositories) for more information. +7. Can be placed in Auto Scaling Groups (ASGs) as the component doesn't store any [stateful data](index.md#autoscaling-of-stateful-nodes). + However, for GitLab Rails certain processes like [migrations](#gitlab-rails-post-configuration) and [Mailroom](../incoming_email.md) should be run on only one node. NOTE: @@ -170,7 +172,7 @@ To set up GitLab and its components to accommodate up to 10,000 users: environment. 1. [Configure the object storage](#configure-the-object-storage) used for shared data objects. -1. [Configure Advanced Search](#configure-advanced-search) (optional) for faster, +1. [Configure advanced search](#configure-advanced-search) (optional) for faster, more advanced code search across your entire GitLab instance. The servers start on the same 10.6.0.0/24 private network range, and can @@ -309,7 +311,7 @@ Configure your load balancer to pass connections on port 443 as `TCP` rather than `HTTP(S)` protocol. This will pass the connection to the application node's NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. -See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html) for details on managing SSL certificates and configuring NGINX. #### Load balancer terminates SSL without backend SSL @@ -320,7 +322,7 @@ terminating SSL. Since communication between the load balancer and GitLab will not be secure, there is some additional configuration needed. See the -[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) +[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) for details. #### Load balancer terminates SSL with backend SSL @@ -333,7 +335,7 @@ Traffic will also be secure between the load balancers and NGINX in this scenario. There is no need to add configuration for proxied SSL since the connection will be secure all the way. However, configuration will need to be added to GitLab to configure SSL certificates. See -the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html) for details on managing SSL certificates and configuring NGINX.
@@ -417,6 +419,11 @@ spread connections equally in practice. ## Configure Consul +Next, we set up the Consul servers. + +NOTE: +Consul must be deployed in an odd number of 3 nodes or more. This is to ensure the nodes can take votes as part of a quorum. + The following IPs will be used as an example: - `10.6.0.11`: Consul 1 @@ -801,6 +808,9 @@ Using [Redis](https://redis.io/) in scalable environment is possible using a **P topology with a [Redis Sentinel](https://redis.io/docs/manual/sentinel/) service to watch and automatically start the failover procedure. +NOTE: +Redis clusters must each be deployed in an odd number of 3 nodes or more. This is to ensure Redis Sentinel can take votes as part of a quorum. This does not apply when configuring Redis externally, such as a cloud provider service. + Redis requires authentication if used with Sentinel. See [Redis Security](https://redis.io/docs/manual/security/) documentation for more information. We recommend using a combination of a Redis password and tight @@ -834,9 +844,9 @@ to be used with GitLab. The following IPs will be used as an example: Managed Redis from cloud providers (such as AWS ElastiCache) will work. If these services support high availability, be sure it _isn't_ of the Redis Cluster type. -Redis version 5.0 or higher is required, which is included with Omnibus GitLab -packages starting with GitLab 13.0. Older Redis versions don't support an -optional count argument to SPOP, which is required for [Merge Trains](../../ci/pipelines/merge_trains.md). + +Because Omnibus GitLab packages ship with Redis 6.0 or later, Redis 6.0 or later is required. Older Redis versions have reached end-of-life. + Note the Redis node's IP address or hostname, port, and password (if required). These will be necessary later when configuring the [GitLab application servers](#configure-gitlab-rails). @@ -1189,7 +1199,7 @@ For more advanced setups refer to the [standalone Gitaly Cluster documentation]( Praefect, the routing and transaction manager for Gitaly Cluster, requires its own database server to store data on Gitaly Cluster status. If you want to have a highly available setup, Praefect requires a third-party PostgreSQL database. -A built-in solution is being [worked on](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5919). +A built-in solution is being [worked on](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/7292). #### Praefect non-HA PostgreSQL standalone using Omnibus GitLab @@ -1336,6 +1346,9 @@ This is how this would work with a Omnibus GitLab PostgreSQL setup: Praefect is the router and transaction manager for Gitaly Cluster and all connections to Gitaly go through it. This section details how to configure it. +NOTE: +Consul must be deployed in an odd number of 3 nodes or more. This is to ensure the nodes can take votes as part of a quorum. + Praefect requires several secret tokens to secure communications across the Cluster: - ``: Used for repositories hosted on your Gitaly cluster and can only be accessed by Gitaly clients that carry this token. @@ -1389,7 +1402,6 @@ Updates to example must be made at: # Praefect Configuration praefect['enable'] = true - praefect['listen_addr'] = '0.0.0.0:2305' # Prevent database migrations from running on upgrade automatically praefect['auto_migrate'] = false @@ -1404,45 +1416,63 @@ Updates to example must be made at: # Please set the real values as explained in Required Information section # - # Praefect External Token - # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster - praefect['auth_token'] = '' - - # Praefect Database Settings - praefect['database_host'] = '10.6.0.141' - praefect['database_port'] = 5432 - # `no_proxy` settings must always be a direct connection for caching - praefect['database_direct_host'] = '10.6.0.141' - praefect['database_direct_port'] = 5432 - praefect['database_dbname'] = 'praefect_production' - praefect['database_user'] = 'praefect' - praefect['database_password'] = '' - - # Praefect Virtual Storage config - # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') - praefect['virtual_storages'] = { - 'default' => { - 'nodes' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '' - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' + praefect['configuration'] = { + # ... + listen_addr: '0.0.0.0:2305', + auth: { + # ... + # + # Praefect External Token + # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster + token: '', + }, + # Praefect Database Settings + database: { + # ... + host: '10.6.0.141', + port: 5432, + # `no_proxy` settings must always be a direct connection for caching + session_pooled: { + # ... + host: '10.6.0.141', + port: 5432, + dbname: 'praefect_production', + user: 'praefect', + password: '', + }, + }, + # Praefect Virtual Storage config + # Name of storage hash must match storage name in git_data_dirs on GitLab + # server ('praefect') and in gitaly['configuration'][:storage] on Gitaly nodes ('gitaly-1') + virtual_storage: [ + { + # ... + name: 'default', + node: [ + { + storage: 'gitaly-1', + address: 'tcp://10.6.0.91:8075', + token: '' + }, + { + storage: 'gitaly-2', + address: 'tcp://10.6.0.92:8075', + token: '' + }, + { + storage: 'gitaly-3', + address: 'tcp://10.6.0.93:8075', + token: '' + }, + ], }, - } - } + ], + # Set the network address Praefect will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9652', } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - praefect['prometheus_listen_addr'] = '0.0.0.0:9652' ## The IPs of the Consul server nodes ## You can also use FQDNs and intermix them with IPs @@ -1493,9 +1523,7 @@ Due to Gitaly having notable input and output requirements, we strongly recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs should have a throughput of at least 8,000 input/output operations per second (IOPS) for read operations and 2,000 IOPS for -write operations. These IOPS values are initial recommendations, and may be -adjusted to greater or lesser values depending on the scale of your -environment's workload. If you're running the environment on a Cloud provider, +write operations. If you're running the environment on a Cloud provider, refer to their documentation about how to configure IOPS correctly. Gitaly servers must not be exposed to the public internet, as Gitaly's network @@ -1505,7 +1533,7 @@ to restrict access to the Gitaly server. Another option is to For configuring Gitaly you should note the following: -- `git_data_dirs` should be configured to reflect the storage path for the specific Gitaly node +- `gitaly['configuration'][:storage]` should be configured to reflect the storage path for the specific Gitaly node - `auth_token` should be the same as `praefect_internal_token` The following IPs will be used as an example: @@ -1554,20 +1582,6 @@ Updates to example must be made at: # Gitaly gitaly['enable'] = true - # Make Gitaly accept connections on all network interfaces. You must use - # firewalls to restrict access to this address/port. - # Comment out following line if you only want to support TLS connections - gitaly['listen_addr'] = "0.0.0.0:8075" - - # Gitaly Auth Token - # Should be the same as praefect_internal_token - gitaly['auth_token'] = '' - - # Gitaly Pack-objects cache - # Recommended to be enabled for improved performance but can notably increase disk I/O - # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info - gitaly['pack_objects_cache_enabled'] = true - # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus @@ -1582,9 +1596,29 @@ Updates to example must be made at: retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address that the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - gitaly['prometheus_listen_addr'] = '0.0.0.0:9236' + + gitaly['configuration'] = { + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + # Comment out following line if you only want to support TLS connections + listen_addr: '0.0.0.0:8075', + # Set the network address that Gitaly will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9236', + auth: { + # Gitaly Auth Token + # Should be the same as praefect_internal_token + token: '', + }, + pack_objects_cache: { + # Gitaly Pack-objects cache + # Recommended to be enabled for improved performance but can notably increase disk I/O + # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info + enabled: true, + }, + } + # # END user configuration ``` @@ -1593,31 +1627,43 @@ Updates to example must be made at: - On Gitaly node 1: ```ruby - git_data_dirs({ - "gitaly-1" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-1', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 2: ```ruby - git_data_dirs({ - "gitaly-2" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-2', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 3: ```ruby - git_data_dirs({ - "gitaly-3" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-3', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace @@ -1637,7 +1683,7 @@ for secure connections, you must: Additionally the certificate, or its certificate authority, must be installed on all Gitaly servers and on all Praefect clients that communicate with it following the procedure described in -[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) (and repeated below). +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl/index.html#install-custom-public-certificates) (and repeated below). Note the following: @@ -1646,7 +1692,7 @@ Note the following: - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. + necessary. To disable the unencrypted listener, set `praefect['configuration'][:listen_addr] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS pass-through. Refer to the load balancers documentation on how to configure this. @@ -1668,9 +1714,15 @@ To configure Praefect with TLS: 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - praefect['tls_listen_addr'] = "0.0.0.0:3305" - praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - praefect['key_path'] = "/etc/gitlab/ssl/key.pem" + praefect['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:3305', + tls: { + # ... + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -1751,7 +1803,7 @@ Updates to example must be made at: # Redis ## Redis connection details - ## First cluster that will host the cache + ## First cluster that will host the cache data gitlab_rails['redis_cache_instance'] = 'redis://:@gitlab-redis-cache' gitlab_rails['redis_cache_sentinels'] = [ @@ -1760,22 +1812,11 @@ Updates to example must be made at: {host: '10.6.0.53', port: 26379}, ] - ## Second cluster that will host the persistent queues, shared state, and actioncable - gitlab_rails['redis_queues_instance'] = 'redis://:@gitlab-redis-persistent' - gitlab_rails['redis_shared_state_instance'] = 'redis://:@gitlab-redis-persistent' - gitlab_rails['redis_actioncable_instance'] = 'redis://:@gitlab-redis-persistent' + ## Second cluster that hosts all other persistent data + redis['master_name'] = 'gitlab-redis-persistent' + redis['master_password'] = '' - gitlab_rails['redis_queues_sentinels'] = [ - {host: '10.6.0.61', port: 26379}, - {host: '10.6.0.62', port: 26379}, - {host: '10.6.0.63', port: 26379}, - ] - gitlab_rails['redis_shared_state_sentinels'] = [ - {host: '10.6.0.61', port: 26379}, - {host: '10.6.0.62', port: 26379}, - {host: '10.6.0.63', port: 26379}, - ] - gitlab_rails['redis_actioncable_sentinels'] = [ + gitlab_rails['redis_sentinels'] = [ {host: '10.6.0.61', port: 26379}, {host: '10.6.0.62', port: 26379}, {host: '10.6.0.63', port: 26379}, @@ -1936,7 +1977,7 @@ On each node perform the following: gitlab_rails['auto_migrate'] = false ## Redis connection details - ## First cluster that will host the cache + ## First cluster that will host the cache data gitlab_rails['redis_cache_instance'] = 'redis://:@gitlab-redis-cache' gitlab_rails['redis_cache_sentinels'] = [ @@ -1945,22 +1986,11 @@ On each node perform the following: {host: '10.6.0.53', port: 26379}, ] - ## Second cluster that will host the persistent queues, shared state, and actioncable - gitlab_rails['redis_queues_instance'] = 'redis://:@gitlab-redis-persistent' - gitlab_rails['redis_shared_state_instance'] = 'redis://:@gitlab-redis-persistent' - gitlab_rails['redis_actioncable_instance'] = 'redis://:@gitlab-redis-persistent' + ## Second cluster that hosts all other persistent data + redis['master_name'] = 'gitlab-redis-persistent' + redis['master_password'] = '' - gitlab_rails['redis_queues_sentinels'] = [ - {host: '10.6.0.61', port: 26379}, - {host: '10.6.0.62', port: 26379}, - {host: '10.6.0.63', port: 26379}, - ] - gitlab_rails['redis_shared_state_sentinels'] = [ - {host: '10.6.0.61', port: 26379}, - {host: '10.6.0.62', port: 26379}, - {host: '10.6.0.63', port: 26379}, - ] - gitlab_rails['redis_actioncable_sentinels'] = [ + gitlab_rails['redis_sentinels'] = [ {host: '10.6.0.61', port: 26379}, {host: '10.6.0.62', port: 26379}, {host: '10.6.0.63', port: 26379}, @@ -2058,7 +2088,7 @@ On each node perform the following: When you specify `https` in the `external_url`, as in the previous example, GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the certificates aren't present, NGINX will fail to start. For more information, see -the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html). +the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html). ### GitLab Rails post-configuration @@ -2070,11 +2100,9 @@ the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html). sudo gitlab-rake gitlab:db:configure ``` - If you encounter a `rake aborted!` error message stating that PgBouncer is - failing to connect to PostgreSQL, it may be that your PgBouncer node's IP - address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` - on your database nodes. Before proceeding, see - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). + Note that this requires the Rails node to be configured to connect to the primary database + directly, [bypassing PgBouncer](../postgresql/pgbouncer.md#procedure-for-bypassing-pgbouncer). + After migrations have completed, you must configure the node to pass through PgBouncer again. 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). @@ -2170,15 +2198,15 @@ and scalable. There are two ways of specifying object storage configuration in GitLab: -- [Consolidated form](../object_storage.md#consolidated-object-storage-configuration): A single credential is +- [Consolidated form](../object_storage.md#configure-a-single-storage-connection-for-all-object-types-consolidated-form): A single credential is shared by all supported object types. -- [Storage-specific form](../object_storage.md#storage-specific-configuration): Every object defines its - own object storage [connection and configuration](../object_storage.md#connection-settings). +- [Storage-specific form](../object_storage.md#configure-each-object-type-to-define-its-own-storage-connection-storage-specific-form): Every object defines its + own object storage [connection and configuration](../object_storage.md#configure-the-connection-settings). The consolidated form is used in the following examples when available. NOTE: -When using the [storage-specific form](../object_storage.md#storage-specific-configuration) +When using the [storage-specific form](../object_storage.md#configure-each-object-type-to-define-its-own-storage-connection-storage-specific-form) in GitLab 14.x and earlier, you should enable [direct upload mode](../../development/uploads/index.md#direct-upload). The previous [background upload](../../development/uploads/index.md#direct-upload) mode, which was deprecated in 14.9, requires shared storage such as NFS. @@ -2200,9 +2228,9 @@ GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily While sharing the job logs through NFS is supported, it's recommended to avoid the need to use NFS by enabling [incremental logging](../job_logs.md#incremental-logging-architecture) (required when no NFS node has been deployed). Incremental logging uses Redis instead of disk space for temporary caching of job logs. -## Configure Advanced Search +## Configure advanced search -You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) +You can leverage Elasticsearch and [enable advanced search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific diff --git a/doc/administration/reference_architectures/1k_users.md b/doc/administration/reference_architectures/1k_users.md index d3aa6fef51e..adcf8eeb4c6 100644 --- a/doc/administration/reference_architectures/1k_users.md +++ b/doc/administration/reference_architectures/1k_users.md @@ -78,9 +78,9 @@ You can also optionally configure GitLab to use an [external PostgreSQL service] or an [external object storage service](../object_storage.md) for added performance and reliability at an increased complexity cost. -## Configure Advanced Search **(PREMIUM SELF)** +## Configure advanced search **(PREMIUM SELF)** -You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) +You can leverage Elasticsearch and [enable advanced search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index 71fe8b301e2..7984b9dd6c7 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -31,8 +31,8 @@ full list of reference architectures, see | Gitaly5 6 | 3 | 32 vCPU, 120 GB memory | `n1-standard-32` | `m5.8xlarge` | | Praefect5 | 3 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | | Praefect PostgreSQL1 | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | -| GitLab Rails | 5 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | +| Sidekiq7 | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | +| GitLab Rails7 | 5 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | | Monitoring node | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | | Object storage4 | - | - | - | - | @@ -53,6 +53,8 @@ full list of reference architectures, see 6. Gitaly has been designed and tested with repositories of varying sizes that follow best practices. However, large repositories or monorepos that don't follow these practices can significantly impact Gitaly requirements. Refer to [Large repositories](index.md#large-repositories) for more information. +7. Can be placed in Auto Scaling Groups (ASGs) as the component doesn't store any [stateful data](index.md#autoscaling-of-stateful-nodes). + However, for GitLab Rails certain processes like [migrations](#gitlab-rails-post-configuration) and [Mailroom](../incoming_email.md) should be run on only one node. NOTE: @@ -170,7 +172,7 @@ To set up GitLab and its components to accommodate up to 25,000 users: environment. 1. [Configure the object storage](#configure-the-object-storage) used for shared data objects. -1. [Configure Advanced Search](#configure-advanced-search) (optional) for faster, +1. [Configure advanced search](#configure-advanced-search) (optional) for faster, more advanced code search across your entire GitLab instance. The servers start on the same 10.6.0.0/24 private network range, and can @@ -320,7 +322,7 @@ Configure your load balancer to pass connections on port 443 as `TCP` rather than `HTTP(S)` protocol. This will pass the connection to the application node's NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. -See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html) for details on managing SSL certificates and configuring NGINX. #### Load balancer terminates SSL without backend SSL @@ -331,7 +333,7 @@ terminating SSL. Since communication between the load balancer and GitLab will not be secure, there is some additional configuration needed. See the -[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) +[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) for details. #### Load balancer terminates SSL with backend SSL @@ -344,7 +346,7 @@ Traffic will also be secure between the load balancers and NGINX in this scenario. There is no need to add configuration for proxied SSL since the connection will be secure all the way. However, configuration will need to be added to GitLab to configure SSL certificates. See -the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html) for details on managing SSL certificates and configuring NGINX.
@@ -434,6 +436,11 @@ spread connections equally in practice. ## Configure Consul +Next, we set up the Consul servers. + +NOTE: +Consul must be deployed in an odd number of 3 nodes or more. This is to ensure the nodes can take votes as part of a quorum. + The following IPs will be used as an example: - `10.6.0.11`: Consul 1 @@ -818,6 +825,9 @@ Using [Redis](https://redis.io/) in scalable environment is possible using a **P topology with a [Redis Sentinel](https://redis.io/docs/manual/sentinel/) service to watch and automatically start the failover procedure. +NOTE: +Redis clusters must each be deployed in an odd number of 3 nodes or more. This is to ensure Redis Sentinel can take votes as part of a quorum. This does not apply when configuring Redis externally, such as a cloud provider service. + Redis requires authentication if used with Sentinel. See [Redis Security](https://redis.io/docs/manual/security/) documentation for more information. We recommend using a combination of a Redis password and tight @@ -851,9 +861,9 @@ to be used with GitLab. The following IPs will be used as an example: Managed Redis from cloud providers (such as AWS ElastiCache) will work. If these services support high availability, be sure it _isn't_ of the Redis Cluster type. -Redis version 5.0 or higher is required, which is included with Omnibus GitLab -packages starting with GitLab 13.0. Older Redis versions don't support an -optional count argument to SPOP, which is required for [Merge Trains](../../ci/pipelines/merge_trains.md). + +Because Omnibus GitLab packages ship with Redis 6.0 or later, Redis 6.0 or later is required. Older Redis versions have reached end-of-life. + Note the Redis node's IP address or hostname, port, and password (if required). These will be necessary later when configuring the [GitLab application servers](#configure-gitlab-rails). @@ -1208,7 +1218,7 @@ For more advanced setups refer to the [standalone Gitaly Cluster documentation]( Praefect, the routing and transaction manager for Gitaly Cluster, requires its own database server to store data on Gitaly Cluster status. If you want to have a highly available setup, Praefect requires a third-party PostgreSQL database. -A built-in solution is being [worked on](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5919). +A built-in solution is being [worked on](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/7292). #### Praefect non-HA PostgreSQL standalone using Omnibus GitLab @@ -1353,6 +1363,9 @@ This is how this would work with a Omnibus GitLab PostgreSQL setup: Praefect is the router and transaction manager for Gitaly Cluster and all connections to Gitaly go through it. This section details how to configure it. +NOTE: +Praefect must be deployed in an odd number of 3 nodes or more. This is to ensure the nodes can take votes as part of a quorum. + Praefect requires several secret tokens to secure communications across the Cluster: - ``: Used for repositories hosted on your Gitaly cluster and can only be accessed by Gitaly clients that carry this token. @@ -1406,7 +1419,6 @@ Updates to example must be made at: # Praefect Configuration praefect['enable'] = true - praefect['listen_addr'] = '0.0.0.0:2305' # Prevent database migrations from running on upgrade automatically praefect['auto_migrate'] = false @@ -1415,51 +1427,69 @@ Updates to example must be made at: # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus - consul['monitoring_service_discovery'] = true + consul['monitoring_service_discovery'] = true # START user configuration # Please set the real values as explained in Required Information section # - # Praefect External Token - # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster - praefect['auth_token'] = '' - - # Praefect Database Settings - praefect['database_host'] = '10.6.0.141' - praefect['database_port'] = 5432 - # `no_proxy` settings must always be a direct connection for caching - praefect['database_direct_host'] = '10.6.0.141' - praefect['database_direct_port'] = 5432 - praefect['database_dbname'] = 'praefect_production' - praefect['database_user'] = 'praefect' - praefect['database_password'] = '' - - # Praefect Virtual Storage config - # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') - praefect['virtual_storages'] = { - 'default' => { - 'nodes' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '' - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' + praefect['configuration'] = { + # ... + listen_addr: '0.0.0.0:2305', + auth: { + # ... + # + # Praefect External Token + # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster + token: '', + }, + # Praefect Database Settings + database: { + # ... + host: '10.6.0.141', + port: 5432, + # `no_proxy` settings must always be a direct connection for caching + session_pooled: { + # ... + host: '10.6.0.141', + port: 5432, + dbname: 'praefect_production', + user: 'praefect', + password: '', + }, + }, + # Praefect Virtual Storage config + # Name of storage hash must match storage name in git_data_dirs on GitLab + # server ('praefect') and in gitaly['configuration'][:storage] on Gitaly nodes ('gitaly-1') + virtual_storage: [ + { + # ... + name: 'default', + node: [ + { + storage: 'gitaly-1', + address: 'tcp://10.6.0.91:8075', + token: '' + }, + { + storage: 'gitaly-2', + address: 'tcp://10.6.0.92:8075', + token: '' + }, + { + storage: 'gitaly-3', + address: 'tcp://10.6.0.93:8075', + token: '' + }, + ], }, - } - } + ], + # Set the network address Praefect will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9652', } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - praefect['prometheus_listen_addr'] = '0.0.0.0:9652' ## The IPs of the Consul server nodes ## You can also use FQDNs and intermix them with IPs @@ -1510,9 +1540,7 @@ Due to Gitaly having notable input and output requirements, we strongly recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs should have a throughput of at least 8,000 input/output operations per second (IOPS) for read operations and 2,000 IOPS for -write operations. These IOPS values are initial recommendations, and may be -adjusted to greater or lesser values depending on the scale of your -environment's workload. If you're running the environment on a Cloud provider, +write operations. If you're running the environment on a Cloud provider, refer to their documentation about how to configure IOPS correctly. Gitaly servers must not be exposed to the public internet, as Gitaly's network @@ -1522,7 +1550,7 @@ to restrict access to the Gitaly server. Another option is to For configuring Gitaly you should note the following: -- `git_data_dirs` should be configured to reflect the storage path for the specific Gitaly node +- `gitaly['configuration'][:storage]` should be configured to reflect the storage path for the specific Gitaly node - `auth_token` should be the same as `praefect_internal_token` The following IPs will be used as an example: @@ -1571,20 +1599,6 @@ Updates to example must be made at: # Gitaly gitaly['enable'] = true - # Make Gitaly accept connections on all network interfaces. You must use - # firewalls to restrict access to this address/port. - # Comment out following line if you only want to support TLS connections - gitaly['listen_addr'] = "0.0.0.0:8075" - - # Gitaly Auth Token - # Should be the same as praefect_internal_token - gitaly['auth_token'] = '' - - # Gitaly Pack-objects cache - # Recommended to be enabled for improved performance but can notably increase disk I/O - # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info - gitaly['pack_objects_cache_enabled'] = true - # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus @@ -1599,9 +1613,29 @@ Updates to example must be made at: retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address that the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - gitaly['prometheus_listen_addr'] = '0.0.0.0:9236' + + gitaly['configuration'] = { + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + # Comment out following line if you only want to support TLS connections + listen_addr: '0.0.0.0:8075', + # Set the network address that Gitaly will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9236', + auth: { + # Gitaly Auth Token + # Should be the same as praefect_internal_token + token: '', + }, + pack_objects_cache: { + # Gitaly Pack-objects cache + # Recommended to be enabled for improved performance but can notably increase disk I/O + # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info + enabled: true, + }, + } + # # END user configuration ``` @@ -1610,31 +1644,43 @@ Updates to example must be made at: - On Gitaly node 1: ```ruby - git_data_dirs({ - "gitaly-1" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-1', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 2: ```ruby - git_data_dirs({ - "gitaly-2" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-2', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 3: ```ruby - git_data_dirs({ - "gitaly-3" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-3', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace @@ -1654,7 +1700,7 @@ for secure connections, you must: Additionally the certificate, or its certificate authority, must be installed on all Gitaly servers and on all Praefect clients that communicate with it following the procedure described in -[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) (and repeated below). +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl/index.html#install-custom-public-certificates) (and repeated below). Note the following: @@ -1663,7 +1709,7 @@ Note the following: - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. + necessary. To disable the unencrypted listener, set `praefect['configuration'][:listen_addr] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS passthrough. Refer to the load balancers documentation on how to configure this. @@ -1685,9 +1731,15 @@ To configure Praefect with TLS: 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - praefect['tls_listen_addr'] = "0.0.0.0:3305" - praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - praefect['key_path'] = "/etc/gitlab/ssl/key.pem" + praefect['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:3305', + tls: { + # ... + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -1768,7 +1820,7 @@ Updates to example must be made at: # Redis ## Redis connection details - ## First cluster that will host the cache + ## First cluster that will host the cache data gitlab_rails['redis_cache_instance'] = 'redis://:@gitlab-redis-cache' gitlab_rails['redis_cache_sentinels'] = [ @@ -1777,22 +1829,11 @@ Updates to example must be made at: {host: '10.6.0.53', port: 26379}, ] - ## Second cluster that will host the persistent queues, shared state, and actioncable - gitlab_rails['redis_queues_instance'] = 'redis://:@gitlab-redis-persistent' - gitlab_rails['redis_shared_state_instance'] = 'redis://:@gitlab-redis-persistent' - gitlab_rails['redis_actioncable_instance'] = 'redis://:@gitlab-redis-persistent' + ## Second cluster that hosts all other persistent data + redis['master_name'] = 'gitlab-redis-persistent' + redis['master_password'] = '' - gitlab_rails['redis_queues_sentinels'] = [ - {host: '10.6.0.61', port: 26379}, - {host: '10.6.0.62', port: 26379}, - {host: '10.6.0.63', port: 26379}, - ] - gitlab_rails['redis_shared_state_sentinels'] = [ - {host: '10.6.0.61', port: 26379}, - {host: '10.6.0.62', port: 26379}, - {host: '10.6.0.63', port: 26379}, - ] - gitlab_rails['redis_actioncable_sentinels'] = [ + gitlab_rails['redis_sentinels'] = [ {host: '10.6.0.61', port: 26379}, {host: '10.6.0.62', port: 26379}, {host: '10.6.0.63', port: 26379}, @@ -1955,7 +1996,7 @@ On each node perform the following: gitlab_rails['auto_migrate'] = false ## Redis connection details - ## First cluster that will host the cache + ## First cluster that will host the cache data gitlab_rails['redis_cache_instance'] = 'redis://:@gitlab-redis-cache' gitlab_rails['redis_cache_sentinels'] = [ @@ -1964,22 +2005,11 @@ On each node perform the following: {host: '10.6.0.53', port: 26379}, ] - ## Second cluster that will host the queues, shared state, and actionable - gitlab_rails['redis_queues_instance'] = 'redis://:@gitlab-redis-persistent' - gitlab_rails['redis_shared_state_instance'] = 'redis://:@gitlab-redis-persistent' - gitlab_rails['redis_actioncable_instance'] = 'redis://:@gitlab-redis-persistent' + ## Second cluster that hosts all other persistent data + redis['master_name'] = 'gitlab-redis-persistent' + redis['master_password'] = '' - gitlab_rails['redis_queues_sentinels'] = [ - {host: '10.6.0.61', port: 26379}, - {host: '10.6.0.62', port: 26379}, - {host: '10.6.0.63', port: 26379}, - ] - gitlab_rails['redis_shared_state_sentinels'] = [ - {host: '10.6.0.61', port: 26379}, - {host: '10.6.0.62', port: 26379}, - {host: '10.6.0.63', port: 26379}, - ] - gitlab_rails['redis_actioncable_sentinels'] = [ + gitlab_rails['redis_sentinels'] = [ {host: '10.6.0.61', port: 26379}, {host: '10.6.0.62', port: 26379}, {host: '10.6.0.63', port: 26379}, @@ -2077,7 +2107,7 @@ On each node perform the following: When you specify `https` in the `external_url`, as in the previous example, GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the certificates aren't present, NGINX will fail to start. For more information, see -the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html). +the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html). ### GitLab Rails post-configuration @@ -2089,11 +2119,9 @@ the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html). sudo gitlab-rake gitlab:db:configure ``` - If you encounter a `rake aborted!` error message stating that PgBouncer is - failing to connect to PostgreSQL, it may be that your PgBouncer node's IP - address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` - on your database nodes. Before proceeding, see - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). + Note that this requires the Rails node to be configured to connect to the primary database + directly, [bypassing PgBouncer](../postgresql/pgbouncer.md#procedure-for-bypassing-pgbouncer). + After migrations have completed, you must configure the node to pass through PgBouncer again. 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). @@ -2188,15 +2216,15 @@ and scalable. There are two ways of specifying object storage configuration in GitLab: -- [Consolidated form](../object_storage.md#consolidated-object-storage-configuration): A single credential is +- [Consolidated form](../object_storage.md#configure-a-single-storage-connection-for-all-object-types-consolidated-form): A single credential is shared by all supported object types. -- [Storage-specific form](../object_storage.md#storage-specific-configuration): Every object defines its - own object storage [connection and configuration](../object_storage.md#connection-settings). +- [Storage-specific form](../object_storage.md#configure-each-object-type-to-define-its-own-storage-connection-storage-specific-form): Every object defines its + own object storage [connection and configuration](../object_storage.md#configure-the-connection-settings). The consolidated form is used in the following examples when available. NOTE: -When using the [storage-specific form](../object_storage.md#storage-specific-configuration) +When using the [storage-specific form](../object_storage.md#configure-each-object-type-to-define-its-own-storage-connection-storage-specific-form) in GitLab 14.x and earlier, you should enable [direct upload mode](../../development/uploads/index.md#direct-upload). The previous [background upload](../../development/uploads/index.md#direct-upload) mode, which was deprecated in 14.9, requires shared storage such as NFS. @@ -2218,9 +2246,9 @@ GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily While sharing the job logs through NFS is supported, it's recommended to avoid the need to use NFS by enabling [incremental logging](../job_logs.md#incremental-logging-architecture) (required when no NFS node has been deployed). Incremental logging uses Redis instead of disk space for temporary caching of job logs. -## Configure Advanced Search +## Configure advanced search -You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) +You can leverage Elasticsearch and [enable advanced search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index ee26504902c..14dc9d26293 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -26,7 +26,7 @@ For a full list of reference architectures, see | PostgreSQL1 | 1 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | `D2s v3` | | Redis2 | 1 | 1 vCPU, 3.75 GB memory | `n1-standard-1` | `m5.large` | `D2s v3` | | Gitaly5 | 1 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | `D4s v3` | -| GitLab Rails | 2 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | +| GitLab Rails6 | 2 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | `F8s v2` | | Monitoring node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | `F2s v2` | | Object storage4 | - | - | - | - | - | @@ -45,6 +45,8 @@ For a full list of reference architectures, see 5. Gitaly has been designed and tested with repositories of varying sizes that follow best practices. However, large repositories or monorepos that don't follow these practices can significantly impact Gitaly requirements. Refer to [Large repositories](index.md#large-repositories) for more information. +6. Can be placed in Auto Scaling Groups (ASGs) as the component doesn't store any [stateful data](index.md#autoscaling-of-stateful-nodes). + However, for GitLab Rails certain processes like [migrations](#gitlab-rails-post-configuration) and [Mailroom](../incoming_email.md) should be run on only one node. NOTE: @@ -101,7 +103,7 @@ To set up GitLab and its components to accommodate up to 2,000 users: environment. 1. [Configure the object storage](#configure-the-object-storage) used for shared data objects. -1. [Configure Advanced Search](#configure-advanced-search) (optional) for faster, +1. [Configure advanced search](#configure-advanced-search) (optional) for faster, more advanced code search across your entire GitLab instance. ## Configure the external load balancer @@ -211,7 +213,7 @@ Configure your load balancer to pass connections on port 443 as `TCP` rather than `HTTP(S)` protocol. This will pass the connection to the application node's NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. -See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html) for details on managing SSL certificates and configuring NGINX. #### Load balancer terminates SSL without backend SSL @@ -222,7 +224,7 @@ terminating SSL. Since communication between the load balancer and GitLab will not be secure, there is some additional configuration needed. See the -[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) +[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) for details. #### Load balancer terminates SSL with backend SSL @@ -235,7 +237,7 @@ Traffic will also be secure between the load balancers and NGINX in this scenario. There is no need to add configuration for proxied SSL since the connection will be secure all the way. However, configuration will need to be added to GitLab to configure SSL certificates. See -the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html) for details on managing SSL certificates and configuring NGINX.
@@ -339,13 +341,7 @@ to be used with GitLab. ### Provide your own Redis instance -Redis version 5.0 or higher is required, as this is what ships with -Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions -do not support an optional count argument to SPOP which is now required for -[Merge Trains](../../ci/pipelines/merge_trains.md). - -In addition, GitLab makes use of certain commands like `UNLINK` and `USAGE` which -were introduced only in Redis 4. +Because Omnibus GitLab packages ship with Redis 6.0 or later, Redis 6.0 or later is required. Older Redis versions have reached end-of-life. Managed Redis from cloud providers such as AWS ElastiCache will work. If these services support high availability, be sure it is not the Redis Cluster type. @@ -423,9 +419,7 @@ Due to Gitaly having notable input and output requirements, we strongly recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs should have a throughput of at least 8,000 input/output operations per second (IOPS) for read operations and 2,000 IOPS -for write operations. These IOPS values are initial recommendations, and may be -adjusted to greater or lesser values depending on the scale of your -environment's workload. If you're running the environment on a Cloud provider, +for write operations. If you're running the environment on a Cloud provider, refer to their documentation about how to configure IOPS correctly. Be sure to note the following items: @@ -459,7 +453,7 @@ To configure the Gitaly server, on the server node you want to use for Gitaly: storage paths, enable the network listener, and to configure the token: NOTE: - You can't remove the `default` entry from `git_data_dirs` because [GitLab requires it](../gitaly/configure_gitaly.md#gitlab-requires-a-default-repository-storage). + You can't remove the `default` entry from `gitaly['configuration'][:storage]` because [GitLab requires it](../gitaly/configure_gitaly.md#gitlab-requires-a-default-repository-storage). ```ruby - gitaly['tls_listen_addr'] = "0.0.0.0:9999" - gitaly['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - gitaly['key_path'] = "/etc/gitlab/ssl/key.pem" + gitaly['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:9999', + tls: { + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Delete `gitaly['listen_addr']` to allow only encrypted connections. @@ -739,7 +756,7 @@ On each node perform the following: When you specify `https` in the `external_url`, as in the previous example, GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the certificates aren't present, NGINX will fail to start. For more information, see -the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html). +the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html). ### GitLab Rails post-configuration @@ -751,11 +768,9 @@ the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html). sudo gitlab-rake gitlab:db:configure ``` - If you encounter a `rake aborted!` error message stating that PgBouncer is - failing to connect to PostgreSQL, it may be that your PgBouncer node's IP - address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` - on your database nodes. Before proceeding, see - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). + Note that this requires the Rails node to be configured to connect to the primary database + directly, [bypassing PgBouncer](../postgresql/pgbouncer.md#procedure-for-bypassing-pgbouncer). + After migrations have completed, you must configure the node to pass through PgBouncer again. 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). @@ -881,15 +896,15 @@ and scalable. There are two ways of specifying object storage configuration in GitLab: -- [Consolidated form](../object_storage.md#consolidated-object-storage-configuration): A single credential is +- [Consolidated form](../object_storage.md#configure-a-single-storage-connection-for-all-object-types-consolidated-form): A single credential is shared by all supported object types. -- [Storage-specific form](../object_storage.md#storage-specific-configuration): Every object defines its - own object storage [connection and configuration](../object_storage.md#connection-settings). +- [Storage-specific form](../object_storage.md#configure-each-object-type-to-define-its-own-storage-connection-storage-specific-form): Every object defines its + own object storage [connection and configuration](../object_storage.md#configure-the-connection-settings). The consolidated form is used in the following examples when available. NOTE: -When using the [storage-specific form](../object_storage.md#storage-specific-configuration) +When using the [storage-specific form](../object_storage.md#configure-each-object-type-to-define-its-own-storage-connection-storage-specific-form) in GitLab 14.x and earlier, you should enable [direct upload mode](../../development/uploads/index.md#direct-upload). The previous [background upload](../../development/uploads/index.md#direct-upload) mode, which was deprecated in 14.9, requires shared storage such as NFS. @@ -911,9 +926,9 @@ GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily While sharing the job logs through NFS is supported, it's recommended to avoid the need to use NFS by enabling [incremental logging](../job_logs.md#incremental-logging-architecture) (required when no NFS node has been deployed). Incremental logging uses Redis instead of disk space for temporary caching of job logs. -## Configure Advanced Search **(PREMIUM SELF)** +## Configure advanced search **(PREMIUM SELF)** -You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) +You can leverage Elasticsearch and [enable advanced search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index 1d5dad02b18..191e5218c43 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -40,8 +40,8 @@ For a full list of reference architectures, see | Gitaly5 6 | 3 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | | Praefect5 | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Praefect PostgreSQL1 | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Sidekiq | 4 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | -| GitLab Rails | 3 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | +| Sidekiq7 | 4 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | +| GitLab Rails7 | 3 | 8 vCPU, 7.2 GB memory | `n1-highcpu-8` | `c5.2xlarge` | | Monitoring node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Object storage4 | - | - | - | - | @@ -62,6 +62,8 @@ For a full list of reference architectures, see 6. Gitaly has been designed and tested with repositories of varying sizes that follow best practices. However, large repositories or monorepos that don't follow these practices can significantly impact Gitaly requirements. Refer to [Large repositories](index.md#large-repositories) for more information. +7. Can be placed in Auto Scaling Groups (ASGs) as the component doesn't store any [stateful data](index.md#autoscaling-of-stateful-nodes). + However, for GitLab Rails certain processes like [migrations](#gitlab-rails-post-configuration) and [Mailroom](../incoming_email.md) should be run on only one node. NOTE: @@ -176,7 +178,7 @@ To set up GitLab and its components to accommodate up to 3,000 users: environment. 1. [Configure the object storage](#configure-the-object-storage) used for shared data objects. -1. [Configure Advanced Search](#configure-advanced-search) (optional) for faster, +1. [Configure advanced search](#configure-advanced-search) (optional) for faster, more advanced code search across your entire GitLab instance. The servers start on the same 10.6.0.0/24 private network range, and can @@ -321,7 +323,7 @@ Configure your load balancer to pass connections on port 443 as `TCP` rather than `HTTP(S)` protocol. This will pass the connection to the application node's NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. -See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html) for details on managing SSL certificates and configuring NGINX. #### Load balancer terminates SSL without backend SSL @@ -332,7 +334,7 @@ terminating SSL. Since communication between the load balancer and GitLab will not be secure, there is some additional configuration needed. See the -[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) +[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) for details. #### Load balancer terminates SSL with backend SSL @@ -345,7 +347,7 @@ Traffic will also be secure between the load balancers and NGINX in this scenario. There is no need to add configuration for proxied SSL since the connection will be secure all the way. However, configuration will need to be added to GitLab to configure SSL certificates. See -the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html) for details on managing SSL certificates and configuring NGINX.
@@ -453,10 +455,7 @@ to be used with GitLab. The following IPs will be used as an example: Managed Redis from cloud providers such as AWS ElastiCache will work. If these services support high availability, be sure it is **not** the Redis Cluster type. -Redis version 5.0 or higher is required, as this is what ships with -Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions -do not support an optional count argument to SPOP which is now required for -[Merge Trains](../../ci/pipelines/merge_trains.md). +Because Omnibus GitLab packages ship with Redis 6.0 or later, Redis 6.0 or later is required. Older Redis versions have reached end-of-life. Note the Redis node's IP address or hostname, port, and password (if required). These will be necessary when configuring the @@ -640,8 +639,13 @@ are supported and can be added if needed. ## Configure Consul and Sentinel -Now that the Redis servers are all set up, let's configure the Sentinel -servers. The following IPs will be used as an example: +Now that the Redis servers are all set up, let's configure the Consul + Sentinel +servers. + +NOTE: +Consul and Redis Sentinel must be deployed in an odd number of 3 nodes or more. This is to ensure the nodes can take votes as part of a quorum. + +The following IPs will be used as an example: - `10.6.0.11`: Consul/Sentinel 1 - `10.6.0.12`: Consul/Sentinel 2 @@ -1144,7 +1148,7 @@ For more advanced setups refer to the [standalone Gitaly Cluster documentation]( Praefect, the routing and transaction manager for Gitaly Cluster, requires its own database server to store data on Gitaly Cluster status. If you want to have a highly available setup, Praefect requires a third-party PostgreSQL database. -A built-in solution is being [worked on](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5919). +A built-in solution is being [worked on](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/7292). #### Praefect non-HA PostgreSQL standalone using Omnibus GitLab @@ -1288,6 +1292,9 @@ This is how this would work with a Omnibus GitLab PostgreSQL setup: Praefect is the router and transaction manager for Gitaly Cluster and all connections to Gitaly go through it. This section details how to configure it. +NOTE: +Praefect must be deployed in an odd number of 3 nodes or more. This is to ensure the nodes can take votes as part of a quorum. + Praefect requires several secret tokens to secure communications across the Cluster: - ``: Used for repositories hosted on your Gitaly cluster and can only be accessed by Gitaly clients that carry this token. @@ -1341,7 +1348,6 @@ Updates to example must be made at: # Praefect Configuration praefect['enable'] = true - praefect['listen_addr'] = '0.0.0.0:2305' # Prevent database migrations from running on upgrade automatically praefect['auto_migrate'] = false @@ -1350,51 +1356,69 @@ Updates to example must be made at: # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus - consul['monitoring_service_discovery'] = true + consul['monitoring_service_discovery'] = true # START user configuration # Please set the real values as explained in Required Information section # - # Praefect External Token - # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster - praefect['auth_token'] = '' - - # Praefect Database Settings - praefect['database_host'] = '10.6.0.141' - praefect['database_port'] = 5432 - # `no_proxy` settings must always be a direct connection for caching - praefect['database_direct_host'] = '10.6.0.141' - praefect['database_direct_port'] = 5432 - praefect['database_dbname'] = 'praefect_production' - praefect['database_user'] = 'praefect' - praefect['database_password'] = '' - - # Praefect Virtual Storage config - # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') - praefect['virtual_storages'] = { - 'default' => { - 'nodes' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '' - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' + praefect['configuration'] = { + # ... + listen_addr: '0.0.0.0:2305', + auth: { + # ... + # + # Praefect External Token + # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster + token: '', + }, + # Praefect Database Settings + database: { + # ... + host: '10.6.0.141', + port: 5432, + # `no_proxy` settings must always be a direct connection for caching + session_pooled: { + # ... + host: '10.6.0.141', + port: 5432, + dbname: 'praefect_production', + user: 'praefect', + password: '', + }, + }, + # Praefect Virtual Storage config + # Name of storage hash must match storage name in git_data_dirs on GitLab + # server ('praefect') and in gitaly['configuration'][:storage] on Gitaly nodes ('gitaly-1') + virtual_storage: [ + { + # ... + name: 'default', + node: [ + { + storage: 'gitaly-1', + address: 'tcp://10.6.0.91:8075', + token: '' + }, + { + storage: 'gitaly-2', + address: 'tcp://10.6.0.92:8075', + token: '' + }, + { + storage: 'gitaly-3', + address: 'tcp://10.6.0.93:8075', + token: '' + }, + ], }, - } - } + ], + # Set the network address Praefect will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9652', } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - praefect['prometheus_listen_addr'] = '0.0.0.0:9652' ## The IPs of the Consul server nodes ## You can also use FQDNs and intermix them with IPs @@ -1445,9 +1469,7 @@ Due to Gitaly having notable input and output requirements, we strongly recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs should have a throughput of at least 8,000 input/output operations per second (IOPS) for read operations and 2,000 IOPS for -write operations. These IOPS values are initial recommendations, and may be -adjusted to greater or lesser values depending on the scale of your -environment's workload. If you're running the environment on a Cloud provider, +write operations. If you're running the environment on a Cloud provider, refer to their documentation about how to configure IOPS correctly. Gitaly servers must not be exposed to the public internet, as Gitaly's network @@ -1457,7 +1479,7 @@ to restrict access to the Gitaly server. Another option is to For configuring Gitaly you should note the following: -- `git_data_dirs` should be configured to reflect the storage path for the specific Gitaly node +- `gitaly['configuration'][:storage]` should be configured to reflect the storage path for the specific Gitaly node - `auth_token` should be the same as `praefect_internal_token` The following IPs will be used as an example: @@ -1506,20 +1528,6 @@ Updates to example must be made at: # balancer. gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' - # Make Gitaly accept connections on all network interfaces. You must use - # firewalls to restrict access to this address/port. - # Comment out following line if you only want to support TLS connections - gitaly['listen_addr'] = "0.0.0.0:8075" - - # Gitaly Auth Token - # Should be the same as praefect_internal_token - gitaly['auth_token'] = '' - - # Gitaly Pack-objects cache - # Recommended to be enabled for improved performance but can notably increase disk I/O - # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info - gitaly['pack_objects_cache_enabled'] = true - # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus @@ -1534,9 +1542,33 @@ Updates to example must be made at: retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address that the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - gitaly['prometheus_listen_addr'] = '0.0.0.0:9236' + + gitaly['configuration'] = { + # ... + # + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + # Comment out following line if you only want to support TLS connections + listen_addr: '0.0.0.0:8075', + # Set the network address that Gitaly will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9236', + # Gitaly Auth Token + # Should be the same as praefect_internal_token + auth: { + # ... + token: '', + }, + # Gitaly Pack-objects cache + # Recommended to be enabled for improved performance but can notably increase disk I/O + # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info + pack_objects_cache: { + # ... + enabled: true, + }, + } + # # END user configuration ``` @@ -1545,31 +1577,43 @@ Updates to example must be made at: - On Gitaly node 1: ```ruby - git_data_dirs({ - "gitaly-1" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-1', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 2: ```ruby - git_data_dirs({ - "gitaly-2" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-2', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 3: ```ruby - git_data_dirs({ - "gitaly-3" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-3', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace @@ -1589,7 +1633,7 @@ for secure connections, you must: Additionally the certificate, or its certificate authority, must be installed on all Gitaly servers and on all Praefect clients that communicate with it following the procedure described in -[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) (and repeated below). +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl/index.html#install-custom-public-certificates) (and repeated below). Note the following: @@ -1598,7 +1642,7 @@ Note the following: - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. + necessary. To disable the unencrypted listener, set `praefect['configuration'][:listen_addr] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS passthrough. Refer to the load balancers documentation on how to configure this. @@ -1620,9 +1664,15 @@ To configure Praefect with TLS: 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - praefect['tls_listen_addr'] = "0.0.0.0:3305" - praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - praefect['key_path'] = "/etc/gitlab/ssl/key.pem" + praefect['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:3305', + tls: { + # ... + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -2018,7 +2068,7 @@ On each node perform the following: When you specify `https` in the `external_url`, as in the previous example, GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the certificates aren't present, NGINX will fail to start. For more information, see -the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html). +the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html). ### GitLab Rails post-configuration @@ -2028,11 +2078,9 @@ the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html). gitlab-rake gitlab:db:configure ``` - If you encounter a `rake aborted!` error message stating that PgBouncer is - failing to connect to PostgreSQL, it may be that your PgBouncer node's IP - address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` - on your database nodes. Before proceeding, see - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). + Note that this requires the Rails node to be configured to connect to the primary database + directly, [bypassing PgBouncer](../postgresql/pgbouncer.md#procedure-for-bypassing-pgbouncer). + After migrations have completed, you must configure the node to pass through PgBouncer again. 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). @@ -2135,15 +2183,15 @@ and scalable. There are two ways of specifying object storage configuration in GitLab: -- [Consolidated form](../object_storage.md#consolidated-object-storage-configuration): A single credential is +- [Consolidated form](../object_storage.md#configure-a-single-storage-connection-for-all-object-types-consolidated-form): A single credential is shared by all supported object types. -- [Storage-specific form](../object_storage.md#storage-specific-configuration): Every object defines its - own object storage [connection and configuration](../object_storage.md#connection-settings). +- [Storage-specific form](../object_storage.md#configure-each-object-type-to-define-its-own-storage-connection-storage-specific-form): Every object defines its + own object storage [connection and configuration](../object_storage.md#configure-the-connection-settings). The consolidated form is used in the following examples when available. NOTE: -When using the [storage-specific form](../object_storage.md#storage-specific-configuration) +When using the [storage-specific form](../object_storage.md#configure-each-object-type-to-define-its-own-storage-connection-storage-specific-form) in GitLab 14.x and earlier, you should enable [direct upload mode](../../development/uploads/index.md#direct-upload). The previous [background upload](../../development/uploads/index.md#direct-upload) mode, which was deprecated in 14.9, requires shared storage such as NFS. @@ -2165,9 +2213,9 @@ GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily While sharing the job logs through NFS is supported, it's recommended to avoid the need to use NFS by enabling [incremental logging](../job_logs.md#incremental-logging-architecture) (required when no NFS node has been deployed). Incremental logging uses Redis instead of disk space for temporary caching of job logs. -## Configure Advanced Search +## Configure advanced search -You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) +You can leverage Elasticsearch and [enable advanced search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index 3bcffa8f606..7d3ddf7578c 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -31,8 +31,8 @@ full list of reference architectures, see | Gitaly5 6 | 3 | 64 vCPU, 240 GB memory | `n1-standard-64` | `m5.16xlarge` | | Praefect5 | 3 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | | Praefect PostgreSQL1 | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Sidekiq | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | -| GitLab Rails | 12 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | +| Sidekiq7 | 4 | 4 vCPU, 15 GB memory | `n1-standard-4` | `m5.xlarge` | +| GitLab Rails7 | 12 | 32 vCPU, 28.8 GB memory | `n1-highcpu-32` | `c5.9xlarge` | | Monitoring node | 1 | 4 vCPU, 3.6 GB memory | `n1-highcpu-4` | `c5.xlarge` | | Object storage4 | - | - | - | - | @@ -53,6 +53,8 @@ full list of reference architectures, see 6. Gitaly has been designed and tested with repositories of varying sizes that follow best practices. However, large repositories or monorepos that don't follow these practices can significantly impact Gitaly requirements. Refer to [Large repositories](index.md#large-repositories) for more information. +7. Can be placed in Auto Scaling Groups (ASGs) as the component doesn't store any [stateful data](index.md#autoscaling-of-stateful-nodes). + However, for GitLab Rails certain processes like [migrations](#gitlab-rails-post-configuration) and [Mailroom](../incoming_email.md) should be run on only one node. NOTE: @@ -170,7 +172,7 @@ To set up GitLab and its components to accommodate up to 50,000 users: environment. 1. [Configure the object storage](#configure-the-object-storage) used for shared data objects. -1. [Configure Advanced Search](#configure-advanced-search) (optional) for faster, +1. [Configure advanced search](#configure-advanced-search) (optional) for faster, more advanced code search across your entire GitLab instance. The servers start on the same 10.6.0.0/24 private network range, and can @@ -318,7 +320,7 @@ Configure your load balancer to pass connections on port 443 as `TCP` rather than `HTTP(S)` protocol. This will pass the connection to the application node's NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. -See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html) for details on managing SSL certificates and configuring NGINX. #### Load balancer terminates SSL without backend SSL @@ -329,7 +331,7 @@ terminating SSL. Since communication between the load balancer and GitLab will not be secure, there is some additional configuration needed. See the -[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) +[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) for details. #### Load balancer terminates SSL with backend SSL @@ -342,7 +344,7 @@ Traffic will also be secure between the load balancers and NGINX in this scenario. There is no need to add configuration for proxied SSL since the connection will be secure all the way. However, configuration will need to be added to GitLab to configure SSL certificates. See -the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html) for details on managing SSL certificates and configuring NGINX.
@@ -426,6 +428,11 @@ spread connections equally in practice. ## Configure Consul +Next, we set up the Consul servers. + +NOTE: +Consul must be deployed in an odd number of 3 nodes or more. This is to ensure the nodes can take votes as part of a quorum. + The following IPs will be used as an example: - `10.6.0.11`: Consul 1 @@ -811,6 +818,9 @@ Using [Redis](https://redis.io/) in scalable environment is possible using a **P topology with a [Redis Sentinel](https://redis.io/docs/manual/sentinel/) service to watch and automatically start the failover procedure. +NOTE: +Redis clusters must each be deployed in an odd number of 3 nodes or more. This is to ensure Redis Sentinel can take votes as part of a quorum. This does not apply when configuring Redis externally, such as a cloud provider service. + Redis requires authentication if used with Sentinel. See [Redis Security](https://redis.io/docs/manual/security/) documentation for more information. We recommend using a combination of a Redis password and tight @@ -844,9 +854,9 @@ to be used with GitLab. The following IPs will be used as an example: Managed Redis from cloud providers (such as AWS ElastiCache) will work. If these services support high availability, be sure it _isn't_ of the Redis Cluster type. -Redis version 5.0 or higher is required, which is included with Omnibus GitLab -packages starting with GitLab 13.0. Older Redis versions don't support an -optional count argument to SPOP, which is required for [Merge Trains](../../ci/pipelines/merge_trains.md). + +Because Omnibus GitLab packages ship with Redis 6.0 or later, Redis 6.0 or later is required. Older Redis versions have reached end-of-life. + Note the Redis node's IP address or hostname, port, and password (if required). These will be necessary later when configuring the [GitLab application servers](#configure-gitlab-rails). @@ -1202,7 +1212,7 @@ For more advanced setups refer to the [standalone Gitaly Cluster documentation]( Praefect, the routing and transaction manager for Gitaly Cluster, requires its own database server to store data on Gitaly Cluster status. If you want to have a highly available setup, Praefect requires a third-party PostgreSQL database. -A built-in solution is being [worked on](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5919). +A built-in solution is being [worked on](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/7292). #### Praefect non-HA PostgreSQL standalone using Omnibus GitLab @@ -1349,6 +1359,9 @@ This is how this would work with a Omnibus GitLab PostgreSQL setup: Praefect is the router and transaction manager for Gitaly Cluster and all connections to Gitaly go through it. This section details how to configure it. +NOTE: +Praefect must be deployed in an odd number of 3 nodes or later. This is to ensure the nodes can take votes as part of a quorum. + Praefect requires several secret tokens to secure communications across the Cluster: - ``: Used for repositories hosted on your Gitaly cluster and can only be accessed by Gitaly clients that carry this token. @@ -1402,7 +1415,6 @@ Updates to example must be made at: # Praefect Configuration praefect['enable'] = true - praefect['listen_addr'] = '0.0.0.0:2305' # Prevent database migrations from running on upgrade automatically praefect['auto_migrate'] = false @@ -1411,51 +1423,69 @@ Updates to example must be made at: # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus - consul['monitoring_service_discovery'] = true + consul['monitoring_service_discovery'] = true # START user configuration # Please set the real values as explained in Required Information section # - # Praefect External Token - # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster - praefect['auth_token'] = '' - - # Praefect Database Settings - praefect['database_host'] = '10.6.0.141' - praefect['database_port'] = 5432 - # `no_proxy` settings must always be a direct connection for caching - praefect['database_direct_host'] = '10.6.0.141' - praefect['database_direct_port'] = 5432 - praefect['database_dbname'] = 'praefect_production' - praefect['database_user'] = 'praefect' - praefect['database_password'] = '' - - # Praefect Virtual Storage config - # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') - praefect['virtual_storages'] = { - 'default' => { - 'nodes' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '' - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' + praefect['configuration'] = { + # ... + listen_addr: '0.0.0.0:2305', + auth: { + # ... + # + # Praefect External Token + # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster + token: '', + }, + # Praefect Database Settings + database: { + # ... + host: '10.6.0.141', + port: 5432, + # `no_proxy` settings must always be a direct connection for caching + session_pooled: { + # ... + host: '10.6.0.141', + port: 5432, + dbname: 'praefect_production', + user: 'praefect', + password: '', + }, + }, + # Praefect Virtual Storage config + # Name of storage hash must match storage name in git_data_dirs on GitLab + # server ('praefect') and in gitaly['configuration'][:storage] on Gitaly nodes ('gitaly-1') + virtual_storage: [ + { + # ... + name: 'default', + node: [ + { + storage: 'gitaly-1', + address: 'tcp://10.6.0.91:8075', + token: '' + }, + { + storage: 'gitaly-2', + address: 'tcp://10.6.0.92:8075', + token: '' + }, + { + storage: 'gitaly-3', + address: 'tcp://10.6.0.93:8075', + token: '' + }, + ], }, - } - } + ], + # Set the network address Praefect will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9652', } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - praefect['prometheus_listen_addr'] = '0.0.0.0:9652' ## The IPs of the Consul server nodes ## You can also use FQDNs and intermix them with IPs @@ -1506,9 +1536,7 @@ Due to Gitaly having notable input and output requirements, we strongly recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs should have a throughput of at least 8,000 input/output operations per second (IOPS) for read operations and 2,000 IOPS for -write operations. These IOPS values are initial recommendations, and may be -adjusted to greater or lesser values depending on the scale of your -environment's workload. If you're running the environment on a Cloud provider, +write operations. If you're running the environment on a Cloud provider, refer to their documentation about how to configure IOPS correctly. Gitaly servers must not be exposed to the public internet, as Gitaly's network @@ -1518,7 +1546,7 @@ to restrict access to the Gitaly server. Another option is to For configuring Gitaly you should note the following: -- `git_data_dirs` should be configured to reflect the storage path for the specific Gitaly node +- `gitaly['configuration'][:storage]` should be configured to reflect the storage path for the specific Gitaly node - `auth_token` should be the same as `praefect_internal_token` The following IPs will be used as an example: @@ -1567,20 +1595,6 @@ Updates to example must be made at: # Gitaly gitaly['enable'] = true - # Make Gitaly accept connections on all network interfaces. You must use - # firewalls to restrict access to this address/port. - # Comment out following line if you only want to support TLS connections - gitaly['listen_addr'] = "0.0.0.0:8075" - - # Gitaly Auth Token - # Should be the same as praefect_internal_token - gitaly['auth_token'] = '' - - # Gitaly Pack-objects cache - # Recommended to be enabled for improved performance but can notably increase disk I/O - # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info - gitaly['pack_objects_cache_enabled'] = true - # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus @@ -1595,9 +1609,29 @@ Updates to example must be made at: retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address that the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - gitaly['prometheus_listen_addr'] = '0.0.0.0:9236' + + gitaly['configuration'] = { + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + # Comment out following line if you only want to support TLS connections + listen_addr: '0.0.0.0:8075', + # Set the network address that Gitaly will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9236', + auth: { + # Gitaly Auth Token + # Should be the same as praefect_internal_token + token: '', + }, + pack_objects_cache: { + # Gitaly Pack-objects cache + # Recommended to be enabled for improved performance but can notably increase disk I/O + # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info + enabled: true, + }, + } + # # END user configuration ``` @@ -1606,31 +1640,43 @@ Updates to example must be made at: - On Gitaly node 1: ```ruby - git_data_dirs({ - "gitaly-1" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-1', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 2: ```ruby - git_data_dirs({ - "gitaly-2" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-2', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 3: ```ruby - git_data_dirs({ - "gitaly-3" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-3', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace @@ -1650,7 +1696,7 @@ for secure connections, you must: Additionally the certificate, or its certificate authority, must be installed on all Gitaly servers and on all Praefect clients that communicate with it following the procedure described in -[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) (and repeated below). +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl/index.html#install-custom-public-certificates) (and repeated below). Note the following: @@ -1659,7 +1705,7 @@ Note the following: - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. + necessary. To disable the unencrypted listener, set `praefect['configuration'][:listen_addr] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS passthrough. Refer to the load balancers documentation on how to configure this. @@ -1681,9 +1727,15 @@ To configure Praefect with TLS: 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - praefect['tls_listen_addr'] = "0.0.0.0:3305" - praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - praefect['key_path'] = "/etc/gitlab/ssl/key.pem" + praefect['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:3305', + tls: { + # ... + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -1764,7 +1816,7 @@ Updates to example must be made at: # Redis ## Redis connection details - ## First cluster that will host the cache + ## First cluster that will host the cache data gitlab_rails['redis_cache_instance'] = 'redis://:@gitlab-redis-cache' gitlab_rails['redis_cache_sentinels'] = [ @@ -1773,22 +1825,11 @@ Updates to example must be made at: {host: '10.6.0.53', port: 26379}, ] - ## Second cluster that will host the persistent queues, shared state, and actioncable - gitlab_rails['redis_queues_instance'] = 'redis://:@gitlab-redis-persistent' - gitlab_rails['redis_shared_state_instance'] = 'redis://:@gitlab-redis-persistent' - gitlab_rails['redis_actioncable_instance'] = 'redis://:@gitlab-redis-persistent' + ## Second cluster that hosts all other persistent data + redis['master_name'] = 'gitlab-redis-persistent' + redis['master_password'] = '' - gitlab_rails['redis_queues_sentinels'] = [ - {host: '10.6.0.61', port: 26379}, - {host: '10.6.0.62', port: 26379}, - {host: '10.6.0.63', port: 26379}, - ] - gitlab_rails['redis_shared_state_sentinels'] = [ - {host: '10.6.0.61', port: 26379}, - {host: '10.6.0.62', port: 26379}, - {host: '10.6.0.63', port: 26379}, - ] - gitlab_rails['redis_actioncable_sentinels'] = [ + gitlab_rails['redis_sentinels'] = [ {host: '10.6.0.61', port: 26379}, {host: '10.6.0.62', port: 26379}, {host: '10.6.0.63', port: 26379}, @@ -1958,7 +1999,7 @@ On each node perform the following: gitlab_rails['auto_migrate'] = false ## Redis connection details - ## First cluster that will host the cache + ## First cluster that will host the cache data gitlab_rails['redis_cache_instance'] = 'redis://:@gitlab-redis-cache' gitlab_rails['redis_cache_sentinels'] = [ @@ -1967,22 +2008,11 @@ On each node perform the following: {host: '10.6.0.53', port: 26379}, ] - ## Second cluster that will host the persistent queues, shared state, and actionable - gitlab_rails['redis_queues_instance'] = 'redis://:@gitlab-redis-persistent' - gitlab_rails['redis_shared_state_instance'] = 'redis://:@gitlab-redis-persistent' - gitlab_rails['redis_actioncable_instance'] = 'redis://:@gitlab-redis-persistent' + ## Second cluster that hosts all other persistent data + redis['master_name'] = 'gitlab-redis-persistent' + redis['master_password'] = '' - gitlab_rails['redis_queues_sentinels'] = [ - {host: '10.6.0.61', port: 26379}, - {host: '10.6.0.62', port: 26379}, - {host: '10.6.0.63', port: 26379}, - ] - gitlab_rails['redis_shared_state_sentinels'] = [ - {host: '10.6.0.61', port: 26379}, - {host: '10.6.0.62', port: 26379}, - {host: '10.6.0.63', port: 26379}, - ] - gitlab_rails['redis_actioncable_sentinels'] = [ + gitlab_rails['redis_sentinels'] = [ {host: '10.6.0.61', port: 26379}, {host: '10.6.0.62', port: 26379}, {host: '10.6.0.63', port: 26379}, @@ -2076,7 +2106,7 @@ On each node perform the following: When you specify `https` in the `external_url`, as in the previous example, GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the certificates aren't present, NGINX will fail to start. For more information, see -the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html). +the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html). ### GitLab Rails post-configuration @@ -2088,11 +2118,9 @@ the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html). sudo gitlab-rake gitlab:db:configure ``` - If you encounter a `rake aborted!` error message stating that PgBouncer is - failing to connect to PostgreSQL, it may be that your PgBouncer node's IP - address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` - on your database nodes. Before proceeding, see - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). + Note that this requires the Rails node to be configured to connect to the primary database + directly, [bypassing PgBouncer](../postgresql/pgbouncer.md#procedure-for-bypassing-pgbouncer). + After migrations have completed, you must configure the node to pass through PgBouncer again. 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). @@ -2187,15 +2215,15 @@ and scalable. There are two ways of specifying object storage configuration in GitLab: -- [Consolidated form](../object_storage.md#consolidated-object-storage-configuration): A single credential is +- [Consolidated form](../object_storage.md#configure-a-single-storage-connection-for-all-object-types-consolidated-form): A single credential is shared by all supported object types. -- [Storage-specific form](../object_storage.md#storage-specific-configuration): Every object defines its - own object storage [connection and configuration](../object_storage.md#connection-settings). +- [Storage-specific form](../object_storage.md#configure-each-object-type-to-define-its-own-storage-connection-storage-specific-form): Every object defines its + own object storage [connection and configuration](../object_storage.md#configure-the-connection-settings). The consolidated form is used in the following examples when available. NOTE: -When using the [storage-specific form](../object_storage.md#storage-specific-configuration) +When using the [storage-specific form](../object_storage.md#configure-each-object-type-to-define-its-own-storage-connection-storage-specific-form) in GitLab 14.x and earlier, you should enable [direct upload mode](../../development/uploads/index.md#direct-upload). The previous [background upload](../../development/uploads/index.md#direct-upload) mode, which was deprecated in 14.9, requires shared storage such as NFS. @@ -2217,9 +2245,9 @@ GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily While sharing the job logs through NFS is supported, it's recommended to avoid the need to use NFS by enabling [incremental logging](../job_logs.md#incremental-logging-architecture) (required when no NFS node has been deployed). Incremental logging uses Redis instead of disk space for temporary caching of job logs. -## Configure Advanced Search +## Configure advanced search -You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) +You can leverage Elasticsearch and [enable advanced search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 691f71289c3..82087c91910 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -37,8 +37,8 @@ costly-to-operate environment by using the | Gitaly5 6 | 3 | 8 vCPU, 30 GB memory | `n1-standard-8` | `m5.2xlarge` | | Praefect5 | 3 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Praefect PostgreSQL1 | 1+ | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | -| Sidekiq | 4 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | -| GitLab Rails | 3 | 16 vCPU, 14.4 GB memory | `n1-highcpu-16` | `c5.4xlarge` | +| Sidekiq7 | 4 | 2 vCPU, 7.5 GB memory | `n1-standard-2` | `m5.large` | +| GitLab Rails7 | 3 | 16 vCPU, 14.4 GB memory | `n1-highcpu-16` | `c5.4xlarge` | | Monitoring node | 1 | 2 vCPU, 1.8 GB memory | `n1-highcpu-2` | `c5.large` | | Object storage4 | - | - | - | - | @@ -59,6 +59,8 @@ costly-to-operate environment by using the 6. Gitaly has been designed and tested with repositories of varying sizes that follow best practices. However, large repositories or monorepos that don't follow these practices can significantly impact Gitaly requirements. Refer to [Large repositories](index.md#large-repositories) for more information. +7. Can be placed in Auto Scaling Groups (ASGs) as the component doesn't store any [stateful data](index.md#autoscaling-of-stateful-nodes). + However, for GitLab Rails certain processes like [migrations](#gitlab-rails-post-configuration) and [Mailroom](../incoming_email.md) should be run on only one node. NOTE: @@ -173,7 +175,7 @@ To set up GitLab and its components to accommodate up to 5,000 users: environment. 1. [Configure the object storage](#configure-the-object-storage) used for shared data objects. -1. [Configure Advanced Search](#configure-advanced-search) (optional) for faster, +1. [Configure advanced search](#configure-advanced-search) (optional) for faster, more advanced code search across your entire GitLab instance. The servers start on the same 10.6.0.0/24 private network range, and can @@ -318,7 +320,7 @@ Configure your load balancer to pass connections on port 443 as `TCP` rather than `HTTP(S)` protocol. This will pass the connection to the application node's NGINX service untouched. NGINX will have the SSL certificate and listen on port 443. -See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +See the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html) for details on managing SSL certificates and configuring NGINX. #### Load balancer terminates SSL without backend SSL @@ -329,7 +331,7 @@ terminating SSL. Since communication between the load balancer and GitLab will not be secure, there is some additional configuration needed. See the -[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) +[proxied SSL documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) for details. #### Load balancer terminates SSL with backend SSL @@ -342,7 +344,7 @@ Traffic will also be secure between the load balancers and NGINX in this scenario. There is no need to add configuration for proxied SSL since the connection will be secure all the way. However, configuration will need to be added to GitLab to configure SSL certificates. See -the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html) +the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html) for details on managing SSL certificates and configuring NGINX.
@@ -450,10 +452,7 @@ to be used with GitLab. The following IPs are used as an example: Managed Redis from cloud providers such as AWS ElastiCache works. If these services support high availability, be sure it is **not** the Redis Cluster type. -Redis version 5.0 or higher is required, as this is what ships with -Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions -do not support an optional count argument to SPOP which is now required for -[Merge Trains](../../ci/pipelines/merge_trains.md). +Because Omnibus GitLab packages ship with Redis 6.0 or later, Redis 6.0 or later is required. Older Redis versions have reached end-of-life. Note the Redis node's IP address or hostname, port, and password (if required). These are necessary when configuring the @@ -637,8 +636,13 @@ are supported and can be added if needed. ## Configure Consul and Sentinel -Now that the Redis servers are all set up, let's configure the Sentinel -servers. The following IPs are used as an example: +Now that the Redis servers are all set up, let's configure the Consul + Sentinel +servers. + +NOTE: +Consul and Redis Sentinel must be deployed in an odd number of 3 nodes or later. This is to ensure the nodes can take votes as part of a quorum. + +The following IPs will be used as an example: - `10.6.0.11`: Consul/Sentinel 1 - `10.6.0.12`: Consul/Sentinel 2 @@ -1140,7 +1144,7 @@ For more advanced setups refer to the [standalone Gitaly Cluster documentation]( Praefect, the routing and transaction manager for Gitaly Cluster, requires its own database server to store data on Gitaly Cluster status. If you want to have a highly available setup, Praefect requires a third-party PostgreSQL database. -A built-in solution is being [worked on](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5919). +A built-in solution is being [worked on](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/7292). #### Praefect non-HA PostgreSQL standalone using Omnibus GitLab @@ -1285,6 +1289,9 @@ This is how this would work with a Omnibus GitLab PostgreSQL setup: Praefect is the router and transaction manager for Gitaly Cluster and all connections to Gitaly go through it. This section details how to configure it. +NOTE: +Praefect must be deployed in an odd number of 3 nodes or later. This is to ensure the nodes can take votes as part of a quorum. + Praefect requires several secret tokens to secure communications across the Cluster: - ``: Used for repositories hosted on your Gitaly cluster and can only be accessed by Gitaly clients that carry this token. @@ -1338,7 +1345,6 @@ Updates to example must be made at: # Praefect Configuration praefect['enable'] = true - praefect['listen_addr'] = '0.0.0.0:2305' # Prevent database migrations from running on upgrade automatically praefect['auto_migrate'] = false @@ -1347,51 +1353,69 @@ Updates to example must be made at: # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus - consul['monitoring_service_discovery'] = true + consul['monitoring_service_discovery'] = true # START user configuration # Please set the real values as explained in Required Information section # - # Praefect External Token - # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster - praefect['auth_token'] = '' - - # Praefect Database Settings - praefect['database_host'] = '10.6.0.141' - praefect['database_port'] = 5432 - # `no_proxy` settings must always be a direct connection for caching - praefect['database_direct_host'] = '10.6.0.141' - praefect['database_direct_port'] = 5432 - praefect['database_dbname'] = 'praefect_production' - praefect['database_user'] = 'praefect' - praefect['database_password'] = '' - - # Praefect Virtual Storage config - # Name of storage hash must match storage name in git_data_dirs on GitLab - # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1') - praefect['virtual_storages'] = { - 'default' => { - 'nodes' => { - 'gitaly-1' => { - 'address' => 'tcp://10.6.0.91:8075', - 'token' => '' - }, - 'gitaly-2' => { - 'address' => 'tcp://10.6.0.92:8075', - 'token' => '' - }, - 'gitaly-3' => { - 'address' => 'tcp://10.6.0.93:8075', - 'token' => '' + praefect['configuration'] = { + # ... + listen_addr: '0.0.0.0:2305', + auth: { + # ... + # + # Praefect External Token + # This is needed by clients outside the cluster (like GitLab Shell) to communicate with the Praefect cluster + token: '', + }, + # Praefect Database Settings + database: { + # ... + host: '10.6.0.141', + port: 5432, + # `no_proxy` settings must always be a direct connection for caching + session_pooled: { + # ... + host: '10.6.0.141', + port: 5432, + dbname: 'praefect_production', + user: 'praefect', + password: '', + }, + }, + # Praefect Virtual Storage config + # Name of storage hash must match storage name in git_data_dirs on GitLab + # server ('praefect') and in gitaly['configuration'][:storage] on Gitaly nodes ('gitaly-1') + virtual_storage: [ + { + # ... + name: 'default', + node: [ + { + storage: 'gitaly-1', + address: 'tcp://10.6.0.91:8075', + token: '' + }, + { + storage: 'gitaly-2', + address: 'tcp://10.6.0.92:8075', + token: '' + }, + { + storage: 'gitaly-3', + address: 'tcp://10.6.0.93:8075', + token: '' + }, + ], }, - } - } + ], + # Set the network address Praefect will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9652', } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - praefect['prometheus_listen_addr'] = '0.0.0.0:9652' ## The IPs of the Consul server nodes ## You can also use FQDNs and intermix them with IPs @@ -1442,9 +1466,7 @@ Due to Gitaly having notable input and output requirements, we strongly recommend that all Gitaly nodes use solid-state drives (SSDs). These SSDs should have a throughput of at least 8,000 input/output operations per second (IOPS) for read operations and 2,000 IOPS for -write operations. These IOPS values are initial recommendations, and may be -adjusted to greater or lesser values depending on the scale of your -environment's workload. If you're running the environment on a Cloud provider, +write operations. If you're running the environment on a Cloud provider, refer to their documentation about how to configure IOPS correctly. Gitaly servers must not be exposed to the public internet, as Gitaly's network @@ -1454,7 +1476,7 @@ to restrict access to the Gitaly server. Another option is to For configuring Gitaly you should note the following: -- `git_data_dirs` should be configured to reflect the storage path for the specific Gitaly node +- `gitaly['configuration'][:storage]` should be configured to reflect the storage path for the specific Gitaly node - `auth_token` should be the same as `praefect_internal_token` The following IPs are used as an example: @@ -1503,20 +1525,6 @@ Updates to example must be made at: # Gitaly gitaly['enable'] = true - # Make Gitaly accept connections on all network interfaces. You must use - # firewalls to restrict access to this address/port. - # Comment out following line if you only want to support TLS connections - gitaly['listen_addr'] = "0.0.0.0:8075" - - # Gitaly Auth Token - # Should be the same as praefect_internal_token - gitaly['auth_token'] = '' - - # Gitaly Pack-objects cache - # Recommended to be enabled for improved performance but can notably increase disk I/O - # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info - gitaly['pack_objects_cache_enabled'] = true - # Configure the Consul agent consul['enable'] = true ## Enable service discovery for Prometheus @@ -1531,9 +1539,29 @@ Updates to example must be made at: retry_join: %w(10.6.0.11 10.6.0.12 10.6.0.13), } - # Set the network addresses that the exporters will listen on for monitoring + # Set the network address that the node exporter will listen on for monitoring node_exporter['listen_address'] = '0.0.0.0:9100' - gitaly['prometheus_listen_addr'] = '0.0.0.0:9236' + + gitaly['configuration'] = { + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + # Comment out following line if you only want to support TLS connections + listen_addr: '0.0.0.0:8075', + # Set the network address that Gitaly will listen on for monitoring + prometheus_listen_addr: '0.0.0.0:9236', + auth: { + # Gitaly Auth Token + # Should be the same as praefect_internal_token + token: '', + }, + pack_objects_cache: { + # Gitaly Pack-objects cache + # Recommended to be enabled for improved performance but can notably increase disk I/O + # Refer to https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#pack-objects-cache for more info + enabled: true, + }, + } + # # END user configuration ``` @@ -1542,31 +1570,43 @@ Updates to example must be made at: - On Gitaly node 1: ```ruby - git_data_dirs({ - "gitaly-1" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-1', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 2: ```ruby - git_data_dirs({ - "gitaly-2" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-2', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` - On Gitaly node 3: ```ruby - git_data_dirs({ - "gitaly-3" => { - "path" => "/var/opt/gitlab/git-data" - } - }) + gitaly['configuration'] = { + # ... + storage: [ + { + name: 'gitaly-3', + path: '/var/opt/gitlab/git-data', + }, + ], + } ``` 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the first Omnibus node you configured and add or replace @@ -1586,7 +1626,7 @@ for secure connections, you must: Additionally the certificate, or its certificate authority, must be installed on all Gitaly servers and on all Praefect clients that communicate with it following the procedure described in -[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates) (and repeated below). +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl/index.html#install-custom-public-certificates) (and repeated below). Note the following: @@ -1595,7 +1635,7 @@ Note the following: - You can configure Praefect servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if - necessary. To disable the unencrypted listener, set `praefect['listen_addr'] = nil`. + necessary. To disable the unencrypted listener, set `praefect['configuration'][:listen_addr] = nil`. - The Internal Load Balancer will also access to the certificates and need to be configured to allow for TLS passthrough. Refer to the load balancers documentation on how to configure this. @@ -1617,9 +1657,15 @@ To configure Praefect with TLS: 1. Edit `/etc/gitlab/gitlab.rb` and add: ```ruby - praefect['tls_listen_addr'] = "0.0.0.0:3305" - praefect['certificate_path'] = "/etc/gitlab/ssl/cert.pem" - praefect['key_path'] = "/etc/gitlab/ssl/key.pem" + praefect['configuration'] = { + # ... + tls_listen_addr: '0.0.0.0:3305', + tls: { + # ... + certificate_path: '/etc/gitlab/ssl/cert.pem', + key_path: '/etc/gitlab/ssl/key.pem', + }, + } ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). @@ -2017,7 +2063,7 @@ On each node perform the following: When you specify `https` in the `external_url`, as in the previous example, GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the certificates aren't present, NGINX fails to start. For more information, see -the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html). +the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html). ### GitLab Rails post-configuration @@ -2027,11 +2073,9 @@ the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/ssl.html). gitlab-rake gitlab:db:configure ``` - If you encounter a `rake aborted!` error message stating that PgBouncer is - failing to connect to PostgreSQL, it may be that your PgBouncer node's IP - address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` - on your database nodes. Before proceeding, see - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](../postgresql/replication_and_failover.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). + Note that this requires the Rails node to be configured to connect to the primary database + directly, [bypassing PgBouncer](../postgresql/pgbouncer.md#procedure-for-bypassing-pgbouncer). + After migrations have completed, you must configure the node to pass through PgBouncer again. 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). @@ -2134,15 +2178,15 @@ and scalable. There are two ways of specifying object storage configuration in GitLab: -- [Consolidated form](../object_storage.md#consolidated-object-storage-configuration): A single credential is +- [Consolidated form](../object_storage.md#configure-a-single-storage-connection-for-all-object-types-consolidated-form): A single credential is shared by all supported object types. -- [Storage-specific form](../object_storage.md#storage-specific-configuration): Every object defines its - own object storage [connection and configuration](../object_storage.md#connection-settings). +- [Storage-specific form](../object_storage.md#configure-each-object-type-to-define-its-own-storage-connection-storage-specific-form): Every object defines its + own object storage [connection and configuration](../object_storage.md#configure-the-connection-settings). The consolidated form is used in the following examples when available. NOTE: -When using the [storage-specific form](../object_storage.md#storage-specific-configuration) +When using the [storage-specific form](../object_storage.md#configure-each-object-type-to-define-its-own-storage-connection-storage-specific-form) in GitLab 14.x and earlier, you should enable [direct upload mode](../../development/uploads/index.md#direct-upload). The previous [background upload](../../development/uploads/index.md#direct-upload) mode, which was deprecated in 14.9, requires shared storage such as NFS. @@ -2164,9 +2208,9 @@ GitLab Runner returns job logs in chunks which Omnibus GitLab caches temporarily While sharing the job logs through NFS is supported, it's recommended to avoid the need to use NFS by enabling [incremental logging](../job_logs.md#incremental-logging-architecture) (required when no NFS node has been deployed). Incremental logging uses Redis instead of disk space for temporary caching of job logs. -## Configure Advanced Search +## Configure advanced search -You can leverage Elasticsearch and [enable Advanced Search](../../integration/advanced_search/elasticsearch.md) +You can leverage Elasticsearch and [enable advanced search](../../integration/advanced_search/elasticsearch.md) for faster, more advanced code search across your entire GitLab instance. Elasticsearch cluster design and requirements are dependent on your specific diff --git a/doc/administration/reference_architectures/index.md b/doc/administration/reference_architectures/index.md index 7b01efa183b..4b9c26e8626 100644 --- a/doc/administration/reference_architectures/index.md +++ b/doc/administration/reference_architectures/index.md @@ -47,9 +47,9 @@ The following Cloud Native Hybrid reference architectures, where select recommen The first choice to consider is whether a Self Managed approach is correct for you and your requirements. -Running any application in production is complex, and the same applies for GitLab. While we aim to make this as smooth as possible, there are still the general complexities. This depends on the design chosen, but typically you'll need to manage all aspects such as hardware, operating systems, networking, storage, security, GitLab itself, and more. +Running any application in production is complex, and the same applies for GitLab. While we aim to make this as smooth as possible, there are still the general complexities. This depends on the design chosen, but typically you'll need to manage all aspects such as hardware, operating systems, networking, storage, security, GitLab itself, and more. This includes both the initial setup of the environment and the longer term maintenance. -As such, it's recommended that you have a working knowledge of running applications in production when deciding on going down this route. For users who want a more managed solution it's recommended to instead explore our other offerings such as [GitLab SaaS](../../subscriptions/gitlab_com/index.md) or [GitLab Dedicated](../../subscriptions/gitlab_dedicated/index.md). +As such, it's recommended that you have a working knowledge of running and maintaining applications in production when deciding on going down this route. If you aren't in this position, our [Professional Services](https://about.gitlab.com/services/#implementation-services) team offers implementation services, but for those who want a more managed solution long term, it's recommended to instead explore our other offerings such as [GitLab SaaS](../../subscriptions/gitlab_com/index.md) or [GitLab Dedicated](../../subscriptions/gitlab_dedicated/index.md). ## Deciding which architecture to use @@ -72,15 +72,13 @@ With standalone setups, especially single node environments, there are [various ### High Availability (HA) -High Availability ensures every component in the GitLab setup can handle failures through various mechanisms. To achieve this however is complex, and the environments required can be sizable. +High Availability ensures every component in the GitLab setup can handle failures through various mechanisms. However, to achieve this is complex and the environments required can be sizable. For environments serving 3,000 or more users we generally recommend that a HA strategy is used as at this level outages have a bigger impact against more users. All the architectures in this range have HA built in by design for this reason. -For users who still need to have HA for a lower number of users this can also be achieved with an adjusted [3K architecture](3k_users.md#supported-modifications-for-lower-user-counts-ha). - #### Do you need High Availability (HA)? -As mentioned above, achieving HA does come at a cost. The environment's required are sizable as each component needs to be multiplied, which comes with additional actual and maintenance costs. +As mentioned above, achieving HA does come at a cost. The environment requirements are sizable as each component needs to be multiplied, which comes with additional actual and maintenance costs. For a lot of our customers with fewer than 3,000 users, we've found a backup strategy is sufficient and even preferable. While this does have a slower recovery time, it also means you have a much smaller architecture and less maintenance costs as a result. @@ -89,13 +87,17 @@ In general then, we'd only recommend you employ HA in the following scenarios: - When you have 3,000 or more users. - When GitLab being down would critically impact your workflow. +#### Scaled-down High Availability (HA) approaches + +If you still need to have HA for a lower number of users, this can be achieved with an adjusted [3K architecture](3k_users.md#supported-modifications-for-lower-user-counts-ha). + #### Zero Downtime Upgrades [Zero Downtime Upgrades](../../update/zero_downtime.md) are available for standard Reference Architecture environments with HA (Cloud Native Hybrid is not supported at this time). This allows for an environment to stay up during an upgrade, but the process is more complex as a result and has some limitations as detailed in the documentation. -When going through this process it's worth noting that there may still be brief moments of downtime when the HA mechanisms tale effect. +When going through this process it's worth noting that there may still be brief moments of downtime when the HA mechanisms take effect. -In most cases the downtime required for doing an upgrade in general shouldn't be substantial, so this is only recommended if it's a key requirement for you. +In most cases the downtime required for doing an upgrade shouldn't be substantial, so this is only recommended if it's a key requirement for you. ### Cloud Native Hybrid (Kubernetes HA) @@ -163,7 +165,7 @@ Before implementing a reference architecture, refer to the following requirement These reference architectures were built and tested on Google Cloud Platform (GCP) using the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) -CPU platform as a baseline ([Sysbench benchmark](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks)). +CPU platform as a lowest common denominator baseline ([Sysbench benchmark](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks)). Newer, similarly-sized CPUs are supported and may have improved performance as a result. For Omnibus GitLab environments, ARM-based equivalents are also supported. @@ -237,10 +239,10 @@ for more information and guidance. [Praefect requires its own database server](../gitaly/praefect.md#postgresql) and that to achieve full High Availability, a third-party PostgreSQL database solution is required. -We hope to offer a built in solutions for these restrictions in the future but, in the meantime, a non HA PostgreSQL server -can be set up using Omnibus GitLab, the specifications reflect. Refer to the following issues for more information: +We hope to offer a built-in solution for these restrictions in the future. In the meantime, a non-HA PostgreSQL server +can be set up using Omnibus GitLab as the specifications reflect. Refer to the following issues for more information: -- [`omnibus-gitlab#5919`](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5919). +- [`omnibus-gitlab#7292`](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/7292). - [`gitaly#3398`](https://gitlab.com/gitlab-org/gitaly/-/issues/3398). ## Recommended cloud providers and services @@ -324,10 +326,13 @@ When selecting a database service, it should run a standard, performant, and [su - Read Replicas for [Database Load Balancing](../postgresql/database_load_balancing.md). - Cross Region replication for [GitLab Geo](../geo/index.md). -Several cloud provider services are known not to support the above or have been found to have other issues and aren't recommended: +#### Unsupported database services + +Several database cloud provider services are known not to support the above or have been found to have other issues and aren't recommended: - [Amazon Aurora](https://aws.amazon.com/rds/aurora/) is incompatible and not supported. See [14.4.0](../../update/index.md#1440) for more details. -- [Azure Database for PostgreSQL Single Server](https://azure.microsoft.com/en-gb/products/postgresql/#overview) (Single / Flexible) is **strongly not recommended** for use due to notable performance / stability issues or missing functionality. See [Recommendation Notes for Azure](#recommendation-notes-for-azure) for more details. +- [Azure Database for PostgreSQL Single Server](https://azure.microsoft.com/en-gb/products/postgresql/#overview) (Single / Flexible) is not supported for use due to notable performance / stability issues or missing functionality. See [Recommendation Notes for Azure](#recommendation-notes-for-azure) for more details. +- Azure Database for PostgreSQL Flexible Server uses Microsoft Azure Active Directory (Azure AD) as authentication mechanism, which is incompatible with GitLab database integration. - [Google AlloyDB](https://cloud.google.com/alloydb) and [Amazon RDS Multi-AZ DB cluster](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html) have not been tested and are not recommended. Both solutions are specifically not expected to work with GitLab Geo. - [Amazon RDS Multi-AZ DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZSingleStandby.html) is a separate product and is supported. @@ -337,10 +342,55 @@ Due to performance issues that we found with several key Azure services, we only In addition to the above, you should be aware of the additional specific guidance for Azure: -- **We outright strongly do not recommend [Azure Database for PostgreSQL Single Server](https://learn.microsoft.com/en-us/azure/postgresql/single-server/overview-single-server)** specifically due to significant performance and stability issues found. **For GitLab 14.0 and higher the service is not supported** due to it only supporting up to PostgreSQL 11. - - A new service, [Azure Database for PostgreSQL Flexible Server](https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/) has been released. [Internal testing](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/91) has shown that it does look to perform as expected, but this hasn't been validated in production, so generally isn't recommended at this time. Additionally, as it's a new service, you may find that it's missing some functionality depending on your requirements. +- [Azure Database for PostgreSQL Single Server](https://azure.microsoft.com/en-gb/products/postgresql/#overview) (Single / Flexible) is not supported for use due to notable performance / stability issues or missing functionality. +- A new service, [Azure Database for PostgreSQL Flexible Server](https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/) has been released. [Internal testing](https://gitlab.com/gitlab-org/quality/reference-architectures/-/issues/91) has shown that it does look to perform as expected, but this hasn't been validated in production, so generally isn't recommended at this time. Additionally, as it's a new service, you may find that it's missing some functionality depending on your requirements. - [Azure Blob Storage](https://azure.microsoft.com/en-gb/products/storage/blobs/) has been found to have [performance limits that can impact production use at certain times](https://gitlab.com/gitlab-org/gitlab/-/issues/344861). However, this has only been seen in our largest architectures (25k+) so far. +## Deviating from the suggested reference architectures + +As a general guideline, the further away you move from the reference architectures, +the harder it is to get support for it. With any deviation, you're introducing +a layer of complexity that adds challenges to finding out where potential +issues might lie. + +The reference architectures use the official GitLab Linux packages (Omnibus +GitLab) or [Helm Charts](https://docs.gitlab.com/charts/) to install and configure the various components. The components are +installed on separate machines (virtualized or bare metal), with machine hardware +requirements listed in the "Configuration" column and equivalent VM standard sizes listed +in GCP/AWS/Azure columns of each [available reference architecture](#available-reference-architectures). + +Running components on Docker (including Docker Compose) with the same specs should be fine, as Docker is well known in terms of support. +However, it is still an additional layer and may still add some support complexities, such as not being able to run `strace` easily in containers. + +### Unsupported designs + +While we endeavour to try and have a good range of support for GitLab environment designs, there are certain approaches we know definitively not to work, and as a result are not supported. Those approaches are detailed in the following sections. + +#### Stateful components in Kubernetes + +[Running stateful components in Kubernetes, such as Gitaly Cluster, is not supported](https://docs.gitlab.com/charts/installation/#configure-the-helm-chart-to-use-external-stateful-data). + +Gitaly Cluster is only supported to be run on VMs as Git itself doesn't match well with the Kubernetes design and attempting to run it can lead to significant and complex issues. +[Refer to epic 6127 for more information](https://gitlab.com/groups/gitlab-org/-/epics/6127). + +This also applies to other third-party stateful components such as Postgres and Redis, but you can explore other third-party solutions for those components if desired such as supported Cloud Provider services unless called out specifically as unsupported. + +#### Autoscaling of stateful nodes + +As a general guidance, only _stateless_ components of GitLab can be run in Autoscaling groups, namely GitLab Rails +and Sidekiq. + +Other components that have state, such as Gitaly, are not supported in this fashion (for more information, see [issue 2997](https://gitlab.com/gitlab-org/gitaly/-/issues/2997)). + +This also applies to other third-party stateful components such as Postgres and Redis, but you can explore other third-party solutions for those components if desired such as supported Cloud Provider services unless called out specifically as unsupported. + +#### Spreading one environment over multiple data centers + +Deploying one GitLab environment over multiple data centers is not supported due to potential split brain edge cases +if a data center were to go down. For example, several components of the GitLab setup, namely Consul, Redis Sentinel and Praefect require an odd number quorum to function correctly and splitting over multiple data centers can impact this notably. + +For deploying GitLab over multiple data centers or regions we offer [GitLab Geo](https://about.gitlab.com/solutions/geo/) as a comprehensive solution. + ## Validation and test results The [Quality Engineering team](https://about.gitlab.com/handbook/engineering/quality/quality-engineering/) @@ -427,34 +477,34 @@ table.test-coverage th { 1k Weekly - - - - + + + + 2k Weekly - - - - + + + + Planned 3k Weekly - - + + Weekly - + 5k Weekly - - - - + + + + 10k @@ -462,29 +512,33 @@ table.test-coverage th { Weekly Weekly Weekly - Ad-Hoc + 25k Weekly - - - - Ad-Hoc + + + + 50k Weekly - - Ad-Hoc (inc Cloud Services) - - + + + + ## Cost to run -The following table details the cost to run the different reference architectures across GCP, AWS, and Azure. Bare-metal costs are not included here as it varies widely depending on each customer configuration. +As a starting point, the following table details initial costs for the different reference architectures across GCP, AWS, and Azure via Omnibus. + +NOTE: +Due to the nature of Cloud Native Hybrid, it's not possible to give a static cost calculation. +Bare-metal costs are also not included here as it varies widely depending on each configuration. @@ -492,91 +546,93 @@ The following table details the cost to run the different reference architecture - - - + + + - - - - - - - - - + + + - - - - - + + + - - - - - + + + - - - - - + + + - - - - - + + + - - - - - + + +
Reference
Architecture		GCPAWSAzureGCPAWSAzure
OmnibusCloud Native Hybrid OmnibusCloud Native Hybrid Omnibus
1k Calculated cost Calculated cost Calculated cost
2kCalculated costCalculated costCalculated costCalculated costCalculated costCalculated cost
3kCalculated costCalculated costCalculated costCalculated costCalculated costCalculated cost
5kCalculated costCalculated costCalculated costCalculated costCalculated costCalculated cost
10kCalculated costCalculated costCalculated costCalculated costCalculated costCalculated cost
25kCalculated costCalculated costCalculate costCalculated costCalculated costCalculate cost
50kCalculated costCalculated costCalculated costCalculated costCalculated costCalculated cost
-## Deviating from the suggested reference architectures +## Maintaining a Reference Architecture environment -As a general guideline, the further away you move from the Reference Architectures, -the harder it is to get support for it. With any deviation, you're introducing -a layer of complexity that adds challenges to finding out where potential -issues might lie. +Maintaining a Reference Architecture environment is generally the same as any other GitLab environment is generally covered in other sections of this documentation. -The reference architectures use the official GitLab Linux packages (Omnibus -GitLab) or [Helm Charts](https://docs.gitlab.com/charts/) to install and configure the various components. The components are -installed on separate machines (virtualized or bare metal), with machine hardware -requirements listed in the "Configuration" column and equivalent VM standard sizes listed -in GCP/AWS/Azure columns of each [available reference architecture](#available-reference-architectures). +In this section you'll find links to documentation for relevant areas as well as any specific Reference Architecture notes. -Running components on Docker (including Compose) with the same specs should be fine, as Docker is well known in terms of support. -However, it is still an additional layer and may still add some support complexities, such as not being able to run `strace` easily in containers. +### Upgrades + +Upgrades for a Reference Architecture environment is the same as any other GitLab environment. +The main [Upgrade GitLab](../../update/index.md) section has detailed steps on how to approach this. + +[Zero-downtime upgrades](#zero-downtime-upgrades) are also available. + +NOTE: +You should upgrade a Reference Architecture in the same order as you created it. + +### Scaling an environment + +Scaling a GitLab environment is designed to be as seamless as possible. + +In terms of the Reference Architectures, you would look to the next size and adjust accordingly. +Most setups would only need vertical scaling, but there are some specific areas that can be adjusted depending on the setup: + +- If you're scaling from a non-HA environment to an HA environment, various components are recommended to be deployed in their HA forms: + - Redis to multi-node Redis w/ Redis Sentinel + - Postgres to multi-node Postgres w/ Consul + PgBouncer + - Gitaly to Gitaly Cluster w/ Praefect +- From 10k users and higher, Redis is recommended to be split into multiple HA servers as it's single threaded. + +Conversely, if you have robust metrics in place that show the environment is over-provisioned you can apply the same process for +scaling downloads. It's recommended to take an iterative approach when scaling downwards however to ensure there are no issues. + +### How to monitor your environment -Other technologies, like [Docker swarm](https://docs.docker.com/engine/swarm/) -are not officially supported, but can be implemented at your own risk. In that -case, GitLab Support is not able to help you. +To monitor your GitLab environment, you can use the tools +[bundled with GitLab](../monitoring/index.md), but it's also possible to use third-party +options if desired. diff --git a/doc/administration/reply_by_email.md b/doc/administration/reply_by_email.md index 6e97ffc3b47..b632108b103 100644 --- a/doc/administration/reply_by_email.md +++ b/doc/administration/reply_by_email.md @@ -1,6 +1,6 @@ --- -stage: Plan -group: Certify +stage: Monitor +group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w GitLab can be set up to allow users to comment on issues and merge requests by replying to notification emails. -## Requirement +## Prerequisite Make sure [incoming email](incoming_email.md) is set up. diff --git a/doc/administration/repository_checks.md b/doc/administration/repository_checks.md index 08c1df3d5eb..3f715e451a8 100644 --- a/doc/administration/repository_checks.md +++ b/doc/administration/repository_checks.md @@ -9,9 +9,9 @@ info: To determine the technical writer assigned to the Stage/Group associated w You can use [`git fsck`](https://git-scm.com/docs/git-fsck) to verify the integrity of all data committed to a repository. GitLab administrators can: -- Manually trigger this check for a project, using the GitLab UI. -- Schedule this check to run automatically for all projects. -- Run this check from the command line. +- [Manually trigger this check for a project](#check-a-projects-repository-using-gitlab-ui). +- [Schedule this check](#enable-repository-checks-for-all-projects) to run automatically for all projects. +- [Run this check from the command line](#run-a-check-using-the-command-line). - Run a [Rake task](raketasks/check.md#repository-integrity) for checking Git repositories, which can be used to run `git fsck` against all repositories and generate repository checksums, as a way to compare repositories on different servers. @@ -68,9 +68,13 @@ You can run [`git fsck`](https://git-scm.com/docs/git-fsck) using the command li 1. Run the check. For example: ```shell - sudo -u git /opt/gitlab/embedded/bin/git -C /var/opt/gitlab/git-data/repositories/@hashed/0b/91/0b91...f9.git fsck + sudo -u git /opt/gitlab/embedded/bin/git \ + -C /var/opt/gitlab/git-data/repositories/@hashed/0b/91/0b91...f9.git fsck --no-dangling ``` + The error `fatal: detected dubious ownership in repository` means you're running the command + using the wrong account. For example, `root`. + ## What to do if a check failed If a repository check fails, locate the error in the [`repocheck.log` file](logs/index.md#repochecklog) on disk at: @@ -93,3 +97,26 @@ of date. The `commit-graph` cache is an auxiliary cache and is not required for While the message can be safely ignored, see the issue [error: Could not read from object database for commit-graph](https://gitlab.com/gitlab-org/gitaly/-/issues/2359) for more details. + +### Dangling commit, tag, or blob messages + +Repository check output often includes tags, blobs, and commits that must be pruned: + +```plaintext +dangling tag 5c6886c774b713a43158aae35c4effdb03a3ceca +dangling blob 3e268c23fcd736db92e89b31d9f267dd4a50ac4b +dangling commit 919ff61d8d78c2e3ea9a32701dff70ecbefdd1d7 +``` + +This is common in Git repositories. They're generated by operations like +force pushing to branches, because this generates a commit in the repository +that is not longer referred to by a ref or by another commit. + +If a repository check fails, the output is likely to include these warnings. + +Ignore these messages, and identify the root cause of the repository check failure +from the other output. + +[GitLab 15.8 and later](https://gitlab.com/gitlab-org/gitaly/-/merge_requests/5230) no +longer includes these messages in the check output. Use the `--no-dangling` option +to suppress then when run from the command line. diff --git a/doc/administration/restart_gitlab.md b/doc/administration/restart_gitlab.md index 7996db3d1e1..e86541b7ced 100644 --- a/doc/administration/restart_gitlab.md +++ b/doc/administration/restart_gitlab.md @@ -126,3 +126,17 @@ kubectl delete pods -l release=,app= ``` The release name can be obtained from the output of the `helm list` command. + +## Docker installation + +If you change the configuration on your [Docker installation](../install/docker.md), for that change to take effect you must restart: + +- The main `gitlab` container. +- Any separate component containers. + +For example, if you deployed Sidekiq on a separate container, to restart the containers, run: + +```shell +sudo docker restart gitlab +sudo docker restart sidekiq +``` diff --git a/doc/administration/server_hooks.md b/doc/administration/server_hooks.md index 3d4f39b5ff0..b167412075b 100644 --- a/doc/administration/server_hooks.md +++ b/doc/administration/server_hooks.md @@ -2,7 +2,6 @@ stage: Systems group: Gitaly info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -disqus_identifier: 'https://docs.gitlab.com/ee/administration/custom_hooks.html' --- # Git server hooks **(FREE SELF)** @@ -28,7 +27,45 @@ alternatives to server hooks include: [Geo](geo/index.md) doesn't replicate server hooks to secondary nodes. -## Create server hooks for a repository +## Set server hooks for a repository + +::Tabs + +:::TabTitle GitLab 15.11 and later + +> [Introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/4629) in GitLab 15.11, `hooks set` command replaces direct file system access. + +Prerequisites: + +- The [storage name](gitaly/configure_gitaly.md#gitlab-requires-a-default-repository-storage), path to the Gitaly configuration file + (default is `/var/opt/gitlab/gitaly/config.toml` on Omnibus GitLab instances), and the + [repository relative path](repository_storage_types.md#from-project-name-to-hashed-path) for the repository. + +To set server hooks for a repository: + +1. Create tarball containing custom hooks: + 1. Write the code to make the server hook function as expected. Git server hooks can be in any programming language. + Ensure the [shebang](https://en.wikipedia.org/wiki/Shebang_(Unix)) at the top reflects the language type. For + example, if the script is in Ruby the shebang is probably `#!/usr/bin/env ruby`. + + - To create a single server hook, create a file with a name that matches the hook type. For example, for a + `pre-receive` server hook, the filename should be `pre-receive` with no extension. + - To create many server hooks, create a directory for the hooks that matches the hook type. For example, for a + `pre-receive` server hook, the directory name should be `pre-receive.d`. Put the files for the hook in that + directory. + + 1. Ensure the server hook files are executable and do not match the backup file pattern (`*~`). The server hooks be + in a `custom_hooks` directory that is at the root of the tarball. + 1. Create the custom hooks archive with the tar command. For example, `tar -cf custom_hooks.tar custom_hooks`. +1. Run the `hooks set` command with required options to set the Git hooks for the repository. For example, + `cat hooks.tar | gitaly hooks set --storage --repository --config `. + + - A path to a valid Gitaly configuration for the node is required to connect to the node and provided to the `--config` flag. + - Custom hooks tarball must be passed via `stdin`. For example, `cat hooks.tar | gitaly hooks set --storage --repository --config `. + +If you implemented the server hook code correctly, it should execute when the Git hook is next triggered. + +:::TabTitle GitLab 15.10 and earlier To create server hooks for a repository: @@ -56,6 +93,8 @@ To create server hooks for a repository: If the server hook code is properly implemented, it should execute when the Git hook is next triggered. +::EndTabs + ### Gitaly Cluster If you use [Gitaly Cluster](gitaly/index.md), the scripts must be copied to every Gitaly node that has a replica of the repository. Every Gitaly node @@ -84,7 +123,7 @@ To create a Git hook that applies to all repositories, set a global server hook. Before creating a global server hook, you must choose a directory for it. -For Omnibus GitLab installations, the directory is set in `gitlab.rb` under `gitaly['custom_hooks_dir']`. You can either: +For Omnibus GitLab installations, the directory is set in `gitlab.rb` under `gitaly['configuration'][:hooks][:custom_hooks_dir]`. You can either: - Use the default suggestion of the `/var/opt/gitlab/gitaly/custom_hooks` directory by uncommenting it. - Add your own setting. @@ -113,6 +152,33 @@ To create a global server hook for all repositories: If the server hook code is properly implemented, it should execute when the Git hook is next triggered. Hooks are executed in alphabetical order by filename in the hook type subdirectories. +## Remove server hooks for a repository + +::Tabs + +:::TabTitle GitLab 15.11 and later + +> [Introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/4629) in GitLab 15.11, `hooks set` command replaces direct file system access. + +Prerequisites: + +- The [storage name and relative path](repository_storage_types.md#from-project-name-to-hashed-path) for the repository. + +To remove server hooks, pass an empty tarball to `hook set` to indicate that the repository should contain no hooks. For example: + +```shell +cat empty_hooks.tar | gitaly hooks set --storage --repository --config `. +``` + +:::TabTitle GitLab 15.10 and earlier + +To remove server hooks: + +1. Go to the location of the repository on disk. +1. Delete the server hooks in the `custom_hooks` directory. + +::EndTabs + ## Chained server hooks GitLab can execute server hooks in a chain. GitLab searches for and executes server hooks in the following order: diff --git a/doc/administration/sidekiq/extra_sidekiq_routing.md b/doc/administration/sidekiq/extra_sidekiq_routing.md deleted file mode 100644 index d1d65498fcc..00000000000 --- a/doc/administration/sidekiq/extra_sidekiq_routing.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'processing_specific_job_classes.md#routing-rules' -remove_date: '2023-02-01' ---- - -This document was moved to [another location](processing_specific_job_classes.md#routing-rules). - - - - - diff --git a/doc/administration/sidekiq/index.md b/doc/administration/sidekiq/index.md index bf858476c0c..6fb12aa6ef9 100644 --- a/doc/administration/sidekiq/index.md +++ b/doc/administration/sidekiq/index.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w You can configure an external Sidekiq instance by using the Sidekiq that's bundled in the GitLab package. Sidekiq requires connection to the Redis, PostgreSQL, and Gitaly instances. -## Configure TCP access for PostgreSQL, Gitaly, and Redis +## Configure TCP access for PostgreSQL, Gitaly, and Redis on the GitLab instance By default, GitLab uses UNIX sockets and is not set up to communicate via TCP. To change this: @@ -32,12 +32,20 @@ By default, GitLab uses UNIX sockets and is not set up to communicate via TCP. T ## Gitaly - # Make Gitaly accept connections on all network interfaces - gitaly['listen_addr'] = "0.0.0.0:8075" - ## Set up the Gitaly token as a form of authentication since you are accessing Gitaly over the network - ## https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#about-the-gitaly-token - gitaly['auth_token'] = 'abc123secret' - praefect['auth_token'] = 'abc123secret' + gitaly['configuration'] = { + # ... + # + # Make Gitaly accept connections on all network interfaces + listen_addr: '0.0.0.0:8075', + auth: { + ## Set up the Gitaly token as a form of authentication since you are accessing Gitaly over the network + ## https://docs.gitlab.com/ee/administration/gitaly/configure_gitaly.html#about-the-gitaly-token + token: 'abc123secret', + }, + } + + gitaly['auth_token'] = '' + praefect['configuration'][:auth][:token] = 'abc123secret' gitlab_rails['gitaly_token'] = 'abc123secret' ## Redis configuration @@ -48,7 +56,6 @@ By default, GitLab uses UNIX sockets and is not set up to communicate via TCP. T redis['password'] = 'redis-password-goes-here' gitlab_rails['redis_password'] = 'redis-password-goes-here' - gitlab_rails['auto_migrate'] = false ``` 1. Run `reconfigure`: @@ -63,18 +70,6 @@ By default, GitLab uses UNIX sockets and is not set up to communicate via TCP. T sudo gitlab-ctl restart postgresql ``` -1. After the restart, set `auto_migrate` to `true` or comment to use the default settings: - - ```ruby - gitlab_rails['auto_migrate'] = true - ``` - -1. Run `reconfigure` again: - - ```shell - sudo gitlab-ctl reconfigure - ``` - ## Set up Sidekiq instance 1. SSH into the Sidekiq server. @@ -170,7 +165,7 @@ Updates to example must be made at: # Replace and gitlab_rails['db_host'] = '' - gitlab_rails['db_port'] = '5432' + gitlab_rails['db_port'] = 5432 gitlab_rails['db_password'] = '' ## Prevent database migrations from running on upgrade automatically gitlab_rails['auto_migrate'] = false @@ -257,7 +252,7 @@ To configure the metrics server: ```ruby sidekiq['metrics_enabled'] = true sidekiq['listen_address'] = "localhost" - sidekiq['listen_port'] = "8082" + sidekiq['listen_port'] = 8082 # Optionally log all the metrics server logs to log/sidekiq_exporter.log sidekiq['exporter_log_enabled'] = true @@ -299,7 +294,7 @@ To make health checks available from `localhost:8092`: ```ruby sidekiq['health_checks_enabled'] = true sidekiq['health_checks_listen_address'] = "localhost" - sidekiq['health_checks_listen_port'] = "8092" + sidekiq['health_checks_listen_port'] = 8092 ``` 1. Reconfigure GitLab: @@ -325,7 +320,7 @@ To enable LDAP with the synchronization worker for Sidekiq: 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby + ```ruby gitlab_rails['ldap_enabled'] = true gitlab_rails['prevent_ldap_sign_in'] = false gitlab_rails['ldap_servers'] = { diff --git a/doc/administration/sidekiq/processing_specific_job_classes.md b/doc/administration/sidekiq/processing_specific_job_classes.md index 61a154c8db2..26c2804f130 100644 --- a/doc/administration/sidekiq/processing_specific_job_classes.md +++ b/doc/administration/sidekiq/processing_specific_job_classes.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w WARNING: These are advanced settings. While they are used on GitLab.com, most GitLab -instances should add more processes that all listen to all queues. This is the +instances should only add more processes that listen to all queues. This is the same approach we take in our [Reference Architectures](../reference_architectures/index.md). GitLab has two options for creating Sidekiq processes that only handle specific @@ -17,7 +17,7 @@ job classes: 1. [Routing rules](#routing-rules) are used on GitLab.com. They direct jobs inside the application to queue names configured by administrators. This lowers the load on Redis, which is important on very large-scale deployments. -1. [Queue selectors](#queue-selectors) perform the job selection outside the +1. [Queue selectors](#queue-selectors-deprecated) perform the job selection outside the application, when starting the Sidekiq process. This was used on GitLab.com until September 2021, and is retained for compatibility reasons. @@ -106,11 +106,19 @@ not a recommendation. sudo gitlab-ctl reconfigure ``` -## Queue selectors + + +## Queue selectors (deprecated) > - [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/45) in GitLab 12.8. > - [Sidekiq cluster, including queue selector, moved](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/181) to GitLab Free in 12.10. > - [Renamed from `experimental_queue_selector` to `queue_selector`](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/147) in GitLab 13.6. +> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/390787) in GitLab 15.9. + +WARNING: +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/390787) in GitLab 15.9 +and is planned for removal in 17.0. Most instances should have [all processes to listen to all queues](extra_sidekiq_processes.md#start-multiple-processes). +Another alternative is to use [routing rules](#routing-rules) (be warned this is an advanced setting). This change is a breaking change. The `queue_selector` option allows queue groups to be selected in a more general way using a [worker matching query](#worker-matching-query). After @@ -141,7 +149,12 @@ syntax. sudo gitlab-ctl reconfigure ``` -### Negate settings +### Negate settings (deprecated) + +WARNING: +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/390787) in GitLab 15.9 +and is planned for removal in 17.0. Most instances should have [all processes to listen to all queues](extra_sidekiq_processes.md#start-multiple-processes). +Another alternative is to use [routing rules](#routing-rules) (be warned this is an advanced setting). This change is a breaking change. This allows you to have the Sidekiq process work on every queue **except** the ones you list. This is generally only used when there are multiple Sidekiq @@ -168,7 +181,7 @@ nodes. In this example, we exclude all import-related jobs from a Sidekiq node. We recommend GitLab deployments add more Sidekiq processes listening to all queues, as in the [Reference Architectures](../reference_architectures/index.md). For very large-scale deployments, we recommend -[routing rules](#routing-rules) instead of [queue selectors](#queue-selectors). We use routing rules on GitLab.com as +[routing rules](#routing-rules) instead of [queue selectors](#queue-selectors-deprecated). We use routing rules on GitLab.com as it helps to lower the load on Redis. To migrate from queue selectors to routing rules: @@ -255,11 +268,13 @@ in a queue group entry is 1, while `min_concurrency` is set to `0`, and `max_con concurrency is set to `2` instead. A concurrency of `2` might be too low in most cases, except for very highly-CPU bound tasks. + + ## Worker matching query GitLab provides a query syntax to match a worker based on its attributes. This query syntax is employed by both [routing rules](#routing-rules) and -[queue selectors](#queue-selectors). A query includes two components: +[queue selectors](#queue-selectors-deprecated). A query includes two components: - Attributes that can be selected. - Operators used to construct a query. diff --git a/doc/administration/sidekiq/sidekiq_memory_killer.md b/doc/administration/sidekiq/sidekiq_memory_killer.md index cb27d44a2e6..63d93919129 100644 --- a/doc/administration/sidekiq/sidekiq_memory_killer.md +++ b/doc/administration/sidekiq/sidekiq_memory_killer.md @@ -56,9 +56,7 @@ Sidekiq memory limits are controlled using environment variables. If jobs do not finish during that time, all currently running jobs are interrupted with a `SIGTERM` signal sent to the Sidekiq process. -- `GITLAB_MEMORY_WATCHDOG_ENABLED`: enabled by default. Set the `GITLAB_MEMORY_WATCHDOG_ENABLED` to false, to use legacy - Daemon Sidekiq Memory Killer implementation used prior GitLab 15.9. Support for setting `GITLAB_MEMORY_WATCHDOG_ENABLED` - will be removed in GitLab 16.0. +- `GITLAB_MEMORY_WATCHDOG_ENABLED`: enabled by default. Set the `GITLAB_MEMORY_WATCHDOG_ENABLED` to false, to disable Watchdog from running. ### Monitor worker restarts diff --git a/doc/administration/silent_mode/index.md b/doc/administration/silent_mode/index.md new file mode 100644 index 00000000000..e98aaaf4e0a --- /dev/null +++ b/doc/administration/silent_mode/index.md @@ -0,0 +1,64 @@ +--- +stage: Systems +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# GitLab Silent Mode (Experiment) **(FREE SELF)** + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/9826) in GitLab 15.11. This feature is an [Experiment](../../policy/alpha-beta-support.md#experiment). + +Silent Mode allows you to suppress outbound communication, such as emails, from GitLab. Silent Mode is not intended to be used on environments which are in-use. Two use-cases are: + +- Validating Geo site promotion. You have a secondary Geo site as part of your [disaster recovery](../geo/disaster_recovery/index.md) solution. You want to regularly test promoting it to become a primary Geo site, as a best practice to ensure your disaster recovery plan actually works. But you don't want to actually perform an entire failover, since the primary site lives in a region which provides the lowest latency to your users. And you don't want to take downtime during every regular test. So, you let the primary site remain up, while you promote the secondary site. You start smoke testing the promoted site. But, the promoted site starts emailing users, the push mirrors push changes to external Git repositories, etc. This is where Silent Mode comes in. You can enable it as part of site promotion, to avoid this issue. +- Validating GitLab backups. You set up a testing instance to test that your backups restore successfully. As part of the restore, you enable Silent Mode, for example to avoid sending invalid emails to users. + +## Enable Silent Mode + +Prerequisites: + +- You must have administrator access. + +There are two ways to enable Silent Mode: + +- [**API**](../../api/settings.md): + + ```shell + curl --request PUT --header "PRIVATE-TOKEN:$ADMIN_TOKEN" "/api/v4/application/settings?silent_mode_enabled=true" + ``` + +- [**Rails console**](../operations/rails_console.md#starting-a-rails-console-session): + + ```ruby + ::Gitlab::CurrentSettings.update!(silent_mode_enabled: true) + ``` + +## Disable Silent Mode + +Prerequisites: + +- You must have administrator access. + +There are two ways to disable Silent Mode: + +- [**API**](../../api/settings.md): + + ```shell + curl --request PUT --header "PRIVATE-TOKEN:$ADMIN_TOKEN" "/api/v4/application/settings?silent_mode_enabled=false" + ``` + +- [**Rails console**](../operations/rails_console.md#starting-a-rails-console-session): + + ```ruby + ::Gitlab::CurrentSettings.update!(silent_mode_enabled: false) + ``` + +## Behavior of GitLab features in Silent Mode + +### Service Desk + +Incoming emails still raise issues, but the users who sent the emails to [Service Desk](../../user/project/service_desk.md) are not notified of issue creation or comments on their issues. + +### Outbound emails + +Outbound emails are suppressed. It may take up to a minute to take effect after enabling Silent Mode. [Issue 405433](https://gitlab.com/gitlab-org/gitlab/-/issues/405433) proposes removing this delay. diff --git a/doc/administration/static_objects_external_storage.md b/doc/administration/static_objects_external_storage.md index ef89e38be6f..73f44ed3889 100644 --- a/doc/administration/static_objects_external_storage.md +++ b/doc/administration/static_objects_external_storage.md @@ -1,6 +1,6 @@ --- stage: Create -group: Editor +group: IDE info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments" type: reference --- diff --git a/doc/administration/system_hooks.md b/doc/administration/system_hooks.md index ddfa9fe9860..8d343e7c541 100644 --- a/doc/administration/system_hooks.md +++ b/doc/administration/system_hooks.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: reference --- diff --git a/doc/administration/terraform_state.md b/doc/administration/terraform_state.md index d3b941bd129..3ac9a5fdce8 100644 --- a/doc/administration/terraform_state.md +++ b/doc/administration/terraform_state.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -22,9 +22,23 @@ Use [external object storage](https://docs.gitlab.com/charts/advanced/external-o ## Disabling Terraform state -To disable terraform state site-wide, follow the steps below. -A GitLab administrator may want to disable Terraform state to reduce disk space or if Terraform is not used in your instance. -To do so, follow the steps below according to your installation's type. +You can disable Terraform state across the entire instance. You might want to disable Terraform to reduce disk space, +or because your instance doesn't use Terraform. + +When Terraform state administration is disabled: + +- On the left sidebar, you cannot select **Infrastructure > Terraform states**. +- Any CI/CD jobs that access the Terraform state fail with this error: + + ```shell + Error refreshing state: HTTP remote state endpoint invalid auth + ``` + +To disable Terraform administration, follow the steps below according to your installation. + +Prerequisite: + +- You must be an administrator. **In Omnibus installations:** @@ -79,7 +93,7 @@ Terraform state files are stored locally, follow the steps below. ## Using object storage **(FREE SELF)** Instead of storing Terraform state files on disk, we recommend the use of -[one of the supported object storage options](object_storage.md#options). +[one of the supported object storage options](object_storage.md#supported-object-storage-providers). This configuration relies on valid credentials to be configured already. [Read more about using object storage with GitLab](object_storage.md). @@ -161,10 +175,10 @@ sudo find /var/opt/gitlab/gitlab-rails/shared/terraform_state -type f | grep -v ### S3-compatible connection settings In GitLab 13.2 and later, you should use the -[consolidated object storage settings](object_storage.md#consolidated-object-storage-configuration). +[consolidated object storage settings](object_storage.md#configure-a-single-storage-connection-for-all-object-types-consolidated-form). This section describes the earlier configuration format. -See [the available connection settings for different providers](object_storage.md#connection-settings). +See [the available connection settings for different providers](object_storage.md#configure-the-connection-settings). **In Omnibus installations:** diff --git a/doc/administration/troubleshooting/index.md b/doc/administration/troubleshooting/index.md index db572f202ea..8be6abc180d 100644 --- a/doc/administration/troubleshooting/index.md +++ b/doc/administration/troubleshooting/index.md @@ -27,3 +27,28 @@ that you can check for feature-specific help, including helpful Rails commands. If you need a testing environment to troubleshoot, see the [apps for a testing environment](test_environments.md). + +## Support team troubleshooting info + +The GitLab Support Team has collected a lot of information about troubleshooting GitLab. +The following documents are used by the Support Team or by customers +with direct guidance from a Support Team member. GitLab administrators may find the +information useful for troubleshooting. However, if you are experiencing trouble with your +GitLab instance, you should check your [support options](https://about.gitlab.com/support/) +before referring to these documents. + +WARNING: +The commands in the following documentation might result in data loss or +other damage to a GitLab instance. They should be used only by experienced administrators +who are aware of the risks. + +- [Diagnostics tools](diagnostics_tools.md) +- [Linux commands](linux_cheat_sheet.md) +- [Troubleshooting Kubernetes](https://docs.gitlab.com/charts/troubleshooting/kubernetes_cheat_sheet.html) +- [Troubleshooting PostgreSQL](postgresql.md) +- [Guide to test environments](test_environments.md) (for Support Engineers) +- [Troubleshooting SSL](ssl.md) +- Related links: + - [Repairing and recovering broken Git repositories](https://git.seveas.net/repairing-and-recovering-broken-git-repositories.html) + - [Testing with OpenSSL](https://www.feistyduck.com/library/openssl-cookbook/online/testing-with-openssl/index.html) + - [`strace` zine](https://wizardzines.com/zines/strace/) diff --git a/doc/administration/uploads.md b/doc/administration/uploads.md index ff0b8ecf178..ab900d7ea9f 100644 --- a/doc/administration/uploads.md +++ b/doc/administration/uploads.md @@ -8,6 +8,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w Uploads represent all user data that may be sent to GitLab as a single file. For example, avatars and note attachments are uploads. Uploads are integral to GitLab functionality and therefore cannot be disabled. +NOTE: +Attachments added to comments or descriptions are deleted **only** when the parent project or group +is deleted. Attachments remain in file storage even when the comment or resource (like issue, merge +request, epic) where they were uploaded is deleted. + ## Using local storage This is the default configuration. To change the location where the uploads are @@ -60,7 +65,7 @@ This configuration relies on valid AWS credentials to be configured already. ### Object Storage Settings In GitLab 13.2 and later, you should use the -[consolidated object storage settings](object_storage.md#consolidated-object-storage-configuration). +[consolidated object storage settings](object_storage.md#configure-a-single-storage-connection-for-all-object-types-consolidated-form). This section describes the earlier configuration format. For source installations the following settings are nested under `uploads:` and then `object_store:`. On Omnibus GitLab installs they are prefixed by `uploads_object_store_`. @@ -74,7 +79,7 @@ For source installations the following settings are nested under `uploads:` and #### Connection settings -See [the available connection settings for different providers](object_storage.md#connection-settings). +See [the available connection settings for different providers](object_storage.md#configure-the-connection-settings). **In Omnibus installations:** @@ -114,7 +119,7 @@ _The uploads are stored by default in `/home/git/gitlab/public/uploads`._ 1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following - lines, making sure to use the [appropriate ones for your provider](object_storage.md#connection-settings): + lines, making sure to use the [appropriate ones for your provider](object_storage.md#configure-the-connection-settings): ```yaml uploads: diff --git a/doc/administration/wikis/index.md b/doc/administration/wikis/index.md index f3442e23160..330e41ee880 100644 --- a/doc/administration/wikis/index.md +++ b/doc/administration/wikis/index.md @@ -1,7 +1,7 @@ --- type: reference, howto -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/api/api_resources.md b/doc/api/api_resources.md index b7c1def0ba4..4a70786b6ee 100644 --- a/doc/api/api_resources.md +++ b/doc/api/api_resources.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -26,7 +26,6 @@ The following API resources are available in the project context: | [Access requests](access_requests.md) | `/projects/:id/access_requests` (also available for groups) | | [Access tokens](project_access_tokens.md) | `/projects/:id/access_tokens` (also available for groups) | | [Agents](cluster_agents.md) | `/projects/:id/cluster_agents` | -| [Award emoji](award_emoji.md) | `/projects/:id/issues/.../award_emoji`, `/projects/:id/merge_requests/.../award_emoji`, `/projects/:id/snippets/.../award_emoji` | | [Branches](branches.md) | `/projects/:id/repository/branches/`, `/projects/:id/repository/merged_branches` | | [Commits](commits.md) | `/projects/:id/repository/commits`, `/projects/:id/statuses` | | [Container Registry](container_registry.md) | `/projects/:id/registry/repositories` | @@ -41,11 +40,12 @@ The following API resources are available in the project context: | [Deployments](deployments.md) | `/projects/:id/deployments` | | [Discussions](discussions.md) (threaded comments) | `/projects/:id/issues/.../discussions`, `/projects/:id/snippets/.../discussions`, `/projects/:id/merge_requests/.../discussions`, `/projects/:id/commits/.../discussions` (also available for groups) | | [Draft Notes](draft_notes.md) (comments) | `/projects/:id/merge_requests/.../draft_notes` +| [Emoji reactions](award_emoji.md) | `/projects/:id/issues/.../award_emoji`, `/projects/:id/merge_requests/.../award_emoji`, `/projects/:id/snippets/.../award_emoji` | | [Environments](environments.md) | `/projects/:id/environments` | | [Error Tracking](error_tracking.md) | `/projects/:id/error_tracking/settings` | | [Events](events.md) | `/projects/:id/events` (also available for users and standalone) | -| [Feature Flag User Lists](feature_flag_user_lists.md) | `/projects/:id/feature_flags_user_lists` | -| [Feature Flags](feature_flags.md) | `/projects/:id/feature_flags` | +| [Feature flag User Lists](feature_flag_user_lists.md) | `/projects/:id/feature_flags_user_lists` | +| [Feature flags](feature_flags.md) | `/projects/:id/feature_flags` | | [Freeze Periods](freeze_periods.md) | `/projects/:id/freeze_periods` | | [Go Proxy](packages/go_proxy.md) | `/projects/:id/packages/go` | | [Helm repository](packages/helm.md) | `/projects/:id/packages/helm_repository` | @@ -163,7 +163,8 @@ The following API resources are available outside of project and group contexts | [Geo Nodes](geo_nodes.md) **(PREMIUM SELF)** | `/geo_nodes` | | [Group Activity Analytics](group_activity_analytics.md) | `/analytics/group_activity/{issues_count}` | | [Group repository storage moves](group_repository_storage_moves.md) **(PREMIUM SELF)** | `/group_repository_storage_moves` | -| [Import repository from GitHub](import.md) | `/import/github` | +| [Import repository from GitHub](import.md#import-repository-from-github) | `/import/github` | +| [Import repository from Bitbucket Server](import.md#import-repository-from-bitbucket-server) | `/import/bitbucket_server` | | [Instance clusters](instance_clusters.md) **(FREE SELF)** | `/admin/clusters` | | [Instance-level CI/CD variables](instance_level_ci_variables.md) **(FREE SELF)** | `/admin/ci/variables` | | [Issues Statistics](issues_statistics.md) | `/issues_statistics` (also available for groups and projects) | diff --git a/doc/api/audit_events.md b/doc/api/audit_events.md index fec719b189c..c4b3d99c742 100644 --- a/doc/api/audit_events.md +++ b/doc/api/audit_events.md @@ -8,6 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/121) in GitLab 12.4. > - [Author Email added to the response body](https://gitlab.com/gitlab-org/gitlab/-/issues/386322) in GitLab 15.9. +> - Support for keyset pagination [added](https://gitlab.com/gitlab-org/gitlab/-/issues/367528) in GitLab 15.11. ## Instance Audit Events **(PREMIUM SELF)** @@ -29,8 +30,8 @@ GET /audit_events | `entity_type` | string | no | Return audit events for the given entity type. Valid values are: `User`, `Group`, or `Project`. | | `entity_id` | integer | no | Return audit events for the given entity ID. Requires `entity_type` attribute to be present. | -By default, `GET` requests return 20 results at a time because the API results -are paginated. +This endpoint supports both offset-based and [keyset-based](rest/index.md#keyset-based-pagination) pagination. You should use keyset-based +pagination when requesting consecutive pages of results. Read more on [pagination](rest/index.md#pagination). @@ -137,7 +138,7 @@ Example response: ## Group Audit Events > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34078) in GitLab 12.5. -> - [Support for keyset pagination added](https://gitlab.com/gitlab-org/gitlab/-/issues/333968) in GitLab 15.2. +> - Support for keyset pagination [added](https://gitlab.com/gitlab-org/gitlab/-/issues/333968) in GitLab 15.2. The Group Audit Events API allows you to retrieve [group audit events](../administration/audit_events.md#group-events). This API cannot retrieve project audit events. @@ -253,13 +254,16 @@ Example response: ## Project Audit Events -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/219238) in GitLab 13.1. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/219238) in GitLab 13.1. +> - Support for keyset pagination [added](https://gitlab.com/gitlab-org/gitlab/-/issues/367528) in GitLab 15.10. The Project Audit Events API allows you to retrieve [project audit events](../administration/audit_events.md#project-events). A user with a Maintainer role (or above) can retrieve project audit events of all users. A user with a Developer role is limited to project audit events based on their individual actions. +When requesting consecutive pages of results, you should use [keyset pagination](rest/index.md#keyset-based-pagination). + ### Retrieve all project audit events ```plaintext diff --git a/doc/api/award_emoji.md b/doc/api/award_emoji.md index a669c6d00c3..a22c61b3391 100644 --- a/doc/api/award_emoji.md +++ b/doc/api/award_emoji.md @@ -4,28 +4,30 @@ group: Project Management info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Award emojis API **(FREE)** +# Emoji reactions API **(FREE)** -An [awarded emoji](../user/award_emojis.md) tells a thousand words. +> [Renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/409884) from "award emojis" to "emoji reactions" in GitLab 16.0. -We call GitLab objects on which you can award an emoji "awardables". You can award emojis on the following: +An [emoji reaction](../user/award_emojis.md) tells a thousand words. + +We call GitLab objects on which you can react with an emoji "awardables". +You can react with emojis on the following: - [Epics](../user/group/epics/index.md) ([API](epics.md)). **(PREMIUM)** - [Issues](../user/project/issues/index.md) ([API](issues.md)). - [Merge requests](../user/project/merge_requests/index.md) ([API](merge_requests.md)). - [Snippets](../user/snippets.md) ([API](snippets.md)). - -Emojis can also [be awarded](../user/award_emojis.md#award-emojis-for-comments) on comments (also known as notes). See also [Notes API](notes.md). +- [Comments](../user/award_emojis.md#emoji-reactions-for-comments) ([API](notes.md)). ## Issues, merge requests, and snippets -See [Award emojis on comments](#award-emojis-on-comments) for information on using these endpoints with comments. +For information on using these endpoints with comments, see [Add reactions to comments](#add-reactions-to-comments). -### List an awardable's award emojis +### List an awardable's emoji reactions > [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/335068) in GitLab 15.1 to allow unauthenticated access to public awardables. -Get a list of all award emojis for a specified awardable. This endpoint can +Get a list of all emoji reactions for a specified awardable. This endpoint can be accessed without authentication if the awardable is publicly accessible. ```plaintext @@ -86,11 +88,11 @@ Example response: ] ``` -### Get single award emoji +### Get single emoji reaction > [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/335068) in GitLab 15.1 to allow unauthenticated access to public awardables. -Get a single award emoji from an issue, snippet, or merge request. This endpoint can +Get a single emoji reaction from an issue, snippet, or merge request. This endpoint can be accessed without authentication if the awardable is publicly accessible. ```plaintext @@ -105,7 +107,7 @@ Parameters: |:---------------|:---------------|:---------|:-----------------------------------------------------------------------------| | `id` | integer/string | yes | ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | | `issue_iid`/`merge_request_iid`/`snippet_id` | integer | yes | ID (`iid` for merge requests/issues, `id` for snippets) of an awardable. | -| `award_id` | integer | yes | ID of the award emoji. | +| `award_id` | integer | yes | ID of the emoji reaction. | Example request: @@ -134,9 +136,9 @@ Example response: } ``` -### Award a new emoji +### Add a new emoji reaction -Create an award emoji on the specified awardable. +Add an emoji reaction on the specified awardable. ```plaintext POST /projects/:id/issues/:issue_iid/award_emoji @@ -177,11 +179,11 @@ Example Response: } ``` -### Delete an award emoji +### Delete an emoji reaction -Sometimes it's just not meant to be and you need to remove the award. +Sometimes it's just not meant to be and you need to remove your reaction. -Only an administrator or the author of the award can delete an award emoji. +Only an administrator or the author of the reaction can delete an emoji reaction. ```plaintext DELETE /projects/:id/issues/:issue_iid/award_emoji/:award_id @@ -195,26 +197,26 @@ Parameters: |:---------------|:---------------|:---------|:-----------------------------------------------------------------------------| | `id` | integer/string | yes | ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | | `issue_iid`/`merge_request_iid`/`snippet_id` | integer | yes | ID (`iid` for merge requests/issues, `id` for snippets) of an awardable. | -| `award_id` | integer | yes | ID of an award emoji. | +| `award_id` | integer | yes | ID of an emoji reaction. | ```shell curl --request DELETE --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/issues/80/award_emoji/344" ``` -## Award emojis on comments +## Add reactions to comments Comments (also known as notes) are a sub-resource of issues, merge requests, and snippets. NOTE: -The examples below describe working with award emojis on an issue's comments, but can be +The examples below describe working with emoji reactions on an issue's comments, but can be adapted to comments on merge requests and snippets. Therefore, you have to replace `issue_iid` either with `merge_request_iid` or with the `snippet_id`. -### List a comment's award emojis +### List a comment's emoji reactions > [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/335068) in GitLab 15.1 to allow unauthenticated access to public comments. -Get all award emojis for a comment (note). This endpoint can +Get all emoji reactions for a comment (note). This endpoint can be accessed without authentication if the comment is publicly accessible. ```plaintext @@ -258,11 +260,11 @@ Example response: ] ``` -### Get an award emoji for a comment +### Get an emoji reaction for a comment > [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/335068) in GitLab 15.1 to allow unauthenticated access to public comments. -Get a single award emoji for a comment (note). This endpoint can +Get a single emoji reaction for a comment (note). This endpoint can be accessed without authentication if the comment is publicly accessible. ```plaintext @@ -276,7 +278,7 @@ Parameters: | `id` | integer/string | yes | ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | | `issue_iid` | integer | yes | Internal ID of an issue. | | `note_id` | integer | yes | ID of a comment (note). | -| `award_id` | integer | yes | ID of the award emoji. | +| `award_id` | integer | yes | ID of the emoji reaction. | Example request: @@ -307,7 +309,7 @@ Example response: ### Award a new emoji on a comment -Create an award emoji on the specified comment (note). +Create an emoji reaction on the specified comment (note). ```plaintext POST /projects/:id/issues/:issue_iid/notes/:note_id/award_emoji @@ -349,11 +351,11 @@ Example response: } ``` -### Delete an award emoji from a comment +### Delete an emoji reaction from a comment -Sometimes it's just not meant to be and you need to remove the award. +Sometimes it's just not meant to be and you need to remove the reaction. -Only an administrator or the author of the award can delete an award emoji. +Only an administrator or the author of the reaction can delete an emoji reaction. ```plaintext DELETE /projects/:id/issues/:issue_iid/notes/:note_id/award_emoji/:award_id @@ -366,7 +368,7 @@ Parameters: | `id` | integer/string | yes | ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | | `issue_iid` | integer | yes | Internal ID of an issue. | | `note_id` | integer | yes | ID of a comment (note). | -| `award_id` | integer | yes | ID of an award emoji. | +| `award_id` | integer | yes | ID of an emoji reaction. | Example request: diff --git a/doc/api/branches.md b/doc/api/branches.md index f99c4443ac8..fa508292e5c 100644 --- a/doc/api/branches.md +++ b/doc/api/branches.md @@ -27,7 +27,7 @@ Parameters: | Attribute | Type | Required | Description | |:----------|:---------------|:---------|:------------| | `id` | integer/string | yes | ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user.| -| `search` | string | no | Return list of branches containing the search string. You can use `^term` and `term$` to find branches that begin and end with `term` respectively. | +| `search` | string | no | Return list of branches containing the search string. You can use `^term` to find branches that begin with `term`, and `term$` to find branches that end with `term`. | | `regex` | string | no | Return list of branches with names matching a [re2](https://github.com/google/re2/wiki/Syntax) regular expression. | Example request: @@ -231,3 +231,9 @@ Example request: ```shell curl --request DELETE --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/repository/merged_branches" ``` + +## Related topics + +- [Branches](../user/project/repository/branches/index.md) +- [Protected branches](../user/project/protected_branches.md) +- [Protected branches API](protected_branches.md) diff --git a/doc/api/bulk_imports.md b/doc/api/bulk_imports.md index 6a33bd1bc95..31445240e1f 100644 --- a/doc/api/bulk_imports.md +++ b/doc/api/bulk_imports.md @@ -1,19 +1,35 @@ --- stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Group migration by direct transfer API **(FREE)** +# Group and project migration by direct transfer API **(FREE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64335) in GitLab 14.1. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64335) in GitLab 14.1. +> - Project migration [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/390515) in GitLab 15.11. With the group migration by direct transfer API, you can start and view the progress of migrations initiated with [group migration by direct transfer](../user/group/import/index.md#migrate-groups-by-direct-transfer-recommended). -## Start a new group migration +WARNING: +Migrating projects with this API is in [Beta](../policy/alpha-beta-support.md#beta). This feature is not +ready for production use. -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/66353) in GitLab 14.2. +## Prerequisites + +For information on prerequisites for group migration by direct transfer API, see +prerequisites for [migrating groups by direct transfer](../user/group/import/index.md#prerequisites). + +## Start a new group or project migration + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/66353) in GitLab 14.2. +> - `project_entity` source type [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/390515) in GitLab 15.11. + +Use this endpoint to start a new group or project migration. Specify: + +- `entities[group_entity]` to migrate a group. +- `entities[project_entity]` to migrate a project (Beta). ```plaintext POST /bulk_imports @@ -25,7 +41,7 @@ POST /bulk_imports | `configuration[url]` | String | yes | Source GitLab instance URL. | | `configuration[access_token]` | String | yes | Access token to the source GitLab instance. | | `entities` | Array | yes | List of entities to import. | -| `entities[source_type]` | String | yes | Source entity type (only `group_entity` is supported). | +| `entities[source_type]` | String | yes | Source entity type. Valid values are `group_entity` (GitLab 14.2 and later) and `project_entity` (GitLab 15.11 and later). | | `entities[source_full_path]` | String | yes | Source full path of the entity to import. | | `entities[destination_name]` | String | yes | Deprecated: Use :destination_slug instead. Destination slug for the entity. | | `entities[destination_slug]` | String | yes | Destination slug for the entity. | @@ -54,18 +70,18 @@ curl --request POST --header "PRIVATE-TOKEN: " "https://gitla { "id": 1, "status": "created", "source_type": "gitlab", "created_at": "2021-06-18T09:45:55.358Z", "updated_at": "2021-06-18T09:46:27.003Z" } ``` -## List all group migrations +## List all group or project migrations ```plaintext GET /bulk_imports ``` -| Attribute | Type | Required | Description | -|:-----------|:--------|:---------|:--------------------------------------------------------------------------------------------| -| `per_page` | integer | no | Number of records to return per page. | -| `page` | integer | no | Page to retrieve. | -| `sort` | string | no | Return GitLab migration sorted in `asc` or `desc` order by creation date. Default is `desc` | -| `status` | string | no | Import status. | +| Attribute | Type | Required | Description | +|:-----------|:--------|:---------|:-----------------------------------------------------------------------------------| +| `per_page` | integer | no | Number of records to return per page. | +| `page` | integer | no | Page to retrieve. | +| `sort` | string | no | Return records sorted in `asc` or `desc` order by creation date. Default is `desc` | +| `status` | string | no | Import status. | The status can be one of the following: @@ -97,18 +113,18 @@ curl --request GET --header "PRIVATE-TOKEN: " "https://gitlab ] ``` -## List all group migrations' entities +## List all group or project migrations' entities ```plaintext GET /bulk_imports/entities ``` -| Attribute | Type | Required | Description | -|:-----------|:--------|:---------|:-----------------------------------------------------------------------------------------------------| -| `per_page` | integer | no | Number of records to return per page. | -| `page` | integer | no | Page to retrieve. | -| `sort` | string | no | Return GitLab migration entities sorted in `asc` or `desc` order by creation date. Default is `desc` | -| `status` | string | no | Import status. | +| Attribute | Type | Required | Description | +|:-----------|:--------|:---------|:-----------------------------------------------------------------------------------| +| `per_page` | integer | no | Number of records to return per page. | +| `page` | integer | no | Page to retrieve. | +| `sort` | string | no | Return records sorted in `asc` or `desc` order by creation date. Default is `desc` | +| `status` | string | no | Import status. | The status can be one of the following: @@ -165,7 +181,7 @@ curl --request GET --header "PRIVATE-TOKEN: " "https://gitlab ] ``` -## Get group migration details +## Get group or project migration details ```plaintext GET /bulk_imports/:id @@ -185,18 +201,18 @@ curl --request GET --header "PRIVATE-TOKEN: " "https://gitlab } ``` -## List group migration entities +## List group or project migration entities ```plaintext GET /bulk_imports/:id/entities ``` -| Attribute | Type | Required | Description | -|:-----------|:--------|:---------|:--------------------------------------------------------------------------------------------| -| `per_page` | integer | no | Number of records to return per page. | -| `page` | integer | no | Page to retrieve. | -| `sort` | string | no | Return GitLab migration sorted in `asc` or `desc` order by creation date. Default is `desc` | -| `status` | string | no | Import status. | +| Attribute | Type | Required | Description | +|:-----------|:--------|:---------|:-----------------------------------------------------------------------------------| +| `per_page` | integer | no | Number of records to return per page. | +| `page` | integer | no | Page to retrieve. | +| `sort` | string | no | Return records sorted in `asc` or `desc` order by creation date. Default is `desc` | +| `status` | string | no | Import status. | The status can be one of the following: @@ -221,7 +237,7 @@ curl --request GET --header "PRIVATE-TOKEN: " "https://gitlab ] ``` -## Get group migration entity details +## Get group or project migration entity details ```plaintext GET /bulk_imports/:id/entities/:entity_id diff --git a/doc/api/cluster_agents.md b/doc/api/cluster_agents.md index 2622d6782aa..4bd16b88d92 100644 --- a/doc/api/cluster_agents.md +++ b/doc/api/cluster_agents.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -242,7 +242,7 @@ curl --request DELETE --header "Private-Token: " "https://git > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/347046) in GitLab 15.0. -Returns a list of tokens for an agent. +Returns a list of active tokens for an agent. You must have at least the Developer role to use this endpoint. @@ -313,6 +313,8 @@ Gets a single agent token. You must have at least the Developer role to use this endpoint. +Returns a `404` if the agent token has been revoked. + ```plaintext GET /projects/:id/cluster_agents/:agent_id/tokens/:token_id ``` diff --git a/doc/api/commits.md b/doc/api/commits.md index 5a3481ee086..7c4d15e5d80 100644 --- a/doc/api/commits.md +++ b/doc/api/commits.md @@ -20,6 +20,8 @@ information: ## List repository commits +> Commits by author [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/114417) in GitLab 15.10. + Get a list of repository commits in a project. ```plaintext @@ -33,6 +35,7 @@ GET /projects/:id/repository/commits | `since` | string | no | Only commits after or on this date are returned in ISO 8601 format `YYYY-MM-DDTHH:MM:SSZ` | | `until` | string | no | Only commits before or on this date are returned in ISO 8601 format `YYYY-MM-DDTHH:MM:SSZ` | | `path` | string | no | The file path | +| `author` | string | no | Search commits by commit author.| | `all` | boolean | no | Retrieve every commit from the repository | | `with_stats` | boolean | no | Stats about each commit are added to the response | | `first_parent` | boolean | no | Follow only the first parent commit upon seeing a merge commit | @@ -525,6 +528,11 @@ cases below is valid: In any of the above cases, the response of `line`, `line_type` and `path` is set to `null`. +For other approaches to commenting on a merge request, see +[Create new merge request note](notes.md#create-new-merge-request-note) in the Notes API, +and [Create a new thread in the merge request diff](discussions.md#create-a-new-thread-in-the-merge-request-diff) +in the Discussions API. + ```plaintext POST /projects/:id/repository/commits/:sha/comments ``` diff --git a/doc/api/dependencies.md b/doc/api/dependencies.md index b99d9e2d91d..5ae36926868 100644 --- a/doc/api/dependencies.md +++ b/doc/api/dependencies.md @@ -7,7 +7,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Dependencies API **(ULTIMATE)** WARNING: -This API is in an [Alpha](../policy/alpha-beta-support.md#alpha-features) stage and considered unstable. +This API is in an [Experiment](../policy/alpha-beta-support.md#experiment) and considered unstable. The response payload may be subject to change or breakage across GitLab releases. @@ -34,7 +34,7 @@ GET /projects/:id/dependencies?package_manager=yarn,bundler | Attribute | Type | Required | Description | | ------------- | -------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | -| `package_manager` | string array | no | Returns dependencies belonging to specified package manager. Valid values: `bundler`, `composer`, `conan`, `go`, `gradle`, `maven`, `npm`, `nuget`, `pip`, `pipenv`, `yarn`, `sbt`, or `setuptools`. | +| `package_manager` | string array | no | Returns dependencies belonging to specified package manager. Valid values: `bundler`, `composer`, `conan`, `go`, `gradle`, `maven`, `npm`, `nuget`, `pip`, `pipenv`, `pnpm`, `yarn`, `sbt`, or `setuptools`. | ```shell curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/4/dependencies" diff --git a/doc/api/deploy_keys.md b/doc/api/deploy_keys.md index 684df9fdfdc..61e93b78067 100644 --- a/doc/api/deploy_keys.md +++ b/doc/api/deploy_keys.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -43,6 +43,7 @@ Example response: "fingerprint": "4a:9d:64:15:ed:3a:e6:07:6e:89:36:b3:3b:03:05:d9", "fingerprint_sha256": "SHA256:Jrs3LD1Ji30xNLtTVf9NDCj7kkBgPBb2pjvTZ3HfIgU", "created_at": "2013-10-02T10:12:29Z", + "expires_at": null, "projects_with_write_access": [ { "id": 73, @@ -71,6 +72,7 @@ Example response: "fingerprint": "0b:cf:58:40:b9:23:96:c7:ba:44:df:0e:9e:87:5e:75", "fingerprint_sha256": "SHA256:lGI/Ys/Wx7PfMhUO1iuBH92JQKYN+3mhJZvWO4Q5ims", "created_at": "2013-10-02T11:12:29Z", + "expires_at": null, "projects_with_write_access": [] } ] @@ -103,6 +105,7 @@ Example response: "fingerprint": "4a:9d:64:15:ed:3a:e6:07:6e:89:36:b3:3b:03:05:d9", "fingerprint_sha256": "SHA256:Jrs3LD1Ji30xNLtTVf9NDCj7kkBgPBb2pjvTZ3HfIgU", "created_at": "2013-10-02T10:12:29Z", + "expires_at": null, "can_push": false }, { @@ -112,6 +115,7 @@ Example response: "fingerprint": "0b:cf:58:40:b9:23:96:c7:ba:44:df:0e:9e:87:5e:75", "fingerprint_sha256": "SHA256:lGI/Ys/Wx7PfMhUO1iuBH92JQKYN+3mhJZvWO4Q5ims", "created_at": "2013-10-02T11:12:29Z", + "expires_at": null, "can_push": false } ] @@ -205,6 +209,7 @@ Example response: "fingerprint": "4a:9d:64:15:ed:3a:e6:07:6e:89:36:b3:3b:03:05:d9", "fingerprint_sha256": "SHA256:Jrs3LD1Ji30xNLtTVf9NDCj7kkBgPBb2pjvTZ3HfIgU", "created_at": "2013-10-02T10:12:29Z", + "expires_at": null, "can_push": false } ``` @@ -220,12 +225,13 @@ project only if the original one is accessible by the same user. POST /projects/:id/deploy_keys ``` -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | -| `title` | string | yes | New deploy key's title | -| `key` | string | yes | New deploy key | -| `can_push` | boolean | no | Can deploy key push to the project's repository | +| Attribute | Type | Required | Description | +| ----------- | ---- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | +| `key` | string | yes | New deploy key | +| `title` | string | yes | New deploy key's title | +| `can_push` | boolean | no | Can deploy key push to the project's repository | +| `expires_at` | datetime | no | Expiration date for the deploy key. Does not expire if no value is provided. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`) | ```shell curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ @@ -241,7 +247,8 @@ Example response: "id" : 12, "title" : "My deploy key", "can_push": true, - "created_at" : "2015-08-29T12:44:31.550Z" + "created_at" : "2015-08-29T12:44:31.550Z", + "expires_at": null } ``` @@ -256,8 +263,8 @@ PUT /projects/:id/deploy_keys/:key_id | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | -| `title` | string | no | New deploy key's title | | `can_push` | boolean | no | Can deploy key push to the project's repository | +| `title` | string | no | New deploy key's title | ```shell curl --request PUT --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ @@ -272,6 +279,7 @@ Example response: "title": "New deploy key", "key": "ssh-rsa AAAA...", "created_at": "2015-08-29T12:44:31.550Z", + "expires_at": null, "can_push": true } ``` @@ -317,7 +325,8 @@ Example response: "key" : "ssh-rsa AAAA...", "id" : 12, "title" : "My deploy key", - "created_at" : "2015-08-29T12:44:31.550Z" + "created_at" : "2015-08-29T12:44:31.550Z", + "expires_at": null } ``` diff --git a/doc/api/deploy_tokens.md b/doc/api/deploy_tokens.md index 2cfb58c25e1..ec9da949e48 100644 --- a/doc/api/deploy_tokens.md +++ b/doc/api/deploy_tokens.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -49,7 +49,7 @@ Example response: ## Project deploy tokens -Project deploy token API endpoints require the Maintainer role or higher +Project deploy token API endpoints require at least the Maintainer role for the project. ### List project deploy tokens @@ -150,9 +150,9 @@ Parameters: | ------------ | ---------------- | ---------------------- | ----------- | | `id` | integer/string | **{check-circle}** Yes | ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | | `name` | string | **{check-circle}** Yes | New deploy token's name | +| `scopes` | array of strings | **{check-circle}** Yes | Indicates the deploy token scopes. Must be at least one of `read_repository`, `read_registry`, `write_registry`, `read_package_registry`, or `write_package_registry`. | | `expires_at` | datetime | **{dotted-circle}** No | Expiration date for the deploy token. Does not expire if no value is provided. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`) | | `username` | string | **{dotted-circle}** No | Username for deploy token. Default is `gitlab+deploy-token-{n}` | -| `scopes` | array of strings | **{check-circle}** Yes | Indicates the deploy token scopes. Must be at least one of `read_repository`, `read_registry`, `write_registry`, `read_package_registry`, or `write_package_registry`. | Example request: @@ -306,9 +306,9 @@ Parameters: | ------------ | ---- | --------- | ----------- | | `id` | integer/string | **{check-circle}** Yes | ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding) owned by the authenticated user | | `name` | string | **{check-circle}** Yes | New deploy token's name | +| `scopes` | array of strings | **{check-circle}** Yes | Indicates the deploy token scopes. Must be at least one of `read_repository`, `read_registry`, `write_registry`, `read_package_registry`, or `write_package_registry`. | | `expires_at` | datetime | **{dotted-circle}** No | Expiration date for the deploy token. Does not expire if no value is provided. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`) | | `username` | string | **{dotted-circle}** No | Username for deploy token. Default is `gitlab+deploy-token-{n}` | -| `scopes` | array of strings | **{check-circle}** Yes | Indicates the deploy token scopes. Must be at least one of `read_repository`, `read_registry`, `write_registry`, `read_package_registry`, or `write_package_registry`. | Example request: diff --git a/doc/api/deployments.md b/doc/api/deployments.md index c6537493eab..cbcf06e1a11 100644 --- a/doc/api/deployments.md +++ b/doc/api/deployments.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: concepts, howto --- @@ -279,7 +279,7 @@ Example response: } ``` -When the [unified approval setting](../ci/environments/deployment_approvals.md#unified-approval-setting) is configured, deployments created by users on GitLab Premium or higher include the `approvals` and `pending_approval_count` properties: +When the [unified approval setting](../ci/environments/deployment_approvals.md#unified-approval-setting) is configured, deployments created by users on GitLab Premium or Ultimate include the `approvals` and `pending_approval_count` properties: ```json { @@ -304,7 +304,7 @@ When the [unified approval setting](../ci/environments/deployment_approvals.md#u } ``` -When the [multiple approval rules](../ci/environments/deployment_approvals.md#multiple-approval-rules) is configured, deployments created by users on GitLab Premium or higher include the `approval_summary` property: +When the [multiple approval rules](../ci/environments/deployment_approvals.md#multiple-approval-rules) is configured, deployments created by users on GitLab Premium or Ultimate include the `approval_summary` property: ```json { @@ -393,7 +393,7 @@ Example response: } ``` -Deployments created by users on GitLab Premium or higher include the `approvals` and `pending_approval_count` properties: +Deployments created by users on GitLab Premium or Ultimate include the `approvals` and `pending_approval_count` properties: ```json { @@ -447,7 +447,7 @@ Example response: } ``` -Deployments created by users on GitLab Premium or higher include the `approvals` and `pending_approval_count` properties: +Deployments created by users on GitLab Premium or Ultimate include the `approvals` and `pending_approval_count` properties: ```json { diff --git a/doc/api/discussions.md b/doc/api/discussions.md index ccef57dab7f..15bbc802817 100644 --- a/doc/api/discussions.md +++ b/doc/api/discussions.md @@ -38,8 +38,8 @@ GET /projects/:id/issues/:issue_iid/discussions | Attribute | Type | Required | Description | | ------------------- | ---------------- | ---------- | ------------ | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `issue_iid` | integer | yes | The IID of an issue | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `issue_iid` | integer | yes | The IID of an issue. | ```json [ @@ -65,6 +65,7 @@ GET /projects/:id/issues/:issue_iid/discussions "system": false, "noteable_id": 3, "noteable_type": "Issue", + "project_id": 5, "noteable_iid": null }, { @@ -85,6 +86,7 @@ GET /projects/:id/issues/:issue_iid/discussions "system": false, "noteable_id": 3, "noteable_type": "Issue", + "project_id": 5, "noteable_iid": null, "resolvable": false } @@ -112,6 +114,7 @@ GET /projects/:id/issues/:issue_iid/discussions "system": false, "noteable_id": 3, "noteable_type": "Issue", + "project_id": 5, "noteable_iid": null, "resolvable": false } @@ -127,7 +130,7 @@ curl --header "PRIVATE-TOKEN: "\ ### Get single issue discussion item -Returns a single discussion item for a specific project issue +Returns a single discussion item for a specific project issue. ```plaintext GET /projects/:id/issues/:issue_iid/discussions/:discussion_id @@ -137,18 +140,18 @@ Parameters: | Attribute | Type | Required | Description | | --------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `issue_iid` | integer | yes | The IID of an issue | -| `discussion_id` | integer | yes | The ID of a discussion item | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `issue_iid` | integer | yes | The IID of an issue. | +| `discussion_id` | integer | yes | The ID of a discussion item. | ```shell -curl --header "PRIVATE-TOKEN: "\ +curl --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/issues/11/discussions/" ``` ### Create new issue thread -Creates a new thread to a single project issue. This is similar to creating a note but other comments (replies) can be added to it later. +Creates a new thread to a single project issue. Similar to creating a note, but other comments (replies) can be added to it later. ```plaintext POST /projects/:id/issues/:issue_iid/discussions @@ -158,13 +161,13 @@ Parameters: | Attribute | Type | Required | Description | | --------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `issue_iid` | integer | yes | The IID of an issue | -| `body` | string | yes | The content of the thread | -| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | +| `body` | string | yes | The content of the thread. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `issue_iid` | integer | yes | The IID of an issue. | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z`. Requires administrator or project/group owner rights. | ```shell -curl --request POST --header "PRIVATE-TOKEN: "\ +curl --request POST --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/issues/11/discussions?body=comment" ``` @@ -172,7 +175,7 @@ curl --request POST --header "PRIVATE-TOKEN: "\ Adds a new note to the thread. This can also [create a thread from a single comment](../user/discussions/index.md#create-a-thread-by-replying-to-a-standard-comment). -**WARNING** +WARNING: Notes can be added to other items than comments, such as system notes, making them threads. ```plaintext @@ -183,15 +186,15 @@ Parameters: | Attribute | Type | Required | Description | | --------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `issue_iid` | integer | yes | The IID of an issue | -| `discussion_id` | integer | yes | The ID of a thread | -| `note_id` | integer | yes | The ID of a thread note | -| `body` | string | yes | The content of the note/reply | -| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | +| `body` | string | yes | The content of the note or reply. | +| `discussion_id` | integer | yes | The ID of a thread. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `issue_iid` | integer | yes | The IID of an issue. | +| `note_id` | integer | yes | The ID of a thread note. | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z`. Requires administrator or project/group owner rights. ```shell -curl --request POST --header "PRIVATE-TOKEN: "\ +curl --request POST --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/issues/11/discussions//notes?body=comment" ``` @@ -207,14 +210,14 @@ Parameters: | Attribute | Type | Required | Description | | --------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `issue_iid` | integer | yes | The IID of an issue | -| `discussion_id` | integer | yes | The ID of a thread | -| `note_id` | integer | yes | The ID of a thread note | -| `body` | string | yes | The content of the note/reply | +| `body` | string | yes | The content of the note or reply. | +| `discussion_id` | integer | yes | The ID of a thread. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `issue_iid` | integer | yes | The IID of an issue. | +| `note_id` | integer | yes | The ID of a thread note. | ```shell -curl --request PUT --header "PRIVATE-TOKEN: "\ +curl --request PUT --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/issues/11/discussions//notes/1108?body=comment" ``` @@ -230,13 +233,13 @@ Parameters: | Attribute | Type | Required | Description | | --------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `issue_iid` | integer | yes | The IID of an issue | -| `discussion_id` | integer | yes | The ID of a discussion | -| `note_id` | integer | yes | The ID of a discussion note | +| `discussion_id` | integer | yes | The ID of a discussion. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `issue_iid` | integer | yes | The IID of an issue. | +| `note_id` | integer | yes | The ID of a discussion note. | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: "\ +curl --request DELETE --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/issues/11/discussions/636" ``` @@ -252,8 +255,8 @@ GET /projects/:id/snippets/:snippet_id/discussions | Attribute | Type | Required | Description | | ------------------- | ---------------- | ---------- | ------------| -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `snippet_id` | integer | yes | The ID of an snippet | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `snippet_id` | integer | yes | The ID of an snippet. | ```json [ @@ -279,6 +282,7 @@ GET /projects/:id/snippets/:snippet_id/discussions "system": false, "noteable_id": 3, "noteable_type": "Snippet", + "project_id": 5, "noteable_iid": null }, { @@ -299,6 +303,7 @@ GET /projects/:id/snippets/:snippet_id/discussions "system": false, "noteable_id": 3, "noteable_type": "Snippet", + "project_id": 5, "noteable_iid": null, "resolvable": false } @@ -326,6 +331,7 @@ GET /projects/:id/snippets/:snippet_id/discussions "system": false, "noteable_id": 3, "noteable_type": "Snippet", + "project_id": 5, "noteable_iid": null, "resolvable": false } @@ -335,13 +341,13 @@ GET /projects/:id/snippets/:snippet_id/discussions ``` ```shell -curl --header "PRIVATE-TOKEN: "\ +curl --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/snippets/11/discussions" ``` ### Get single snippet discussion item -Returns a single discussion item for a specific project snippet +Returns a single discussion item for a specific project snippet. ```plaintext GET /projects/:id/snippets/:snippet_id/discussions/:discussion_id @@ -351,19 +357,19 @@ Parameters: | Attribute | Type | Required | Description | | --------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `snippet_id` | integer | yes | The ID of an snippet | -| `discussion_id` | integer | yes | The ID of a discussion item | +| `discussion_id` | integer | yes | The ID of a discussion item. +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `snippet_id` | integer | yes | The ID of an snippet. | ```shell -curl --request POST --header "PRIVATE-TOKEN: "\ +curl --request POST --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/snippets/11/discussions/" ``` ### Create new snippet thread -Creates a new thread to a single project snippet. This is similar to creating -a note but other comments (replies) can be added to it later. +Creates a new thread to a single project snippet. Similar to creating +a note, but other comments (replies) can be added to it later. ```plaintext POST /projects/:id/snippets/:snippet_id/discussions @@ -373,10 +379,10 @@ Parameters: | Attribute | Type | Required | Description | | --------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `snippet_id` | integer | yes | The ID of an snippet | -| `body` | string | yes | The content of a discussion | -| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | +| `body` | string | yes | The content of a discussion. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `snippet_id` | integer | yes | The ID of an snippet. | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z`. Requires administrator or project/group owner rights. | ```shell curl --request POST --header "PRIVATE-TOKEN: "\ @@ -395,15 +401,15 @@ Parameters: | Attribute | Type | Required | Description | | --------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `snippet_id` | integer | yes | The ID of an snippet | -| `discussion_id` | integer | yes | The ID of a thread | -| `note_id` | integer | yes | The ID of a thread note | -| `body` | string | yes | The content of the note/reply | -| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | +| `body` | string | yes | The content of the note or reply. | +| `discussion_id` | integer | yes | The ID of a thread. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `note_id` | integer | yes | The ID of a thread note. | +| `snippet_id` | integer | yes | The ID of an snippet. | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z`. Requires administrator or project/group owner rights. | ```shell -curl --request POST --header "PRIVATE-TOKEN: "\ +curl --request POST --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/snippets/11/discussions//notes?body=comment" ``` @@ -419,14 +425,14 @@ Parameters: | Attribute | Type | Required | Description | | --------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `snippet_id` | integer | yes | The ID of an snippet | -| `discussion_id` | integer | yes | The ID of a thread | -| `note_id` | integer | yes | The ID of a thread note | -| `body` | string | yes | The content of the note/reply | +| `body` | string | yes | The content of the note or reply. | +| `discussion_id` | integer | yes | The ID of a thread. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `note_id` | integer | yes | The ID of a thread note. | +| `snippet_id` | integer | yes | The ID of an snippet. | ```shell -curl --request PUT --header "PRIVATE-TOKEN: "\ +curl --request PUT --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/snippets/11/discussions//notes/1108?body=comment" ``` @@ -442,13 +448,13 @@ Parameters: | Attribute | Type | Required | Description | | --------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `snippet_id` | integer | yes | The ID of an snippet | -| `discussion_id` | integer | yes | The ID of a discussion | -| `note_id` | integer | yes | The ID of a discussion note | +| `discussion_id` | integer | yes | The ID of a discussion. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `note_id` | integer | yes | The ID of a discussion note. | +| `snippet_id` | integer | yes | The ID of an snippet. | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: "\ +curl --request DELETE --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/snippets/11/discussions/636" ``` @@ -464,8 +470,8 @@ GET /groups/:id/epics/:epic_id/discussions | Attribute | Type | Required | Description | | ------------------- | ---------------- | ---------- | ------------ | -| `id` | integer/string | yes | The ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding) | -| `epic_id` | integer | yes | The ID of an epic | +| `epic_id` | integer | yes | The ID of an epic. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding). | ```json [ @@ -491,6 +497,7 @@ GET /groups/:id/epics/:epic_id/discussions "system": false, "noteable_id": 3, "noteable_type": "Epic", + "project_id": 5, "noteable_iid": null, "resolvable": false }, @@ -512,6 +519,7 @@ GET /groups/:id/epics/:epic_id/discussions "system": false, "noteable_id": 3, "noteable_type": "Epic", + "project_id": 5, "noteable_iid": null, "resolvable": false } @@ -539,6 +547,7 @@ GET /groups/:id/epics/:epic_id/discussions "system": false, "noteable_id": 3, "noteable_type": "Epic", + "project_id": 5, "noteable_iid": null, "resolvable": false } @@ -554,7 +563,7 @@ curl --header "PRIVATE-TOKEN: "\ ### Get single epic discussion item -Returns a single discussion item for a specific group epic +Returns a single discussion item for a specific group epic. ```plaintext GET /groups/:id/epics/:epic_id/discussions/:discussion_id @@ -564,19 +573,19 @@ Parameters: | Attribute | Type | Required | Description | | --------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding) | -| `epic_id` | integer | yes | The ID of an epic | -| `discussion_id` | integer | yes | The ID of a discussion item | +| `discussion_id` | integer | yes | The ID of a discussion item. | +| `epic_id` | integer | yes | The ID of an epic. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding). | ```shell -curl --request POST --header "PRIVATE-TOKEN: "\ +curl --request POST --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/groups/5/epics/11/discussions/" ``` ### Create new epic thread -Creates a new thread to a single group epic. This is similar to creating -a note but other comments (replies) can be added to it later. +Creates a new thread to a single group epic. Similar to creating +a note, but other comments (replies) can be added to it later. ```plaintext POST /groups/:id/epics/:epic_id/discussions @@ -586,13 +595,13 @@ Parameters: | Attribute | Type | Required | Description | | --------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding) | -| `epic_id` | integer | yes | The ID of an epic | -| `body` | string | yes | The content of the thread | -| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | +| `body` | string | yes | The content of the thread. | +| `epic_id` | integer | yes | The ID of an epic. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding). | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z`. Requires administrator or project/group owner rights. | ```shell -curl --request POST --header "PRIVATE-TOKEN: "\ +curl --request POST --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/groups/5/epics/11/discussions?body=comment" ``` @@ -609,15 +618,15 @@ Parameters: | Attribute | Type | Required | Description | | --------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding) | -| `epic_id` | integer | yes | The ID of an epic | -| `discussion_id` | integer | yes | The ID of a thread | -| `note_id` | integer | yes | The ID of a thread note | -| `body` | string | yes | The content of the note/reply | -| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | +| `body` | string | yes | The content of the note or reply. | +| `discussion_id` | integer | yes | The ID of a thread. | +| `epic_id` | integer | yes | The ID of an epic. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding). | +| `note_id` | integer | yes | The ID of a thread note. | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z`. Requires administrator or project/group owner rights. | ```shell -curl --request POST --header "PRIVATE-TOKEN: "\ +curl --request POST --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/groups/5/epics/11/discussions//notes?body=comment" ``` @@ -633,14 +642,14 @@ Parameters: | Attribute | Type | Required | Description | | --------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding) | -| `epic_id` | integer | yes | The ID of an epic | -| `discussion_id` | integer | yes | The ID of a thread | -| `note_id` | integer | yes | The ID of a thread note | -| `body` | string | yes | The content of note/reply | +| `body` | string | yes | The content of note or reply. | +| `discussion_id` | integer | yes | The ID of a thread. | +| `epic_id` | integer | yes | The ID of an epic. | +| `id` | integer/string | yes | The ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding). | +| `note_id` | integer | yes | The ID of a thread note. | ```shell -curl --request PUT --header "PRIVATE-TOKEN: "\ +curl --request PUT --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/groups/5/epics/11/discussions//notes/1108?body=comment" ``` @@ -656,13 +665,13 @@ Parameters: | Attribute | Type | Required | Description | | --------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding) | -| `epic_id` | integer | yes | The ID of an epic | -| `discussion_id` | integer | yes | The ID of a thread | -| `note_id` | integer | yes | The ID of a thread note | +| `discussion_id` | integer | yes | The ID of a thread. | +| `epic_id` | integer | yes | The ID of an epic. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding). | +| `note_id` | integer | yes | The ID of a thread note. | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: "\ +curl --request DELETE --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/groups/5/epics/11/discussions/636" ``` @@ -678,8 +687,8 @@ GET /projects/:id/merge_requests/:merge_request_iid/discussions | Attribute | Type | Required | Description | | ------------------- | ---------------- | ---------- | ------------ | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `merge_request_iid` | integer | yes | The IID of a merge request | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `merge_request_iid` | integer | yes | The IID of a merge request. | ```json [ @@ -705,6 +714,7 @@ GET /projects/:id/merge_requests/:merge_request_iid/discussions "system": false, "noteable_id": 3, "noteable_type": "Merge request", + "project_id": 5, "noteable_iid": null, "resolved": false, "resolvable": true, @@ -729,6 +739,7 @@ GET /projects/:id/merge_requests/:merge_request_iid/discussions "system": false, "noteable_id": 3, "noteable_type": "Merge request", + "project_id": 5, "noteable_iid": null, "resolved": false, "resolvable": true, @@ -758,6 +769,7 @@ GET /projects/:id/merge_requests/:merge_request_iid/discussions "system": false, "noteable_id": 3, "noteable_type": "Merge request", + "project_id": 5, "noteable_iid": null, "resolved": false, "resolvable": true, @@ -794,6 +806,7 @@ Diff comments also contain position: "system": false, "noteable_id": 3, "noteable_type": "Merge request", + "project_id": 5, "noteable_iid": null, "commit_id": "4803c71e6b1833ca72b8b26ef2ecd5adc8a38031", "position": { @@ -826,13 +839,13 @@ Diff comments also contain position: ``` ```shell -curl --header "PRIVATE-TOKEN: "\ +curl --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions" ``` ### Get single merge request discussion item -Returns a single discussion item for a specific project merge request +Returns a single discussion item for a specific project merge request. ```plaintext GET /projects/:id/merge_requests/:merge_request_iid/discussions/:discussion_id @@ -842,12 +855,12 @@ Parameters: | Attribute | Type | Required | Description | | ------------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `merge_request_iid` | integer | yes | The IID of a merge request | -| `discussion_id` | integer | yes | The ID of a discussion item | +| `discussion_id` | string | yes | The ID of a discussion item. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `merge_request_iid` | integer | yes | The IID of a merge request. | ```shell -curl --header "PRIVATE-TOKEN: "\ +curl --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions/" ``` @@ -855,8 +868,10 @@ curl --header "PRIVATE-TOKEN: "\ > The `commit id` entry was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47130) in GitLab 13.7. -Creates a new thread to a single project merge request. This is similar to creating -a note but other comments (replies) can be added to it later. +Creates a new thread to a single project merge request. Similar to creating +a note but other comments (replies) can be added to it later. For other approaches, +see [Post comment to commit](commits.md#post-comment-to-commit) in the Commits API, +and [Create new merge request note](notes.md#create-new-merge-request-note) in the Notes API. ```plaintext POST /projects/:id/merge_requests/:merge_request_iid/discussions @@ -866,30 +881,30 @@ Parameters for all comments: | Attribute | Type | Required | Description | | ---------------------------------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `merge_request_iid` | integer | yes | The IID of a merge request | -| `body` | string | yes | The content of the thread | -| `commit_id` | string | no | SHA referencing commit to start this thread on | -| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | -| `position` | hash | no | Position when creating a diff note | -| `position[base_sha]` | string | yes | Base commit SHA in the source branch | -| `position[start_sha]` | string | yes | SHA referencing commit in target branch | -| `position[head_sha]` | string | yes | SHA referencing HEAD of this merge request | -| `position[position_type]` | string | yes | Type of the position reference', allowed values: `text` or `image` | -| `position[new_path]` | string | yes (if the position type is `text`) | File path after change | -| `position[new_line]` | integer | no | Line number after change (for `text` diff notes) | -| `position[old_path]` | string | yes (if the position type is `text`) | File path before change | -| `position[old_line]` | integer | no | Line number before change (for `text` diff notes) | -| `position[line_range]` | hash | no | Line range for a multi-line diff note | -| `position[width]` | integer | no | Width of the image (for `image` diff notes) | -| `position[height]` | integer | no | Height of the image (for `image` diff notes) | -| `position[x]` | float | no | X coordinate (for `image` diff notes) | -| `position[y]` | float | no | Y coordinate (for `image` diff notes) | +| `body` | string | yes | The content of the thread. | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `merge_request_iid` | integer | yes | The IID of a merge request. | +| `position[base_sha]` | string | yes | Base commit SHA in the source branch. | +| `position[head_sha]` | string | yes | SHA referencing HEAD of this merge request. | +| `position[start_sha]` | string | yes | SHA referencing commit in target branch. | +| `position[new_path]` | string | yes (if the position type is `text`) | File path after change. | +| `position[old_path]` | string | yes (if the position type is `text`) | File path before change. | +| `position[position_type]` | string | yes | Type of the position reference. Allowed values: `text` or `image`. | +| `commit_id` | string | no | SHA referencing commit to start this thread on. | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z`. Requires administrator or project/group owner rights. | +| `position` | hash | no | Position when creating a diff note. | +| `position[new_line]` | integer | no | For `text` diff notes, the line number after change. | +| `position[old_line]` | integer | no | For `text` diff notes, the line number before change. | +| `position[line_range]` | hash | no | Line range for a multi-line diff note. | +| `position[width]` | integer | no | For `image` diff notes, width of the image. | +| `position[height]` | integer | no | For `image` diff notes, height of the image. | +| `position[x]` | float | no | For `image` diff notes, X coordinate. | +| `position[y]` | float | no | For `image` diff notes, Y coordinate. | #### Create a new thread on the overview page ```shell -curl --request POST --header "PRIVATE-TOKEN: "\ +curl --request POST --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions?body=comment" ``` @@ -903,18 +918,17 @@ curl --request POST --header "PRIVATE-TOKEN: "\ use `position[old_line]` and don't include `position[new_line]`. - To create a thread on an unchanged line, include both `position[new_line]` and `position[old_line]` for the line. These positions might not be the same if earlier - changes in the file changed the line number. This is a bug that we plan to fix in - [GraphQL `createDiffNote` forces clients to compute redundant information (#325161)](https://gitlab.com/gitlab-org/gitlab/-/issues/325161). -- If you specify incorrect `base`/`head`/`start` `SHA` parameters, you might run - into the following bug: - [Merge request comments receive "download" link instead of inline code (#296829)](https://gitlab.com/gitlab-org/gitlab/-/issues/296829). + changes in the file changed the line number. For the discussion about a fix, see + [issue 32516](https://gitlab.com/gitlab-org/gitlab/-/issues/325161). +- If you specify incorrect `base`, `head`, `start`, or `SHA` parameters, you might run + into the bug described in [issue #296829)](https://gitlab.com/gitlab-org/gitlab/-/issues/296829). To create a new thread: 1. [Get the latest merge request version](merge_requests.md#get-merge-request-diff-versions): ```shell - curl --header "PRIVATE-TOKEN: "\ + curl --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/versions" ``` @@ -957,12 +971,12 @@ Parameters for multiline comments only: | Attribute | Type | Required | Description | | ---------------------------------------- | -------------- | -------- | ----------- | -| `position[line_range][start]` | hash | no | Multiline note starting line | -| `position[line_range][start][line_code]` | string | yes | [Line code](#line-code) for the start line | -| `position[line_range][start][type]` | string | yes | Use `new` for lines added by this commit, otherwise `old`. | -| `position[line_range][end]` | hash | no | Multiline note ending line | -| `position[line_range][end][line_code]` | string | yes | [Line code](#line-code) for the end line | +| `position[line_range][end][line_code]` | string | yes | [Line code](#line-code) for the end line. | | `position[line_range][end][type]` | string | yes | Use `new` for lines added by this commit, otherwise `old`. | +| `position[line_range][start][line_code]` | string | yes | [Line code](#line-code) for the start line. | +| `position[line_range][start][type]` | string | yes | Use `new` for lines added by this commit, otherwise `old`. | +| `position[line_range][end]` | hash | no | Multiline note ending line. | +| `position[line_range][start]` | hash | no | Multiline note starting line. | #### Line code @@ -976,9 +990,9 @@ For example, if a commit (``) deletes line 463 in the README, you can on the deletion by referencing line 463 in the *old* file: ```shell -curl --request POST --header "PRIVATE-TOKEN: [ACCESS_TOKEN]"\ - --form "note=Very clever to remove this unnecessary line!"\ - --form "path=README" --form "line=463" --form "line_type=old"\ +curl --request POST --header "PRIVATE-TOKEN: [ACCESS_TOKEN]" \ + --form "note=Very clever to remove this unnecessary line!" \ + --form "path=README" --form "line=463" --form "line_type=old" \ "https://gitlab.com/api/v4/projects/47/repository/commits//comments" ``` @@ -986,9 +1000,9 @@ If a commit (``) adds line 157 to `hello.rb`, you can comment on the addition by referencing line 157 in the *new* file: ```shell -curl --request POST --header "PRIVATE-TOKEN: [ACCESS_TOKEN]"\ - --form "note=This is brilliant!" --form "path=hello.rb"\ - --form "line=157" --form "line_type=new"\ +curl --request POST --header "PRIVATE-TOKEN: [ACCESS_TOKEN]" \ + --form "note=This is brilliant!" --form "path=hello.rb" \ + --form "line=157" --form "line_type=new" \ "https://gitlab.com/api/v4/projects/47/repository/commits//comments" ``` @@ -1008,13 +1022,13 @@ Parameters: | Attribute | Type | Required | Description | | ------------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `merge_request_iid` | integer | yes | The IID of a merge request | -| `discussion_id` | integer | yes | The ID of a thread | -| `resolved` | boolean | yes | Resolve/unresolve the discussion | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `discussion_id` | string | yes | The ID of a thread. | +| `merge_request_iid` | integer | yes | The IID of a merge request. | +| `resolved` | boolean | yes | Resolve or unresolve the discussion. | ```shell -curl --request PUT --header "PRIVATE-TOKEN: "\ +curl --request PUT --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions/?resolved=true" ``` @@ -1031,15 +1045,15 @@ Parameters: | Attribute | Type | Required | Description | | ------------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `merge_request_iid` | integer | yes | The IID of a merge request | -| `discussion_id` | integer | yes | The ID of a thread | -| `note_id` | integer | yes | The ID of a thread note | -| `body` | string | yes | The content of the note/reply | -| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | +| `body` | string | yes | The content of the note or reply. | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `discussion_id` | string | yes | The ID of a thread. | +| `merge_request_iid` | integer | yes | The IID of a merge request. | +| `note_id` | integer | yes | The ID of a thread note. | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z`. Requires administrator or project/group owner rights. | ```shell -curl --request POST --header "PRIVATE-TOKEN: "\ +curl --request POST --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions//notes?body=comment" ``` @@ -1055,22 +1069,22 @@ Parameters: | Attribute | Type | Required | Description | | ------------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `merge_request_iid` | integer | yes | The IID of a merge request | -| `discussion_id` | integer | yes | The ID of a thread | -| `note_id` | integer | yes | The ID of a thread note | -| `body` | string | no | The content of the note/reply (exactly one of `body` or `resolved` must be set | -| `resolved` | boolean | no | Resolve/unresolve the note (exactly one of `body` or `resolved` must be set | +| `discussion_id` | string | yes | The ID of a thread. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `merge_request_iid` | integer | yes | The IID of a merge request. | +| `note_id` | integer | yes | The ID of a thread note. | +| `body` | string | no | The content of the note or reply. Exactly one of `body` or `resolved` must be set. | +| `resolved` | boolean | no | Resolve or unresolve the note. Exactly one of `body` or `resolved` must be set. | ```shell -curl --request PUT --header "PRIVATE-TOKEN: "\ +curl --request PUT --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions//notes/1108?body=comment" ``` Resolving a note: ```shell -curl --request PUT --header "PRIVATE-TOKEN: "\ +curl --request PUT --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions//notes/1108?resolved=true" ``` @@ -1086,13 +1100,13 @@ Parameters: | Attribute | Type | Required | Description | | ------------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `merge_request_iid` | integer | yes | The IID of a merge request | -| `discussion_id` | integer | yes | The ID of a thread | -| `note_id` | integer | yes | The ID of a thread note | +| `discussion_id` | string | yes | The ID of a thread. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `merge_request_iid` | integer | yes | The IID of a merge request. | +| `note_id` | integer | yes | The ID of a thread note. | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: "\ +curl --request DELETE --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/discussions/636" ``` @@ -1108,8 +1122,8 @@ GET /projects/:id/repository/commits/:commit_id/discussions | Attribute | Type | Required | Description | | ------------------- | ---------------- | ---------- | ------------ | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `commit_id` | string | yes | The SHA of a commit | +| `commit_id` | string | yes | The SHA of a commit. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | ```json [ @@ -1135,6 +1149,7 @@ GET /projects/:id/repository/commits/:commit_id/discussions "system": false, "noteable_id": 3, "noteable_type": "Commit", + "project_id": 5, "noteable_iid": null, "resolvable": false }, @@ -1156,6 +1171,7 @@ GET /projects/:id/repository/commits/:commit_id/discussions "system": false, "noteable_id": 3, "noteable_type": "Commit", + "project_id": 5, "noteable_iid": null, "resolvable": false } @@ -1183,6 +1199,7 @@ GET /projects/:id/repository/commits/:commit_id/discussions "system": false, "noteable_id": 3, "noteable_type": "Commit", + "project_id": 5, "noteable_iid": null, "resolvable": false } @@ -1217,6 +1234,7 @@ Diff comments contain also position: "system": false, "noteable_id": 3, "noteable_type": "Commit", + "project_id": 5, "noteable_iid": null, "position": { "base_sha": "b5d6e7b1613fca24d250fa8e5bc7bcc3dd6002ef", @@ -1236,7 +1254,7 @@ Diff comments contain also position: ``` ```shell -curl --header "PRIVATE-TOKEN: "\ +curl --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/repository/commits//discussions" ``` @@ -1252,18 +1270,18 @@ Parameters: | Attribute | Type | Required | Description | | ------------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `commit_id` | string | yes | The SHA of a commit | -| `discussion_id` | string | yes | The ID of a discussion item | +| `commit_id` | string | yes | The SHA of a commit. | +| `discussion_id` | string | yes | The ID of a discussion item. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | ```shell -curl --header "PRIVATE-TOKEN: "\ +curl --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/repository/commits//discussions/" ``` ### Create new commit thread -Creates a new thread to a single project commit. This is similar to creating +Creates a new thread to a single project commit. Similar to creating a note but other comments (replies) can be added to it later. ```plaintext @@ -1274,32 +1292,37 @@ Parameters: | Attribute | Type | Required | Description | | ------------------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `commit_id` | string | yes | The SHA of a commit | -| `body` | string | yes | The content of the thread | -| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | -| `position` | hash | no | Position when creating a diff note | -| `position[base_sha]` | string | yes | SHA of the parent commit| -| `position[start_sha]` | string | yes | SHA of the parent commit | -| `position[head_sha]` | string | yes | The SHA of this commit (same as `commit_id`) | -| `position[position_type]` | string | yes | Type of the position reference', allowed values: `text` or `image` | -| `position[new_path]` | string | no | File path after change | -| `position[new_line]` | integer | no | Line number after change | -| `position[old_path]` | string | no | File path before change | -| `position[old_line]` | integer | no | Line number before change | -| `position[width]` | integer | no | Width of the image (for `image` diff notes) | -| `position[height]` | integer | no | Height of the image (for `image` diff notes) | -| `position[x]` | integer | no | X coordinate (for `image` diff notes) | -| `position[y]` | integer | no | Y coordinate (for `image` diff notes) | +| `body` | string | yes | The content of the thread. | +| `commit_id` | string | yes | The SHA of a commit. | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `position[base_sha]` | string | yes | SHA of the parent commit. | +| `position[head_sha]` | string | yes | The SHA of this commit. Same as `commit_id`. | +| `position[start_sha]` | string | yes | SHA of the parent commit. | +| `position[position_type]` | string | yes | Type of the position reference. Allowed values: `text` or `image`. | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such as `2016-03-11T03:45:40Z`. Requires administrator or project/group owner rights. | +| `position` | hash | no | Position when creating a diff note. | + +| `position[new_path]` | string | no | File path after change. | +| `position[new_line]` | integer | no | Line number after change. | +| `position[old_path]` | string | no | File path before change. | +| `position[old_line]` | integer | no | Line number before change. | +| `position[height]` | integer | no | For `image` diff notes, image height. | +| `position[width]` | integer | no | For `image` diff notes, image width. | +| `position[x]` | integer | no | For `image` diff notes, X coordinate. | +| `position[y]` | integer | no | For `image` diff notes, Y coordinate. | ```shell -curl --request POST --header "PRIVATE-TOKEN: "\ +curl --request POST --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/repository/commits//discussions?body=comment" ``` The rules for creating the API request are the same as when -[creating a new thread in the merge request diff](#create-a-new-thread-in-the-merge-request-diff), -with the exception of `base_sha`, `start_sha`, and `head_sha` attributes. +[creating a new thread in the merge request diff](#create-a-new-thread-in-the-merge-request-diff). +The exceptions: + +- `base_sha` +- `head_sha` +- `start_sha` ### Add note to existing commit thread @@ -1313,15 +1336,15 @@ Parameters: | Attribute | Type | Required | Description | | ------------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `commit_id` | string | yes | The SHA of a commit | -| `discussion_id` | string | yes | The ID of a thread | -| `note_id` | integer | yes | The ID of a thread note | -| `body` | string | yes | The content of the note/reply | -| `created_at` | string | no | Date time string, ISO 8601 formatted, such `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | +| `body` | string | yes | The content of the note or reply. | +| `commit_id` | string | yes | The SHA of a commit. | +| `discussion_id` | string | yes | The ID of a thread. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `note_id` | integer | yes | The ID of a thread note. | +| `created_at` | string | no | Date time string, ISO 8601 formatted, such `2016-03-11T03:45:40Z`. Requires administrator or project/group owner rights. | ```shell -curl --request POST --header "PRIVATE-TOKEN: "\ +curl --request POST --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/repository/commits//discussions//notes?body=comment ``` @@ -1337,21 +1360,21 @@ Parameters: | Attribute | Type | Required | Description | | ------------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `commit_id` | string | yes | The SHA of a commit | -| `discussion_id` | string | yes | The ID of a thread | -| `note_id` | integer | yes | The ID of a thread note | -| `body` | string | no | The content of a note | +| `commit_id` | string | yes | The SHA of a commit. | +| `discussion_id` | string | yes | The ID of a thread. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `note_id` | integer | yes | The ID of a thread note. | +| `body` | string | no | The content of a note. | ```shell -curl --request PUT --header "PRIVATE-TOKEN: "\ +curl --request PUT --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/repository/commits//discussions//notes/1108?body=comment" ``` Resolving a note: ```shell -curl --request PUT --header "PRIVATE-TOKEN: "\ +curl --request PUT --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/repository/commits//discussions//notes/1108?resolved=true" ``` @@ -1367,12 +1390,12 @@ Parameters: | Attribute | Type | Required | Description | | ------------------- | -------------- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | -| `commit_id` | string | yes | The SHA of a commit | -| `discussion_id` | string | yes | The ID of a thread | -| `note_id` | integer | yes | The ID of a thread note | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `commit_id` | string | yes | The SHA of a commit. | +| `discussion_id` | string | yes | The ID of a thread. | +| `note_id` | integer | yes | The ID of a thread note. | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: "\ +curl --request DELETE --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/5/repository/commits//discussions//notes/636" ``` diff --git a/doc/api/dora/metrics.md b/doc/api/dora/metrics.md index f0e90234ff4..d30194c3da0 100644 --- a/doc/api/dora/metrics.md +++ b/doc/api/dora/metrics.md @@ -11,6 +11,8 @@ type: reference, api > - The legacy key/value pair `{ "" => "" }` was removed from the payload in GitLab 14.0. > `time_to_restore_service` metric was introduced in GitLab 14.9. +You can also retrieve [DORA metrics](../../user/analytics/dora_metrics.md) with the [GraphQL API](../../api/graphql/reference/index.md). + All methods require at least the Reporter role. ## Get project-level DORA metrics @@ -26,7 +28,6 @@ GET /projects/:id/dora/metrics | `id` | integer/string | yes | The ID or [URL-encoded path of the project](../rest/index.md#namespaced-path-encoding) can be accessed by the authenticated user. | | `metric` | string | yes | One of `deployment_frequency`, `lead_time_for_changes`, `time_to_restore_service` or `change_failure_rate`. | | `end_date` | string | no | Date range to end at. ISO 8601 Date format, for example `2021-03-01`. Default is the current date. | -| `environment_tier` | string | no | The [tier of the environment](../../ci/environments/index.md#deployment-tier-of-environments). Default is `production`. Deprecated, use `environment_tiers`. | | `environment_tiers` | array of strings | no | The [tiers of the environments](../../ci/environments/index.md#deployment-tier-of-environments). Default is `production`. | | `interval` | string | no | The bucketing interval. One of `all`, `monthly` or `daily`. Default is `daily`. | | `start_date` | string | no | Date range to start from. ISO 8601 Date format, for example `2021-03-01`. Default is 3 months ago. | @@ -67,7 +68,6 @@ GET /groups/:id/dora/metrics | `id` | integer/string | yes | The ID or [URL-encoded path of the project](../rest/index.md#namespaced-path-encoding) can be accessed by the authenticated user. | | `metric` | string | yes | One of `deployment_frequency`, `lead_time_for_changes`, `time_to_restore_service` or `change_failure_rate`. | | `end_date` | string | no | Date range to end at. ISO 8601 Date format, for example `2021-03-01`. Default is the current date. | -| `environment_tier` | string | no | The [tier of the environment](../../ci/environments/index.md#deployment-tier-of-environments). Default is `production`. Deprecated, use `environment_tiers`. | | `environment_tiers` | array of strings | no | The [tiers of the environments](../../ci/environments/index.md#deployment-tier-of-environments). Default is `production`. | | `interval` | string | no | The bucketing interval. One of `all`, `monthly` or `daily`. Default is `daily`. | | `start_date` | string | no | Date range to start from. ISO 8601 Date format, for example `2021-03-01`. Default is 3 months ago. | @@ -101,8 +101,8 @@ parameter: | `metric` query parameter | Description of `value` in response | |:---------------------------|:-----------------------------------| +| `deployment_frequency` | The API returns the total number of successful deployments during the time period. [Issue 371271](https://gitlab.com/gitlab-org/gitlab/-/issues/371271) proposes to update the API to return the daily average instead of the total number. | | `change_failure_rate` | The number of incidents divided by the number of deployments during the time period. Available only for production environment. | -| `deployment_frequency` | The number of successful deployments during the time period. | | `lead_time_for_changes` | The median number of seconds between the merge of the merge request (MR) and the deployment of the MR commits for all MRs deployed during the time period. | | `time_to_restore_service` | The median number of seconds an incident was open during the time period. Available only for production environment. | diff --git a/doc/api/draft_notes.md b/doc/api/draft_notes.md index a168c41092c..e532de6a502 100644 --- a/doc/api/draft_notes.md +++ b/doc/api/draft_notes.md @@ -94,6 +94,46 @@ GET /projects/:id/merge_requests/:merge_request_iid/draft_notes/:draft_note_id curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/14/merge_requests/11/draft_notes/5" ``` +## Create a draft note + +Create a draft note for a given merge request. + +```plaintext +POST /projects/:id/merge_requests/:merge_request_iid/draft_notes +``` + +| Attribute | Type | Required | Description | +| --------------------------- | ----------------- | ----------- | --------------------- | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). +| `merge_request_iid` | integer | yes | The IID of a project merge request. +| `note` | string | yes | The content of a note. +| `commit_id` | string | no | The SHA of a commit to associate the draft note to. +| `in_reply_to_discussion_id` | integer | no | The ID of a discussion the draft note replies to. +| `resolve_discussion` | boolean | no | The associated discussion should be resolved. + +```shell +curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/14/merge_requests/11/draft_notes?note=note +``` + +## Modify existing draft note + +Modify a draft note for a given merge request. + +```plaintext +PUT /projects/:id/merge_requests/:merge_request_iid/draft_notes/:draft_note_id +``` + +| Attribute | Type | Required | Description | +| ------------------- | ----------------- | ----------- | --------------------- | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). +| `draft_note_id` | integer | yes | The ID of a draft note. +| `merge_request_iid` | integer | yes | The IID of a project merge request. +| `note` | string | no | The content of a note. + +```shell +curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/14/merge_requests/11/draft_notes/5" +``` + ## Delete a draft note Deletes an existing draft note for a given merge request. @@ -129,3 +169,20 @@ PUT /projects/:id/merge_requests/:merge_request_iid/draft_notes/:draft_note_id/p ```shell curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/14/merge_requests/11/draft_notes/5/publish" ``` + +## Publish all pending draft notes + +Bulk publishes all existing draft notes for a given merge request that belong to the user. + +```plaintext +POST /projects/:id/merge_requests/:merge_request_iid/draft_notes/bulk_publish +``` + +| Attribute | Type | Required | Description | +| ------------------- | ----------------- | -------- | --------------------- | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). +| `merge_request_iid` | integer | yes | The IID of a project merge request. + +```shell +curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/14/merge_requests/11/draft_notes/bulk_publish" +``` diff --git a/doc/api/environments.md b/doc/api/environments.md index bbf6c5fee99..3cbb6076300 100644 --- a/doc/api/environments.md +++ b/doc/api/environments.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: concepts, howto --- @@ -281,6 +281,8 @@ Example response: ## Update an existing environment +> Parameter `name` [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/338897) in GitLab 16.0. + Updates an existing environment's name and/or `external_url`. It returns `200` if the environment was successfully updated. In case of an error, a status code `400` is returned. @@ -293,7 +295,6 @@ PUT /projects/:id/environments/:environments_id |------------------|----------------|----------|---------------------------------------------------------------------------------------------------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | | `environment_id` | integer | yes | The ID of the environment. | -| `name` | string | no | [Deprecated and will be removed in GitLab 16.0](https://gitlab.com/gitlab-org/gitlab/-/issues/338897). | | `external_url` | string | no | The new `external_url`. | | `tier` | string | no | The tier of the new environment. Allowed values are `production`, `staging`, `testing`, `development`, and `other`. | @@ -339,7 +340,7 @@ curl --request DELETE --header "PRIVATE-TOKEN: " "https://git > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/296625) in GitLab 14.2. It schedules for deletion multiple environments that have already been -[stopped](../ci/environments/index.md#stop-an-environment) and +[stopped](../ci/environments/index.md#stopping-an-environment) and are [in the review app folder](../ci/review_apps/index.md). The actual deletion is performed after 1 week from the time of execution. By default, it only deletes environments 30 days or older. You can change this default using the `before` parameter. @@ -415,7 +416,7 @@ Example response: ## Stop stale environments -Issue stop request to all environments that were last modified or deployed to before a specified date. Excludes protected environments. Returns `200` if stop request was successful and `400` if the before date is invalid. For details of exactly when the environment is stopped, see [Stop an environment](../ci/environments/index.md#stop-an-environment). +Issue stop request to all environments that were last modified or deployed to before a specified date. Excludes protected environments. Returns `200` if stop request was successful and `400` if the before date is invalid. For details of exactly when the environment is stopped, see [Stop an environment](../ci/environments/index.md#stopping-an-environment). ```plaintext POST /projects/:id/environments/stop_stale diff --git a/doc/api/epic_issues.md b/doc/api/epic_issues.md index 9d3a32beb07..506d6d4b339 100644 --- a/doc/api/epic_issues.md +++ b/doc/api/epic_issues.md @@ -11,7 +11,7 @@ Every API call to the epic issues API endpoint must be authenticated. If a user is not a member of a group and the group is private, a `GET` request on that group results in a `404` status code. -Epics are available only in GitLab [Premium and higher](https://about.gitlab.com/pricing/). +Epics are available only in GitLab [Premium and Ultimate](https://about.gitlab.com/pricing/). If the Epics feature is not available, a `403` status code is returned. ## Epic Issues pagination diff --git a/doc/api/error_tracking.md b/doc/api/error_tracking.md index 36bcbb30d4b..3515b080b12 100644 --- a/doc/api/error_tracking.md +++ b/doc/api/error_tracking.md @@ -39,6 +39,45 @@ Example response: } ``` +### Create Error Tracking settings + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/393035/) in GitLab 15.10. +The API allows you to create Error Tracking settings for a project. Only for users with Maintainer role for +the project. + +NOTE: +This API is only available when used with [integrated error tracking](../operations/error_tracking.md#integrated-error-tracking). + +```plaintext +PUT /projects/:id/error_tracking/settings +``` + +Supported attributes: + +| Attribute | Type | Required | Description | +| ------------ | ------- |----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | integer | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | +| `active` | boolean | yes | Pass `true` to enable the error tracking setting configuration or `false` to disable it. | +| `integrated` | boolean | yes | Pass `true` to enable the integrated error tracking backend. [Available in](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68260) GitLab 14.2 and later. | + +Example request: + +```shell +curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/error_tracking/settings?active=true&integrated=true" +``` + +Example response: + +```json +{ + "active": true, + "project_name": null, + "sentry_external_url": null, + "api_url": null, + "integrated": true +} +``` + ### Enable or disable the Error Tracking project settings The API allows you to enable or disable the Error Tracking settings for a project. Only for users with the @@ -55,7 +94,7 @@ PATCH /projects/:id/error_tracking/settings | `integrated` | boolean | no | Pass `true` to enable the integrated error tracking backend. [Available in](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68260) GitLab 14.2 and later. | ```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/error_tracking/settings?active=true" +curl --request PATCH --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/error_tracking/settings?active=true" ``` Example response: diff --git a/doc/api/feature_flag_specs.md b/doc/api/feature_flag_specs.md deleted file mode 100644 index 960d00278d6..00000000000 --- a/doc/api/feature_flag_specs.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -stage: Release -group: Release -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -remove_date: '2023-02-14' -redirect_to: 'feature_flags.md' ---- - -# Feature Flag Specs API (removed) **(PREMIUM)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9566) in GitLab 12.5. - -This API was removed in [GitLab 14.0](https://gitlab.com/gitlab-org/gitlab/-/issues/213369). -Use [the new API](feature_flags.md) instead. diff --git a/doc/api/feature_flag_user_lists.md b/doc/api/feature_flag_user_lists.md index 27e2e925506..0152ec62c7f 100644 --- a/doc/api/feature_flag_user_lists.md +++ b/doc/api/feature_flag_user_lists.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -9,9 +9,9 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/205409) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.10. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212318) to GitLab Free in 13.5. -API for accessing GitLab Feature Flag User Lists. +API for accessing GitLab feature flag user lists. -Users with Developer or higher [permissions](../user/permissions.md) can access the Feature Flag User Lists API. +Users with at least the Developer [role](../user/permissions.md) can access the feature flag user lists API. NOTE: `GET` requests return twenty results at a time because the API results diff --git a/doc/api/feature_flags.md b/doc/api/feature_flags.md index cd9907c8032..e83bb2eb300 100644 --- a/doc/api/feature_flags.md +++ b/doc/api/feature_flags.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -11,7 +11,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w API for accessing resources of [GitLab feature flags](../operations/feature_flags.md). -Users with Developer or higher [permissions](../user/permissions.md) can access the feature flag API. +Users with at least the Developer [role](../user/permissions.md) can access the feature flag API. ## Feature flags pagination diff --git a/doc/api/features.md b/doc/api/features.md index c3db1e53f68..742d4527be5 100644 --- a/doc/api/features.md +++ b/doc/api/features.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -107,6 +107,9 @@ Set a feature's gate value. If a feature with the given name doesn't exist yet, it's created. The value can be a boolean, or an integer to indicate percentage of time. +WARNING: +Before you enable a feature still in development, you should understand the [security and stability risks](../administration/feature_flags.md#risks-when-enabling-features-still-in-development). + ```plaintext POST /features/:name ``` diff --git a/doc/api/freeze_periods.md b/doc/api/freeze_periods.md index ce7377e1e35..c2cf2d84056 100644 --- a/doc/api/freeze_periods.md +++ b/doc/api/freeze_periods.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: concepts, howto --- diff --git a/doc/api/geo_nodes.md b/doc/api/geo_nodes.md index 0b0dd543503..038b7d633a6 100644 --- a/doc/api/geo_nodes.md +++ b/doc/api/geo_nodes.md @@ -439,6 +439,18 @@ Example response: "snippet_repositories_verification_failed_count": null, "snippet_repositories_synced_in_percentage": "0.00%", "snippet_repositories_verified_in_percentage": "0.00%", + "project_wiki_repositories_count": 3, + "project_wiki_repositories_checksum_total_count": 3, + "project_wiki_repositories_checksummed_count": 3, + "project_wiki_repositories_checksum_failed_count": 0, + "project_wiki_repositories_synced_count": null, + "project_wiki_repositories_failed_count": null, + "project_wiki_repositories_registry_count": null, + "project_wiki_repositories_verification_total_count": null, + "project_wiki_repositories_verified_count": null, + "project_wiki_repositories_verification_failed_count": null, + "project_wiki_repositories_synced_in_percentage": "0.00%", + "project_wiki_repositories_verified_in_percentage": "0.00%", "group_wiki_repositories_count": 5, "group_wiki_repositories_checksum_total_count": 5, "group_wiki_repositories_checksummed_count": 5, @@ -520,6 +532,13 @@ Example response: "container_repositories_registry_count": 5, "container_repositories_synced_in_percentage": "100.00%", "container_repositories_synced_missing_on_primary_count": 0, + "container_repositories_checksum_total_count": 0, + "container_repositories_checksummed_count": 0, + "container_repositories_checksum_failed_count": 0, + "container_repositories_verification_total_count": 0, + "container_repositories_verified_count": 0, + "container_repositories_verification_failed_count": 0, + "container_repositories_verified_in_percentage": "100.00%", "dependency_proxy_manifests_count": 5, "dependency_proxy_manifests_checksum_total_count": 5, "dependency_proxy_manifests_checksummed_count": 5, @@ -716,6 +735,13 @@ Example response: "container_repositories_registry_count": 5, "container_repositories_synced_in_percentage": "100.00%", "container_repositories_synced_missing_on_primary_count": 0, + "container_repositories_checksum_total_count": 0, + "container_repositories_checksummed_count": 0, + "container_repositories_checksum_failed_count": 0, + "container_repositories_verification_total_count": 0, + "container_repositories_verified_count": 0, + "container_repositories_verification_failed_count": 0, + "container_repositories_verified_in_percentage": "100.00%", "dependency_proxy_manifests_count": 5, "dependency_proxy_manifests_checksum_total_count": 5, "dependency_proxy_manifests_checksummed_count": 5, @@ -922,6 +948,13 @@ Example response: "container_repositories_registry_count": 5, "container_repositories_synced_in_percentage": "100.00%", "container_repositories_synced_missing_on_primary_count": 0, + "container_repositories_checksum_total_count": 0, + "container_repositories_checksummed_count": 0, + "container_repositories_checksum_failed_count": 0, + "container_repositories_verification_total_count": 0, + "container_repositories_verified_count": 0, + "container_repositories_verification_failed_count": 0, + "container_repositories_verified_in_percentage": "100.00%", "dependency_proxy_manifests_count": 5, "dependency_proxy_manifests_checksum_total_count": 5, "dependency_proxy_manifests_checksummed_count": 5, diff --git a/doc/api/geo_sites.md b/doc/api/geo_sites.md new file mode 100644 index 00000000000..40410cdf540 --- /dev/null +++ b/doc/api/geo_sites.md @@ -0,0 +1,1111 @@ +--- +stage: Systems +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Geo sites API **(PREMIUM SELF)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/369140) in GitLab 16.0. + +Use the Geo sites API to manage Geo site endpoints. + +Prerequisites: + +- You must be an administrator. + +## Create a new Geo site + +Creates a new Geo site. + +```plaintext +POST /geo_sites +``` + +```shell +curl --header "PRIVATE-TOKEN: " "https://primary.example.com/api/v4/geo_sites" \ + --request POST \ + -d "name=himynameissomething" \ + -d "url=https://another-node.example.com/" +``` + +| Attribute | Type | Required | Description | +|---------------------------------------|---------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------| +| `primary` | boolean | no | Specifying whether this site should be primary. Defaults to false. | +| `enabled` | boolean | no | Flag indicating if the Geo site is enabled. Defaults to true. | +| `name` | string | yes | The unique identifier for the Geo site. Must match `geo_node_name` if it is set in `gitlab.rb`, otherwise it must match `external_url` | +| `url` | string | yes | The user-facing URL for the Geo site. | +| `internal_url` | string | no | The URL defined on the primary site that secondary sites should use to contact it. Returns `url` if not set. | +| `files_max_capacity` | integer | no | Control the maximum concurrency of LFS/attachment backfill for this secondary site. Defaults to 10. | +| `repos_max_capacity` | integer | no | Control the maximum concurrency of repository backfill for this secondary site. Defaults to 25. | +| `verification_max_capacity` | integer | no | Control the maximum concurrency of repository verification for this site. Defaults to 100. | +| `container_repositories_max_capacity` | integer | no | Control the maximum concurrency of container repository sync for this site. Defaults to 10. | +| `sync_object_storage` | boolean | no | Flag indicating if the secondary Geo site should replicate blobs in Object Storage. Defaults to false. | +| `selective_sync_type` | string | no | Limit syncing to only specific groups or shards. Valid values: `"namespaces"`, `"shards"`, or `null`. | +| `selective_sync_shards` | array | no | The repository storage for the projects synced if `selective_sync_type` == `shards`. | +| `selective_sync_namespace_ids` | array | no | The IDs of groups that should be synced, if `selective_sync_type` == `namespaces`. | +| `minimum_reverification_interval` | integer | no | The interval (in days) in which the repository verification is valid. Once expired, it is reverified. This has no effect when set on a secondary site. | + +Example response: + +```json +{ + "id": 3, + "name": "Test Site 1", + "url": "https://secondary.example.com/", + "internal_url": "https://secondary.example.com/", + "primary": false, + "enabled": true, + "current": false, + "files_max_capacity": 10, + "repos_max_capacity": 25, + "verification_max_capacity": 100, + "container_repositories_max_capacity": 10, + "selective_sync_type": "namespaces", + "selective_sync_shards": [], + "selective_sync_namespace_ids": [1, 25], + "minimum_reverification_interval": 7, + "sync_object_storage": false, + "web_edit_url": "https://primary.example.com/admin/geo/sites/3/edit", + "web_geo_replication_details_url": "https://secondary.example.com/admin/geo/sites/3/replication/lfs_objects", + "_links": { + "self": "https://primary.example.com/api/v4/geo_sites/3", + "status": "https://primary.example.com/api/v4/geo_sites/3/status", + "repair": "https://primary.example.com/api/v4/geo_sites/3/repair" + } +} +``` + +## Retrieve configuration about all Geo sites + +```plaintext +GET /geo_sites +``` + +```shell +curl --header "PRIVATE-TOKEN: " "https://primary.example.com/api/v4/geo_sites" +``` + +Example response: + +```json +[ + { + "id": 1, + "name": "us-site", + "url": "https://primary.example.com/", + "internal_url": "https://internal.example.com/", + "primary": true, + "enabled": true, + "current": true, + "files_max_capacity": 10, + "repos_max_capacity": 25, + "verification_max_capacity": 100, + "container_repositories_max_capacity": 10, + "selective_sync_type": "namespaces", + "selective_sync_shards": [], + "selective_sync_namespace_ids": [1, 25], + "minimum_reverification_interval": 7, + "web_edit_url": "https://primary.example.com/admin/geo/sites/1/edit", + "_links": { + "self": "https://primary.example.com/api/v4/geo_sites/1", + "status":"https://primary.example.com/api/v4/geo_sites/1/status", + "repair":"https://primary.example.com/api/v4/geo_sites/1/repair" + } + }, + { + "id": 2, + "name": "cn-site", + "url": "https://secondary.example.com/", + "internal_url": "https://secondary.example.com/", + "primary": false, + "enabled": true, + "current": false, + "files_max_capacity": 10, + "repos_max_capacity": 25, + "verification_max_capacity": 100, + "container_repositories_max_capacity": 10, + "selective_sync_type": "namespaces", + "selective_sync_shards": [], + "selective_sync_namespace_ids": [1, 25], + "minimum_reverification_interval": 7, + "sync_object_storage": true, + "web_edit_url": "https://primary.example.com/admin/geo/sites/2/edit", + "web_geo_replication_details_url": "https://secondary.example.com/admin/geo/sites/2/replication/lfs_objects", + "_links": { + "self":"https://primary.example.com/api/v4/geo_sites/2", + "status":"https://primary.example.com/api/v4/geo_sites/2/status", + "repair":"https://primary.example.com/api/v4/geo_sites/2/repair" + } + } +] +``` + +## Retrieve configuration about a specific Geo site + +```plaintext +GET /geo_sites/:id +``` + +```shell +curl --header "PRIVATE-TOKEN: " "https://primary.example.com/api/v4/geo_sites/1" +``` + +Example response: + +```json +{ + "id": 1, + "name": "us-site", + "url": "https://primary.example.com/", + "internal_url": "https://primary.example.com/", + "primary": true, + "enabled": true, + "current": true, + "files_max_capacity": 10, + "repos_max_capacity": 25, + "verification_max_capacity": 100, + "container_repositories_max_capacity": 10, + "selective_sync_type": "namespaces", + "selective_sync_shards": [], + "selective_sync_namespace_ids": [1, 25], + "minimum_reverification_interval": 7, + "web_edit_url": "https://primary.example.com/admin/geo/sites/1/edit", + "_links": { + "self": "https://primary.example.com/api/v4/geo_sites/1", + "status":"https://primary.example.com/api/v4/geo_sites/1/status", + "repair":"https://primary.example.com/api/v4/geo_sites/1/repair" + } +} +``` + +## Edit a primary Geo site + +Updates settings of an existing primary Geo site. + +```plaintext +PUT /geo_sites/:id +``` + +| Attribute | Type | Required | Description | +|---------------------------------------|---------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | integer | yes | The ID of the Geo site. | +| `enabled` | boolean | no | Flag indicating if the Geo site is enabled. | +| `name` | string | no | The unique identifier for the Geo site. Must match `geo_node_name` if it is set in `gitlab.rb`, otherwise it must match `external_url`. | +| `url` | string | no | The user-facing URL of the Geo site. | +| `internal_url` | string | no | The URL defined on the primary site that secondary sites should use to contact it. Returns `url` if not set. | +| `files_max_capacity` | integer | no | Control the maximum concurrency of LFS/attachment backfill for this secondary site. | +| `repos_max_capacity` | integer | no | Control the maximum concurrency of repository backfill for this secondary site. | +| `verification_max_capacity` | integer | no | Control the maximum concurrency of verification for this site. | +| `container_repositories_max_capacity` | integer | no | Control the maximum concurrency of container repository sync for this site. | +| `selective_sync_type` | string | no | Limit syncing to only specific groups or shards. Valid values: `"namespaces"`, `"shards"`, or `null`. | +| `selective_sync_shards` | array | no | The repository storage for the projects synced if `selective_sync_type` == `shards`. | +| `selective_sync_namespace_ids` | array | no | The IDs of groups that should be synced, if `selective_sync_type` == `namespaces`. | +| `minimum_reverification_interval` | integer | no | The interval (in days) in which the repository verification is valid. Once expired, it is reverified. This has no effect when set on a secondary site. | + +Example response: + +```json +{ + "id": 1, + "name": "us-site", + "url": "https://primary.example.com/", + "internal_url": "https://internal.example.com/", + "primary": true, + "enabled": true, + "current": true, + "files_max_capacity": 10, + "repos_max_capacity": 25, + "verification_max_capacity": 100, + "container_repositories_max_capacity": 10, + "selective_sync_type": "namespaces", + "selective_sync_shards": [], + "selective_sync_namespace_ids": [1, 25], + "minimum_reverification_interval": 7, + "web_edit_url": "https://primary.example.com/admin/geo/sites/1/edit", + "_links": { + "self": "https://primary.example.com/api/v4/geo_sites/1", + "status": "https://primary.example.com/api/v4/geo_sites/1/status", + "repair": "https://primary.example.com/api/v4/geo_sites/1/repair" + } +} + +``` + +## Delete a Geo site + +Removes the Geo site. + +```plaintext +DELETE /geo_sites/:id +``` + +| Attribute | Type | Required | Description | +|-----------|---------|----------|-------------------------| +| `id` | integer | yes | The ID of the Geo site. | + +## Repair a primary Geo site + +Repairs the OAuth authentication of a primary Geo site. + +```plaintext +POST /geo_sites/:id/repair +``` + +Example response: + +```json +{ + "id": 1, + "name": "us-site", + "url": "https://primary.example.com/", + "internal_url": "https://primary.example.com/", + "primary": true, + "enabled": true, + "current": true, + "files_max_capacity": 10, + "repos_max_capacity": 25, + "verification_max_capacity": 100, + "container_repositories_max_capacity": 10, + "web_edit_url": "https://primary.example.com/admin/geo/sites/1/edit", + "_links": { + "self": "https://primary.example.com/api/v4/geo_sites/1", + "status":"https://primary.example.com/api/v4/geo_sites/1/status", + "repair":"https://primary.example.com/api/v4/geo_sites/1/repair" + } +} +``` + +## Retrieve status about all Geo sites + +```plaintext +GET /geo_sites/status +``` + +```shell +curl --header "PRIVATE-TOKEN: " "https://primary.example.com/api/v4/geo_sites/status" +``` + +Example response: + +```json +[ + { + "geo_node_id": 1, + "repository_verification_enabled": true, + "repositories_replication_enabled": null, + "repositories_synced_count": null, + "repositories_failed_count": null, + "wikis_synced_count": null, + "wikis_failed_count": null, + "repositories_verified_count": null, + "repositories_verification_failed_count": null, + "repositories_verification_total_count": null, + "wikis_verified_count": null, + "wikis_verification_failed_count": null, + "wikis_verification_total_count": null, + "job_artifacts_synced_missing_on_primary_count": null, + "repositories_checksummed_count": 19, + "repositories_checksum_failed_count": 0, + "repositories_checksum_mismatch_count": null, + "repositories_checksum_total_count": 19, + "wikis_checksummed_count": 0, + "wikis_checksum_failed_count": 0, + "wikis_checksum_mismatch_count": null, + "wikis_checksum_total_count": 19, + "repositories_retrying_verification_count": null, + "wikis_retrying_verification_count": null, + "projects_count": 19, + "container_repositories_replication_enabled": null, + "design_repositories_replication_enabled": null, + "design_repositories_count": null, + "design_repositories_synced_count": null, + "design_repositories_failed_count": null, + "lfs_objects_count": 0, + "lfs_objects_checksum_total_count": 0, + "lfs_objects_checksummed_count": 0, + "lfs_objects_checksum_failed_count": 0, + "lfs_objects_synced_count": null, + "lfs_objects_failed_count": null, + "lfs_objects_registry_count": null, + "lfs_objects_verification_total_count": null, + "lfs_objects_verified_count": null, + "lfs_objects_verification_failed_count": null, + "merge_request_diffs_count": 0, + "merge_request_diffs_checksum_total_count": 0, + "merge_request_diffs_checksummed_count": 0, + "merge_request_diffs_checksum_failed_count": 0, + "merge_request_diffs_synced_count": null, + "merge_request_diffs_failed_count": null, + "merge_request_diffs_registry_count": null, + "merge_request_diffs_verification_total_count": null, + "merge_request_diffs_verified_count": null, + "merge_request_diffs_verification_failed_count": null, + "package_files_count": 25, + "package_files_checksum_total_count": 25, + "package_files_checksummed_count": 25, + "package_files_checksum_failed_count": 0, + "package_files_synced_count": null, + "package_files_failed_count": null, + "package_files_registry_count": null, + "package_files_verification_total_count": null, + "package_files_verified_count": null, + "package_files_verification_failed_count": null, + "terraform_state_versions_count": 18, + "terraform_state_versions_checksum_total_count": 18, + "terraform_state_versions_checksummed_count": 18, + "terraform_state_versions_checksum_failed_count": 0, + "terraform_state_versions_synced_count": null, + "terraform_state_versions_failed_count": null, + "terraform_state_versions_registry_count": null, + "terraform_state_versions_verification_total_count": null, + "terraform_state_versions_verified_count": null, + "terraform_state_versions_verification_failed_count": null, + "snippet_repositories_count": 20, + "snippet_repositories_checksum_total_count": 20, + "snippet_repositories_checksummed_count": 20, + "snippet_repositories_checksum_failed_count": 0, + "snippet_repositories_synced_count": null, + "snippet_repositories_failed_count": null, + "snippet_repositories_registry_count": null, + "snippet_repositories_verification_total_count": null, + "snippet_repositories_verified_count": null, + "snippet_repositories_verification_failed_count": null, + "group_wiki_repositories_count": 0, + "group_wiki_repositories_checksum_total_count": null, + "group_wiki_repositories_checksummed_count": null, + "group_wiki_repositories_checksum_failed_count": null, + "group_wiki_repositories_synced_count": null, + "group_wiki_repositories_failed_count": null, + "group_wiki_repositories_registry_count": null, + "group_wiki_repositories_verification_total_count": null, + "group_wiki_repositories_verified_count": null, + "group_wiki_repositories_verification_failed_count": null, + "pipeline_artifacts_count": 0, + "pipeline_artifacts_checksum_total_count": 0, + "pipeline_artifacts_checksummed_count": 0, + "pipeline_artifacts_checksum_failed_count": 0, + "pipeline_artifacts_synced_count": null, + "pipeline_artifacts_failed_count": null, + "pipeline_artifacts_registry_count": null, + "pipeline_artifacts_verification_total_count": null, + "pipeline_artifacts_verified_count": null, + "pipeline_artifacts_verification_failed_count": null, + "pages_deployments_count": 0, + "pages_deployments_checksum_total_count": 0, + "pages_deployments_checksummed_count": 0, + "pages_deployments_checksum_failed_count": 0, + "pages_deployments_synced_count": null, + "pages_deployments_failed_count": null, + "pages_deployments_registry_count": null, + "pages_deployments_verification_total_count": null, + "pages_deployments_verified_count": null, + "pages_deployments_verification_failed_count": null, + "uploads_count": 51, + "uploads_checksum_total_count": 51, + "uploads_checksummed_count": 51, + "uploads_checksum_failed_count": 0, + "uploads_synced_count": null, + "uploads_failed_count": null, + "uploads_registry_count": null, + "uploads_verification_total_count": null, + "uploads_verified_count": null, + "uploads_verification_failed_count": null, + "job_artifacts_count": 205, + "job_artifacts_checksum_total_count": 205, + "job_artifacts_checksummed_count": 205, + "job_artifacts_checksum_failed_count": 0, + "job_artifacts_synced_count": null, + "job_artifacts_failed_count": null, + "job_artifacts_registry_count": null, + "job_artifacts_verification_total_count": null, + "job_artifacts_verified_count": null, + "job_artifacts_verification_failed_count": null, + "ci_secure_files_count": 0, + "ci_secure_files_checksum_total_count": 0, + "ci_secure_files_checksummed_count": 0, + "ci_secure_files_checksum_failed_count": 0, + "ci_secure_files_synced_count": null, + "ci_secure_files_failed_count": null, + "ci_secure_files_registry_count": null, + "ci_secure_files_verification_total_count": null, + "ci_secure_files_verified_count": null, + "ci_secure_files_verification_failed_count": null, + "container_repositories_count": 0, + "container_repositories_checksum_total_count": 0, + "container_repositories_checksummed_count": 0, + "container_repositories_checksum_failed_count": 0, + "container_repositories_synced_count": null, + "container_repositories_failed_count": null, + "container_repositories_registry_count": null, + "container_repositories_verification_total_count": null, + "container_repositories_verified_count": null, + "container_repositories_verification_failed_count": null, + "dependency_proxy_blobs_count": 0, + "dependency_proxy_blobs_checksum_total_count": 0, + "dependency_proxy_blobs_checksummed_count": 0, + "dependency_proxy_blobs_checksum_failed_count": 0, + "dependency_proxy_blobs_synced_count": null, + "dependency_proxy_blobs_failed_count": null, + "dependency_proxy_blobs_registry_count": null, + "dependency_proxy_blobs_verification_total_count": null, + "dependency_proxy_blobs_verified_count": null, + "dependency_proxy_blobs_verification_failed_count": null, + "dependency_proxy_manifests_count": 0, + "dependency_proxy_manifests_checksum_total_count": 0, + "dependency_proxy_manifests_checksummed_count": 0, + "dependency_proxy_manifests_checksum_failed_count": 0, + "dependency_proxy_manifests_synced_count": null, + "dependency_proxy_manifests_failed_count": null, + "dependency_proxy_manifests_registry_count": null, + "dependency_proxy_manifests_verification_total_count": null, + "dependency_proxy_manifests_verified_count": null, + "dependency_proxy_manifests_verification_failed_count": null, + "project_wiki_repositories_count": 19, + "project_wiki_repositories_checksum_total_count": 19, + "project_wiki_repositories_checksummed_count": 19, + "project_wiki_repositories_checksum_failed_count": 0, + "project_wiki_repositories_synced_count": null, + "project_wiki_repositories_failed_count": null, + "project_wiki_repositories_registry_count": null, + "project_wiki_repositories_verification_total_count": null, + "project_wiki_repositories_verified_count": null, + "project_wiki_repositories_verification_failed_count": null, + "git_fetch_event_count_weekly": null, + "git_push_event_count_weekly": null, + "proxy_remote_requests_event_count_weekly": null, + "proxy_local_requests_event_count_weekly": null, + "repositories_synced_in_percentage": "0.00%", + "repositories_checksummed_in_percentage": "100.00%", + "repositories_verified_in_percentage": "0.00%", + "repositories_checked_in_percentage": "0.00%", + "wikis_synced_in_percentage": "0.00%", + "wikis_checksummed_in_percentage": "0.00%", + "wikis_verified_in_percentage": "0.00%", + "replication_slots_used_in_percentage": "100.00%", + "design_repositories_synced_in_percentage": "0.00%", + "lfs_objects_synced_in_percentage": "0.00%", + "lfs_objects_verified_in_percentage": "0.00%", + "merge_request_diffs_synced_in_percentage": "0.00%", + "merge_request_diffs_verified_in_percentage": "0.00%", + "package_files_synced_in_percentage": "0.00%", + "package_files_verified_in_percentage": "0.00%", + "terraform_state_versions_synced_in_percentage": "0.00%", + "terraform_state_versions_verified_in_percentage": "0.00%", + "snippet_repositories_synced_in_percentage": "0.00%", + "snippet_repositories_verified_in_percentage": "0.00%", + "group_wiki_repositories_synced_in_percentage": "0.00%", + "group_wiki_repositories_verified_in_percentage": "0.00%", + "pipeline_artifacts_synced_in_percentage": "0.00%", + "pipeline_artifacts_verified_in_percentage": "0.00%", + "pages_deployments_synced_in_percentage": "0.00%", + "pages_deployments_verified_in_percentage": "0.00%", + "uploads_synced_in_percentage": "0.00%", + "uploads_verified_in_percentage": "0.00%", + "job_artifacts_synced_in_percentage": "0.00%", + "job_artifacts_verified_in_percentage": "0.00%", + "ci_secure_files_synced_in_percentage": "0.00%", + "ci_secure_files_verified_in_percentage": "0.00%", + "container_repositories_synced_in_percentage": "0.00%", + "container_repositories_verified_in_percentage": "0.00%", + "dependency_proxy_blobs_synced_in_percentage": "0.00%", + "dependency_proxy_blobs_verified_in_percentage": "0.00%", + "dependency_proxy_manifests_synced_in_percentage": "0.00%", + "dependency_proxy_manifests_verified_in_percentage": "0.00%", + "project_wiki_repositories_synced_in_percentage": "0.00%", + "project_wiki_repositories_verified_in_percentage": "0.00%", + "repositories_count": 19, + "wikis_count": 19, + "replication_slots_count": 1, + "replication_slots_used_count": 1, + "healthy": true, + "health": "Healthy", + "health_status": "Healthy", + "missing_oauth_application": false, + "db_replication_lag_seconds": null, + "replication_slots_max_retained_wal_bytes": 0, + "repositories_checked_count": null, + "repositories_checked_failed_count": null, + "last_event_id": 357, + "last_event_timestamp": 1683127088, + "cursor_last_event_id": null, + "cursor_last_event_timestamp": 0, + "last_successful_status_check_timestamp": 1683129788, + "version": "16.0.0-pre", + "revision": "129eb954664", + "selective_sync_type": null, + "namespaces": [], + "updated_at": "2023-05-03T16:03:10.117Z", + "storage_shards_match": true, + "_links": { + "self": "https://primary.example.com/api/v4/geo_sites/1/status", + "site": "https://primary.example.com/api/v4/geo_sites/1" + } + }, + { + "geo_node_id": 2, + "repository_verification_enabled": true, + "repositories_replication_enabled": true, + "repositories_synced_count": 18, + "repositories_failed_count": 0, + "wikis_synced_count": 18, + "wikis_failed_count": 0, + "repositories_verified_count": 0, + "repositories_verification_failed_count": 0, + "repositories_verification_total_count": 19, + "wikis_verified_count": 0, + "wikis_verification_failed_count": 0, + "wikis_verification_total_count": 19, + "job_artifacts_synced_missing_on_primary_count": null, + "repositories_checksummed_count": null, + "repositories_checksum_failed_count": null, + "repositories_checksum_mismatch_count": 0, + "repositories_checksum_total_count": null, + "wikis_checksummed_count": null, + "wikis_checksum_failed_count": null, + "wikis_checksum_mismatch_count": 0, + "wikis_checksum_total_count": null, + "repositories_retrying_verification_count": 0, + "wikis_retrying_verification_count": 0, + "projects_count": 19, + "container_repositories_replication_enabled": null, + "design_repositories_replication_enabled": true, + "design_repositories_count": 0, + "design_repositories_synced_count": 0, + "design_repositories_failed_count": 0, + "lfs_objects_count": 0, + "lfs_objects_checksum_total_count": null, + "lfs_objects_checksummed_count": null, + "lfs_objects_checksum_failed_count": null, + "lfs_objects_synced_count": 0, + "lfs_objects_failed_count": 0, + "lfs_objects_registry_count": 0, + "lfs_objects_verification_total_count": 0, + "lfs_objects_verified_count": 0, + "lfs_objects_verification_failed_count": 0, + "merge_request_diffs_count": 0, + "merge_request_diffs_checksum_total_count": null, + "merge_request_diffs_checksummed_count": null, + "merge_request_diffs_checksum_failed_count": null, + "merge_request_diffs_synced_count": 0, + "merge_request_diffs_failed_count": 0, + "merge_request_diffs_registry_count": 0, + "merge_request_diffs_verification_total_count": 0, + "merge_request_diffs_verified_count": 0, + "merge_request_diffs_verification_failed_count": 0, + "package_files_count": 25, + "package_files_checksum_total_count": null, + "package_files_checksummed_count": null, + "package_files_checksum_failed_count": null, + "package_files_synced_count": 1, + "package_files_failed_count": 24, + "package_files_registry_count": 25, + "package_files_verification_total_count": 1, + "package_files_verified_count": 1, + "package_files_verification_failed_count": 0, + "terraform_state_versions_count": 18, + "terraform_state_versions_checksum_total_count": null, + "terraform_state_versions_checksummed_count": null, + "terraform_state_versions_checksum_failed_count": null, + "terraform_state_versions_synced_count": 0, + "terraform_state_versions_failed_count": 0, + "terraform_state_versions_registry_count": 18, + "terraform_state_versions_verification_total_count": 0, + "terraform_state_versions_verified_count": 0, + "terraform_state_versions_verification_failed_count": 0, + "snippet_repositories_count": 20, + "snippet_repositories_checksum_total_count": null, + "snippet_repositories_checksummed_count": null, + "snippet_repositories_checksum_failed_count": null, + "snippet_repositories_synced_count": 20, + "snippet_repositories_failed_count": 0, + "snippet_repositories_registry_count": 20, + "snippet_repositories_verification_total_count": 20, + "snippet_repositories_verified_count": 20, + "snippet_repositories_verification_failed_count": 0, + "group_wiki_repositories_count": 0, + "group_wiki_repositories_checksum_total_count": null, + "group_wiki_repositories_checksummed_count": null, + "group_wiki_repositories_checksum_failed_count": null, + "group_wiki_repositories_synced_count": 0, + "group_wiki_repositories_failed_count": 0, + "group_wiki_repositories_registry_count": 0, + "group_wiki_repositories_verification_total_count": null, + "group_wiki_repositories_verified_count": null, + "group_wiki_repositories_verification_failed_count": null, + "pipeline_artifacts_count": 0, + "pipeline_artifacts_checksum_total_count": null, + "pipeline_artifacts_checksummed_count": null, + "pipeline_artifacts_checksum_failed_count": null, + "pipeline_artifacts_synced_count": 0, + "pipeline_artifacts_failed_count": 0, + "pipeline_artifacts_registry_count": 0, + "pipeline_artifacts_verification_total_count": 0, + "pipeline_artifacts_verified_count": 0, + "pipeline_artifacts_verification_failed_count": 0, + "pages_deployments_count": 0, + "pages_deployments_checksum_total_count": null, + "pages_deployments_checksummed_count": null, + "pages_deployments_checksum_failed_count": null, + "pages_deployments_synced_count": 0, + "pages_deployments_failed_count": 0, + "pages_deployments_registry_count": 0, + "pages_deployments_verification_total_count": 0, + "pages_deployments_verified_count": 0, + "pages_deployments_verification_failed_count": 0, + "uploads_count": 51, + "uploads_checksum_total_count": null, + "uploads_checksummed_count": null, + "uploads_checksum_failed_count": null, + "uploads_synced_count": 0, + "uploads_failed_count": 1, + "uploads_registry_count": 51, + "uploads_verification_total_count": 0, + "uploads_verified_count": 0, + "uploads_verification_failed_count": 0, + "job_artifacts_count": 0, + "job_artifacts_checksum_total_count": null, + "job_artifacts_checksummed_count": null, + "job_artifacts_checksum_failed_count": null, + "job_artifacts_synced_count": 0, + "job_artifacts_failed_count": 0, + "job_artifacts_registry_count": 0, + "job_artifacts_verification_total_count": 0, + "job_artifacts_verified_count": 0, + "job_artifacts_verification_failed_count": 0, + "ci_secure_files_count": 0, + "ci_secure_files_checksum_total_count": null, + "ci_secure_files_checksummed_count": null, + "ci_secure_files_checksum_failed_count": null, + "ci_secure_files_synced_count": 0, + "ci_secure_files_failed_count": 0, + "ci_secure_files_registry_count": 0, + "ci_secure_files_verification_total_count": 0, + "ci_secure_files_verified_count": 0, + "ci_secure_files_verification_failed_count": 0, + "container_repositories_count": null, + "container_repositories_checksum_total_count": null, + "container_repositories_checksummed_count": null, + "container_repositories_checksum_failed_count": null, + "container_repositories_synced_count": null, + "container_repositories_failed_count": null, + "container_repositories_registry_count": null, + "container_repositories_verification_total_count": null, + "container_repositories_verified_count": null, + "container_repositories_verification_failed_count": null, + "dependency_proxy_blobs_count": 0, + "dependency_proxy_blobs_checksum_total_count": null, + "dependency_proxy_blobs_checksummed_count": null, + "dependency_proxy_blobs_checksum_failed_count": null, + "dependency_proxy_blobs_synced_count": 0, + "dependency_proxy_blobs_failed_count": 0, + "dependency_proxy_blobs_registry_count": 0, + "dependency_proxy_blobs_verification_total_count": 0, + "dependency_proxy_blobs_verified_count": 0, + "dependency_proxy_blobs_verification_failed_count": 0, + "dependency_proxy_manifests_count": 0, + "dependency_proxy_manifests_checksum_total_count": null, + "dependency_proxy_manifests_checksummed_count": null, + "dependency_proxy_manifests_checksum_failed_count": null, + "dependency_proxy_manifests_synced_count": 0, + "dependency_proxy_manifests_failed_count": 0, + "dependency_proxy_manifests_registry_count": 0, + "dependency_proxy_manifests_verification_total_count": 0, + "dependency_proxy_manifests_verified_count": 0, + "dependency_proxy_manifests_verification_failed_count": 0, + "project_wiki_repositories_count": 19, + "project_wiki_repositories_checksum_total_count": null, + "project_wiki_repositories_checksummed_count": null, + "project_wiki_repositories_checksum_failed_count": null, + "project_wiki_repositories_synced_count": 19, + "project_wiki_repositories_failed_count": 0, + "project_wiki_repositories_registry_count": 19, + "project_wiki_repositories_verification_total_count": 19, + "project_wiki_repositories_verified_count": 19, + "project_wiki_repositories_verification_failed_count": 0, + "git_fetch_event_count_weekly": null, + "git_push_event_count_weekly": null, + "proxy_remote_requests_event_count_weekly": null, + "proxy_local_requests_event_count_weekly": null, + "repositories_synced_in_percentage": "94.74%", + "repositories_checksummed_in_percentage": "0.00%", + "repositories_verified_in_percentage": "0.00%", + "repositories_checked_in_percentage": "0.00%", + "wikis_synced_in_percentage": "94.74%", + "wikis_checksummed_in_percentage": "0.00%", + "wikis_verified_in_percentage": "0.00%", + "replication_slots_used_in_percentage": "0.00%", + "design_repositories_synced_in_percentage": "0.00%", + "lfs_objects_synced_in_percentage": "0.00%", + "lfs_objects_verified_in_percentage": "0.00%", + "merge_request_diffs_synced_in_percentage": "0.00%", + "merge_request_diffs_verified_in_percentage": "0.00%", + "package_files_synced_in_percentage": "4.00%", + "package_files_verified_in_percentage": "4.00%", + "terraform_state_versions_synced_in_percentage": "0.00%", + "terraform_state_versions_verified_in_percentage": "0.00%", + "snippet_repositories_synced_in_percentage": "100.00%", + "snippet_repositories_verified_in_percentage": "100.00%", + "group_wiki_repositories_synced_in_percentage": "0.00%", + "group_wiki_repositories_verified_in_percentage": "0.00%", + "pipeline_artifacts_synced_in_percentage": "0.00%", + "pipeline_artifacts_verified_in_percentage": "0.00%", + "pages_deployments_synced_in_percentage": "0.00%", + "pages_deployments_verified_in_percentage": "0.00%", + "uploads_synced_in_percentage": "0.00%", + "uploads_verified_in_percentage": "0.00%", + "job_artifacts_synced_in_percentage": "0.00%", + "job_artifacts_verified_in_percentage": "0.00%", + "ci_secure_files_synced_in_percentage": "0.00%", + "ci_secure_files_verified_in_percentage": "0.00%", + "container_repositories_synced_in_percentage": "0.00%", + "container_repositories_verified_in_percentage": "0.00%", + "dependency_proxy_blobs_synced_in_percentage": "0.00%", + "dependency_proxy_blobs_verified_in_percentage": "0.00%", + "dependency_proxy_manifests_synced_in_percentage": "0.00%", + "dependency_proxy_manifests_verified_in_percentage": "0.00%", + "project_wiki_repositories_synced_in_percentage": "100.00%", + "project_wiki_repositories_verified_in_percentage": "100.00%", + "repositories_count": 19, + "wikis_count": 19, + "replication_slots_count": null, + "replication_slots_used_count": null, + "healthy": false, + "health": "An existing tracking database cannot be reused..", + "health_status": "Unhealthy", + "missing_oauth_application": false, + "db_replication_lag_seconds": 0, + "replication_slots_max_retained_wal_bytes": null, + "repositories_checked_count": null, + "repositories_checked_failed_count": null, + "last_event_id": 357, + "last_event_timestamp": 1683127088, + "cursor_last_event_id": 357, + "cursor_last_event_timestamp": 1683127088, + "last_successful_status_check_timestamp": 1683127146, + "version": "16.0.0-pre", + "revision": "129eb954664", + "selective_sync_type": "", + "namespaces": [], + "updated_at": "2023-05-03T15:19:06.174Z", + "storage_shards_match": true, + "_links": { + "self": "https://primary.example.com/api/v4/geo_sites/2/status", + "site": "https://primary.example.com/api/v4/geo_sites/2" + } + } +] +``` + +## Retrieve status about a specific Geo site + +```plaintext +GET /geo_sites/:id/status +``` + +```shell +curl --header "PRIVATE-TOKEN: " "https://primary.example.com/api/v4/geo_sites/2/status" +``` + +Example response: + +```json + { + "geo_node_id": 2, + "repository_verification_enabled": true, + "repositories_replication_enabled": true, + "repositories_synced_count": 18, + "repositories_failed_count": 0, + "wikis_synced_count": 18, + "wikis_failed_count": 0, + "repositories_verified_count": 0, + "repositories_verification_failed_count": 0, + "repositories_verification_total_count": 19, + "wikis_verified_count": 0, + "wikis_verification_failed_count": 0, + "wikis_verification_total_count": 19, + "job_artifacts_synced_missing_on_primary_count": null, + "repositories_checksummed_count": null, + "repositories_checksum_failed_count": null, + "repositories_checksum_mismatch_count": 0, + "repositories_checksum_total_count": null, + "wikis_checksummed_count": null, + "wikis_checksum_failed_count": null, + "wikis_checksum_mismatch_count": 0, + "wikis_checksum_total_count": null, + "repositories_retrying_verification_count": 0, + "wikis_retrying_verification_count": 0, + "projects_count": 19, + "container_repositories_replication_enabled": null, + "design_repositories_replication_enabled": true, + "design_repositories_count": 0, + "design_repositories_synced_count": 0, + "design_repositories_failed_count": 0, + "lfs_objects_count": 0, + "lfs_objects_checksum_total_count": null, + "lfs_objects_checksummed_count": null, + "lfs_objects_checksum_failed_count": null, + "lfs_objects_synced_count": 0, + "lfs_objects_failed_count": 0, + "lfs_objects_registry_count": 0, + "lfs_objects_verification_total_count": 0, + "lfs_objects_verified_count": 0, + "lfs_objects_verification_failed_count": 0, + "merge_request_diffs_count": 0, + "merge_request_diffs_checksum_total_count": null, + "merge_request_diffs_checksummed_count": null, + "merge_request_diffs_checksum_failed_count": null, + "merge_request_diffs_synced_count": 0, + "merge_request_diffs_failed_count": 0, + "merge_request_diffs_registry_count": 0, + "merge_request_diffs_verification_total_count": 0, + "merge_request_diffs_verified_count": 0, + "merge_request_diffs_verification_failed_count": 0, + "package_files_count": 25, + "package_files_checksum_total_count": null, + "package_files_checksummed_count": null, + "package_files_checksum_failed_count": null, + "package_files_synced_count": 1, + "package_files_failed_count": 24, + "package_files_registry_count": 25, + "package_files_verification_total_count": 1, + "package_files_verified_count": 1, + "package_files_verification_failed_count": 0, + "terraform_state_versions_count": 18, + "terraform_state_versions_checksum_total_count": null, + "terraform_state_versions_checksummed_count": null, + "terraform_state_versions_checksum_failed_count": null, + "terraform_state_versions_synced_count": 0, + "terraform_state_versions_failed_count": 0, + "terraform_state_versions_registry_count": 18, + "terraform_state_versions_verification_total_count": 0, + "terraform_state_versions_verified_count": 0, + "terraform_state_versions_verification_failed_count": 0, + "snippet_repositories_count": 20, + "snippet_repositories_checksum_total_count": null, + "snippet_repositories_checksummed_count": null, + "snippet_repositories_checksum_failed_count": null, + "snippet_repositories_synced_count": 20, + "snippet_repositories_failed_count": 0, + "snippet_repositories_registry_count": 20, + "snippet_repositories_verification_total_count": 20, + "snippet_repositories_verified_count": 20, + "snippet_repositories_verification_failed_count": 0, + "group_wiki_repositories_count": 0, + "group_wiki_repositories_checksum_total_count": null, + "group_wiki_repositories_checksummed_count": null, + "group_wiki_repositories_checksum_failed_count": null, + "group_wiki_repositories_synced_count": 0, + "group_wiki_repositories_failed_count": 0, + "group_wiki_repositories_registry_count": 0, + "group_wiki_repositories_verification_total_count": null, + "group_wiki_repositories_verified_count": null, + "group_wiki_repositories_verification_failed_count": null, + "pipeline_artifacts_count": 0, + "pipeline_artifacts_checksum_total_count": null, + "pipeline_artifacts_checksummed_count": null, + "pipeline_artifacts_checksum_failed_count": null, + "pipeline_artifacts_synced_count": 0, + "pipeline_artifacts_failed_count": 0, + "pipeline_artifacts_registry_count": 0, + "pipeline_artifacts_verification_total_count": 0, + "pipeline_artifacts_verified_count": 0, + "pipeline_artifacts_verification_failed_count": 0, + "pages_deployments_count": 0, + "pages_deployments_checksum_total_count": null, + "pages_deployments_checksummed_count": null, + "pages_deployments_checksum_failed_count": null, + "pages_deployments_synced_count": 0, + "pages_deployments_failed_count": 0, + "pages_deployments_registry_count": 0, + "pages_deployments_verification_total_count": 0, + "pages_deployments_verified_count": 0, + "pages_deployments_verification_failed_count": 0, + "uploads_count": 51, + "uploads_checksum_total_count": null, + "uploads_checksummed_count": null, + "uploads_checksum_failed_count": null, + "uploads_synced_count": 0, + "uploads_failed_count": 1, + "uploads_registry_count": 51, + "uploads_verification_total_count": 0, + "uploads_verified_count": 0, + "uploads_verification_failed_count": 0, + "job_artifacts_count": 0, + "job_artifacts_checksum_total_count": null, + "job_artifacts_checksummed_count": null, + "job_artifacts_checksum_failed_count": null, + "job_artifacts_synced_count": 0, + "job_artifacts_failed_count": 0, + "job_artifacts_registry_count": 0, + "job_artifacts_verification_total_count": 0, + "job_artifacts_verified_count": 0, + "job_artifacts_verification_failed_count": 0, + "ci_secure_files_count": 0, + "ci_secure_files_checksum_total_count": null, + "ci_secure_files_checksummed_count": null, + "ci_secure_files_checksum_failed_count": null, + "ci_secure_files_synced_count": 0, + "ci_secure_files_failed_count": 0, + "ci_secure_files_registry_count": 0, + "ci_secure_files_verification_total_count": 0, + "ci_secure_files_verified_count": 0, + "ci_secure_files_verification_failed_count": 0, + "container_repositories_count": null, + "container_repositories_checksum_total_count": null, + "container_repositories_checksummed_count": null, + "container_repositories_checksum_failed_count": null, + "container_repositories_synced_count": null, + "container_repositories_failed_count": null, + "container_repositories_registry_count": null, + "container_repositories_verification_total_count": null, + "container_repositories_verified_count": null, + "container_repositories_verification_failed_count": null, + "dependency_proxy_blobs_count": 0, + "dependency_proxy_blobs_checksum_total_count": null, + "dependency_proxy_blobs_checksummed_count": null, + "dependency_proxy_blobs_checksum_failed_count": null, + "dependency_proxy_blobs_synced_count": 0, + "dependency_proxy_blobs_failed_count": 0, + "dependency_proxy_blobs_registry_count": 0, + "dependency_proxy_blobs_verification_total_count": 0, + "dependency_proxy_blobs_verified_count": 0, + "dependency_proxy_blobs_verification_failed_count": 0, + "dependency_proxy_manifests_count": 0, + "dependency_proxy_manifests_checksum_total_count": null, + "dependency_proxy_manifests_checksummed_count": null, + "dependency_proxy_manifests_checksum_failed_count": null, + "dependency_proxy_manifests_synced_count": 0, + "dependency_proxy_manifests_failed_count": 0, + "dependency_proxy_manifests_registry_count": 0, + "dependency_proxy_manifests_verification_total_count": 0, + "dependency_proxy_manifests_verified_count": 0, + "dependency_proxy_manifests_verification_failed_count": 0, + "project_wiki_repositories_count": 19, + "project_wiki_repositories_checksum_total_count": null, + "project_wiki_repositories_checksummed_count": null, + "project_wiki_repositories_checksum_failed_count": null, + "project_wiki_repositories_synced_count": 19, + "project_wiki_repositories_failed_count": 0, + "project_wiki_repositories_registry_count": 19, + "project_wiki_repositories_verification_total_count": 19, + "project_wiki_repositories_verified_count": 19, + "project_wiki_repositories_verification_failed_count": 0, + "git_fetch_event_count_weekly": null, + "git_push_event_count_weekly": null, + "proxy_remote_requests_event_count_weekly": null, + "proxy_local_requests_event_count_weekly": null, + "repositories_synced_in_percentage": "94.74%", + "repositories_checksummed_in_percentage": "0.00%", + "repositories_verified_in_percentage": "0.00%", + "repositories_checked_in_percentage": "0.00%", + "wikis_synced_in_percentage": "94.74%", + "wikis_checksummed_in_percentage": "0.00%", + "wikis_verified_in_percentage": "0.00%", + "replication_slots_used_in_percentage": "0.00%", + "design_repositories_synced_in_percentage": "0.00%", + "lfs_objects_synced_in_percentage": "0.00%", + "lfs_objects_verified_in_percentage": "0.00%", + "merge_request_diffs_synced_in_percentage": "0.00%", + "merge_request_diffs_verified_in_percentage": "0.00%", + "package_files_synced_in_percentage": "4.00%", + "package_files_verified_in_percentage": "4.00%", + "terraform_state_versions_synced_in_percentage": "0.00%", + "terraform_state_versions_verified_in_percentage": "0.00%", + "snippet_repositories_synced_in_percentage": "100.00%", + "snippet_repositories_verified_in_percentage": "100.00%", + "group_wiki_repositories_synced_in_percentage": "0.00%", + "group_wiki_repositories_verified_in_percentage": "0.00%", + "pipeline_artifacts_synced_in_percentage": "0.00%", + "pipeline_artifacts_verified_in_percentage": "0.00%", + "pages_deployments_synced_in_percentage": "0.00%", + "pages_deployments_verified_in_percentage": "0.00%", + "uploads_synced_in_percentage": "0.00%", + "uploads_verified_in_percentage": "0.00%", + "job_artifacts_synced_in_percentage": "0.00%", + "job_artifacts_verified_in_percentage": "0.00%", + "ci_secure_files_synced_in_percentage": "0.00%", + "ci_secure_files_verified_in_percentage": "0.00%", + "container_repositories_synced_in_percentage": "0.00%", + "container_repositories_verified_in_percentage": "0.00%", + "dependency_proxy_blobs_synced_in_percentage": "0.00%", + "dependency_proxy_blobs_verified_in_percentage": "0.00%", + "dependency_proxy_manifests_synced_in_percentage": "0.00%", + "dependency_proxy_manifests_verified_in_percentage": "0.00%", + "project_wiki_repositories_synced_in_percentage": "100.00%", + "project_wiki_repositories_verified_in_percentage": "100.00%", + "repositories_count": 19, + "wikis_count": 19, + "replication_slots_count": null, + "replication_slots_used_count": null, + "healthy": false, + "health": "An existing tracking database cannot be reused..", + "health_status": "Unhealthy", + "missing_oauth_application": false, + "db_replication_lag_seconds": 0, + "replication_slots_max_retained_wal_bytes": null, + "repositories_checked_count": null, + "repositories_checked_failed_count": null, + "last_event_id": 357, + "last_event_timestamp": 1683127088, + "cursor_last_event_id": 357, + "cursor_last_event_timestamp": 1683127088, + "last_successful_status_check_timestamp": 1683127146, + "version": "16.0.0-pre", + "revision": "129eb954664", + "selective_sync_type": "", + "namespaces": [], + "updated_at": "2023-05-03T15:19:06.174Z", + "storage_shards_match": true, + "_links": { + "self": "https://primary.example.com/api/v4/geo_sites/2/status", + "site": "https://primary.example.com/api/v4/geo_sites/2" + } +} +``` + +NOTE: +The `health_status` parameter can only be in an "Healthy" or "Unhealthy" state, while the `health` parameter can be empty, "Healthy", or contain the actual error message. + +## Retrieve project sync or verification failures that occurred on the current site + +This only works on a secondary site. + +```plaintext +GET /geo_sites/current/failures +``` + +| Attribute | Type | Required | Description | +|----------------|--------|----------|--------------------------------------------------------------| +| `type` | string | no | Type of failed objects (`repository`/`wiki`) | +| `failure_type` | string | no | Type of failures (`sync`/`checksum_mismatch`/`verification`) | + +This endpoint uses [Pagination](rest/index.md#pagination). + +```shell +curl --header "PRIVATE-TOKEN: " "https://primary.example.com/api/v4/geo_sites/current/failures" +``` + +Example response: + +```json +[ + { + "project_id": 3, + "last_repository_synced_at": "2017-10-31 14:25:55 UTC", + "last_repository_successful_sync_at": "2017-10-31 14:26:04 UTC", + "last_wiki_synced_at": "2017-10-31 14:26:04 UTC", + "last_wiki_successful_sync_at": "2017-10-31 14:26:11 UTC", + "repository_retry_count": null, + "wiki_retry_count": 1, + "last_repository_sync_failure": null, + "last_wiki_sync_failure": "Error syncing Wiki repository", + "last_repository_verification_failure": "", + "last_wiki_verification_failure": "", + "repository_verification_checksum_sha": "da39a3ee5e6b4b0d32e5bfef9a601890afd80709", + "wiki_verification_checksum_sha": "da39a3ee5e6b4b0d3255bfef9ef0189aafd80709", + "repository_checksum_mismatch": false, + "wiki_checksum_mismatch": false + } +] +``` diff --git a/doc/api/graphql/branch_rules.md b/doc/api/graphql/branch_rules.md index c38ebd673d0..5709ce9fafb 100644 --- a/doc/api/graphql/branch_rules.md +++ b/doc/api/graphql/branch_rules.md @@ -130,7 +130,7 @@ Instead of requesting access, it may be easier for you to run the query in the project(fullPath: "flightjs/Flight") { ``` -1. Visit the [GraphiQL explorer tool](http://gdk.test:3000/-/graphql-explorer) for your GDK instance. +1. In your GDK instance, visit the GraphiQL explorer tool: `http://gdk.test:3000/-/graphql-explorer`. 1. Paste the `query` listed above into the left window of your GraphiQL explorer tool. 1. Select **Play** to view the result. diff --git a/doc/api/graphql/custom_emoji.md b/doc/api/graphql/custom_emoji.md index 9c794a080c9..25ae37b75a9 100644 --- a/doc/api/graphql/custom_emoji.md +++ b/doc/api/graphql/custom_emoji.md @@ -14,13 +14,13 @@ On self-managed GitLab, by default this feature is not available. To make it ava On GitLab.com, this feature is available. This feature is ready for production use. -To use custom emojis in comments and descriptions, you can add them to a group using the GraphQL API. +To use custom emojis in comments and descriptions, you can add them to a top-level group using the GraphQL API. Parameters: | Attribute | Type | Required | Description | | :----------- | :------------- | :--------------------- | :------------------------------------------------------------------------ | -| `group_path` | integer/string | **{check-circle}** Yes | ID or [URL-encoded path of the group](../rest/index.md#namespaced-path-encoding) | +| `group_path` | integer/string | **{check-circle}** Yes | ID or [URL-encoded path of the top-level group](../rest/index.md#namespaced-path-encoding) | | `name` | string | **{check-circle}** Yes | Name of the custom emoji. | | `file` | string | **{check-circle}** Yes | URL of the custom emoji image. | diff --git a/doc/api/graphql/getting_started.md b/doc/api/graphql/getting_started.md index 57d7880988b..237c0cc6934 100644 --- a/doc/api/graphql/getting_started.md +++ b/doc/api/graphql/getting_started.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -16,9 +16,9 @@ the API itself. The examples documented here can be run using: -- The command line. -- GraphiQL. -- Rails console. +- [Command line](#command-line). +- [GraphiQL](#graphiql). +- [Rails console](#rails-console). ### Command line @@ -79,6 +79,7 @@ If you are running GitLab 12.0, enable the `graphql` GraphQL queries can be run in a [Rails console session](../../administration/operations/rails_console.md#starting-a-rails-console-session). For example, to search projects: ```ruby +current_user = User.find_by_id(1) query = <<~EOQ query securityGetProjects($search: String!) { projects(search: $search) { @@ -182,7 +183,7 @@ Mutations generally use InputTypes and variables, neither of which appear here. Mutations have: -- Inputs. For example, arguments, such as which emoji you'd like to award, +- Inputs. For example, arguments, such as which emoji reaction you'd like to add, and to which object. - Return statements. That is, what you'd like to get back when it's successful. - Errors. Always ask for what went wrong, just in case. @@ -285,6 +286,24 @@ We've asked for the note details, but it doesn't exist anymore, so we get `null` More about mutations: [GraphQL Documentation](https://graphql.org/learn/queries/#mutations). +### Update project settings + +You can update multiple project settings in a single GraphQL mutation. +This example is a workaround for [the major change](../../update/deprecations.md#default-cicd-job-token-ci_job_token-scope-changed) +in `CI_JOB_TOKEN` scoping behavior. + +```graphql +mutation DisableCI_JOB_TOKENscope { + projectCiCdSettingsUpdate(input:{fullPath: "/", inboundJobTokenScopeEnabled: false, jobTokenScopeEnabled: false}) { + ciCdSettings { + inboundJobTokenScopeEnabled + jobTokenScopeEnabled + } + errors + } +} +``` + ### Introspective queries Clients can query the GraphQL endpoint for information about its own schema. diff --git a/doc/api/graphql/index.md b/doc/api/graphql/index.md index 4286d459e54..abedb63d575 100644 --- a/doc/api/graphql/index.md +++ b/doc/api/graphql/index.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 7dc5b557d33..c52d14f59fe 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -55,6 +55,26 @@ CI related settings that apply to the entire instance. Returns [`CiApplicationSettings`](#ciapplicationsettings). +### `Query.ciCatalogResources` + +CI Catalog resources visible to the current user. + +WARNING: +**Introduced** in 15.11. +This feature is an Experiment. It can be changed or removed at any time. + +Returns [`CiCatalogResourceConnection`](#cicatalogresourceconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `projectPath` | [`ID`](#id) | Project with the namespace catalog. | + ### `Query.ciConfig` Linted and processed contents of a CI config. @@ -85,8 +105,21 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `date` | [`Date`](#date) | Date for which to retrieve the usage data, should be the first day of a month. | | `namespaceId` | [`NamespaceID`](#namespaceid) | Global ID of the Namespace for the monthly CI/CD minutes usage. | +### `Query.ciPipelineStage` + +Stage belonging to a CI pipeline. + +Returns [`CiStage`](#cistage). + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `id` | [`CiStageID!`](#cistageid) | Global ID of the CI stage. | + ### `Query.ciVariables` List of the instance's CI/CD variables. @@ -218,6 +251,16 @@ four standard [pagination arguments](#connection-pagination-arguments): | ---- | ---- | ----------- | | `search` | [`String`](#string) | Search query for group name or group full path. | +### `Query.instanceExternalAuditEventDestinations` + +Instance level external audit event destinations. + +Returns [`InstanceExternalAuditEventDestinationConnection`](#instanceexternalauditeventdestinationconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + ### `Query.instanceSecurityDashboard` Fields related to Instance Security Dashboard. @@ -242,7 +285,7 @@ Find issues visible to the current user. At least one filter must be provided. WARNING: **Introduced** in 15.6. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Returns [`IssueConnection`](#issueconnection). @@ -270,6 +313,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | `iid` | [`String`](#string) | IID of the issue. For example, "1". | | `iids` | [`[String!]`](#string) | List of IIDs of issues. For example, `["1", "2"]`. | | `in` | [`[IssuableSearchableField!]`](#issuablesearchablefield) | Specify the fields to perform the search in. Defaults to `[TITLE, DESCRIPTION]`. Requires the `search` argument.'. | +| `includeArchived` | [`Boolean`](#boolean) | Whether to include issues from archived projects. Defaults to `false`. | | `includeSubepics` | [`Boolean`](#boolean) | Whether to include subepics when filtering issues by epicId. | | `iterationId` | [`[ID]`](#id) | List of iteration Global IDs applied to the issue. | | `iterationWildcardId` | [`IterationWildcardId`](#iterationwildcardid) | Filter by iteration ID wildcard. | @@ -373,7 +417,7 @@ Find a note. WARNING: **Introduced** in 15.9. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Returns [`Note`](#note). @@ -541,7 +585,7 @@ Find a synthetic note. WARNING: **Introduced** in 15.9. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Returns [`Note`](#note). @@ -716,7 +760,7 @@ Find a work item. WARNING: **Introduced** in 15.1. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Returns [`WorkItem`](#workitem). @@ -726,6 +770,42 @@ Returns [`WorkItem`](#workitem). | ---- | ---- | ----------- | | `id` | [`WorkItemID!`](#workitemid) | Global ID of the work item. | +### `Query.workspace` + +Find a workspace. + +WARNING: +**Introduced** in 16.0. +This feature is an Experiment. It can be changed or removed at any time. + +Returns [`Workspace`](#workspace). + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `id` | [`RemoteDevelopmentWorkspaceID!`](#remotedevelopmentworkspaceid) | Find a workspace by its ID. | + +### `Query.workspaces` + +Find workspaces owned by the current user by their IDs. + +WARNING: +**Introduced** in 16.0. +This feature is an Experiment. It can be changed or removed at any time. + +Returns [`WorkspaceConnection`](#workspaceconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `ids` | [`[RemoteDevelopmentWorkspaceID!]`](#remotedevelopmentworkspaceid) | Array of global workspace IDs. For example, `["gid://gitlab/RemoteDevelopment::Workspace/1"]`. | + ## `Mutation` type The `Mutation` type contains all the mutations you can execute. @@ -745,8 +825,36 @@ mutation($id: NoteableID!, $body: String!) { } ``` +### `Mutation.achievementsAward` + +WARNING: +**Introduced** in 15.10. +This feature is an Experiment. It can be changed or removed at any time. + +Input type: `AchievementsAwardInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `achievementId` | [`AchievementsAchievementID!`](#achievementsachievementid) | Global ID of the achievement being awarded. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `userId` | [`UserID!`](#userid) | Global ID of the user being awarded the achievement. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `userAchievement` | [`UserAchievement`](#userachievement) | Achievement award. | + ### `Mutation.achievementsCreate` +WARNING: +**Introduced** in 15.8. +This feature is an Experiment. It can be changed or removed at any time. + Input type: `AchievementsCreateInput` #### Arguments @@ -767,6 +875,78 @@ Input type: `AchievementsCreateInput` | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +### `Mutation.achievementsDelete` + +WARNING: +**Introduced** in 15.11. +This feature is an Experiment. It can be changed or removed at any time. + +Input type: `AchievementsDeleteInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `achievementId` | [`AchievementsAchievementID!`](#achievementsachievementid) | Global ID of the achievement being deleted. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `achievement` | [`Achievement`](#achievement) | Achievement. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + +### `Mutation.achievementsRevoke` + +WARNING: +**Introduced** in 15.10. +This feature is an Experiment. It can be changed or removed at any time. + +Input type: `AchievementsRevokeInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `userAchievementId` | [`AchievementsUserAchievementID!`](#achievementsuserachievementid) | Global ID of the user achievement being revoked. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `userAchievement` | [`UserAchievement`](#userachievement) | Achievement award. | + +### `Mutation.achievementsUpdate` + +WARNING: +**Introduced** in 15.11. +This feature is an Experiment. It can be changed or removed at any time. + +Input type: `AchievementsUpdateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `achievementId` | [`AchievementsAchievementID!`](#achievementsachievementid) | Global ID of the achievement being updated. | +| `avatar` | [`Upload`](#upload) | Avatar for the achievement. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `description` | [`String`](#string) | Description of or notes for the achievement. | +| `name` | [`String`](#string) | Name for the achievement. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `achievement` | [`Achievement`](#achievement) | Achievement. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + ### `Mutation.addProjectToSecurityDashboard` Input type: `AddProjectToSecurityDashboardInput` @@ -803,6 +983,7 @@ Input type: `AdminSidekiqQueuesDeleteJobsInput` | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `featureCategory` | [`String`](#string) | Delete jobs matching feature_category in the context metadata. | | `jobId` | [`String`](#string) | Delete jobs matching job_id in the context metadata. | +| `mergeActionStatus` | [`String`](#string) | Delete jobs matching merge_action_status in the context metadata. | | `pipelineId` | [`String`](#string) | Delete jobs matching pipeline_id in the context metadata. | | `project` | [`String`](#string) | Delete jobs matching project in the context metadata. | | `queueName` | [`String!`](#string) | Name of the queue to delete jobs from. | @@ -823,6 +1004,35 @@ Input type: `AdminSidekiqQueuesDeleteJobsInput` | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `result` | [`DeleteJobsResponse`](#deletejobsresponse) | Information about the status of the deletion request. | +### `Mutation.aiAction` + +WARNING: +**Introduced** in 15.11. +This feature is an Experiment. It can be changed or removed at any time. + +Input type: `AiActionInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `explainCode` | [`AiExplainCodeInput`](#aiexplaincodeinput) | Input for explain_code AI action. | +| `explainVulnerability` | [`AiExplainVulnerabilityInput`](#aiexplainvulnerabilityinput) | Input for explain_vulnerability AI action. | +| `generateDescription` | [`AiGenerateDescriptionInput`](#aigeneratedescriptioninput) | Input for generate_description AI action. | +| `generateTestFile` | [`GenerateTestFileInput`](#generatetestfileinput) | Input for generate_test_file AI action. | +| `markupFormat` | [`MarkupFormat`](#markupformat) | Indicates the response format. | +| `summarizeComments` | [`AiSummarizeCommentsInput`](#aisummarizecommentsinput) | Input for summarize_comments AI action. | +| `tanukiBot` | [`AiTanukiBotInput`](#aitanukibotinput) | Input for tanuki_bot AI action. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `requestId` | [`String`](#string) | ID of the request. | + ### `Mutation.alertSetAssignees` Input type: `AlertSetAssigneesInput` @@ -869,36 +1079,6 @@ Input type: `AlertTodoCreateInput` | `issue` | [`Issue`](#issue) | Issue created after mutation. | | `todo` | [`Todo`](#todo) | To-do item after mutation. | -### `Mutation.apiFuzzingCiConfigurationCreate` - -WARNING: -**Deprecated** in 15.1. -The configuration snippet is now generated client-side. - -Input type: `ApiFuzzingCiConfigurationCreateInput` - -#### Arguments - -| Name | Type | Description | -| ---- | ---- | ----------- | -| `apiSpecificationFile` | [`String!`](#string) | File path or URL to the file that defines the API surface for scanning. Must be in the format specified by the `scanMode` argument. | -| `authPassword` | [`String`](#string) | CI variable containing the password for authenticating with the target API. | -| `authUsername` | [`String`](#string) | CI variable containing the username for authenticating with the target API. | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `projectPath` | [`ID!`](#id) | Full path of the project. | -| `scanMode` | [`ApiFuzzingScanMode!`](#apifuzzingscanmode) | Mode for API fuzzing scans. | -| `scanProfile` | [`String`](#string) | Name of a default profile to use for scanning. Ex: Quick-10. | -| `target` | [`String!`](#string) | URL for the target of API fuzzing scans. | - -#### Fields - -| Name | Type | Description | -| ---- | ---- | ----------- | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `configurationYaml` **{warning-solid}** | [`String`](#string) | **Deprecated:** The configuration snippet is now generated client-side. Deprecated in 14.6. | -| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -| `gitlabCiYamlEditPath` **{warning-solid}** | [`String`](#string) | **Deprecated:** The configuration snippet is now generated client-side. Deprecated in 14.6. | - ### `Mutation.approveDeployment` Input type: `ApproveDeploymentInput` @@ -1168,6 +1348,31 @@ Input type: `BoardListUpdateLimitMetricsInput` | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `list` | [`BoardList`](#boardlist) | Updated list. | +### `Mutation.bulkDestroyJobArtifacts` + +WARNING: +**Introduced** in 15.10. +This feature is an Experiment. It can be changed or removed at any time. + +Input type: `BulkDestroyJobArtifactsInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `ids` | [`[CiJobArtifactID!]!`](#cijobartifactid) | Global IDs of the job artifacts to destroy. | +| `projectId` | [`ProjectID!`](#projectid) | Global Project ID of the job artifacts to destroy. Incompatible with projectPath. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `destroyedCount` | [`Int`](#int) | Number of job artifacts deleted. | +| `destroyedIds` | [`[CiJobArtifactID!]`](#cijobartifactid) | IDs of job artifacts that were deleted. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + ### `Mutation.bulkEnableDevopsAdoptionNamespaces` **BETA** This endpoint is subject to change without notice. @@ -1194,7 +1399,7 @@ Input type: `BulkEnableDevopsAdoptionNamespacesInput` WARNING: **Introduced** in 15.3. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Input type: `BulkRunnerDeleteInput` @@ -1214,6 +1419,52 @@ Input type: `BulkRunnerDeleteInput` | `deletedIds` | [`[CiRunnerID!]`](#cirunnerid) | IDs of records effectively deleted. Only present if operation was performed synchronously. | | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +### `Mutation.catalogResourcesCreate` + +WARNING: +**Introduced** in 15.11. +This feature is an Experiment. It can be changed or removed at any time. + +Input type: `CatalogResourcesCreateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `projectPath` | [`ID!`](#id) | Project to convert to a catalog resource. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + +### `Mutation.ciAiGenerateConfig` + +WARNING: +**Introduced** in 16.0. +This feature is an Experiment. It can be changed or removed at any time. + +Input type: `CiAiGenerateConfigInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `projectPath` | [`ID!`](#id) | Project path for the project related to the open config editor. | +| `userContent` | [`String!`](#string) | Content of the user message to be sent to the language model. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `userMessage` | [`AiMessageType`](#aimessagetype) | User chat message. | + ### `Mutation.ciCdSettingsUpdate` WARNING: @@ -1230,11 +1481,10 @@ Input type: `CiCdSettingsUpdateInput` | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `fullPath` | [`ID!`](#id) | Full Path of the project the settings belong to. | | `inboundJobTokenScopeEnabled` | [`Boolean`](#boolean) | Indicates CI/CD job tokens generated in other projects have restricted access to this project. | -| `jobTokenScopeEnabled` | [`Boolean`](#boolean) | Indicates CI/CD job tokens generated in this project have restricted access to other projects. | +| `jobTokenScopeEnabled` **{warning-solid}** | [`Boolean`](#boolean) | **Deprecated:** Outbound job token scope is being removed. This field can now only be set to false. Deprecated in 16.0. | | `keepLatestArtifact` | [`Boolean`](#boolean) | Indicates if the latest artifact should be kept for the project. | | `mergePipelinesEnabled` | [`Boolean`](#boolean) | Indicates if merge pipelines are enabled for the project. | | `mergeTrainsEnabled` | [`Boolean`](#boolean) | Indicates if merge trains are enabled for the project. | -| `optInJwt` | [`Boolean`](#boolean) | When disabled, the JSON Web Token is always available in all jobs in the pipeline. | #### Fields @@ -1253,7 +1503,7 @@ Input type: `CiJobTokenScopeAddProjectInput` | Name | Type | Description | | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `direction` | [`CiJobTokenScopeDirection`](#cijobtokenscopedirection) | Direction of access, which defaults to outbound. | +| `direction` **{warning-solid}** | [`CiJobTokenScopeDirection`](#cijobtokenscopedirection) | **Deprecated:** Outbound job token scope is being removed. This field can now only be set to INBOUND. Deprecated in 16.0. | | `projectPath` | [`ID!`](#id) | Project that the CI job token scope belongs to. | | `targetProjectPath` | [`ID!`](#id) | Project to be added to the CI job token scope. | @@ -1535,6 +1785,10 @@ Input type: `CreateAlertIssueInput` ### `Mutation.createAnnotation` +WARNING: +**Deprecated** in 16.0. +Underlying feature was removed in 16.0. + Input type: `CreateAnnotationInput` #### Arguments @@ -1652,7 +1906,7 @@ Input type: `CreateComplianceFrameworkInput` WARNING: **Introduced** in 13.6. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Input type: `CreateCustomEmojiInput` @@ -2343,6 +2597,10 @@ Input type: `DastSiteValidationRevokeInput` ### `Mutation.deleteAnnotation` +WARNING: +**Deprecated** in 16.0. +Underlying feature was removed in 16.0. + Input type: `DeleteAnnotationInput` #### Arguments @@ -2401,6 +2659,26 @@ Input type: `DesignManagementMoveInput` | `designCollection` | [`DesignCollection`](#designcollection) | Current state of the collection. | | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +### `Mutation.designManagementUpdate` + +Input type: `DesignManagementUpdateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `description` | [`String`](#string) | Description of the design. | +| `id` | [`DesignManagementDesignID!`](#designmanagementdesignid) | ID of the design to update. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `design` | [`Design!`](#design) | Updated design. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + ### `Mutation.designManagementUpload` Input type: `DesignManagementUploadInput` @@ -2522,7 +2800,7 @@ Input type: `DestroyContainerRepositoryTagsInput` WARNING: **Introduced** in 13.6. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Input type: `DestroyCustomEmojiInput` @@ -2760,6 +3038,28 @@ Input type: `EnableDevopsAdoptionNamespaceInput` | `enabledNamespace` | [`DevopsAdoptionEnabledNamespace`](#devopsadoptionenablednamespace) | Enabled namespace after mutation. | | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +### `Mutation.environmentStop` + +Stop an environment. + +Input type: `EnvironmentStopInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `force` | [`Boolean`](#boolean) | Force environment to stop without executing on_stop actions. | +| `id` | [`EnvironmentID!`](#environmentid) | Global ID of the environment to stop. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `environment` | [`Environment`](#environment) | Environment after attempt to stop. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + ### `Mutation.environmentsCanaryIngressUpdate` **Deprecated** This endpoint is planned to be removed along with certificate-based clusters. [See this epic](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) for more information. @@ -2813,6 +3113,7 @@ Input type: `EpicBoardCreateInput` | Name | Type | Description | | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `displayColors` | [`Boolean`](#boolean) | Whether or not display epic colors. Ignored unless `epic_color_highlight` flag is enabled. | | `groupPath` | [`ID`](#id) | Full path of the group with which the resource is associated. | | `hideBacklogList` | [`Boolean`](#boolean) | Whether or not backlog list is hidden. | | `hideClosedList` | [`Boolean`](#boolean) | Whether or not closed list is hidden. | @@ -2879,6 +3180,7 @@ Input type: `EpicBoardUpdateInput` | Name | Type | Description | | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `displayColors` | [`Boolean`](#boolean) | Whether or not display epic colors. Ignored unless `epic_color_highlight` flag is enabled. | | `hideBacklogList` | [`Boolean`](#boolean) | Whether or not backlog list is hidden. | | `hideClosedList` | [`Boolean`](#boolean) | Whether or not closed list is hidden. | | `id` | [`BoardsEpicBoardID!`](#boardsepicboardid) | Epic board global ID. | @@ -3104,6 +3406,33 @@ Input type: `ExternalAuditEventDestinationUpdateInput` | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `externalAuditEventDestination` | [`ExternalAuditEventDestination`](#externalauditeventdestination) | Updated destination. | +### `Mutation.geoRegistriesUpdate` + +Mutates a Geo registry. Does not mutate the registry entry if `geo_registries_update_mutation` feature flag is disabled. + +WARNING: +**Introduced** in 16.1. +This feature is an Experiment. It can be changed or removed at any time. + +Input type: `GeoRegistriesUpdateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `action` | [`GeoRegistryAction!`](#georegistryaction) | Action to be executed on a Geo registry. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `registryClass` | [`GeoRegistryClass!`](#georegistryclass) | Class of the Geo registry to be updated. | +| `registryId` | [`GeoBaseRegistryID!`](#geobaseregistryid) | ID of the Geo registry entry to be updated. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `registry` | [`Registrable`](#registrable) | Updated Geo registry entry. | + ### `Mutation.gitlabSubscriptionActivate` Input type: `GitlabSubscriptionActivateInput` @@ -3136,7 +3465,7 @@ Input type: `GroupMemberBulkUpdateInput` | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `expiresAt` | [`Time`](#time) | Date and time the membership expires. | | `groupId` | [`GroupID!`](#groupid) | Global ID of the group. | -| `userIds` | [`[UserID!]!`](#userid) | Global IDs of the group members. | +| `userIds` | [`[UserID!]!`](#userid) | Global IDs of the members. | #### Fields @@ -3250,6 +3579,63 @@ Input type: `HttpIntegrationUpdateInput` | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `integration` | [`AlertManagementHttpIntegration`](#alertmanagementhttpintegration) | HTTP integration. | +### `Mutation.instanceExternalAuditEventDestinationCreate` + +Input type: `InstanceExternalAuditEventDestinationCreateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `destinationUrl` | [`String!`](#string) | Destination URL. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `instanceExternalAuditEventDestination` | [`InstanceExternalAuditEventDestination`](#instanceexternalauditeventdestination) | Destination created. | + +### `Mutation.instanceExternalAuditEventDestinationDestroy` + +Input type: `InstanceExternalAuditEventDestinationDestroyInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `id` | [`AuditEventsInstanceExternalAuditEventDestinationID!`](#auditeventsinstanceexternalauditeventdestinationid) | ID of the external instance audit event destination to destroy. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + +### `Mutation.instanceExternalAuditEventDestinationUpdate` + +Input type: `InstanceExternalAuditEventDestinationUpdateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `destinationUrl` | [`String`](#string) | Destination URL to change. | +| `id` | [`AuditEventsInstanceExternalAuditEventDestinationID!`](#auditeventsinstanceexternalauditeventdestinationid) | ID of the external instance audit event destination to update. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `instanceExternalAuditEventDestination` | [`InstanceExternalAuditEventDestination`](#instanceexternalauditeventdestination) | Updated destination. | + ### `Mutation.issuableResourceLinkCreate` Input type: `IssuableResourceLinkCreateInput` @@ -3641,7 +4027,7 @@ Allows updating several properties for a set of issues. Does nothing if the `bul WARNING: **Introduced** in 15.9. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Input type: `IssuesBulkUpdateInput` @@ -3649,12 +4035,18 @@ Input type: `IssuesBulkUpdateInput` | Name | Type | Description | | ---- | ---- | ----------- | +| `addLabelIds` | [`[LabelID!]`](#labelid) | Global ID array of the labels that will be added to the issues. | | `assigneeIds` | [`[UserID!]`](#userid) | Global ID array of the users that will be assigned to the given issues. Existing assignees will be replaced with the ones on this list. | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `epicId` | [`EpicID`](#epicid) | Global ID of the epic that will be assigned to the issues. | +| `healthStatus` | [`HealthStatus`](#healthstatus) | Health status that will be assigned to the issues. | | `ids` | [`[IssueID!]!`](#issueid) | Global ID array of the issues that will be updated. IDs that the user can't update will be ignored. A max of 100 can be provided. | | `iterationId` | [`IterationID`](#iterationid) | Global ID of the iteration that will be assigned to the issues. | | `milestoneId` | [`MilestoneID`](#milestoneid) | Global ID of the milestone that will be assigned to the issues. | -| `parentId` | [`IssueParentID!`](#issueparentid) | Global ID of the parent that the bulk update will be scoped to . Example `IssueParentID` are `"gid://gitlab/Project/1"` and `"gid://gitlab/Group/1"`. | +| `parentId` | [`IssueParentID!`](#issueparentid) | Global ID of the parent to which the bulk update will be scoped. The parent can be a project **(FREE)** or a group **(PREMIUM)**. Example `IssueParentID` are `"gid://gitlab/Project/1"` and `"gid://gitlab/Group/1"`. | +| `removeLabelIds` | [`[LabelID!]`](#labelid) | Global ID array of the labels that will be removed from the issues. | +| `stateEvent` | [`IssueStateEvent`](#issuestateevent) | Close or reopen an issue. | +| `subscriptionEvent` | [`IssuableSubscriptionEvent`](#issuablesubscriptionevent) | Subscribe to or unsubscribe from issue notifications. | #### Fields @@ -4198,22 +4590,48 @@ Input type: `MergeRequestUpdateInput` | Name | Type | Description | | ---- | ---- | ----------- | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `description` | [`String`](#string) | Description of the merge request (Markdown rendered as HTML for caching). | -| `iid` | [`String!`](#string) | IID of the merge request to mutate. | -| `projectPath` | [`ID!`](#id) | Project the merge request to mutate is in. | -| `state` | [`MergeRequestNewState`](#mergerequestnewstate) | Action to perform to change the state. | -| `targetBranch` | [`String`](#string) | Target branch of the merge request. | -| `timeEstimate` | [`String`](#string) | Estimated time to complete the merge request, or `0` to remove the current estimate. | -| `title` | [`String`](#string) | Title of the merge request. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `description` | [`String`](#string) | Description of the merge request (Markdown rendered as HTML for caching). | +| `iid` | [`String!`](#string) | IID of the merge request to mutate. | +| `projectPath` | [`ID!`](#id) | Project the merge request to mutate is in. | +| `state` | [`MergeRequestNewState`](#mergerequestnewstate) | Action to perform to change the state. | +| `targetBranch` | [`String`](#string) | Target branch of the merge request. | +| `timeEstimate` | [`String`](#string) | Estimated time to complete the merge request, or `0` to remove the current estimate. | +| `title` | [`String`](#string) | Title of the merge request. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request after mutation. | + +### `Mutation.mergeRequestUpdateApprovalRule` + +Input type: `MergeRequestUpdateApprovalRuleInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `approvalRuleId` | [`Int!`](#int) | ID of an approval rule. | +| `approvalsRequired` | [`Int!`](#int) | Number of required approvals for a given rule. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `groupIds` | [`[String!]`](#string) | IDs of groups as approvers. | +| `iid` | [`String!`](#string) | IID of the merge request to mutate. | +| `name` | [`String!`](#string) | Name of the approval rule. | +| `projectPath` | [`ID!`](#id) | Project the merge request to mutate is in. | +| `removeHiddenGroups` | [`[Boolean!]`](#boolean) | Whether hidden groups should be removed. | +| `userIds` | [`[String!]`](#string) | IDs of users as approvers. | #### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -| `mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request after mutation. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request after mutation. | ### `Mutation.namespaceBanDestroy` @@ -4601,11 +5019,10 @@ Input type: `ProjectCiCdSettingsUpdateInput` | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `fullPath` | [`ID!`](#id) | Full Path of the project the settings belong to. | | `inboundJobTokenScopeEnabled` | [`Boolean`](#boolean) | Indicates CI/CD job tokens generated in other projects have restricted access to this project. | -| `jobTokenScopeEnabled` | [`Boolean`](#boolean) | Indicates CI/CD job tokens generated in this project have restricted access to other projects. | +| `jobTokenScopeEnabled` **{warning-solid}** | [`Boolean`](#boolean) | **Deprecated:** Outbound job token scope is being removed. This field can now only be set to false. Deprecated in 16.0. | | `keepLatestArtifact` | [`Boolean`](#boolean) | Indicates if the latest artifact should be kept for the project. | | `mergePipelinesEnabled` | [`Boolean`](#boolean) | Indicates if merge pipelines are enabled for the project. | | `mergeTrainsEnabled` | [`Boolean`](#boolean) | Indicates if merge trains are enabled for the project. | -| `optInJwt` | [`Boolean`](#boolean) | When disabled, the JSON Web Token is always available in all jobs in the pipeline. | #### Fields @@ -4634,6 +5051,30 @@ Input type: `ProjectInitializeProductAnalyticsInput` | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `project` | [`Project`](#project) | Project on which the initialization took place. | +### `Mutation.projectMemberBulkUpdate` + +Updates multiple members of a project. To use this mutation, you must have at least the Maintainer role. + +Input type: `ProjectMemberBulkUpdateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `accessLevel` | [`MemberAccessLevel!`](#memberaccesslevel) | Access level to update the members to. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `expiresAt` | [`Time`](#time) | Date and time the membership expires. | +| `projectId` | [`ProjectID!`](#projectid) | Global ID of the project. | +| `userIds` | [`[UserID!]!`](#userid) | Global IDs of the members. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `projectMembers` | [`[ProjectMember!]`](#projectmember) | Project members after mutation. | + ### `Mutation.projectSetComplianceFramework` Assign (or unset) a compliance framework to a project. @@ -4677,6 +5118,30 @@ Input type: `ProjectSetLockedInput` | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `project` | [`Project`](#project) | Project after mutation. | +### `Mutation.projectSyncFork` + +WARNING: +**Introduced** in 15.9. +This feature is an Experiment. It can be changed or removed at any time. + +Input type: `ProjectSyncForkInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `projectPath` | [`ID!`](#id) | Full path of the project to initialize. | +| `targetBranch` | [`String!`](#string) | Ref of the fork to fetch into. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `details` | [`ForkDetails`](#forkdetails) | Updated fork details. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + ### `Mutation.prometheusIntegrationCreate` Input type: `PrometheusIntegrationCreateInput` @@ -4937,6 +5402,39 @@ Input type: `RepositionImageDiffNoteInput` | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `note` | [`Note`](#note) | Note after mutation. | +### `Mutation.runnerCreate` + +WARNING: +**Introduced** in 15.10. +This feature is an Experiment. It can be changed or removed at any time. + +Input type: `RunnerCreateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `accessLevel` | [`CiRunnerAccessLevel`](#cirunneraccesslevel) | Access level of the runner. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `description` | [`String`](#string) | Description of the runner. | +| `groupId` | [`GroupID`](#groupid) | Global ID of the group that the runner is created in (valid only for group runner). | +| `locked` | [`Boolean`](#boolean) | Indicates the runner is locked. | +| `maintenanceNote` | [`String`](#string) | Runner's maintenance notes. | +| `maximumTimeout` | [`Int`](#int) | Maximum timeout (in seconds) for jobs processed by the runner. | +| `paused` | [`Boolean`](#boolean) | Indicates the runner is not allowed to receive jobs. | +| `projectId` | [`ProjectID`](#projectid) | Global ID of the project that the runner is created in (valid only for project runner). | +| `runUntagged` | [`Boolean`](#boolean) | Indicates the runner is able to run untagged jobs. | +| `runnerType` | [`CiRunnerType!`](#cirunnertype) | Type of the runner to create. | +| `tagList` | [`[String!]`](#string) | Tags associated with the runner. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `runner` | [`CiRunner`](#cirunner) | Runner after mutation. | + ### `Mutation.runnerDelete` Input type: `RunnerDeleteInput` @@ -5078,7 +5576,7 @@ Input type: `ScanExecutionPolicyCommitInput` | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `fullPath` | [`String`](#string) | Full path of the project. | -| `name` | [`String`](#string) | Name of the policy. If the name is null, the `name` field from `policy_yaml` is used. | +| `name` | [`String!`](#string) | Name of the policy. If the name is null, the `name` field from `policy_yaml` is used. | | `operationMode` | [`MutationOperationMode!`](#mutationoperationmode) | Changes the operation mode. | | `policyYaml` | [`String!`](#string) | YAML snippet of the policy. | | `projectPath` **{warning-solid}** | [`ID`](#id) | **Deprecated:** Use `fullPath`. Deprecated in 14.10. | @@ -5111,6 +5609,25 @@ Input type: `SecurityFindingCreateIssueInput` | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `issue` | [`Issue`](#issue) | Issue created after mutation. | +### `Mutation.securityFindingCreateMergeRequest` + +Input type: `SecurityFindingCreateMergeRequestInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `uuid` | [`String!`](#string) | UUID of the security finding to be used to create a merge request. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `mergeRequest` | [`MergeRequest`](#mergerequest) | Merge Request created after mutation. | + ### `Mutation.securityFindingDismiss` Input type: `SecurityFindingDismissInput` @@ -5130,6 +5647,7 @@ Input type: `SecurityFindingDismissInput` | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `securityFinding` | [`PipelineSecurityReportFinding`](#pipelinesecurityreportfinding) | Dismissed finding. | | `uuid` | [`String`](#string) | UUID of dismissed finding. | ### `Mutation.securityFindingRevertToDetected` @@ -5296,7 +5814,7 @@ Input type: `TerraformStateUnlockInput` WARNING: **Introduced** in 15.6. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Input type: `TimelineEventCreateInput` @@ -6031,6 +6549,7 @@ Input type: `UserPreferencesUpdateInput` | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `issuesSort` | [`IssueSort`](#issuesort) | Sort order for issue lists. | +| `visibilityPipelineIdType` | [`VisibilityPipelineIdType`](#visibilitypipelineidtype) | Determines whether the pipeline list shows ID or IID. | #### Fields @@ -6049,6 +6568,7 @@ Input type: `VulnerabilityConfirmInput` | Name | Type | Description | | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `comment` | [`String`](#string) | Comment why vulnerability was confirmed (maximum 50,000 characters). | | `id` | [`VulnerabilityID!`](#vulnerabilityid) | ID of the vulnerability to be confirmed. | #### Fields @@ -6100,7 +6620,7 @@ Input type: `VulnerabilityDismissInput` | Name | Type | Description | | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `comment` | [`String`](#string) | Comment why vulnerability should be dismissed (max. 50 000 characters). | +| `comment` | [`String`](#string) | Comment why vulnerability was dismissed (maximum 50,000 characters). | | `dismissalReason` | [`VulnerabilityDismissalReason`](#vulnerabilitydismissalreason) | Reason why vulnerability should be dismissed. | | `id` | [`VulnerabilityID!`](#vulnerabilityid) | ID of the vulnerability to be dismissed. | @@ -6110,7 +6630,7 @@ Input type: `VulnerabilityDismissInput` | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -| `vulnerability` | [`Vulnerability`](#vulnerability) | Vulnerability after dismissal. | +| `vulnerability` | [`Vulnerability`](#vulnerability) | Vulnerability after state change. | ### `Mutation.vulnerabilityExternalIssueLinkCreate` @@ -6151,31 +6671,25 @@ Input type: `VulnerabilityExternalIssueLinkDestroyInput` | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -### `Mutation.vulnerabilityFindingDismiss` - -WARNING: -**Deprecated** in 15.5. -Use VulnerabilityDismiss for vulnerabilities or SecurityFindingDismiss for pipeline findings. +### `Mutation.vulnerabilityIssueLinkCreate` -Input type: `VulnerabilityFindingDismissInput` +Input type: `VulnerabilityIssueLinkCreateInput` #### Arguments | Name | Type | Description | | ---- | ---- | ----------- | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `comment` | [`String`](#string) | Comment why finding should be dismissed. | -| `dismissalReason` | [`VulnerabilityDismissalReason`](#vulnerabilitydismissalreason) | Reason why finding should be dismissed. | -| `id` **{warning-solid}** | [`VulnerabilitiesFindingID`](#vulnerabilitiesfindingid) | **Deprecated:** Use `uuid`. Deprecated in 15.2. | -| `uuid` | [`String`](#string) | UUID of the finding to be dismissed. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `issueId` | [`IssueID!`](#issueid) | ID of the issue to link to. | +| `vulnerabilityIds` | [`[VulnerabilityID!]!`](#vulnerabilityid) | IDs of vulnerabilities to link to the given issue. Up to 100 can be provided. | #### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -| `finding` | [`PipelineSecurityReportFinding`](#pipelinesecurityreportfinding) | Finding after dismissal. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `issueLinks` | [`[VulnerabilityIssueLink!]`](#vulnerabilityissuelink) | Created issue links. | ### `Mutation.vulnerabilityResolve` @@ -6186,6 +6700,7 @@ Input type: `VulnerabilityResolveInput` | Name | Type | Description | | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `comment` | [`String`](#string) | Comment why vulnerability was resolved (maximum 50,000 characters). | | `id` | [`VulnerabilityID!`](#vulnerabilityid) | ID of the vulnerability to be resolved. | #### Fields @@ -6205,7 +6720,8 @@ Input type: `VulnerabilityRevertToDetectedInput` | Name | Type | Description | | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `id` | [`VulnerabilityID!`](#vulnerabilityid) | ID of the vulnerability to be reverted. | +| `comment` | [`String`](#string) | Comment why vulnerability was reverted to detected (maximum 50,000 characters). | +| `id` | [`VulnerabilityID!`](#vulnerabilityid) | ID of the vulnerability to be reverted to detected. | #### Fields @@ -6213,7 +6729,33 @@ Input type: `VulnerabilityRevertToDetectedInput` | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | -| `vulnerability` | [`Vulnerability`](#vulnerability) | Vulnerability after revert. | +| `vulnerability` | [`Vulnerability`](#vulnerability) | Vulnerability after state change. | + +### `Mutation.workItemConvert` + +Converts the work item to a new type. + +WARNING: +**Introduced** in 15.11. +This feature is an Experiment. It can be changed or removed at any time. + +Input type: `WorkItemConvertInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `id` | [`WorkItemID!`](#workitemid) | Global ID of the work item. | +| `workItemTypeId` | [`WorkItemsTypeID!`](#workitemstypeid) | Global ID of the new work item type. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `workItem` | [`WorkItem`](#workitem) | Updated work item. | ### `Mutation.workItemCreate` @@ -6221,7 +6763,7 @@ Creates a work item. WARNING: **Introduced** in 15.1. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Input type: `WorkItemCreateInput` @@ -6235,7 +6777,8 @@ Input type: `WorkItemCreateInput` | `hierarchyWidget` | [`WorkItemWidgetHierarchyCreateInput`](#workitemwidgethierarchycreateinput) | Input for hierarchy widget. | | `iterationWidget` | [`WorkItemWidgetIterationInput`](#workitemwidgetiterationinput) | Iteration widget of the work item. | | `milestoneWidget` | [`WorkItemWidgetMilestoneInput`](#workitemwidgetmilestoneinput) | Input for milestone widget. | -| `projectPath` | [`ID!`](#id) | Full path of the project the work item is associated with. | +| `namespacePath` | [`ID`](#id) | Full path of the namespace(project or group) the work item is created in. | +| `projectPath` **{warning-solid}** | [`ID`](#id) | **Deprecated:** Please use namespace_path instead. That will cover for both projects and groups. Deprecated in 15.10. | | `title` | [`String!`](#string) | Title of the work item. | | `workItemTypeId` | [`WorkItemsTypeID!`](#workitemstypeid) | Global ID of a work item type. | @@ -6253,7 +6796,7 @@ Creates a work item from a task in another work item's description. WARNING: **Introduced** in 15.1. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Input type: `WorkItemCreateFromTaskInput` @@ -6280,7 +6823,7 @@ Deletes a work item. WARNING: **Introduced** in 15.1. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Input type: `WorkItemDeleteInput` @@ -6305,7 +6848,7 @@ Deletes a task in a work item's description. WARNING: **Introduced** in 15.1. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Input type: `WorkItemDeleteTaskInput` @@ -6326,13 +6869,43 @@ Input type: `WorkItemDeleteTaskInput` | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `workItem` | [`WorkItem`](#workitem) | Updated work item. | +### `Mutation.workItemExport` + +WARNING: +**Introduced** in 15.10. +This feature is an Experiment. It can be changed or removed at any time. + +Input type: `WorkItemExportInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `authorUsername` **{warning-solid}** | [`String`](#string) | **Deprecated:** This feature is an Experiment. It can be changed or removed at any time. Introduced in 15.9. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `iids` | [`[String!]`](#string) | List of IIDs of work items. For example, `["1", "2"]`. | +| `in` | [`[IssuableSearchableField!]`](#issuablesearchablefield) | Specify the fields to perform the search in. Defaults to `[TITLE, DESCRIPTION]`. Requires the `search` argument.'. | +| `projectPath` | [`ID!`](#id) | Full project path. | +| `search` | [`String`](#string) | Search query for title or description. | +| `selectedFields` | [`[AvailableExportFields!]`](#availableexportfields) | List of selected fields to be exported. Omit to export all available fields. | +| `state` | [`IssuableState`](#issuablestate) | Current state of the work item. | +| `types` | [`[IssueType!]`](#issuetype) | Filter work items by the given work item types. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `message` | [`String`](#string) | Export request result message. | + ### `Mutation.workItemUpdate` Updates a work item by Global ID. WARNING: **Introduced** in 15.1. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Input type: `WorkItemUpdateInput` @@ -6341,8 +6914,10 @@ Input type: `WorkItemUpdateInput` | Name | Type | Description | | ---- | ---- | ----------- | | `assigneesWidget` | [`WorkItemWidgetAssigneesInput`](#workitemwidgetassigneesinput) | Input for assignees widget. | +| `awardEmojiWidget` | [`WorkItemWidgetAwardEmojiUpdateInput`](#workitemwidgetawardemojiupdateinput) | Input for award emoji widget. | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `confidential` | [`Boolean`](#boolean) | Sets the work item confidentiality. | +| `currentUserTodosWidget` | [`WorkItemWidgetCurrentUserTodosInput`](#workitemwidgetcurrentusertodosinput) | Input for to-dos widget. | | `descriptionWidget` | [`WorkItemWidgetDescriptionInput`](#workitemwidgetdescriptioninput) | Input for description widget. | | `healthStatusWidget` | [`WorkItemWidgetHealthStatusInput`](#workitemwidgethealthstatusinput) | Input for health status widget. | | `hierarchyWidget` | [`WorkItemWidgetHierarchyUpdateInput`](#workitemwidgethierarchyupdateinput) | Input for hierarchy widget. | @@ -6350,6 +6925,7 @@ Input type: `WorkItemUpdateInput` | `iterationWidget` | [`WorkItemWidgetIterationInput`](#workitemwidgetiterationinput) | Input for iteration widget. | | `labelsWidget` | [`WorkItemWidgetLabelsUpdateInput`](#workitemwidgetlabelsupdateinput) | Input for labels widget. | | `milestoneWidget` | [`WorkItemWidgetMilestoneInput`](#workitemwidgetmilestoneinput) | Input for milestone widget. | +| `notificationsWidget` | [`WorkItemWidgetNotificationsUpdateInput`](#workitemwidgetnotificationsupdateinput) | Input for notifications widget. | | `progressWidget` | [`WorkItemWidgetProgressInput`](#workitemwidgetprogressinput) | Input for progress widget. | | `startAndDueDateWidget` | [`WorkItemWidgetStartAndDueDateUpdateInput`](#workitemwidgetstartandduedateupdateinput) | Input for start and due date widget. | | `stateEvent` | [`WorkItemStateEvent`](#workitemstateevent) | Close or reopen a work item. | @@ -6371,7 +6947,7 @@ Updates a work item's task by Global ID. WARNING: **Introduced** in 15.1. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Input type: `WorkItemUpdateTaskInput` @@ -6392,6 +6968,59 @@ Input type: `WorkItemUpdateTaskInput` | `task` | [`WorkItem`](#workitem) | Updated task. | | `workItem` | [`WorkItem`](#workitem) | Updated work item. | +### `Mutation.workspaceCreate` + +WARNING: +**Introduced** in 16.0. +This feature is an Experiment. It can be changed or removed at any time. + +Input type: `WorkspaceCreateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `clusterAgentId` | [`ClustersAgentID!`](#clustersagentid) | ID of the cluster agent the created workspace will be associated with. | +| `desiredState` | [`String!`](#string) | Desired state of the created workspace. | +| `devfilePath` | [`String!`](#string) | Project repo git path containing the devfile used to configure the workspace. | +| `devfileRef` | [`String!`](#string) | Project repo git ref containing the devfile used to configure the workspace. | +| `editor` | [`String!`](#string) | Editor to inject into the created workspace. Must match a configured template. | +| `maxHoursBeforeTermination` | [`Int!`](#int) | Maximum hours the workspace can exist before it is automatically terminated. | +| `projectId` | [`ProjectID!`](#projectid) | ID of the project that will provide the Devfile for the created workspace. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `workspace` | [`Workspace`](#workspace) | Created workspace. | + +### `Mutation.workspaceUpdate` + +WARNING: +**Introduced** in 16.0. +This feature is an Experiment. It can be changed or removed at any time. + +Input type: `WorkspaceUpdateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `desiredState` | [`String!`](#string) | Desired state of the created workspace. | +| `id` | [`RemoteDevelopmentWorkspaceID!`](#remotedevelopmentworkspaceid) | Global ID of the workspace. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `workspace` | [`Workspace`](#workspace) | Created workspace. | + ## Connections Some types in our schema are `Connection` types - they represent a paginated @@ -6477,6 +7106,29 @@ The edge type for [`AgentConfiguration`](#agentconfiguration). | `cursor` | [`String!`](#string) | A cursor for use in pagination. | | `node` | [`AgentConfiguration`](#agentconfiguration) | The item at the end of the edge. | +#### `AiMessageTypeConnection` + +The connection type for [`AiMessageType`](#aimessagetype). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[AiMessageTypeEdge]`](#aimessagetypeedge) | A list of edges. | +| `nodes` | [`[AiMessageType]`](#aimessagetype) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `AiMessageTypeEdge` + +The edge type for [`AiMessageType`](#aimessagetype). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`AiMessageType`](#aimessagetype) | The item at the end of the edge. | + #### `AlertManagementAlertConnection` The connection type for [`AlertManagementAlert`](#alertmanagementalert). @@ -6754,6 +7406,30 @@ The edge type for [`CiBuildNeed`](#cibuildneed). | `cursor` | [`String!`](#string) | A cursor for use in pagination. | | `node` | [`CiBuildNeed`](#cibuildneed) | The item at the end of the edge. | +#### `CiCatalogResourceConnection` + +The connection type for [`CiCatalogResource`](#cicatalogresource). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `count` | [`Int!`](#int) | Total count of collection. | +| `edges` | [`[CiCatalogResourceEdge]`](#cicatalogresourceedge) | A list of edges. | +| `nodes` | [`[CiCatalogResource]`](#cicatalogresource) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `CiCatalogResourceEdge` + +The edge type for [`CiCatalogResource`](#cicatalogresource). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`CiCatalogResource`](#cicatalogresource) | The item at the end of the edge. | + #### `CiConfigGroupConnection` The connection type for [`CiConfigGroup`](#ciconfiggroup). @@ -7096,6 +7772,30 @@ The edge type for [`CiRunner`](#cirunner). | `node` | [`CiRunner`](#cirunner) | The item at the end of the edge. | | `webUrl` | [`String`](#string) | Web URL of the runner. The value depends on where you put this field in the query. You can use it for projects or groups. | +#### `CiRunnerManagerConnection` + +The connection type for [`CiRunnerManager`](#cirunnermanager). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `count` | [`Int!`](#int) | Total count of collection. | +| `edges` | [`[CiRunnerManagerEdge]`](#cirunnermanageredge) | A list of edges. | +| `nodes` | [`[CiRunnerManager]`](#cirunnermanager) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `CiRunnerManagerEdge` + +The edge type for [`CiRunnerManager`](#cirunnermanager). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`CiRunnerManager`](#cirunnermanager) | The item at the end of the edge. | + #### `CiSecureFileRegistryConnection` The connection type for [`CiSecureFileRegistry`](#cisecurefileregistry). @@ -7166,6 +7866,52 @@ The edge type for [`ClusterAgentActivityEvent`](#clusteragentactivityevent). | `cursor` | [`String!`](#string) | A cursor for use in pagination. | | `node` | [`ClusterAgentActivityEvent`](#clusteragentactivityevent) | The item at the end of the edge. | +#### `ClusterAgentAuthorizationCiAccessConnection` + +The connection type for [`ClusterAgentAuthorizationCiAccess`](#clusteragentauthorizationciaccess). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[ClusterAgentAuthorizationCiAccessEdge]`](#clusteragentauthorizationciaccessedge) | A list of edges. | +| `nodes` | [`[ClusterAgentAuthorizationCiAccess]`](#clusteragentauthorizationciaccess) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `ClusterAgentAuthorizationCiAccessEdge` + +The edge type for [`ClusterAgentAuthorizationCiAccess`](#clusteragentauthorizationciaccess). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`ClusterAgentAuthorizationCiAccess`](#clusteragentauthorizationciaccess) | The item at the end of the edge. | + +#### `ClusterAgentAuthorizationUserAccessConnection` + +The connection type for [`ClusterAgentAuthorizationUserAccess`](#clusteragentauthorizationuseraccess). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[ClusterAgentAuthorizationUserAccessEdge]`](#clusteragentauthorizationuseraccessedge) | A list of edges. | +| `nodes` | [`[ClusterAgentAuthorizationUserAccess]`](#clusteragentauthorizationuseraccess) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `ClusterAgentAuthorizationUserAccessEdge` + +The edge type for [`ClusterAgentAuthorizationUserAccess`](#clusteragentauthorizationuseraccess). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`ClusterAgentAuthorizationUserAccess`](#clusteragentauthorizationuseraccess) | The item at the end of the edge. | + #### `ClusterAgentConnection` The connection type for [`ClusterAgent`](#clusteragent). @@ -7630,6 +8376,29 @@ The edge type for [`DastSiteValidation`](#dastsitevalidation). | `cursor` | [`String!`](#string) | A cursor for use in pagination. | | `node` | [`DastSiteValidation`](#dastsitevalidation) | The item at the end of the edge. | +#### `DependencyConnection` + +The connection type for [`Dependency`](#dependency). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[DependencyEdge]`](#dependencyedge) | A list of edges. | +| `nodes` | [`[Dependency]`](#dependency) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `DependencyEdge` + +The edge type for [`Dependency`](#dependency). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`Dependency`](#dependency) | The item at the end of the edge. | + #### `DependencyProxyBlobConnection` The connection type for [`DependencyProxyBlob`](#dependencyproxyblob). @@ -8228,54 +8997,100 @@ The edge type for [`IncidentManagementOncallRotation`](#incidentmanagementoncall | Name | Type | Description | | ---- | ---- | ----------- | -| `cursor` | [`String!`](#string) | A cursor for use in pagination. | -| `node` | [`IncidentManagementOncallRotation`](#incidentmanagementoncallrotation) | The item at the end of the edge. | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`IncidentManagementOncallRotation`](#incidentmanagementoncallrotation) | The item at the end of the edge. | + +#### `IncidentManagementOncallScheduleConnection` + +The connection type for [`IncidentManagementOncallSchedule`](#incidentmanagementoncallschedule). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[IncidentManagementOncallScheduleEdge]`](#incidentmanagementoncallscheduleedge) | A list of edges. | +| `nodes` | [`[IncidentManagementOncallSchedule]`](#incidentmanagementoncallschedule) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `IncidentManagementOncallScheduleEdge` + +The edge type for [`IncidentManagementOncallSchedule`](#incidentmanagementoncallschedule). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`IncidentManagementOncallSchedule`](#incidentmanagementoncallschedule) | The item at the end of the edge. | + +#### `IncidentManagementOncallShiftConnection` + +The connection type for [`IncidentManagementOncallShift`](#incidentmanagementoncallshift). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[IncidentManagementOncallShiftEdge]`](#incidentmanagementoncallshiftedge) | A list of edges. | +| `nodes` | [`[IncidentManagementOncallShift]`](#incidentmanagementoncallshift) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `IncidentManagementOncallShiftEdge` + +The edge type for [`IncidentManagementOncallShift`](#incidentmanagementoncallshift). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`IncidentManagementOncallShift`](#incidentmanagementoncallshift) | The item at the end of the edge. | -#### `IncidentManagementOncallScheduleConnection` +#### `InheritedCiVariableConnection` -The connection type for [`IncidentManagementOncallSchedule`](#incidentmanagementoncallschedule). +The connection type for [`InheritedCiVariable`](#inheritedcivariable). ##### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| `edges` | [`[IncidentManagementOncallScheduleEdge]`](#incidentmanagementoncallscheduleedge) | A list of edges. | -| `nodes` | [`[IncidentManagementOncallSchedule]`](#incidentmanagementoncallschedule) | A list of nodes. | -| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | +| `edges` | [`[InheritedCiVariableEdge]`](#inheritedcivariableedge) | A list of edges. | +| `nodes` | [`[InheritedCiVariable]`](#inheritedcivariable) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | -#### `IncidentManagementOncallScheduleEdge` +#### `InheritedCiVariableEdge` -The edge type for [`IncidentManagementOncallSchedule`](#incidentmanagementoncallschedule). +The edge type for [`InheritedCiVariable`](#inheritedcivariable). ##### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| `cursor` | [`String!`](#string) | A cursor for use in pagination. | -| `node` | [`IncidentManagementOncallSchedule`](#incidentmanagementoncallschedule) | The item at the end of the edge. | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`InheritedCiVariable`](#inheritedcivariable) | The item at the end of the edge. | -#### `IncidentManagementOncallShiftConnection` +#### `InstanceExternalAuditEventDestinationConnection` -The connection type for [`IncidentManagementOncallShift`](#incidentmanagementoncallshift). +The connection type for [`InstanceExternalAuditEventDestination`](#instanceexternalauditeventdestination). ##### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| `edges` | [`[IncidentManagementOncallShiftEdge]`](#incidentmanagementoncallshiftedge) | A list of edges. | -| `nodes` | [`[IncidentManagementOncallShift]`](#incidentmanagementoncallshift) | A list of nodes. | -| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | +| `edges` | [`[InstanceExternalAuditEventDestinationEdge]`](#instanceexternalauditeventdestinationedge) | A list of edges. | +| `nodes` | [`[InstanceExternalAuditEventDestination]`](#instanceexternalauditeventdestination) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | -#### `IncidentManagementOncallShiftEdge` +#### `InstanceExternalAuditEventDestinationEdge` -The edge type for [`IncidentManagementOncallShift`](#incidentmanagementoncallshift). +The edge type for [`InstanceExternalAuditEventDestination`](#instanceexternalauditeventdestination). ##### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| `cursor` | [`String!`](#string) | A cursor for use in pagination. | -| `node` | [`IncidentManagementOncallShift`](#incidentmanagementoncallshift) | The item at the end of the edge. | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`InstanceExternalAuditEventDestination`](#instanceexternalauditeventdestination) | The item at the end of the edge. | #### `IssuableResourceLinkConnection` @@ -9276,6 +10091,29 @@ The edge type for [`ProjectMember`](#projectmember). | `cursor` | [`String!`](#string) | A cursor for use in pagination. | | `node` | [`ProjectMember`](#projectmember) | The item at the end of the edge. | +#### `ProjectWikiRepositoryRegistryConnection` + +The connection type for [`ProjectWikiRepositoryRegistry`](#projectwikirepositoryregistry). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[ProjectWikiRepositoryRegistryEdge]`](#projectwikirepositoryregistryedge) | A list of edges. | +| `nodes` | [`[ProjectWikiRepositoryRegistry]`](#projectwikirepositoryregistry) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `ProjectWikiRepositoryRegistryEdge` + +The edge type for [`ProjectWikiRepositoryRegistry`](#projectwikirepositoryregistry). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`ProjectWikiRepositoryRegistry`](#projectwikirepositoryregistry) | The item at the end of the edge. | + #### `ProtectedEnvironmentApprovalRuleConnection` The connection type for [`ProtectedEnvironmentApprovalRule`](#protectedenvironmentapprovalrule). @@ -10099,7 +10937,7 @@ The connection type for [`Timelog`](#timelog). | `edges` | [`[TimelogEdge]`](#timelogedge) | A list of edges. | | `nodes` | [`[Timelog]`](#timelog) | A list of nodes. | | `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | -| `totalSpentTime` | [`Int!`](#int) | Total time spent in seconds. | +| `totalSpentTime` | [`BigInt!`](#bigint) | Total time spent in seconds. | #### `TimelogEdge` @@ -10273,6 +11111,29 @@ The edge type for [`UsageTrendsMeasurement`](#usagetrendsmeasurement). | `cursor` | [`String!`](#string) | A cursor for use in pagination. | | `node` | [`UsageTrendsMeasurement`](#usagetrendsmeasurement) | The item at the end of the edge. | +#### `UserAchievementConnection` + +The connection type for [`UserAchievement`](#userachievement). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[UserAchievementEdge]`](#userachievementedge) | A list of edges. | +| `nodes` | [`[UserAchievement]`](#userachievement) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `UserAchievementEdge` + +The edge type for [`UserAchievement`](#userachievement). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`UserAchievement`](#userachievement) | The item at the end of the edge. | + #### `UserCalloutConnection` The connection type for [`UserCallout`](#usercallout). @@ -10457,6 +11318,29 @@ The edge type for [`VulnerabilityScanner`](#vulnerabilityscanner). | `cursor` | [`String!`](#string) | A cursor for use in pagination. | | `node` | [`VulnerabilityScanner`](#vulnerabilityscanner) | The item at the end of the edge. | +#### `VulnerabilityStateTransitionTypeConnection` + +The connection type for [`VulnerabilityStateTransitionType`](#vulnerabilitystatetransitiontype). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[VulnerabilityStateTransitionTypeEdge]`](#vulnerabilitystatetransitiontypeedge) | A list of edges. | +| `nodes` | [`[VulnerabilityStateTransitionType]`](#vulnerabilitystatetransitiontype) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `VulnerabilityStateTransitionTypeEdge` + +The edge type for [`VulnerabilityStateTransitionType`](#vulnerabilitystatetransitiontype). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`VulnerabilityStateTransitionType`](#vulnerabilitystatetransitiontype) | The item at the end of the edge. | + #### `WorkItemConnection` The connection type for [`WorkItem`](#workitem). @@ -10503,6 +11387,29 @@ The edge type for [`WorkItemType`](#workitemtype). | `cursor` | [`String!`](#string) | A cursor for use in pagination. | | `node` | [`WorkItemType`](#workitemtype) | The item at the end of the edge. | +#### `WorkspaceConnection` + +The connection type for [`Workspace`](#workspace). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[WorkspaceEdge]`](#workspaceedge) | A list of edges. | +| `nodes` | [`[Workspace]`](#workspace) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `WorkspaceEdge` + +The edge type for [`Workspace`](#workspace). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`Workspace`](#workspace) | The item at the end of the edge. | + ## Object types Object types represent the resources that the GitLab GraphQL API can return. @@ -10568,8 +11475,9 @@ Representation of a GitLab user. | `description` | [`String`](#string) | Description or notes for the achievement. | | `id` | [`AchievementsAchievementID!`](#achievementsachievementid) | ID of the achievement. | | `name` | [`String!`](#string) | Name of the achievement. | -| `namespace` | [`Namespace!`](#namespace) | Namespace of the achievement. | +| `namespace` | [`Namespace`](#namespace) | Namespace of the achievement. | | `updatedAt` | [`Time!`](#time) | Timestamp the achievement was last updated. | +| `userAchievements` **{warning-solid}** | [`UserAchievementConnection`](#userachievementconnection) | **Introduced** in 15.10. This feature is an Experiment. It can be changed or removed at any time. Recipients for the achievement. | ### `AgentConfiguration` @@ -10594,6 +11502,28 @@ Information about a connected Agent. | `podNamespace` | [`String`](#string) | Namespace of the pod running the Agent. | | `version` | [`String`](#string) | Agent version tag. | +### `AiMessageType` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `content` | [`String`](#string) | Content of the message or null if loading. | +| `errors` | [`[String!]!`](#string) | Errors that occurred while asynchronously fetching an AI(assistant) response. | +| `id` | [`ID`](#id) | Global ID of the message. | +| `isFetching` | [`Boolean`](#boolean) | Whether the content is still being fetched, for a message with the assistant role. | +| `role` | [`String!`](#string) | Role of the message (system, user, assistant). | + +### `AiResponse` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `errors` | [`[String!]`](#string) | Errors return by AI API as response. | +| `requestId` | [`String`](#string) | ID of the original request. | +| `responseBody` | [`String`](#string) | Response body from AI API. | + ### `AlertManagementAlert` Describes an alert from the project's Alert Management. @@ -10617,7 +11547,7 @@ Describes an alert from the project's Alert Management. | `iid` | [`ID!`](#id) | Internal ID of the alert. | | `issue` | [`Issue`](#issue) | Issue attached to the alert. | | `issueIid` **{warning-solid}** | [`ID`](#id) | **Deprecated** in 13.10. Use issue field. | -| `metricsDashboardUrl` | [`String`](#string) | URL for metrics embed for the alert. | +| `metricsDashboardUrl` **{warning-solid}** | [`String`](#string) | **Deprecated** in 16.0. Returns no data. Underlying feature was removed in 16.0. | | `monitoringTool` | [`String`](#string) | Monitoring tool the alert came from. | | `notes` | [`NoteConnection!`](#noteconnection) | All notes on this noteable. (see [Connections](#connections)) | | `prometheusAlert` | [`PrometheusAlert`](#prometheusalert) | Alert condition for Prometheus. | @@ -10773,6 +11703,7 @@ Describes a rule for who can approve merge requests. | Name | Type | Description | | ---- | ---- | ----------- | +| `allowMergeWhenInvalid` | [`Boolean`](#boolean) | Indicates if the rule can be ignored if it is invalid. | | `approvalsRequired` | [`Int`](#int) | Number of required approvals. | | `approved` | [`Boolean`](#boolean) | Indicates if the rule is satisfied. | | `approvedBy` | [`UserCoreConnection`](#usercoreconnection) | List of users defined in the rule that approved the merge request. (see [Connections](#connections)) | @@ -10781,6 +11712,7 @@ Describes a rule for who can approve merge requests. | `eligibleApprovers` | [`[UserCore!]`](#usercore) | List of all users eligible to approve the merge request (defined explicitly and from associated groups). | | `groups` | [`GroupConnection`](#groupconnection) | List of groups added as approvers for the rule. (see [Connections](#connections)) | | `id` | [`GlobalID!`](#globalid) | ID of the rule. | +| `invalid` | [`Boolean`](#boolean) | Indicates if the rule is invalid and cannot be approved. | | `name` | [`String`](#string) | Name of the rule. | | `overridden` | [`Boolean`](#boolean) | Indicates if the rule was overridden for the merge request. | | `section` | [`String`](#string) | Named section of the Code Owners file that the rule applies to. | @@ -11013,7 +11945,6 @@ four standard [pagination arguments](#connection-pagination-arguments): | `confidential` | [`Boolean`](#boolean) | Filter epics by given confidentiality. | | `createdAfter` | [`Time`](#time) | Epics created after this date. | | `createdBefore` | [`Time`](#time) | Epics created before this date. | -| `endDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.end. | | `iid` | [`ID`](#id) | IID of the epic, e.g., "1". | | `iidStartsWith` | [`String`](#string) | Filter epics by IID for autocomplete. | | `iids` | [`[ID!]`](#id) | List of IIDs of epics, e.g., `[1, 2]`. | @@ -11024,10 +11955,9 @@ four standard [pagination arguments](#connection-pagination-arguments): | `milestoneTitle` | [`String`](#string) | Filter epics by milestone title, computed from epic's issues. | | `myReactionEmoji` | [`String`](#string) | Filter by reaction emoji applied by the current user. | | `not` | [`NegatedEpicFilterInput`](#negatedepicfilterinput) | Negated epic arguments. | -| `or` **{warning-solid}** | [`UnionedEpicFilterInput`](#unionedepicfilterinput) | **Introduced** in 15.9. This feature is in Alpha. It can be changed or removed at any time. List of arguments with inclusive OR. Ignored unless `or_issuable_queries` flag is enabled. | +| `or` **{warning-solid}** | [`UnionedEpicFilterInput`](#unionedepicfilterinput) | **Introduced** in 15.9. This feature is an Experiment. It can be changed or removed at any time. List of arguments with inclusive OR. Ignored unless `or_issuable_queries` flag is enabled. | | `search` | [`String`](#string) | Search query for title or description. | | `sort` | [`EpicSort`](#epicsort) | List epics by sort order. | -| `startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | | `state` | [`EpicState`](#epicstate) | Filter epics by state. | | `timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | | `topLevelHierarchyOnly` | [`Boolean`](#boolean) | Filter epics with a top-level hierarchy. | @@ -11052,7 +11982,6 @@ four standard [pagination arguments](#connection-pagination-arguments): | `confidential` | [`Boolean`](#boolean) | Filter epics by given confidentiality. | | `createdAfter` | [`Time`](#time) | Epics created after this date. | | `createdBefore` | [`Time`](#time) | Epics created before this date. | -| `endDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.end. | | `iid` | [`ID`](#id) | IID of the epic, e.g., "1". | | `iidStartsWith` | [`String`](#string) | Filter epics by IID for autocomplete. | | `iids` | [`[ID!]`](#id) | List of IIDs of epics, e.g., `[1, 2]`. | @@ -11063,10 +11992,9 @@ four standard [pagination arguments](#connection-pagination-arguments): | `milestoneTitle` | [`String`](#string) | Filter epics by milestone title, computed from epic's issues. | | `myReactionEmoji` | [`String`](#string) | Filter by reaction emoji applied by the current user. | | `not` | [`NegatedEpicFilterInput`](#negatedepicfilterinput) | Negated epic arguments. | -| `or` **{warning-solid}** | [`UnionedEpicFilterInput`](#unionedepicfilterinput) | **Introduced** in 15.9. This feature is in Alpha. It can be changed or removed at any time. List of arguments with inclusive OR. Ignored unless `or_issuable_queries` flag is enabled. | +| `or` **{warning-solid}** | [`UnionedEpicFilterInput`](#unionedepicfilterinput) | **Introduced** in 15.9. This feature is an Experiment. It can be changed or removed at any time. List of arguments with inclusive OR. Ignored unless `or_issuable_queries` flag is enabled. | | `search` | [`String`](#string) | Search query for title or description. | | `sort` | [`EpicSort`](#epicsort) | List epics by sort order. | -| `startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | | `state` | [`EpicState`](#epicstate) | Filter epics by state. | | `timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | | `topLevelHierarchyOnly` | [`Boolean`](#boolean) | Filter epics with a top-level hierarchy. | @@ -11224,6 +12152,17 @@ Represents the total number of issues and their weights for a particular day. | `id` | [`ID!`](#id) | ID of the BuildNeed. | | `name` | [`String`](#string) | Name of the job we need to complete. | +### `CiCatalogResource` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `description` **{warning-solid}** | [`String`](#string) | **Introduced** in 15.11. This feature is an Experiment. It can be changed or removed at any time. Description of the catalog resource. | +| `icon` **{warning-solid}** | [`String`](#string) | **Introduced** in 15.11. This feature is an Experiment. It can be changed or removed at any time. Icon for the catalog resource. | +| `id` **{warning-solid}** | [`ID!`](#id) | **Introduced** in 15.11. This feature is an Experiment. It can be changed or removed at any time. ID of the catalog resource. | +| `name` **{warning-solid}** | [`String`](#string) | **Introduced** in 15.11. This feature is an Experiment. It can be changed or removed at any time. Name of the catalog resource. | + ### `CiConfig` #### Fields @@ -11390,6 +12329,7 @@ CI/CD variables for a GitLab instance. | `allowFailure` | [`Boolean!`](#boolean) | Whether the job is allowed to fail. | | `artifacts` | [`CiJobArtifactConnection`](#cijobartifactconnection) | Artifacts generated by the job. (see [Connections](#connections)) | | `browseArtifactsPath` | [`String`](#string) | URL for browsing the artifact's archive. | +| `canPlayJob` | [`Boolean!`](#boolean) | Indicates whether the current user can play the job. | | `cancelable` | [`Boolean!`](#boolean) | Indicates the job can be canceled. | | `commitPath` | [`String`](#string) | Path to the commit that triggered the job. | | `coverage` | [`Float`](#float) | Coverage level of the job. | @@ -11399,6 +12339,7 @@ CI/CD variables for a GitLab instance. | `downstreamPipeline` | [`Pipeline`](#pipeline) | Downstream pipeline for a bridge. | | `duration` | [`Int`](#int) | Duration of the job in seconds. | | `erasedAt` | [`Time`](#time) | When the job was erased. | +| `failureMessage` | [`String`](#string) | Message on why the job failed. | | `finishedAt` | [`Time`](#time) | When a job has finished running. | | `id` | [`JobID`](#jobid) | ID of the job. | | `kind` | [`CiJobKind!`](#cijobkind) | Indicates the type of job. | @@ -11407,6 +12348,7 @@ CI/CD variables for a GitLab instance. | `name` | [`String`](#string) | Name of the job. | | `needs` | [`CiBuildNeedConnection`](#cibuildneedconnection) | References to builds that must complete before the jobs run. (see [Connections](#connections)) | | `pipeline` | [`Pipeline`](#pipeline) | Pipeline the job belongs to. | +| `playPath` | [`String`](#string) | Play path of the job. | | `playable` | [`Boolean!`](#boolean) | Indicates the job can be played. | | `previousStageJobsOrNeeds` | [`JobNeedUnionConnection`](#jobneedunionconnection) | Jobs that must complete before the job runs. Returns `BuildNeed`, which is the needed jobs if the job uses the `needs` keyword, or the previous stage jobs otherwise. (see [Connections](#connections)) | | `project` | [`Project`](#project) | Project that the job belongs to. | @@ -11416,6 +12358,9 @@ CI/CD variables for a GitLab instance. | `refPath` | [`String`](#string) | Path to the ref. | | `retried` | [`Boolean`](#boolean) | Indicates that the job has been retried. | | `retryable` | [`Boolean!`](#boolean) | Indicates the job can be retried. | +| `runner` | [`CiRunner`](#cirunner) | Runner assigned to execute the job. | +| `runnerManager` **{warning-solid}** | [`CiRunnerManager`](#cirunnermanager) | **Introduced** in 15.11. This feature is an Experiment. It can be changed or removed at any time. Runner manager assigned to the job. | +| `scheduled` | [`Boolean!`](#boolean) | Indicates the job is scheduled. | | `scheduledAt` | [`Time`](#time) | Schedule for the build. | | `schedulingType` | [`String`](#string) | Type of job scheduling. Value is `dag` if the job uses the `needs` keyword, and `stage` otherwise. | | `shortSha` | [`String!`](#string) | Short SHA1 ID of the commit. | @@ -11424,6 +12369,7 @@ CI/CD variables for a GitLab instance. | `status` | [`CiJobStatus`](#cijobstatus) | Status of the job. | | `stuck` | [`Boolean!`](#boolean) | Indicates the job is stuck. | | `tags` | [`[String!]`](#string) | Tags for the current job. | +| `trace` | [`CiJobTrace`](#cijobtrace) | Trace generated by the job. | | `triggered` | [`Boolean`](#boolean) | Whether the job was triggered. | | `userPermissions` | [`JobPermissions!`](#jobpermissions) | Permissions for the current user on the resource. | | `webPath` | [`String`](#string) | Web path of the job. | @@ -11451,6 +12397,14 @@ CI/CD variables for a GitLab instance. | `outboundAllowlist` | [`ProjectConnection!`](#projectconnection) | Allow list of projects that are accessible using the current project's CI Job tokens. (see [Connections](#connections)) | | `projects` **{warning-solid}** | [`ProjectConnection!`](#projectconnection) | **Deprecated** in 15.9. The `projects` attribute is being deprecated. Use `outbound_allowlist`. | +### `CiJobTrace` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `htmlSummary` **{warning-solid}** | [`String!`](#string) | **Introduced** in 15.11. This feature is an Experiment. It can be changed or removed at any time. HTML summary containing the last 10 lines of the trace. | + ### `CiJobsDurationStatistics` Representation of duration statistics for a group of CI jobs. @@ -11459,11 +12413,11 @@ Representation of duration statistics for a group of CI jobs. | Name | Type | Description | | ---- | ---- | ----------- | -| `p50` **{warning-solid}** | [`Duration`](#duration) | **Introduced** in 15.8. This feature is in Alpha. It can be changed or removed at any time. 50th percentile. 50% of the durations are lower than this value. | -| `p75` **{warning-solid}** | [`Duration`](#duration) | **Introduced** in 15.8. This feature is in Alpha. It can be changed or removed at any time. 75th percentile. 75% of the durations are lower than this value. | -| `p90` **{warning-solid}** | [`Duration`](#duration) | **Introduced** in 15.8. This feature is in Alpha. It can be changed or removed at any time. 90th percentile. 90% of the durations are lower than this value. | -| `p95` **{warning-solid}** | [`Duration`](#duration) | **Introduced** in 15.8. This feature is in Alpha. It can be changed or removed at any time. 95th percentile. 95% of the durations are lower than this value. | -| `p99` **{warning-solid}** | [`Duration`](#duration) | **Introduced** in 15.8. This feature is in Alpha. It can be changed or removed at any time. 99th percentile. 99% of the durations are lower than this value. | +| `p50` **{warning-solid}** | [`Duration`](#duration) | **Introduced** in 15.8. This feature is an Experiment. It can be changed or removed at any time. 50th percentile. 50% of the durations are lower than this value. | +| `p75` **{warning-solid}** | [`Duration`](#duration) | **Introduced** in 15.8. This feature is an Experiment. It can be changed or removed at any time. 75th percentile. 75% of the durations are lower than this value. | +| `p90` **{warning-solid}** | [`Duration`](#duration) | **Introduced** in 15.8. This feature is an Experiment. It can be changed or removed at any time. 90th percentile. 90% of the durations are lower than this value. | +| `p95` **{warning-solid}** | [`Duration`](#duration) | **Introduced** in 15.8. This feature is an Experiment. It can be changed or removed at any time. 95th percentile. 95% of the durations are lower than this value. | +| `p99` **{warning-solid}** | [`Duration`](#duration) | **Introduced** in 15.8. This feature is an Experiment. It can be changed or removed at any time. 99th percentile. 99% of the durations are lower than this value. | ### `CiJobsStatistics` @@ -11473,7 +12427,7 @@ Statistics for a group of CI jobs. | Name | Type | Description | | ---- | ---- | ----------- | -| `queuedDuration` **{warning-solid}** | [`CiJobsDurationStatistics`](#cijobsdurationstatistics) | **Introduced** in 15.8. This feature is in Alpha. It can be changed or removed at any time. Statistics for amount of time that jobs were waiting to be picked up. The calculation is performed based on the most recent 100 jobs executed by the 5000 most recently created runners in context. If no filter is applied to runners, the calculation is performed based on the most recent 100 jobs globally. | +| `queuedDuration` **{warning-solid}** | [`CiJobsDurationStatistics`](#cijobsdurationstatistics) | **Introduced** in 15.8. This feature is an Experiment. It can be changed or removed at any time. Statistics for amount of time that jobs were waiting to be picked up. The calculation is performed based on the most recent 100 jobs executed by the 5000 most recently created runners in context. If no filter is applied to runners, the calculation is performed based on the most recent 100 jobs globally. | ### `CiManualVariable` @@ -11542,18 +12496,21 @@ CI/CD variables for a project. | `architectureName` | [`String`](#string) | Architecture provided by the the runner. | | `contactedAt` | [`Time`](#time) | Timestamp of last contact from this runner. | | `createdAt` | [`Time`](#time) | Timestamp of creation of this runner. | +| `createdBy` | [`UserCore`](#usercore) | User that created this runner. | | `description` | [`String`](#string) | Description of the runner. | | `editAdminUrl` | [`String`](#string) | Admin form URL of the runner. Only available for administrators. | -| `ephemeralAuthenticationToken` **{warning-solid}** | [`String`](#string) | **Introduced** in 15.9. This feature is in Alpha. It can be changed or removed at any time. Ephemeral authentication token used for runner machine registration. | +| `ephemeralAuthenticationToken` **{warning-solid}** | [`String`](#string) | **Introduced** in 15.9. This feature is an Experiment. It can be changed or removed at any time. Ephemeral authentication token used for runner manager registration. Only available for the creator of the runner for a limited time during registration. | +| `ephemeralRegisterUrl` **{warning-solid}** | [`String`](#string) | **Introduced** in 15.11. This feature is an Experiment. It can be changed or removed at any time. URL of the registration page of the runner manager. Only available for the creator of the runner for a limited time during registration. | | `executorName` | [`String`](#string) | Executor last advertised by the runner. | | `groups` | [`GroupConnection`](#groupconnection) | Groups the runner is associated with. For group runners only. (see [Connections](#connections)) | | `id` | [`CiRunnerID!`](#cirunnerid) | ID of the runner. | | `ipAddress` | [`String`](#string) | IP address of the runner. | | `jobCount` | [`Int`](#int) | Number of jobs processed by the runner (limited to 1000, plus one to indicate that more items exist). | -| `jobExecutionStatus` **{warning-solid}** | [`CiRunnerJobExecutionStatus`](#cirunnerjobexecutionstatus) | **Introduced** in 15.7. This feature is in Alpha. It can be changed or removed at any time. Job execution status of the runner. | +| `jobExecutionStatus` **{warning-solid}** | [`CiRunnerJobExecutionStatus`](#cirunnerjobexecutionstatus) | **Introduced** in 15.7. This feature is an Experiment. It can be changed or removed at any time. Job execution status of the runner. | | `locked` | [`Boolean`](#boolean) | Indicates the runner is locked. | | `maintenanceNote` | [`String`](#string) | Runner's maintenance notes. | | `maintenanceNoteHtml` | [`String`](#string) | GitLab Flavored Markdown rendering of `maintenance_note`. | +| `managers` **{warning-solid}** | [`CiRunnerManagerConnection`](#cirunnermanagerconnection) | **Introduced** in 15.10. This feature is an Experiment. It can be changed or removed at any time. Machines associated with the runner configuration. | | `maximumTimeout` | [`Int`](#int) | Maximum timeout (in seconds) for jobs processed by the runner. | | `ownerProject` | [`Project`](#project) | Project that owns the runner. For project runners only. | | `paused` | [`Boolean!`](#boolean) | Indicates the runner is paused and not available to run jobs. | @@ -11561,13 +12518,14 @@ CI/CD variables for a project. | `privateProjectsMinutesCostFactor` | [`Float`](#float) | Private projects' "minutes cost factor" associated with the runner (GitLab.com only). | | `projectCount` | [`Int`](#int) | Number of projects that the runner is associated with. | | `publicProjectsMinutesCostFactor` | [`Float`](#float) | Public projects' "minutes cost factor" associated with the runner (GitLab.com only). | +| `registerAdminUrl` | [`String`](#string) | URL of the temporary registration page of the runner. Only available before the runner is registered. Only available for administrators. | | `revision` | [`String`](#string) | Revision of the runner. | | `runUntagged` | [`Boolean!`](#boolean) | Indicates the runner is able to run untagged jobs. | | `runnerType` | [`CiRunnerType!`](#cirunnertype) | Type of the runner. | | `shortSha` | [`String`](#string) | First eight characters of the runner's token used to authenticate new job requests. Used as the runner's unique ID. | | `tagList` | [`[String!]`](#string) | Tags associated with the runner. | | `tokenExpiresAt` | [`Time`](#time) | Runner token expiration time. | -| `upgradeStatus` **{warning-solid}** | [`CiRunnerUpgradeStatus`](#cirunnerupgradestatus) | **Introduced** in 14.10. This feature is in Alpha. It can be changed or removed at any time. Availability of upgrades for the runner. | +| `upgradeStatus` **{warning-solid}** | [`CiRunnerUpgradeStatus`](#cirunnerupgradestatus) | **Introduced** in 14.10. This feature is an Experiment. It can be changed or removed at any time. Availability of upgrades for the runner. | | `userPermissions` | [`RunnerPermissions!`](#runnerpermissions) | Permissions for the current user on the resource. | | `version` | [`String`](#string) | Version of the runner. | @@ -11606,7 +12564,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | `membership` | [`Boolean`](#boolean) | Return only projects that the current user is a member of. | | `search` | [`String`](#string) | Search query, which can be for the project name, a path, or a description. | | `searchNamespaces` | [`Boolean`](#boolean) | Include namespace in project search. | -| `sort` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.4. Default sort order will change in 16.0. Specify `"id_asc"` if query results' order is important. | +| `sort` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.4. Default sort order will change in GitLab 17.0. Specify `"id_asc"` if you require the query results to be ordered by ascending IDs. | | `topics` | [`[String!]`](#string) | Filter projects by topics. | ##### `CiRunner.status` @@ -11619,7 +12577,26 @@ Returns [`CiRunnerStatus!`](#cirunnerstatus). | Name | Type | Description | | ---- | ---- | ----------- | -| `legacyMode` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.0. Will be removed in 17.0. In GitLab 16.0 and later, the field will act as if `legacyMode` is null. | +| `legacyMode` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.0. Will be removed in 17.0. | + +### `CiRunnerManager` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `architectureName` | [`String`](#string) | Architecture provided by the runner manager. | +| `contactedAt` | [`Time`](#time) | Timestamp of last contact from the runner manager. | +| `createdAt` | [`Time`](#time) | Timestamp of creation of the runner manager. | +| `executorName` | [`String`](#string) | Executor last advertised by the runner. | +| `id` | [`CiRunnerManagerID!`](#cirunnermanagerid) | ID of the runner manager. | +| `ipAddress` | [`String`](#string) | IP address of the runner manager. | +| `platformName` | [`String`](#string) | Platform provided by the runner manager. | +| `revision` | [`String`](#string) | Revision of the runner. | +| `runner` | [`CiRunner`](#cirunner) | Runner configuration for the runner manager. | +| `status` | [`CiRunnerStatus!`](#cirunnerstatus) | Status of the runner manager. | +| `systemId` | [`String!`](#string) | System ID associated with the runner manager. | +| `version` | [`String`](#string) | Version of the runner. | ### `CiSecureFileRegistry` @@ -11677,39 +12654,40 @@ GitLab CI/CD configuration template. | `id` | [`ID!`](#id) | ID of the cluster agent. | | `name` | [`String`](#string) | Name of the cluster agent. | | `project` | [`Project`](#project) | Project this cluster agent is associated with. | +| `tokens` | [`ClusterAgentTokenConnection`](#clusteragenttokenconnection) | Tokens associated with the cluster agent. (see [Connections](#connections)) | | `updatedAt` | [`Time`](#time) | Timestamp the cluster agent was updated. | | `vulnerabilityImages` | [`VulnerabilityContainerImageConnection`](#vulnerabilitycontainerimageconnection) | Container images reported on the agent vulnerabilities. (see [Connections](#connections)) | | `webPath` | [`String`](#string) | Web path of the cluster agent. | -#### Fields with arguments - -##### `ClusterAgent.tokens` +### `ClusterAgentActivityEvent` -Tokens associated with the cluster agent. +#### Fields -Returns [`ClusterAgentTokenConnection`](#clusteragenttokenconnection). +| Name | Type | Description | +| ---- | ---- | ----------- | +| `agentToken` | [`ClusterAgentToken`](#clusteragenttoken) | Agent token associated with the event. | +| `kind` | [`String`](#string) | Type of event. | +| `level` | [`String`](#string) | Severity of the event. | +| `recordedAt` | [`Time`](#time) | Timestamp the event was recorded. | +| `user` | [`UserCore`](#usercore) | User associated with the event. | -This field returns a [connection](#connections). It accepts the -four standard [pagination arguments](#connection-pagination-arguments): -`before: String`, `after: String`, `first: Int`, `last: Int`. +### `ClusterAgentAuthorizationCiAccess` -###### Arguments +#### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| `status` | [`AgentTokenStatus`](#agenttokenstatus) | Status of the token. | +| `agent` | [`ClusterAgent`](#clusteragent) | Authorized cluster agent. | +| `config` | [`JSON`](#json) | Configuration for the authorized project. | -### `ClusterAgentActivityEvent` +### `ClusterAgentAuthorizationUserAccess` #### Fields | Name | Type | Description | | ---- | ---- | ----------- | -| `agentToken` | [`ClusterAgentToken`](#clusteragenttoken) | Agent token associated with the event. | -| `kind` | [`String`](#string) | Type of event. | -| `level` | [`String`](#string) | Severity of the event. | -| `recordedAt` | [`Time`](#time) | Timestamp the event was recorded. | -| `user` | [`UserCore`](#usercore) | User associated with the event. | +| `agent` | [`ClusterAgent`](#clusteragent) | Authorized cluster agent. | +| `config` | [`JSON`](#json) | Configuration for the authorized project. | ### `ClusterAgentToken` @@ -11834,6 +12812,68 @@ four standard [pagination arguments](#connection-pagination-arguments): | `updatedBefore` | [`Time`](#time) | Pipelines updated before this date. | | `username` | [`String`](#string) | Filter pipelines by the user that triggered the pipeline. | +### `CommitParentNames` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `names` | [`[String!]`](#string) | Names of the commit parent (branch or tag). | + +### `CommitReferences` + +#### Fields with arguments + +##### `CommitReferences.containingBranches` + +Get branch names containing a given commit. + +Returns [`CommitParentNames`](#commitparentnames). + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `excludeTipped` | [`Boolean!`](#boolean) | Exclude tipping refs. WARNING: This argument can be confusing, if there is a limit. for example set the limit to 5 and in the 5 out a total of 25 refs there is 2 tipped refs, then the method will only 3 refs, even though there is more. | +| `limit` | [`Int!`](#int) | Number of ref names to return. | + +##### `CommitReferences.containingTags` + +Get tag names containing a given commit. + +Returns [`CommitParentNames`](#commitparentnames). + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `excludeTipped` | [`Boolean!`](#boolean) | Exclude tipping refs. WARNING: This argument can be confusing, if there is a limit. for example set the limit to 5 and in the 5 out a total of 25 refs there is 2 tipped refs, then the method will only 3 refs, even though there is more. | +| `limit` | [`Int!`](#int) | Number of ref names to return. | + +##### `CommitReferences.tippingBranches` + +Get branch names tipping at a given commit. + +Returns [`CommitParentNames`](#commitparentnames). + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `limit` | [`Int!`](#int) | Number of ref names to return. | + +##### `CommitReferences.tippingTags` + +Get tag names tipping at a given commit. + +Returns [`CommitParentNames`](#commitparentnames). + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `limit` | [`Int!`](#int) | Number of ref names to return. | + ### `ComplianceFramework` Represents a ComplianceFramework associated with a Project. @@ -12169,8 +13209,9 @@ Represents a DAST Pre Scan Verification Step. | Name | Type | Description | | ---- | ---- | ----------- | +| `checkType` | [`DastPreScanVerificationCheckType`](#dastprescanverificationchecktype) | Type of the pre scan verification check. | | `errors` | [`[String!]`](#string) | Errors that occurred in the pre scan verification step. | -| `name` | [`String`](#string) | Name of the pre scan verification step. | +| `name` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.10. This was renamed. Use: [`DastPreScanVerificationStep.checkType`](#dastprescanverificationstepchecktype). | | `success` | [`Boolean!`](#boolean) | Whether or not the pre scan verification step has errors. | ### `DastProfile` @@ -12334,6 +13375,20 @@ The response from the AdminSidekiqQueuesDeleteJobs mutation. | `id` | [`NoteID!`](#noteid) | ID of the deleted note. | | `lastDiscussionNote` | [`Boolean`](#boolean) | Whether deleted note is the last note in the discussion. | +### `Dependency` + +A software dependency used by a project. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `id` | [`GlobalID!`](#globalid) | ID of the dependency. | +| `location` | [`Location`](#location) | Information about where the dependency is located. | +| `name` | [`String!`](#string) | Name of the dependency. | +| `packager` | [`PackageManager`](#packagemanager) | Description of the tool used to manage the dependency. | +| `version` | [`String`](#string) | Version of the dependency. | + ### `DependencyProxyBlob` Dependency proxy blob. @@ -12434,12 +13489,14 @@ The deployment of an environment. | Name | Type | Description | | ---- | ---- | ----------- | | `approvalSummary` | [`DeploymentApprovalSummary`](#deploymentapprovalsummary) | Approval summary of the deployment.This field can only be resolved for one deployment in any single request. | +| `approvals` | [`[DeploymentApproval!]`](#deploymentapproval) | Current approvals of the deployment. | | `commit` | [`Commit`](#commit) | Commit details of the deployment. | | `createdAt` | [`Time`](#time) | When the deployment record was created. | | `finishedAt` | [`Time`](#time) | When the deployment finished. | | `id` | [`ID`](#id) | Global ID of the deployment. | | `iid` | [`ID`](#id) | Project-level internal ID of the deployment. | | `job` | [`CiJob`](#cijob) | Pipeline job of the deployment. | +| `pendingApprovalCount` | [`Int`](#int) | Number of pending unified approvals on the deployment. | | `ref` | [`String`](#string) | Git-Ref that the deployment ran on. | | `sha` | [`String`](#string) | Git-SHA that the deployment ran on. | | `status` | [`DeploymentStatus`](#deploymentstatus) | Status of the deployment. | @@ -12533,6 +13590,8 @@ A single design. | Name | Type | Description | | ---- | ---- | ----------- | | `commenters` | [`UserCoreConnection!`](#usercoreconnection) | All commenters on this noteable. (see [Connections](#connections)) | +| `description` | [`String`](#string) | Description of the design. | +| `descriptionHtml` | [`String`](#string) | GitLab Flavored Markdown rendering of `description`. | | `diffRefs` | [`DiffRefs!`](#diffrefs) | Diff refs for this design. | | `discussions` | [`DiscussionConnection!`](#discussionconnection) | All discussions on this noteable. (see [Connections](#connections)) | | `event` | [`DesignVersionEvent!`](#designversionevent) | How this design was changed in the current version. | @@ -12925,10 +13984,9 @@ Returns [`[DoraMetric!]`](#dorametric). | Name | Type | Description | | ---- | ---- | ----------- | | `endDate` | [`Date`](#date) | Date range to end at. Default is the current date. | -| `environmentTier` | [`DeploymentTier`](#deploymenttier) | Deployment tier of the environments to return. Deprecated, please update to `environment_tiers` param. | | `environmentTiers` | [`[DeploymentTier!]`](#deploymenttier) | Deployment tiers of the environments to return. Defaults to `[PRODUCTION]`. | -| `interval` | [`DoraMetricBucketingInterval`](#dorametricbucketinginterval) | How the metric should be aggregrated. Defaults to `DAILY`. In the case of `ALL`, the `date` field in the response will be `null`. | -| `metric` | [`DoraMetricType!`](#dorametrictype) | Type of metric to return. | +| `interval` | [`DoraMetricBucketingInterval`](#dorametricbucketinginterval) | How the metric should be aggregated. Defaults to `DAILY`. In the case of `ALL`, the `date` field in the response will be `null`. | +| `metric` **{warning-solid}** | [`DoraMetricType`](#dorametrictype) | **Deprecated** in 15.10. Superseded by metrics fields. See `DoraMetric` type. | | `startDate` | [`Date`](#date) | Date range to start from. Default is 3 months ago. | ### `DoraMetric` @@ -12937,8 +13995,12 @@ Returns [`[DoraMetric!]`](#dorametric). | Name | Type | Description | | ---- | ---- | ----------- | +| `changeFailureRate` | [`Float`](#float) | Percentage of deployments that caused incidents in production. | | `date` | [`String`](#string) | Date of the data point. | -| `value` | [`Float`](#float) | Value of the data point. | +| `deploymentFrequency` | [`Float`](#float) | Number of deployments per day. | +| `leadTimeForChanges` | [`Float`](#float) | Median time to deploy a merged merge request. | +| `timeToRestoreService` | [`Float`](#float) | Median time to close an incident. | +| `value` **{warning-solid}** | [`Float`](#float) | **Deprecated** in 15.10. Moved to corresponding metric field. | ### `EgressNode` @@ -12949,7 +14011,7 @@ Returns [`[DoraMetric!]`](#dorametric). | `artifactsEgress` | [`BigInt!`](#bigint) | Artifacts egress for that project in that period of time. | | `date` | [`String!`](#string) | First day of the node range. There is one node per month. | | `packagesEgress` | [`BigInt!`](#bigint) | Packages egress for that project in that period of time. | -| `registryEgress` | [`BigInt!`](#bigint) | Registery egress for that project in that period of time. | +| `registryEgress` | [`BigInt!`](#bigint) | Registry egress for that project in that period of time. | | `repositoryEgress` | [`BigInt!`](#bigint) | Repository egress for that project in that period of time. | | `totalEgress` | [`BigInt!`](#bigint) | Total egress for that project in that period of time. | @@ -13025,6 +14087,10 @@ Returns [`Deployment`](#deployment). Metrics dashboard schema for the environment. +WARNING: +**Deprecated** in 16.0. +Returns no data. Underlying feature was removed in 16.0. + Returns [`MetricsDashboard`](#metricsdashboard). ###### Arguments @@ -13127,7 +14193,6 @@ four standard [pagination arguments](#connection-pagination-arguments): | `confidential` | [`Boolean`](#boolean) | Filter epics by given confidentiality. | | `createdAfter` | [`Time`](#time) | Epics created after this date. | | `createdBefore` | [`Time`](#time) | Epics created before this date. | -| `endDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.end. | | `iid` | [`ID`](#id) | IID of the epic, e.g., "1". | | `iidStartsWith` | [`String`](#string) | Filter epics by IID for autocomplete. | | `iids` | [`[ID!]`](#id) | List of IIDs of epics, e.g., `[1, 2]`. | @@ -13138,10 +14203,9 @@ four standard [pagination arguments](#connection-pagination-arguments): | `milestoneTitle` | [`String`](#string) | Filter epics by milestone title, computed from epic's issues. | | `myReactionEmoji` | [`String`](#string) | Filter by reaction emoji applied by the current user. | | `not` | [`NegatedEpicFilterInput`](#negatedepicfilterinput) | Negated epic arguments. | -| `or` **{warning-solid}** | [`UnionedEpicFilterInput`](#unionedepicfilterinput) | **Introduced** in 15.9. This feature is in Alpha. It can be changed or removed at any time. List of arguments with inclusive OR. Ignored unless `or_issuable_queries` flag is enabled. | +| `or` **{warning-solid}** | [`UnionedEpicFilterInput`](#unionedepicfilterinput) | **Introduced** in 15.9. This feature is an Experiment. It can be changed or removed at any time. List of arguments with inclusive OR. Ignored unless `or_issuable_queries` flag is enabled. | | `search` | [`String`](#string) | Search query for title or description. | | `sort` | [`EpicSort`](#epicsort) | List epics by sort order. | -| `startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | | `state` | [`EpicState`](#epicstate) | Filter epics by state. | | `timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | | `topLevelHierarchyOnly` | [`Boolean`](#boolean) | Filter epics with a top-level hierarchy. | @@ -13166,7 +14230,6 @@ four standard [pagination arguments](#connection-pagination-arguments): | `confidential` | [`Boolean`](#boolean) | Filter epics by given confidentiality. | | `createdAfter` | [`Time`](#time) | Epics created after this date. | | `createdBefore` | [`Time`](#time) | Epics created before this date. | -| `endDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.end. | | `iid` | [`ID`](#id) | IID of the epic, e.g., "1". | | `iidStartsWith` | [`String`](#string) | Filter epics by IID for autocomplete. | | `iids` | [`[ID!]`](#id) | List of IIDs of epics, e.g., `[1, 2]`. | @@ -13177,10 +14240,9 @@ four standard [pagination arguments](#connection-pagination-arguments): | `milestoneTitle` | [`String`](#string) | Filter epics by milestone title, computed from epic's issues. | | `myReactionEmoji` | [`String`](#string) | Filter by reaction emoji applied by the current user. | | `not` | [`NegatedEpicFilterInput`](#negatedepicfilterinput) | Negated epic arguments. | -| `or` **{warning-solid}** | [`UnionedEpicFilterInput`](#unionedepicfilterinput) | **Introduced** in 15.9. This feature is in Alpha. It can be changed or removed at any time. List of arguments with inclusive OR. Ignored unless `or_issuable_queries` flag is enabled. | +| `or` **{warning-solid}** | [`UnionedEpicFilterInput`](#unionedepicfilterinput) | **Introduced** in 15.9. This feature is an Experiment. It can be changed or removed at any time. List of arguments with inclusive OR. Ignored unless `or_issuable_queries` flag is enabled. | | `search` | [`String`](#string) | Search query for title or description. | | `sort` | [`EpicSort`](#epicsort) | List epics by sort order. | -| `startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | | `state` | [`EpicState`](#epicstate) | Filter epics by state. | | `timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | | `topLevelHierarchyOnly` | [`Boolean`](#boolean) | Filter epics with a top-level hierarchy. | @@ -13223,6 +14285,7 @@ Represents an epic board. | Name | Type | Description | | ---- | ---- | ----------- | +| `displayColors` | [`Boolean`](#boolean) | Whether or not display epic colors. | | `hideBacklogList` | [`Boolean`](#boolean) | Whether or not backlog list is hidden. | | `hideClosedList` | [`Boolean`](#boolean) | Whether or not closed list is hidden. | | `id` | [`BoardsEpicBoardID!`](#boardsepicboardid) | Global ID of the epic board. | @@ -13474,7 +14537,7 @@ Represents epic board list metadata. | Name | Type | Description | | ---- | ---- | ----------- | | `epicsCount` | [`Int`](#int) | Count of epics in the list. | -| `totalWeight` **{warning-solid}** | [`Int`](#int) | **Introduced** in 14.7. This feature is in Alpha. It can be changed or removed at any time. Total weight of all issues in the list. | +| `totalWeight` **{warning-solid}** | [`Int`](#int) | **Introduced** in 14.7. This feature is an Experiment. It can be changed or removed at any time. Total weight of all issues in the list. | ### `EpicPermissions` @@ -13597,6 +14660,8 @@ Details of the fork project compared to its upstream project. | ---- | ---- | ----------- | | `ahead` | [`Int`](#int) | Number of commits ahead of upstream. | | `behind` | [`Int`](#int) | Number of commits behind upstream. | +| `hasConflicts` | [`Boolean`](#boolean) | Indicates if the fork conflicts with its upstream project. | +| `isSyncing` | [`Boolean`](#boolean) | Indicates if there is a synchronization in progress. | ### `GeoNode` @@ -13666,7 +14731,7 @@ Find Dependency Proxy Blob registries on this Geo node. WARNING: **Introduced** in 15.6. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Returns [`DependencyProxyBlobRegistryConnection`](#dependencyproxyblobregistryconnection). @@ -13835,6 +14900,25 @@ four standard [pagination arguments](#connection-pagination-arguments): | `replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | | `verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | +##### `GeoNode.projectWikiRepositoryRegistries` + +Find Project Wiki Repository registries on this Geo node. Ignored if `geo_project_wiki_repository_replication` feature flag is disabled. + +Returns [`ProjectWikiRepositoryRegistryConnection`](#projectwikirepositoryregistryconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| `keyword` | [`String`](#string) | Filters registries by their attributes using a keyword. | +| `replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | +| `verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | + ##### `GeoNode.snippetRepositoryRegistries` Find snippet repository registries on this Geo node. @@ -13926,7 +15010,6 @@ GPG signature for a signed commit. | Name | Type | Description | | ---- | ---- | ----------- | -| `achievements` **{warning-solid}** | [`AchievementConnection`](#achievementconnection) | **Introduced** in 15.8. This feature is in Alpha. It can be changed or removed at any time. Achievements for the namespace. Returns `null` if the `achievements` feature flag is disabled. | | `actualRepositorySizeLimit` | [`Float`](#float) | Size limit for repositories in the namespace in bytes. | | `additionalPurchasedStorageSize` | [`Float`](#float) | Additional storage purchased for the root namespace in bytes. | | `allowStaleRunnerPruning` | [`Boolean!`](#boolean) | Indicates whether to regularly prune stale group runners. Defaults to false. | @@ -13935,7 +15018,7 @@ GPG signature for a signed commit. | `containerRepositoriesCount` | [`Int!`](#int) | Number of container repositories in the group. | | `containsLockedProjects` | [`Boolean!`](#boolean) | Includes at least one project where the repository size exceeds the limit. | | `crossProjectPipelineAvailable` | [`Boolean!`](#boolean) | Indicates if the cross_project_pipeline feature is available for the namespace. | -| `customEmoji` **{warning-solid}** | [`CustomEmojiConnection`](#customemojiconnection) | **Introduced** in 13.6. This feature is in Alpha. It can be changed or removed at any time. Custom emoji within this namespace. | +| `customEmoji` **{warning-solid}** | [`CustomEmojiConnection`](#customemojiconnection) | **Introduced** in 13.6. This feature is an Experiment. It can be changed or removed at any time. Custom emoji within this namespace. | | `dependencyProxyBlobCount` | [`Int!`](#int) | Number of dependency proxy blobs cached in the group. | | `dependencyProxyBlobs` | [`DependencyProxyBlobConnection`](#dependencyproxyblobconnection) | Dependency Proxy blobs. (see [Connections](#connections)) | | `dependencyProxyImageCount` | [`Int!`](#int) | Number of dependency proxy images cached in the group. | @@ -13944,6 +15027,7 @@ GPG signature for a signed commit. | `dependencyProxyManifests` | [`DependencyProxyManifestConnection`](#dependencyproxymanifestconnection) | Dependency Proxy manifests. (see [Connections](#connections)) | | `dependencyProxySetting` | [`DependencyProxySetting`](#dependencyproxysetting) | Dependency Proxy settings for the group. | | `dependencyProxyTotalSize` | [`String!`](#string) | Total size of the dependency proxy cached images. | +| `dependencyProxyTotalSizeInBytes` | [`Int!`](#int) | Total size of the dependency proxy cached images in bytes. | | `description` | [`String`](#string) | Description of the namespace. | | `descriptionHtml` | [`String`](#string) | GitLab Flavored Markdown rendering of `description`. | | `dora` | [`Dora`](#dora) | Group's DORA metrics. | @@ -13952,6 +15036,7 @@ GPG signature for a signed commit. | `epicBoards` | [`EpicBoardConnection`](#epicboardconnection) | Find epic boards. (see [Connections](#connections)) | | `epicsEnabled` | [`Boolean`](#boolean) | Indicates if Epics are enabled for namespace. | | `externalAuditEventDestinations` | [`ExternalAuditEventDestinationConnection`](#externalauditeventdestinationconnection) | External locations that receive audit events belonging to the group. (see [Connections](#connections)) | +| `flowMetrics` **{warning-solid}** | [`GroupValueStreamAnalyticsFlowMetrics`](#groupvaluestreamanalyticsflowmetrics) | **Introduced** in 15.10. This feature is an Experiment. It can be changed or removed at any time. Flow metrics for value stream analytics. | | `fullName` | [`String!`](#string) | Full name of the namespace. | | `fullPath` | [`ID!`](#id) | Full path of the namespace. | | `id` | [`ID!`](#id) | ID of the namespace. | @@ -13974,7 +15059,7 @@ GPG signature for a signed commit. | `storageSizeLimit` | [`Float`](#float) | Total storage limit of the root namespace in bytes. | | `subgroupCreationLevel` | [`String`](#string) | Permission level required to create subgroups within the group. | | `temporaryStorageIncreaseEndsOn` | [`Time`](#time) | Date until the temporary storage increase is active. | -| `timelogCategories` **{warning-solid}** | [`TimeTrackingTimelogCategoryConnection`](#timetrackingtimelogcategoryconnection) | **Introduced** in 15.3. This feature is in Alpha. It can be changed or removed at any time. Timelog categories for the namespace. | +| `timelogCategories` **{warning-solid}** | [`TimeTrackingTimelogCategoryConnection`](#timetrackingtimelogcategoryconnection) | **Introduced** in 15.3. This feature is an Experiment. It can be changed or removed at any time. Timelog categories for the namespace. | | `totalRepositorySize` | [`Float`](#float) | Total repository size of all projects in the root namespace in bytes. | | `totalRepositorySizeExcess` | [`Float`](#float) | Total excess repository size of all projects in the root namespace in bytes. | | `twoFactorGracePeriod` | [`Int`](#int) | Time before two-factor authentication is enforced. | @@ -13985,6 +15070,26 @@ GPG signature for a signed commit. #### Fields with arguments +##### `Group.achievements` + +Achievements for the namespace. Returns `null` if the `achievements` feature flag is disabled. + +WARNING: +**Introduced** in 15.8. +This feature is an Experiment. It can be changed or removed at any time. + +Returns [`AchievementConnection`](#achievementconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `ids` | [`[AchievementsAchievementID!]`](#achievementsachievementid) | Filter achievements by IDs. | + ##### `Group.billableMembersCount` Number of billable users in the group. @@ -14055,6 +15160,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `hasRemoteDevelopmentAgentConfig` | [`Boolean`](#boolean) | Returns only cluster agents which have an associated remote development agent config. | | `hasVulnerabilities` | [`Boolean`](#boolean) | Returns only cluster agents which have vulnerabilities. | ##### `Group.codeCoverageActivities` @@ -14153,7 +15259,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | `from` | [`ISO8601Date!`](#iso8601date) | Start date of the reporting time range. | -| `to` | [`ISO8601Date!`](#iso8601date) | End date of the reporting time range. The end date must be within 31 days after the start date. | +| `to` | [`ISO8601Date!`](#iso8601date) | End date of the reporting time range. The end date must be within 93 days after the start date. | ##### `Group.dataTransfer` @@ -14165,7 +15271,7 @@ Returns [`GroupDataTransfer`](#groupdatatransfer). | Name | Type | Description | | ---- | ---- | ----------- | -| `from` | [`Date`](#date) | Retain egress data for 1 year. Current month will increase dynamically as egress occurs. | +| `from` | [`Date`](#date) | Retain egress data for one year. Data for the current month will increase dynamically as egress occurs. | | `to` | [`Date`](#date) | End date for the data. | ##### `Group.descendantGroups` @@ -14200,7 +15306,6 @@ Returns [`Epic`](#epic). | `confidential` | [`Boolean`](#boolean) | Filter epics by given confidentiality. | | `createdAfter` | [`Time`](#time) | Epics created after this date. | | `createdBefore` | [`Time`](#time) | Epics created before this date. | -| `endDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.end. | | `iid` | [`ID`](#id) | IID of the epic, e.g., "1". | | `iidStartsWith` | [`String`](#string) | Filter epics by IID for autocomplete. | | `iids` | [`[ID!]`](#id) | List of IIDs of epics, e.g., `[1, 2]`. | @@ -14211,10 +15316,9 @@ Returns [`Epic`](#epic). | `milestoneTitle` | [`String`](#string) | Filter epics by milestone title, computed from epic's issues. | | `myReactionEmoji` | [`String`](#string) | Filter by reaction emoji applied by the current user. | | `not` | [`NegatedEpicFilterInput`](#negatedepicfilterinput) | Negated epic arguments. | -| `or` **{warning-solid}** | [`UnionedEpicFilterInput`](#unionedepicfilterinput) | **Introduced** in 15.9. This feature is in Alpha. It can be changed or removed at any time. List of arguments with inclusive OR. Ignored unless `or_issuable_queries` flag is enabled. | +| `or` **{warning-solid}** | [`UnionedEpicFilterInput`](#unionedepicfilterinput) | **Introduced** in 15.9. This feature is an Experiment. It can be changed or removed at any time. List of arguments with inclusive OR. Ignored unless `or_issuable_queries` flag is enabled. | | `search` | [`String`](#string) | Search query for title or description. | | `sort` | [`EpicSort`](#epicsort) | List epics by sort order. | -| `startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | | `state` | [`EpicState`](#epicstate) | Filter epics by state. | | `timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | | `topLevelHierarchyOnly` | [`Boolean`](#boolean) | Filter epics with a top-level hierarchy. | @@ -14251,7 +15355,6 @@ four standard [pagination arguments](#connection-pagination-arguments): | `confidential` | [`Boolean`](#boolean) | Filter epics by given confidentiality. | | `createdAfter` | [`Time`](#time) | Epics created after this date. | | `createdBefore` | [`Time`](#time) | Epics created before this date. | -| `endDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.end. | | `iid` | [`ID`](#id) | IID of the epic, e.g., "1". | | `iidStartsWith` | [`String`](#string) | Filter epics by IID for autocomplete. | | `iids` | [`[ID!]`](#id) | List of IIDs of epics, e.g., `[1, 2]`. | @@ -14262,10 +15365,9 @@ four standard [pagination arguments](#connection-pagination-arguments): | `milestoneTitle` | [`String`](#string) | Filter epics by milestone title, computed from epic's issues. | | `myReactionEmoji` | [`String`](#string) | Filter by reaction emoji applied by the current user. | | `not` | [`NegatedEpicFilterInput`](#negatedepicfilterinput) | Negated epic arguments. | -| `or` **{warning-solid}** | [`UnionedEpicFilterInput`](#unionedepicfilterinput) | **Introduced** in 15.9. This feature is in Alpha. It can be changed or removed at any time. List of arguments with inclusive OR. Ignored unless `or_issuable_queries` flag is enabled. | +| `or` **{warning-solid}** | [`UnionedEpicFilterInput`](#unionedepicfilterinput) | **Introduced** in 15.9. This feature is an Experiment. It can be changed or removed at any time. List of arguments with inclusive OR. Ignored unless `or_issuable_queries` flag is enabled. | | `search` | [`String`](#string) | Search query for title or description. | | `sort` | [`EpicSort`](#epicsort) | List epics by sort order. | -| `startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | | `state` | [`EpicState`](#epicstate) | Filter epics by state. | | `timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | | `topLevelHierarchyOnly` | [`Boolean`](#boolean) | Filter epics with a top-level hierarchy. | @@ -14391,7 +15493,6 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | -| `endDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.end. | | `id` | [`ID`](#id) | Global ID of the Iteration to look up. | | `iid` | [`ID`](#id) | Internal ID of the Iteration to look up. | | `in` | [`[IterationSearchableField!]`](#iterationsearchablefield) | Fields in which the fuzzy-search should be performed with the query given in the argument `search`. Defaults to `[title]`. | @@ -14399,7 +15500,6 @@ four standard [pagination arguments](#connection-pagination-arguments): | `iterationCadenceIds` | [`[IterationsCadenceID!]`](#iterationscadenceid) | Global iteration cadence IDs by which to look up the iterations. | | `search` | [`String`](#string) | Query used for fuzzy-searching in the fields selected in the argument `in`. Returns all iterations if empty. | | `sort` | [`IterationSort`](#iterationsort) | List iterations by sort order. If unspecified, an arbitrary order (subject to change) is used. | -| `startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | | `state` | [`IterationState`](#iterationstate) | Filter iterations by state. | | `timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | | `title` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.4. The argument will be removed in 15.4. Please use `search` and `in` fields instead. | @@ -14466,6 +15566,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `assigneeUsername` | [`String`](#string) | Username of the assignee. | | `authorUsername` | [`String`](#string) | Username of the author. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | @@ -14501,13 +15602,11 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | `containingDate` | [`Time`](#time) | Date the milestone contains. | -| `endDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.end. | | `ids` | [`[ID!]`](#id) | Array of global milestone IDs, e.g., `"gid://gitlab/Milestone/1"`. | | `includeAncestors` | [`Boolean`](#boolean) | Include milestones from all parent groups. | | `includeDescendants` | [`Boolean`](#boolean) | Include milestones from all subgroups and subprojects. | | `searchTitle` | [`String`](#string) | Search string for the title. | | `sort` | [`MilestoneSort`](#milestonesort) | Sort milestones by this criteria. | -| `startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | | `state` | [`MilestoneStateEnum`](#milestonestateenum) | Filter milestones by state. | | `timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | | `title` | [`String`](#string) | Title of the milestone. | @@ -14578,6 +15677,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `complianceFrameworkFilters` | [`ComplianceFrameworkFilters`](#complianceframeworkfilters) | Filters applied when selecting a compliance framework. | | `hasCodeCoverage` | [`Boolean`](#boolean) | Returns only the projects which have code coverage. | | `hasVulnerabilities` | [`Boolean`](#boolean) | Returns only the projects which have vulnerabilities. | | `ids` | [`[ID!]`](#id) | Filter projects by IDs. | @@ -14641,7 +15741,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | -| `actionScanTypes` | [`[SecurityReportTypeEnum!]`](#securityreporttypeenum) | Filters policies by the action scan type. Only these scan types are supported: `dast`, `secret_detection`, `cluster_image_scanning`, `container_scanning`, `sast`, `dependency_scanning`. | +| `actionScanTypes` | [`[SecurityReportTypeEnum!]`](#securityreporttypeenum) | Filters policies by the action scan type. Only these scan types are supported: `dast`, `secret_detection`, `cluster_image_scanning`, `container_scanning`, `sast`, `sast_iac`, `dependency_scanning`. | | `relationship` | [`SecurityPolicyRelationType`](#securitypolicyrelationtype) | Filter policies by the given policy relationship. | ##### `Group.scanResultPolicies` @@ -14806,58 +15906,150 @@ Represents a Group Membership. #### Fields with arguments -##### `GroupMember.mergeRequestInteraction` +##### `GroupMember.mergeRequestInteraction` + +Find a merge request. + +Returns [`UserMergeRequestInteraction`](#usermergerequestinteraction). + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `id` | [`MergeRequestID!`](#mergerequestid) | Global ID of the merge request. | + +### `GroupPermissions` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `createProjects` | [`Boolean!`](#boolean) | Indicates the user can perform `create_projects` on this resource. | +| `readGroup` | [`Boolean!`](#boolean) | Indicates the user can perform `read_group` on this resource. | + +### `GroupReleaseStats` + +Contains release-related statistics about a group. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `releasesCount` | [`Int`](#int) | Total number of releases in all descendant projects of the group. | +| `releasesPercentage` | [`Int`](#int) | Percentage of the group's descendant projects that have at least one release. | + +### `GroupSecurityPolicySource` + +Represents the source of a security policy belonging to a group. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `inherited` | [`Boolean!`](#boolean) | Indicates whether this policy is inherited from parent group. | +| `namespace` | [`Namespace`](#namespace) | Project the policy is associated with. | + +### `GroupStats` + +Contains statistics about a group. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `releaseStats` | [`GroupReleaseStats`](#groupreleasestats) | Statistics related to releases within the group. | + +### `GroupValueStreamAnalyticsFlowMetrics` + +Exposes aggregated value stream flow metrics. + +#### Fields with arguments + +##### `GroupValueStreamAnalyticsFlowMetrics.cycleTime` -Find a merge request. +Median time from first commit to issue closed. -Returns [`UserMergeRequestInteraction`](#usermergerequestinteraction). +Returns [`ValueStreamAnalyticsMetric`](#valuestreamanalyticsmetric). ###### Arguments | Name | Type | Description | | ---- | ---- | ----------- | -| `id` | [`MergeRequestID!`](#mergerequestid) | Global ID of the merge request. | +| `assigneeUsernames` | [`[String!]`](#string) | Usernames of users assigned to the issue. | +| `authorUsername` | [`String`](#string) | Username of the author of the issue. | +| `from` | [`Time!`](#time) | Timestamp marking the start date and time. | +| `labelNames` | [`[String!]`](#string) | Labels applied to the issue. | +| `milestoneTitle` | [`String`](#string) | Milestone applied to the issue. | +| `projectIds` | [`[ID!]`](#id) | Project IDs within the group hierarchy. | +| `to` | [`Time!`](#time) | Timestamp marking the end date and time. | -### `GroupPermissions` +##### `GroupValueStreamAnalyticsFlowMetrics.deploymentCount` -#### Fields +Number of production deployments in the given period. + +Returns [`ValueStreamAnalyticsMetric`](#valuestreamanalyticsmetric). + +###### Arguments | Name | Type | Description | | ---- | ---- | ----------- | -| `createProjects` | [`Boolean!`](#boolean) | Indicates the user can perform `create_projects` on this resource. | -| `readGroup` | [`Boolean!`](#boolean) | Indicates the user can perform `read_group` on this resource. | +| `from` | [`Time!`](#time) | Timestamp marking the start date and time. | +| `projectIds` | [`[ID!]`](#id) | Project IDs within the group hierarchy. | +| `to` | [`Time!`](#time) | Timestamp marking the end date and time. | -### `GroupReleaseStats` +##### `GroupValueStreamAnalyticsFlowMetrics.issueCount` -Contains release-related statistics about a group. +Number of issues opened in the given period. -#### Fields +Returns [`ValueStreamAnalyticsMetric`](#valuestreamanalyticsmetric). + +###### Arguments | Name | Type | Description | | ---- | ---- | ----------- | -| `releasesCount` | [`Int`](#int) | Total number of releases in all descendant projects of the group. | -| `releasesPercentage` | [`Int`](#int) | Percentage of the group's descendant projects that have at least one release. | +| `assigneeUsernames` | [`[String!]`](#string) | Usernames of users assigned to the issue. | +| `authorUsername` | [`String`](#string) | Username of the author of the issue. | +| `from` | [`Time!`](#time) | Timestamp marking the start date and time. | +| `labelNames` | [`[String!]`](#string) | Labels applied to the issue. | +| `milestoneTitle` | [`String`](#string) | Milestone applied to the issue. | +| `projectIds` | [`[ID!]`](#id) | Project IDs within the group hierarchy. | +| `to` | [`Time!`](#time) | Timestamp marking the end date and time. | -### `GroupSecurityPolicySource` +##### `GroupValueStreamAnalyticsFlowMetrics.issuesCompletedCount` -Represents the source of a security policy belonging to a group. +Number of open issues closed (completed) in the given period. Maximum value is 10,001. -#### Fields +Returns [`ValueStreamAnalyticsMetric`](#valuestreamanalyticsmetric). + +###### Arguments | Name | Type | Description | | ---- | ---- | ----------- | -| `inherited` | [`Boolean!`](#boolean) | Indicates whether this policy is inherited from parent group. | -| `namespace` | [`Namespace`](#namespace) | Project the policy is associated with. | +| `assigneeUsernames` | [`[String!]`](#string) | Usernames of users assigned to the issue. | +| `authorUsername` | [`String`](#string) | Username of the author of the issue. | +| `from` | [`Time!`](#time) | Timestamp marking the start date and time. | +| `labelNames` | [`[String!]`](#string) | Labels applied to the issue. | +| `milestoneTitle` | [`String`](#string) | Milestone applied to the issue. | +| `projectIds` | [`[ID!]`](#id) | Project IDs within the group hierarchy. | +| `to` | [`Time!`](#time) | Timestamp marking the end date and time. | -### `GroupStats` +##### `GroupValueStreamAnalyticsFlowMetrics.leadTime` -Contains statistics about a group. +Median time from when the issue was created to when it was closed. -#### Fields +Returns [`ValueStreamAnalyticsMetric`](#valuestreamanalyticsmetric). + +###### Arguments | Name | Type | Description | | ---- | ---- | ----------- | -| `releaseStats` | [`GroupReleaseStats`](#groupreleasestats) | Statistics related to releases within the group. | +| `assigneeUsernames` | [`[String!]`](#string) | Usernames of users assigned to the issue. | +| `authorUsername` | [`String`](#string) | Username of the author of the issue. | +| `from` | [`Time!`](#time) | Timestamp marking the start date and time. | +| `labelNames` | [`[String!]`](#string) | Labels applied to the issue. | +| `milestoneTitle` | [`String`](#string) | Milestone applied to the issue. | +| `projectIds` | [`[ID!]`](#id) | Project IDs within the group hierarchy. | +| `to` | [`Time!`](#time) | Timestamp marking the end date and time. | ### `GroupWikiRepositoryRegistry` @@ -14968,6 +16160,36 @@ A block of time for which a participant is on-call. | `participant` | [`OncallParticipantType`](#oncallparticipanttype) | Participant assigned to the on-call shift. | | `startsAt` | [`Time`](#time) | Start time of the on-call shift. | +### `InheritedCiVariable` + +CI/CD variables a project inherites from its parent group and ancestors. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `environmentScope` | [`String`](#string) | Scope defining the environments that can use the variable. | +| `groupCiCdSettingsPath` | [`String`](#string) | Indicates the path to the CI/CD settings of the group the variable belongs to. | +| `groupName` | [`String`](#string) | Indicates group the variable belongs to. | +| `id` | [`ID!`](#id) | ID of the variable. | +| `key` | [`String`](#string) | Name of the variable. | +| `masked` | [`Boolean`](#boolean) | Indicates whether the variable is masked. | +| `protected` | [`Boolean`](#boolean) | Indicates whether the variable is protected. | +| `raw` | [`Boolean`](#boolean) | Indicates whether the variable is raw. | +| `variableType` | [`CiVariableType`](#civariabletype) | Type of the variable. | + +### `InstanceExternalAuditEventDestination` + +Represents an external resource to send instance audit events to. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `destinationUrl` | [`String!`](#string) | External destination to send audit events to. | +| `id` | [`ID!`](#id) | ID of the destination. | +| `verificationToken` | [`String!`](#string) | Verification token to validate source of event. | + ### `InstanceSecurityDashboard` #### Fields @@ -14992,6 +16214,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `hasRemoteDevelopmentAgentConfig` | [`Boolean`](#boolean) | Returns only cluster agents which have an associated remote development agent config. | | `hasVulnerabilities` | [`Boolean`](#boolean) | Returns only cluster agents which have vulnerabilities. | ##### `InstanceSecurityDashboard.projects` @@ -15212,6 +16435,7 @@ Check permissions for the current user on a issue. | `readDesign` | [`Boolean!`](#boolean) | Indicates the user can perform `read_design` on this resource. | | `readIssue` | [`Boolean!`](#boolean) | Indicates the user can perform `read_issue` on this resource. | | `reopenIssue` | [`Boolean!`](#boolean) | Indicates the user can perform `reopen_issue` on this resource. | +| `updateDesign` | [`Boolean!`](#boolean) | Indicates the user can perform `update_design` on this resource. | | `updateIssue` | [`Boolean!`](#boolean) | Indicates the user can perform `update_issue` on this resource. | ### `IssueStatusCountsType` @@ -15456,6 +16680,15 @@ Represents an entry from the Cloud License history. | `type` | [`String!`](#string) | Type of the license. | | `usersInLicenseCount` | [`Int`](#int) | Number of paid users in the license. | +### `Location` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `blobPath` | [`String`](#string) | HTTP URI path to view the input file in GitLab. | +| `path` | [`String`](#string) | Path, relative to the root of the repository, of the filewhich was analyzed to detect the dependency. | + ### `MavenMetadata` Maven metadata. @@ -15502,6 +16735,7 @@ Defines which user roles, users, or groups can merge into a protected branch. | `autoMergeEnabled` | [`Boolean!`](#boolean) | Indicates if auto merge is enabled for the merge request. | | `autoMergeStrategy` | [`String`](#string) | Selected auto merge strategy. | | `availableAutoMergeStrategies` | [`[String!]`](#string) | Array of available auto merge strategies. | +| `awardEmoji` | [`AwardEmojiConnection`](#awardemojiconnection) | List of award emojis associated with the merge request. (see [Connections](#connections)) | | `commenters` | [`UserCoreConnection!`](#usercoreconnection) | All commenters on this noteable. (see [Connections](#connections)) | | `commitCount` | [`Int`](#int) | Number of commits in the merge request. | | `commits` | [`CommitConnection`](#commitconnection) | Merge request commits. (see [Connections](#connections)) | @@ -15546,6 +16780,7 @@ Defines which user roles, users, or groups can merge into a protected branch. | `milestone` | [`Milestone`](#milestone) | Milestone of the merge request. | | `notes` | [`NoteConnection!`](#noteconnection) | All notes on this noteable. (see [Connections](#connections)) | | `participants` | [`MergeRequestParticipantConnection`](#mergerequestparticipantconnection) | Participants in the merge request. This includes the author, assignees, reviewers, and users mentioned in notes. (see [Connections](#connections)) | +| `preparedAt` | [`Time`](#time) | Timestamp of when the merge request was prepared. | | `project` | [`Project!`](#project) | Alias for target_project. | | `projectId` | [`Int!`](#int) | ID of the merge request project. | | `rebaseCommitSha` | [`String`](#string) | Rebase commit SHA of the merge request. | @@ -15564,7 +16799,7 @@ Defines which user roles, users, or groups can merge into a protected branch. | `squashOnMerge` | [`Boolean!`](#boolean) | Indicates if the merge request will be squashed when merged. | | `state` | [`MergeRequestState!`](#mergerequeststate) | State of the merge request. | | `subscribed` | [`Boolean!`](#boolean) | Indicates if the currently logged in user is subscribed to this merge request. | -| `suggestedReviewers` **{warning-solid}** | [`SuggestedReviewersType`](#suggestedreviewerstype) | **Introduced** in 15.4. This feature is in Alpha. It can be changed or removed at any time. Suggested reviewers for merge request. Returns `null` if `suggested_reviewers` feature flag is disabled. This flag is disabled by default and only available on GitLab.com because the feature is experimental and is subject to change without notice. | +| `suggestedReviewers` **{warning-solid}** | [`SuggestedReviewersType`](#suggestedreviewerstype) | **Introduced** in 15.4. This feature is an Experiment. It can be changed or removed at any time. Suggested reviewers for merge request. Returns `null` if `suggested_reviewers` feature flag is disabled. This flag is disabled by default and only available on GitLab.com because the feature is experimental and is subject to change without notice. | | `targetBranch` | [`String!`](#string) | Target branch of the merge request. | | `targetBranchExists` | [`Boolean!`](#boolean) | Indicates if the target branch of the merge request exists. | | `targetProject` | [`Project!`](#project) | Target project of the merge request. | @@ -15690,6 +16925,7 @@ A user assigned to a merge request. | `savedReplies` | [`SavedReplyConnection`](#savedreplyconnection) | Saved replies authored by the user. Will not return saved replies if `saved_replies` feature flag is disabled. (see [Connections](#connections)) | | `state` | [`UserState!`](#userstate) | State of the user. | | `status` | [`UserStatus`](#userstatus) | User status. | +| `userAchievements` **{warning-solid}** | [`UserAchievementConnection`](#userachievementconnection) | **Introduced** in 15.10. This feature is an Experiment. It can be changed or removed at any time. Achievements for the user. Only returns for namespaces where the `achievements` feature flag is enabled. | | `userPermissions` | [`UserPermissions!`](#userpermissions) | Permissions for the current user on the resource. | | `username` | [`String!`](#string) | Username of the user. Unique within this instance of GitLab. | | `webPath` | [`String!`](#string) | Web path of the user. | @@ -15711,6 +16947,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `authorUsername` | [`String`](#string) | Username of the author. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | | `createdBefore` | [`Time`](#time) | Merge requests created before this timestamp. | @@ -15745,6 +16982,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `assigneeUsername` | [`String`](#string) | Username of the assignee. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | | `createdBefore` | [`Time`](#time) | Merge requests created before this timestamp. | @@ -15796,6 +17034,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `assigneeUsername` | [`String`](#string) | Username of the assignee. | | `authorUsername` | [`String`](#string) | Username of the author. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | @@ -15906,6 +17145,22 @@ four standard [pagination arguments](#connection-pagination-arguments): | `state` | [`[TodoStateEnum!]`](#todostateenum) | State of the todo. | | `type` | [`[TodoTargetEnum!]`](#todotargetenum) | Type of the todo. | +##### `MergeRequestAssignee.workspaces` + +Workspaces owned by the current user. + +Returns [`WorkspaceConnection`](#workspaceconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `ids` | [`[RemoteDevelopmentWorkspaceID!]`](#remotedevelopmentworkspaceid) | Array of global workspace IDs. For example, `["gid://gitlab/RemoteDevelopment::Workspace/1"]`. | + ### `MergeRequestAuthor` The author of the merge request. @@ -15936,6 +17191,7 @@ The author of the merge request. | `savedReplies` | [`SavedReplyConnection`](#savedreplyconnection) | Saved replies authored by the user. Will not return saved replies if `saved_replies` feature flag is disabled. (see [Connections](#connections)) | | `state` | [`UserState!`](#userstate) | State of the user. | | `status` | [`UserStatus`](#userstatus) | User status. | +| `userAchievements` **{warning-solid}** | [`UserAchievementConnection`](#userachievementconnection) | **Introduced** in 15.10. This feature is an Experiment. It can be changed or removed at any time. Achievements for the user. Only returns for namespaces where the `achievements` feature flag is enabled. | | `userPermissions` | [`UserPermissions!`](#userpermissions) | Permissions for the current user on the resource. | | `username` | [`String!`](#string) | Username of the user. Unique within this instance of GitLab. | | `webPath` | [`String!`](#string) | Web path of the user. | @@ -15957,6 +17213,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `authorUsername` | [`String`](#string) | Username of the author. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | | `createdBefore` | [`Time`](#time) | Merge requests created before this timestamp. | @@ -15991,6 +17248,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `assigneeUsername` | [`String`](#string) | Username of the assignee. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | | `createdBefore` | [`Time`](#time) | Merge requests created before this timestamp. | @@ -16042,6 +17300,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `assigneeUsername` | [`String`](#string) | Username of the assignee. | | `authorUsername` | [`String`](#string) | Username of the author. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | @@ -16152,6 +17411,22 @@ four standard [pagination arguments](#connection-pagination-arguments): | `state` | [`[TodoStateEnum!]`](#todostateenum) | State of the todo. | | `type` | [`[TodoTargetEnum!]`](#todotargetenum) | Type of the todo. | +##### `MergeRequestAuthor.workspaces` + +Workspaces owned by the current user. + +Returns [`WorkspaceConnection`](#workspaceconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `ids` | [`[RemoteDevelopmentWorkspaceID!]`](#remotedevelopmentworkspaceid) | Array of global workspace IDs. For example, `["gid://gitlab/RemoteDevelopment::Workspace/1"]`. | + ### `MergeRequestDiffRegistry` Represents the Geo sync and verification state of a Merge Request diff. @@ -16201,6 +17476,7 @@ A user participating in a merge request. | `savedReplies` | [`SavedReplyConnection`](#savedreplyconnection) | Saved replies authored by the user. Will not return saved replies if `saved_replies` feature flag is disabled. (see [Connections](#connections)) | | `state` | [`UserState!`](#userstate) | State of the user. | | `status` | [`UserStatus`](#userstatus) | User status. | +| `userAchievements` **{warning-solid}** | [`UserAchievementConnection`](#userachievementconnection) | **Introduced** in 15.10. This feature is an Experiment. It can be changed or removed at any time. Achievements for the user. Only returns for namespaces where the `achievements` feature flag is enabled. | | `userPermissions` | [`UserPermissions!`](#userpermissions) | Permissions for the current user on the resource. | | `username` | [`String!`](#string) | Username of the user. Unique within this instance of GitLab. | | `webPath` | [`String!`](#string) | Web path of the user. | @@ -16222,6 +17498,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `authorUsername` | [`String`](#string) | Username of the author. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | | `createdBefore` | [`Time`](#time) | Merge requests created before this timestamp. | @@ -16256,6 +17533,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `assigneeUsername` | [`String`](#string) | Username of the assignee. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | | `createdBefore` | [`Time`](#time) | Merge requests created before this timestamp. | @@ -16307,6 +17585,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `assigneeUsername` | [`String`](#string) | Username of the assignee. | | `authorUsername` | [`String`](#string) | Username of the author. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | @@ -16417,6 +17696,22 @@ four standard [pagination arguments](#connection-pagination-arguments): | `state` | [`[TodoStateEnum!]`](#todostateenum) | State of the todo. | | `type` | [`[TodoTargetEnum!]`](#todotargetenum) | Type of the todo. | +##### `MergeRequestParticipant.workspaces` + +Workspaces owned by the current user. + +Returns [`WorkspaceConnection`](#workspaceconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `ids` | [`[RemoteDevelopmentWorkspaceID!]`](#remotedevelopmentworkspaceid) | Array of global workspace IDs. For example, `["gid://gitlab/RemoteDevelopment::Workspace/1"]`. | + ### `MergeRequestPermissions` Check permissions for the current user on a merge request. @@ -16466,6 +17761,7 @@ A user assigned to a merge request as a reviewer. | `savedReplies` | [`SavedReplyConnection`](#savedreplyconnection) | Saved replies authored by the user. Will not return saved replies if `saved_replies` feature flag is disabled. (see [Connections](#connections)) | | `state` | [`UserState!`](#userstate) | State of the user. | | `status` | [`UserStatus`](#userstatus) | User status. | +| `userAchievements` **{warning-solid}** | [`UserAchievementConnection`](#userachievementconnection) | **Introduced** in 15.10. This feature is an Experiment. It can be changed or removed at any time. Achievements for the user. Only returns for namespaces where the `achievements` feature flag is enabled. | | `userPermissions` | [`UserPermissions!`](#userpermissions) | Permissions for the current user on the resource. | | `username` | [`String!`](#string) | Username of the user. Unique within this instance of GitLab. | | `webPath` | [`String!`](#string) | Web path of the user. | @@ -16487,6 +17783,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `authorUsername` | [`String`](#string) | Username of the author. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | | `createdBefore` | [`Time`](#time) | Merge requests created before this timestamp. | @@ -16521,6 +17818,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `assigneeUsername` | [`String`](#string) | Username of the assignee. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | | `createdBefore` | [`Time`](#time) | Merge requests created before this timestamp. | @@ -16572,6 +17870,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `assigneeUsername` | [`String`](#string) | Username of the assignee. | | `authorUsername` | [`String`](#string) | Username of the author. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | @@ -16682,6 +17981,22 @@ four standard [pagination arguments](#connection-pagination-arguments): | `state` | [`[TodoStateEnum!]`](#todostateenum) | State of the todo. | | `type` | [`[TodoTargetEnum!]`](#todotargetenum) | Type of the todo. | +##### `MergeRequestReviewer.workspaces` + +Workspaces owned by the current user. + +Returns [`WorkspaceConnection`](#workspaceconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `ids` | [`[RemoteDevelopmentWorkspaceID!]`](#remotedevelopmentworkspaceid) | Array of global workspace IDs. For example, `["gid://gitlab/RemoteDevelopment::Workspace/1"]`. | + ### `Metadata` #### Fields @@ -16803,7 +18118,6 @@ Contains statistics about a milestone. | Name | Type | Description | | ---- | ---- | ----------- | -| `achievements` **{warning-solid}** | [`AchievementConnection`](#achievementconnection) | **Introduced** in 15.8. This feature is in Alpha. It can be changed or removed at any time. Achievements for the namespace. Returns `null` if the `achievements` feature flag is disabled. | | `actualRepositorySizeLimit` | [`Float`](#float) | Size limit for repositories in the namespace in bytes. | | `additionalPurchasedStorageSize` | [`Float`](#float) | Additional storage purchased for the root namespace in bytes. | | `containsLockedProjects` | [`Boolean!`](#boolean) | Includes at least one project where the repository size exceeds the limit. | @@ -16824,13 +18138,33 @@ Contains statistics about a milestone. | `sharedRunnersSetting` | [`SharedRunnersSetting`](#sharedrunnerssetting) | Shared runners availability for the namespace and its descendants. | | `storageSizeLimit` | [`Float`](#float) | Total storage limit of the root namespace in bytes. | | `temporaryStorageIncreaseEndsOn` | [`Time`](#time) | Date until the temporary storage increase is active. | -| `timelogCategories` **{warning-solid}** | [`TimeTrackingTimelogCategoryConnection`](#timetrackingtimelogcategoryconnection) | **Introduced** in 15.3. This feature is in Alpha. It can be changed or removed at any time. Timelog categories for the namespace. | +| `timelogCategories` **{warning-solid}** | [`TimeTrackingTimelogCategoryConnection`](#timetrackingtimelogcategoryconnection) | **Introduced** in 15.3. This feature is an Experiment. It can be changed or removed at any time. Timelog categories for the namespace. | | `totalRepositorySize` | [`Float`](#float) | Total repository size of all projects in the root namespace in bytes. | | `totalRepositorySizeExcess` | [`Float`](#float) | Total excess repository size of all projects in the root namespace in bytes. | | `visibility` | [`String`](#string) | Visibility of the namespace. | #### Fields with arguments +##### `Namespace.achievements` + +Achievements for the namespace. Returns `null` if the `achievements` feature flag is disabled. + +WARNING: +**Introduced** in 15.8. +This feature is an Experiment. It can be changed or removed at any time. + +Returns [`AchievementConnection`](#achievementconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `ids` | [`[AchievementsAchievementID!]`](#achievementsachievementid) | Filter achievements by IDs. | + ##### `Namespace.complianceFrameworks` Compliance frameworks available to projects in this namespace. @@ -16861,6 +18195,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `complianceFrameworkFilters` | [`ComplianceFrameworkFilters`](#complianceframeworkfilters) | Filters applied when selecting a compliance framework. | | `hasCodeCoverage` | [`Boolean`](#boolean) | Returns only the projects which have code coverage. | | `hasVulnerabilities` | [`Boolean`](#boolean) | Returns only the projects which have vulnerabilities. | | `ids` | [`[ID!]`](#id) | Filter projects by IDs. | @@ -16884,7 +18219,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | -| `actionScanTypes` | [`[SecurityReportTypeEnum!]`](#securityreporttypeenum) | Filters policies by the action scan type. Only these scan types are supported: `dast`, `secret_detection`, `cluster_image_scanning`, `container_scanning`, `sast`, `dependency_scanning`. | +| `actionScanTypes` | [`[SecurityReportTypeEnum!]`](#securityreporttypeenum) | Filters policies by the action scan type. Only these scan types are supported: `dast`, `secret_detection`, `cluster_image_scanning`, `container_scanning`, `sast`, `sast_iac`, `dependency_scanning`. | | `relationship` | [`SecurityPolicyRelationType`](#securitypolicyrelationtype) | Filter policies by the given policy relationship. | ##### `Namespace.scanResultPolicies` @@ -17450,6 +18785,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `jobKind` | [`CiJobKind`](#cijobkind) | Filter jobs by kind. | | `retried` | [`Boolean`](#boolean) | Filter jobs by retry-status. | | `securityReportTypes` | [`[SecurityReportTypeEnum!]`](#securityreporttypeenum) | Filter jobs by the type of security report they produce. | | `statuses` | [`[CiJobStatus!]`](#cijobstatus) | Filter jobs by status. | @@ -17615,7 +18951,7 @@ Represents a pipeline schedule. | ---- | ---- | ----------- | | `adminPipelineSchedule` | [`Boolean!`](#boolean) | Indicates the user can perform `admin_pipeline_schedule` on this resource. | | `playPipelineSchedule` | [`Boolean!`](#boolean) | Indicates the user can perform `play_pipeline_schedule` on this resource. | -| `takeOwnershipPipelineSchedule` | [`Boolean!`](#boolean) | Indicates the user can perform `take_ownership_pipeline_schedule` on this resource. | +| `takeOwnershipPipelineSchedule` **{warning-solid}** | [`Boolean!`](#boolean) | **Deprecated** in 15.9. Use admin_pipeline_schedule permission to determine if the user can take ownership of a pipeline schedule. | | `updatePipelineSchedule` | [`Boolean!`](#boolean) | Indicates the user can perform `update_pipeline_schedule` on this resource. | ### `PipelineScheduleVariable` @@ -17639,7 +18975,6 @@ Represents vulnerability finding of a security report on the pipeline. | Name | Type | Description | | ---- | ---- | ----------- | | `assets` | [`[AssetType!]`](#assettype) | List of assets associated with the vulnerability. | -| `confidence` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.4. This field will be removed from the Finding domain model. | | `description` | [`String`](#string) | Description of the vulnerability finding. | | `descriptionHtml` | [`String`](#string) | GitLab Flavored Markdown rendering of `description`. | | `details` | [`[VulnerabilityDetail!]!`](#vulnerabilitydetail) | Details of the security finding. | @@ -17653,9 +18988,8 @@ Represents vulnerability finding of a security report on the pipeline. | `links` | [`[VulnerabilityLink!]`](#vulnerabilitylink) | List of links associated with the vulnerability. | | `location` | [`VulnerabilityLocation`](#vulnerabilitylocation) | Location metadata for the vulnerability. Its fields depend on the type of security scan that found the vulnerability. | | `mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request that fixes the vulnerability. | -| `name` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.1. Use `title`. | | `project` | [`Project`](#project) | Project on which the vulnerability finding was found. | -| `projectFingerprint` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.1. The `project_fingerprint` attribute is being deprecated. Use `uuid` to identify findings. | +| `projectFingerprint` **{warning-solid}** | [`String`](#string) | **Introduced** in 16.0. This feature is an Experiment. It can be changed or removed at any time. Fingerprint of the vulnerability finding. | | `remediations` | [`[VulnerabilityRemediationType!]`](#vulnerabilityremediationtype) | Remediations of the security report finding. | | `reportType` | [`VulnerabilityReportType`](#vulnerabilityreporttype) | Type of the security report that found the vulnerability finding. | | `scanner` | [`VulnerabilityScanner`](#vulnerabilityscanner) | Scanner metadata for the vulnerability. | @@ -17687,6 +19021,7 @@ Represents a product analytics dashboard. | ---- | ---- | ----------- | | `description` | [`String`](#string) | Description of the dashboard. | | `panels` | [`ProductAnalyticsDashboardPanelConnection!`](#productanalyticsdashboardpanelconnection) | Panels shown on the dashboard. (see [Connections](#connections)) | +| `slug` | [`String!`](#string) | Slug of the dashboard. | | `title` | [`String!`](#string) | Title of the dashboard. | ### `ProductAnalyticsDashboardPanel` @@ -17721,12 +19056,14 @@ Represents a product analytics dashboard visualization. | ---- | ---- | ----------- | | `actualRepositorySizeLimit` | [`Float`](#float) | Size limit for the repository in bytes. | | `agentConfigurations` | [`AgentConfigurationConnection`](#agentconfigurationconnection) | Agent configurations defined by the project. (see [Connections](#connections)) | +| `aiConversations` **{warning-solid}** | [`ProjectConversations`](#projectconversations) | **Introduced** in 16.0. This feature is an Experiment. It can be changed or removed at any time. Ai Chat conversations related to a given project. | | `allowMergeOnSkippedPipeline` | [`Boolean`](#boolean) | If `only_allow_merge_if_pipeline_succeeds` is true, indicates if merge requests of the project can also be merged with skipped jobs. | | `apiFuzzingCiConfiguration` | [`ApiFuzzingCiConfiguration`](#apifuzzingciconfiguration) | API fuzzing configuration for the project. | | `archived` | [`Boolean`](#boolean) | Indicates the archived status of the project. | | `autocloseReferencedIssues` | [`Boolean`](#boolean) | Indicates if issues referenced by merge requests and commits within the default branch are closed automatically. | | `avatarUrl` | [`String`](#string) | URL to avatar image file of the project. | | `branchRules` | [`BranchRuleConnection`](#branchruleconnection) | Branch rules configured for the project. (see [Connections](#connections)) | +| `ciAccessAuthorizedAgents` | [`ClusterAgentAuthorizationCiAccessConnection`](#clusteragentauthorizationciaccessconnection) | Authorized cluster agents for the project through ci_access keyword. (see [Connections](#connections)) | | `ciCdSettings` | [`ProjectCiCdSetting`](#projectcicdsetting) | CI/CD settings for the project. | | `ciConfigPathOrDefault` | [`String!`](#string) | Path of the CI configuration file. | | `ciJobTokenScope` | [`CiJobTokenScopeType`](#cijobtokenscopetype) | The CI Job Tokens scope of access. | @@ -17742,23 +19079,27 @@ Represents a product analytics dashboard visualization. | `description` | [`String`](#string) | Short description of the project. | | `descriptionHtml` | [`String`](#string) | GitLab Flavored Markdown rendering of `description`. | | `dora` | [`Dora`](#dora) | Project's DORA metrics. | +| `flowMetrics` **{warning-solid}** | [`ProjectValueStreamAnalyticsFlowMetrics`](#projectvaluestreamanalyticsflowmetrics) | **Introduced** in 15.10. This feature is an Experiment. It can be changed or removed at any time. Flow metrics for value stream analytics. | | `forksCount` | [`Int!`](#int) | Number of times the project has been forked. | | `fullPath` | [`ID!`](#id) | Full path of the project. | | `grafanaIntegration` | [`GrafanaIntegration`](#grafanaintegration) | Grafana integration details for the project. | | `group` | [`Group`](#group) | Group of the project. | +| `hasJiraVulnerabilityIssueCreationEnabled` | [`Boolean!`](#boolean) | Indicates whether Jira issue creation from vulnerabilities is enabled. | | `httpUrlToRepo` | [`String`](#string) | URL to connect to the project via HTTPS. | | `id` | [`ID!`](#id) | ID of the project. | | `importStatus` | [`String`](#string) | Status of import background job of the project. | | `incidentManagementTimelineEventTags` | [`[TimelineEventTagType!]`](#timelineeventtagtype) | Timeline event tags for the project. | +| `inheritedCiVariables` | [`InheritedCiVariableConnection`](#inheritedcivariableconnection) | List of CI/CD variables the project inherited from its parent group and ancestors. (see [Connections](#connections)) | +| `isCatalogResource` **{warning-solid}** | [`Boolean`](#boolean) | **Introduced** in 15.11. This feature is an Experiment. It can be changed or removed at any time. Indicates if a project is a catalog resource. | | `issuesEnabled` | [`Boolean`](#boolean) | Indicates if Issues are enabled for the current user. | | `jiraImportStatus` | [`String`](#string) | Status of Jira import background job of the project. | | `jiraImports` | [`JiraImportConnection`](#jiraimportconnection) | Jira imports into the project. (see [Connections](#connections)) | -| `jitsuKey` **{warning-solid}** | [`String`](#string) | **Introduced** in 15.7. This feature is in Alpha. It can be changed or removed at any time. Jitsu key assigned to the project. | | `jobsEnabled` | [`Boolean`](#boolean) | Indicates if CI/CD pipeline jobs are enabled for the current user. | | `languages` | [`[RepositoryLanguage!]`](#repositorylanguage) | Programming languages used in the project. | | `lastActivityAt` | [`Time`](#time) | Timestamp of the project last activity. | | `lfsEnabled` | [`Boolean`](#boolean) | Indicates if the project has Large File Storage (LFS) enabled. | | `mergeCommitTemplate` | [`String`](#string) | Template used to create merge commit message in merge requests. | +| `mergeRequestsDisableCommittersApproval` | [`Boolean!`](#boolean) | Indicates that committers of the given merge request cannot approve. | | `mergeRequestsEnabled` | [`Boolean`](#boolean) | Indicates if Merge Requests are enabled for the current user. | | `mergeRequestsFfOnlyEnabled` | [`Boolean`](#boolean) | Indicates if no merge commits should be created and all merges should instead be fast-forwarded, which means that merging is only allowed if the branch could be fast-forwarded. | | `name` | [`String!`](#string) | Name of the project (without namespace). | @@ -17773,6 +19114,8 @@ Represents a product analytics dashboard visualization. | `pathLocks` | [`PathLockConnection`](#pathlockconnection) | The project's path locks. (see [Connections](#connections)) | | `pipelineAnalytics` | [`PipelineAnalytics`](#pipelineanalytics) | Pipeline analytics. | | `printingMergeRequestLinkEnabled` | [`Boolean`](#boolean) | Indicates if a link to create or view a merge request should display after a push to Git repositories of the project from the command line. | +| `productAnalyticsInstrumentationKey` **{warning-solid}** | [`String`](#string) | **Introduced** in 16.0. This feature is an Experiment. It can be changed or removed at any time. Product Analytics instrumentation key assigned to the project. | +| `productAnalyticsState` **{warning-solid}** | [`ProductAnalyticsState`](#productanalyticsstate) | **Introduced** in 15.10. This feature is an Experiment. It can be changed or removed at any time. Current state of the product analytics stack for this project.Can only be called for one project in a single request. | | `publicJobs` | [`Boolean`](#boolean) | Indicates if there is public access to pipelines and job details of the project, including output logs and artifacts. | | `pushRules` | [`PushRules`](#pushrules) | Project's push rules settings. | | `recentIssueBoards` | [`BoardConnection`](#boardconnection) | List of recently visited boards of the project. Maximum size is 4. (see [Connections](#connections)) | @@ -17794,11 +19137,14 @@ Represents a product analytics dashboard visualization. | `sshUrlToRepo` | [`String`](#string) | URL to connect to the project via SSH. | | `starCount` | [`Int!`](#int) | Number of times the project has been starred. | | `statistics` | [`ProjectStatistics`](#projectstatistics) | Statistics of the project. | +| `statisticsDetailsPaths` | [`ProjectStatisticsRedirect`](#projectstatisticsredirect) | Redirects for Statistics of the project. | | `suggestionCommitMessage` | [`String`](#string) | Commit message used to apply merge request suggestions. | | `tagList` **{warning-solid}** | [`String`](#string) | **Deprecated** in 13.12. Use `topics`. | | `terraformStates` | [`TerraformStateConnection`](#terraformstateconnection) | Terraform states associated with the project. (see [Connections](#connections)) | -| `timelogCategories` **{warning-solid}** | [`TimeTrackingTimelogCategoryConnection`](#timetrackingtimelogcategoryconnection) | **Introduced** in 15.3. This feature is in Alpha. It can be changed or removed at any time. Timelog categories for the project. | +| `timelogCategories` **{warning-solid}** | [`TimeTrackingTimelogCategoryConnection`](#timetrackingtimelogcategoryconnection) | **Introduced** in 15.3. This feature is an Experiment. It can be changed or removed at any time. Timelog categories for the project. | | `topics` | [`[String!]`](#string) | List of project topics. | +| `trackingKey` **{warning-solid}** | [`String`](#string) | **Introduced** in 16.0. This feature is an Experiment. It can be changed or removed at any time. Tracking key assigned to the project. | +| `userAccessAuthorizedAgents` | [`ClusterAgentAuthorizationUserAccessConnection`](#clusteragentauthorizationuseraccessconnection) | Authorized cluster agents for the project through user_access keyword. (see [Connections](#connections)) | | `userPermissions` | [`ProjectPermissions!`](#projectpermissions) | Permissions for the current user on the resource. | | `visibility` | [`String`](#string) | Visibility of the project. | | `vulnerabilityImages` | [`VulnerabilityContainerImageConnection`](#vulnerabilitycontainerimageconnection) | Container images reported on the project vulnerabilities. (see [Connections](#connections)) | @@ -17937,7 +19283,7 @@ CI/CD config variable. WARNING: **Introduced** in 15.3. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Returns [`[CiConfigVariable!]`](#ciconfigvariable). @@ -17945,7 +19291,7 @@ Returns [`[CiConfigVariable!]`](#ciconfigvariable). | Name | Type | Description | | ---- | ---- | ----------- | -| `sha` | [`String!`](#string) | Sha. | +| `ref` | [`String!`](#string) | Ref. | ##### `Project.ciTemplate` @@ -17985,6 +19331,7 @@ Returns [`ClusterAgent`](#clusteragent). | Name | Type | Description | | ---- | ---- | ----------- | +| `hasRemoteDevelopmentAgentConfig` | [`Boolean`](#boolean) | Returns only cluster agents which have an associated remote development agent config. | | `hasVulnerabilities` | [`Boolean`](#boolean) | Returns only cluster agents which have vulnerabilities. | | `name` | [`String!`](#string) | Name of the cluster agent. | @@ -18002,8 +19349,25 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `hasRemoteDevelopmentAgentConfig` | [`Boolean`](#boolean) | Returns only cluster agents which have an associated remote development agent config. | | `hasVulnerabilities` | [`Boolean`](#boolean) | Returns only cluster agents which have vulnerabilities. | +##### `Project.commitReferences` + +Get tag names containing a given commit. + +WARNING: +**Introduced** in 16.0. +This feature is an Experiment. It can be changed or removed at any time. + +Returns [`CommitReferences`](#commitreferences). + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `commitSha` | [`String!`](#string) | Project commit SHA identifier. For example, `287774414568010855642518513f085491644061`. | + ##### `Project.containerRepositories` Container repositories of the project. @@ -18089,9 +19453,30 @@ Returns [`ProjectDataTransfer`](#projectdatatransfer). | Name | Type | Description | | ---- | ---- | ----------- | -| `from` | [`Date`](#date) | Retain egress data for 1 year. Current month will increase dynamically as egress occurs. | +| `from` | [`Date`](#date) | Retain egress data for one year. Data for the current month will increase dynamically as egress occurs. | | `to` | [`Date`](#date) | End date for the data. | +##### `Project.dependencies` + +Software dependencies used by the project. + +WARNING: +**Introduced** in 15.9. +This feature is an Experiment. It can be changed or removed at any time. + +Returns [`DependencyConnection`](#dependencyconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `packageManagers` | [`[PackageManager!]`](#packagemanager) | Filter dependencies by package managers. | +| `sort` | [`DependencySort`](#dependencysort) | Sort dependencies by given criteria. | + ##### `Project.deployment` Details of the deployment of the project. @@ -18144,7 +19529,7 @@ Details of the fork project compared to its upstream project. WARNING: **Introduced** in 15.7. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Returns [`ForkDetails`](#forkdetails). @@ -18433,7 +19818,6 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | -| `endDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.end. | | `id` | [`ID`](#id) | Global ID of the Iteration to look up. | | `iid` | [`ID`](#id) | Internal ID of the Iteration to look up. | | `in` | [`[IterationSearchableField!]`](#iterationsearchablefield) | Fields in which the fuzzy-search should be performed with the query given in the argument `search`. Defaults to `[title]`. | @@ -18441,7 +19825,6 @@ four standard [pagination arguments](#connection-pagination-arguments): | `iterationCadenceIds` | [`[IterationsCadenceID!]`](#iterationscadenceid) | Global iteration cadence IDs by which to look up the iterations. | | `search` | [`String`](#string) | Query used for fuzzy-searching in the fields selected in the argument `in`. Returns all iterations if empty. | | `sort` | [`IterationSort`](#iterationsort) | List iterations by sort order. If unspecified, an arbitrary order (subject to change) is used. | -| `startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | | `state` | [`IterationState`](#iterationstate) | Filter iterations by state. | | `timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | | `title` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.4. The argument will be removed in 15.4. Please use `search` and `in` fields instead. | @@ -18530,6 +19913,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `assigneeUsername` | [`String`](#string) | Username of the assignee. | | `authorUsername` | [`String`](#string) | Username of the author. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | @@ -18564,12 +19948,10 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | `containingDate` | [`Time`](#time) | Date the milestone contains. | -| `endDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.end. | | `ids` | [`[ID!]`](#id) | Array of global milestone IDs, e.g., `"gid://gitlab/Milestone/1"`. | | `includeAncestors` | [`Boolean`](#boolean) | Also return milestones in the project's parent group and its ancestors. | | `searchTitle` | [`String`](#string) | Search string for the title. | | `sort` | [`MilestoneSort`](#milestonesort) | Sort milestones by this criteria. | -| `startDate` **{warning-solid}** | [`Time`](#time) | **Deprecated** in 13.5. Use timeframe.start. | | `state` | [`MilestoneStateEnum`](#milestonestateenum) | Filter milestones by state. | | `timeframe` | [`Timeframe`](#timeframe) | List items overlapping the given timeframe. | | `title` | [`String`](#string) | Title of the milestone. | @@ -18706,7 +20088,7 @@ Product Analytics dashboards of the project. WARNING: **Introduced** in 15.6. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Returns [`ProductAnalyticsDashboardConnection`](#productanalyticsdashboardconnection). @@ -18847,7 +20229,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | -| `actionScanTypes` | [`[SecurityReportTypeEnum!]`](#securityreporttypeenum) | Filters policies by the action scan type. Only these scan types are supported: `dast`, `secret_detection`, `cluster_image_scanning`, `container_scanning`, `sast`, `dependency_scanning`. | +| `actionScanTypes` | [`[SecurityReportTypeEnum!]`](#securityreporttypeenum) | Filters policies by the action scan type. Only these scan types are supported: `dast`, `secret_detection`, `cluster_image_scanning`, `container_scanning`, `sast`, `sast_iac`, `dependency_scanning`. | | `relationship` | [`SecurityPolicyRelationType`](#securitypolicyrelationtype) | Filter policies by the given policy relationship. | ##### `Project.scanResultPolicies` @@ -18982,7 +20364,7 @@ Visible forks of the project. WARNING: **Introduced** in 15.10. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Returns [`ProjectConnection`](#projectconnection). @@ -19083,7 +20465,7 @@ Work items of the project. WARNING: **Introduced** in 15.1. -This feature is in Alpha. It can be changed or removed at any time. +This feature is an Experiment. It can be changed or removed at any time. Returns [`WorkItemConnection`](#workitemconnection). @@ -19095,14 +20477,14 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | -| `authorUsername` **{warning-solid}** | [`String`](#string) | **Introduced** in 15.9. This feature is in Alpha. It can be changed or removed at any time. Filter work items by author username. | -| `iid` | [`String`](#string) | IID of the issue. For example, "1". | +| `authorUsername` **{warning-solid}** | [`String`](#string) | **Introduced** in 15.9. This feature is an Experiment. It can be changed or removed at any time. Filter work items by author username. | +| `iid` | [`String`](#string) | IID of the work item. For example, "1". | | `iids` | [`[String!]`](#string) | List of IIDs of work items. For example, `["1", "2"]`. | | `in` | [`[IssuableSearchableField!]`](#issuablesearchablefield) | Specify the fields to perform the search in. Defaults to `[TITLE, DESCRIPTION]`. Requires the `search` argument.'. | | `requirementLegacyWidget` **{warning-solid}** | [`RequirementLegacyFilterInput`](#requirementlegacyfilterinput) | **Deprecated** in 15.9. Use work item IID filter instead. | | `search` | [`String`](#string) | Search query for title or description. | -| `sort` | [`WorkItemSort`](#workitemsort) | Sort work items by this criteria. | -| `state` | [`IssuableState`](#issuablestate) | Current state of this work item. | +| `sort` | [`WorkItemSort`](#workitemsort) | Sort work items by criteria. | +| `state` | [`IssuableState`](#issuablestate) | Current state of the work item. | | `statusWidget` | [`StatusFilterInput`](#statusfilterinput) | Input for status widget filter. Ignored if `work_items_mvc_2` is disabled. | | `types` | [`[IssueType!]`](#issuetype) | Filter work items by the given work item types. | @@ -19117,9 +20499,16 @@ four standard [pagination arguments](#connection-pagination-arguments): | `keepLatestArtifact` | [`Boolean`](#boolean) | Whether to keep the latest builds artifacts. | | `mergePipelinesEnabled` | [`Boolean`](#boolean) | Whether merge pipelines are enabled. | | `mergeTrainsEnabled` | [`Boolean`](#boolean) | Whether merge trains are enabled. | -| `optInJwt` | [`Boolean`](#boolean) | When disabled, the JSON Web Token is always available in all jobs in the pipeline. | | `project` | [`Project`](#project) | Project the CI/CD settings belong to. | +### `ProjectConversations` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `ciConfigMessages` **{warning-solid}** | [`AiMessageTypeConnection`](#aimessagetypeconnection) | **Introduced** in 16.0. This feature is an Experiment. It can be changed or removed at any time. Messages generated by open ai and the user. | + ### `ProjectDataTransfer` #### Fields @@ -19254,6 +20643,125 @@ Represents the source of a security policy belonging to a project. | `uploadsSize` | [`Float`](#float) | Uploads size of the project in bytes. | | `wikiSize` | [`Float`](#float) | Wiki size of the project in bytes. | +### `ProjectStatisticsRedirect` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `buildArtifacts` | [`String!`](#string) | Redirection Route for job_artifacts. | +| `containerRegistry` | [`String!`](#string) | Redirection Route for container_registry. | +| `packages` | [`String!`](#string) | Redirection Route for packages. | +| `repository` | [`String!`](#string) | Redirection Route for repository. | +| `snippets` | [`String!`](#string) | Redirection Route for snippets. | +| `wiki` | [`String!`](#string) | Redirection Route for wiki. | + +### `ProjectValueStreamAnalyticsFlowMetrics` + +Exposes aggregated value stream flow metrics. + +#### Fields with arguments + +##### `ProjectValueStreamAnalyticsFlowMetrics.cycleTime` + +Median time from first commit to issue closed. + +Returns [`ValueStreamAnalyticsMetric`](#valuestreamanalyticsmetric). + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `assigneeUsernames` | [`[String!]`](#string) | Usernames of users assigned to the issue. | +| `authorUsername` | [`String`](#string) | Username of the author of the issue. | +| `from` | [`Time!`](#time) | Timestamp marking the start date and time. | +| `labelNames` | [`[String!]`](#string) | Labels applied to the issue. | +| `milestoneTitle` | [`String`](#string) | Milestone applied to the issue. | +| `to` | [`Time!`](#time) | Timestamp marking the end date and time. | + +##### `ProjectValueStreamAnalyticsFlowMetrics.deploymentCount` + +Number of production deployments in the given period. + +Returns [`ValueStreamAnalyticsMetric`](#valuestreamanalyticsmetric). + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `from` | [`Time!`](#time) | Timestamp marking the start date and time. | +| `to` | [`Time!`](#time) | Timestamp marking the end date and time. | + +##### `ProjectValueStreamAnalyticsFlowMetrics.issueCount` + +Number of issues opened in the given period. + +Returns [`ValueStreamAnalyticsMetric`](#valuestreamanalyticsmetric). + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `assigneeUsernames` | [`[String!]`](#string) | Usernames of users assigned to the issue. | +| `authorUsername` | [`String`](#string) | Username of the author of the issue. | +| `from` | [`Time!`](#time) | Timestamp marking the start date and time. | +| `labelNames` | [`[String!]`](#string) | Labels applied to the issue. | +| `milestoneTitle` | [`String`](#string) | Milestone applied to the issue. | +| `to` | [`Time!`](#time) | Timestamp marking the end date and time. | + +##### `ProjectValueStreamAnalyticsFlowMetrics.issuesCompletedCount` + +Number of open issues closed (completed) in the given period. Maximum value is 10,001. + +Returns [`ValueStreamAnalyticsMetric`](#valuestreamanalyticsmetric). + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `assigneeUsernames` | [`[String!]`](#string) | Usernames of users assigned to the issue. | +| `authorUsername` | [`String`](#string) | Username of the author of the issue. | +| `from` | [`Time!`](#time) | Timestamp marking the start date and time. | +| `labelNames` | [`[String!]`](#string) | Labels applied to the issue. | +| `milestoneTitle` | [`String`](#string) | Milestone applied to the issue. | +| `to` | [`Time!`](#time) | Timestamp marking the end date and time. | + +##### `ProjectValueStreamAnalyticsFlowMetrics.leadTime` + +Median time from when the issue was created to when it was closed. + +Returns [`ValueStreamAnalyticsMetric`](#valuestreamanalyticsmetric). + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `assigneeUsernames` | [`[String!]`](#string) | Usernames of users assigned to the issue. | +| `authorUsername` | [`String`](#string) | Username of the author of the issue. | +| `from` | [`Time!`](#time) | Timestamp marking the start date and time. | +| `labelNames` | [`[String!]`](#string) | Labels applied to the issue. | +| `milestoneTitle` | [`String`](#string) | Milestone applied to the issue. | +| `to` | [`Time!`](#time) | Timestamp marking the end date and time. | + +### `ProjectWikiRepositoryRegistry` + +Represents the Geo replication and verification state of a project_wiki_repository. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `createdAt` | [`Time`](#time) | Timestamp when the ProjectWikiRepositoryRegistry was created. | +| `id` | [`ID!`](#id) | ID of the ProjectWikiRepositoryRegistry. | +| `lastSyncFailure` | [`String`](#string) | Error message during sync of the ProjectWikiRepositoryRegistry. | +| `lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the ProjectWikiRepositoryRegistry. | +| `projectWikiRepositoryId` | [`ID!`](#id) | ID of the Project Wiki Repository. | +| `retryAt` | [`Time`](#time) | Timestamp after which the ProjectWikiRepositoryRegistry is resynced. | +| `retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the ProjectWikiRepositoryRegistry. | +| `state` | [`RegistryState`](#registrystate) | Sync state of the ProjectWikiRepositoryRegistry. | +| `verificationRetryAt` | [`Time`](#time) | Timestamp after which the ProjectWikiRepositoryRegistry is reverified. | +| `verifiedAt` | [`Time`](#time) | Timestamp of the most recent successful verification of the ProjectWikiRepositoryRegistry. | + ### `PrometheusAlert` The alert condition for Prometheus. @@ -19278,6 +20786,7 @@ Protected Environments of the environment. | `group` | [`Group`](#group) | Group details. Present if it's group-level protected environment. | | `name` | [`String`](#string) | Name of the environment if it's a project-level protected environment. Tier of the environment if it's a group-level protected environment. | | `project` | [`Project`](#project) | Project details. Present if it's project-level protected environment. | +| `requiredApprovalCount` | [`Int`](#int) | Required approval count for Unified Approval Setting. | ### `ProtectedEnvironmentApprovalRule` @@ -19410,7 +20919,6 @@ Represents an asset link associated with a release. | ---- | ---- | ----------- | | `directAssetPath` | [`String`](#string) | Relative path for the direct asset link. | | `directAssetUrl` | [`String`](#string) | Direct asset URL of the link. | -| `external` **{warning-solid}** | [`Boolean`](#boolean) | **Deprecated** in 15.9. No longer used. | | `id` | [`ID!`](#id) | ID of the link. | | `linkType` | [`ReleaseAssetLinkType`](#releaseassetlinktype) | Type of the link: `other`, `runbook`, `image`, `package`; defaults to `other`. | | `name` | [`String`](#string) | Name of the link. | @@ -19510,6 +21018,18 @@ Returns [`[String!]`](#string). | `offset` | [`Int!`](#int) | Number of branch names to skip. | | `searchPattern` | [`String!`](#string) | Pattern to search for branch names by. | +##### `Repository.codeOwnersPath` + +Path to CODEOWNERS file in a ref. + +Returns [`String`](#string). + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `ref` | [`String`](#string) | Name of the ref. | + ##### `Repository.paginatedTree` Paginated tree of the repository. @@ -19678,6 +21198,7 @@ Counts of requirements by their state. | `lfsObjectsSize` | [`Float!`](#float) | LFS objects size in bytes. | | `packagesSize` | [`Float!`](#float) | Packages size in bytes. | | `pipelineArtifactsSize` | [`Float!`](#float) | CI pipeline artifacts size in bytes. | +| `registrySizeEstimated` | [`Boolean!`](#boolean) | Indicates whether the deduplicated Container Registry size for the namespace is an estimated value or not. | | `repositorySize` | [`Float!`](#float) | Git repository size in bytes. | | `snippetsSize` | [`Float!`](#float) | Snippets size in bytes. | | `storageSize` | [`Float!`](#float) | Total storage in bytes. | @@ -19827,6 +21348,7 @@ Represents the scan result policy. | `enabled` | [`Boolean!`](#boolean) | Indicates whether this policy is enabled. | | `groupApprovers` | [`[Group!]`](#group) | Approvers of the group type. | | `name` | [`String!`](#string) | Name of the policy. | +| `roleApprovers` | [`[MemberAccessLevelName!]`](#memberaccesslevelname) | Approvers of the role type. Users belonging to these role(s) alone will be approvers. | | `source` | [`SecurityPolicySource!`](#securitypolicysource) | Source of the policy. Its fields depend on the source type. | | `updatedAt` | [`Time!`](#time) | Timestamp of when the policy YAML was last updated. | | `userApprovers` | [`[UserCore!]`](#usercore) | Approvers of the user type. | @@ -20199,6 +21721,7 @@ SSH signature for a signed commit. | ---- | ---- | ----------- | | `commitSha` | [`String`](#string) | SHA of the associated commit. | | `key` | [`Key`](#key) | SSH key used for the signature. | +| `keyFingerprintSha256` | [`String`](#string) | Fingerprint of the key. | | `project` | [`Project`](#project) | Project of the associated commit. | | `user` | [`UserCore`](#usercore) | User associated with the key. | | `verificationStatus` | [`VerificationStatus`](#verificationstatus) | Indicates verification status of the associated key or certificate. | @@ -20529,6 +22052,7 @@ Describes an incident management timeline event. | `issue` | [`Issue`](#issue) | Issue that logged time was added to. | | `mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request that logged time was added to. | | `note` | [`Note`](#note) | Note where the quick action was executed to add the logged time. | +| `project` | [`Project!`](#project) | Target project of the timelog merge request or issue. | | `spentAt` | [`Time`](#time) | Timestamp of when the time tracked was spent at. | | `summary` | [`String`](#string) | Summary of how the time was spent. | | `timeSpent` | [`Int!`](#int) | Time spent displayed in seconds. | @@ -20648,6 +22172,21 @@ Represents a recorded measurement (object count) for the Admins. | `identifier` | [`MeasurementIdentifier!`](#measurementidentifier) | Type of objects being measured. | | `recordedAt` | [`Time`](#time) | Time the measurement was recorded. | +### `UserAchievement` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `achievement` | [`Achievement!`](#achievement) | Achievement awarded. | +| `awardedByUser` | [`UserCore!`](#usercore) | Awarded by. | +| `createdAt` | [`Time!`](#time) | Timestamp the achievement was created. | +| `id` | [`AchievementsUserAchievementID!`](#achievementsuserachievementid) | ID of the user achievement. | +| `revokedAt` | [`Time`](#time) | Timestamp the achievement was revoked. | +| `revokedByUser` | [`UserCore`](#usercore) | Revoked by. | +| `updatedAt` | [`Time!`](#time) | Timestamp the achievement was last updated. | +| `user` | [`UserCore!`](#usercore) | Achievement recipient. | + ### `UserCallout` #### Fields @@ -20686,6 +22225,7 @@ Core represention of a GitLab user. | `savedReplies` | [`SavedReplyConnection`](#savedreplyconnection) | Saved replies authored by the user. Will not return saved replies if `saved_replies` feature flag is disabled. (see [Connections](#connections)) | | `state` | [`UserState!`](#userstate) | State of the user. | | `status` | [`UserStatus`](#userstatus) | User status. | +| `userAchievements` **{warning-solid}** | [`UserAchievementConnection`](#userachievementconnection) | **Introduced** in 15.10. This feature is an Experiment. It can be changed or removed at any time. Achievements for the user. Only returns for namespaces where the `achievements` feature flag is enabled. | | `userPermissions` | [`UserPermissions!`](#userpermissions) | Permissions for the current user on the resource. | | `username` | [`String!`](#string) | Username of the user. Unique within this instance of GitLab. | | `webPath` | [`String!`](#string) | Web path of the user. | @@ -20707,6 +22247,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `authorUsername` | [`String`](#string) | Username of the author. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | | `createdBefore` | [`Time`](#time) | Merge requests created before this timestamp. | @@ -20741,6 +22282,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `assigneeUsername` | [`String`](#string) | Username of the assignee. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | | `createdBefore` | [`Time`](#time) | Merge requests created before this timestamp. | @@ -20792,6 +22334,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `assigneeUsername` | [`String`](#string) | Username of the assignee. | | `authorUsername` | [`String`](#string) | Username of the author. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | @@ -20902,6 +22445,22 @@ four standard [pagination arguments](#connection-pagination-arguments): | `state` | [`[TodoStateEnum!]`](#todostateenum) | State of the todo. | | `type` | [`[TodoTargetEnum!]`](#todotargetenum) | Type of the todo. | +##### `UserCore.workspaces` + +Workspaces owned by the current user. + +Returns [`WorkspaceConnection`](#workspaceconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `ids` | [`[RemoteDevelopmentWorkspaceID!]`](#remotedevelopmentworkspaceid) | Array of global workspace IDs. For example, `["gid://gitlab/RemoteDevelopment::Workspace/1"]`. | + ### `UserMergeRequestInteraction` Information about a merge request given a specific user. @@ -20935,6 +22494,7 @@ fields relate to interactions between the two entities. | Name | Type | Description | | ---- | ---- | ----------- | | `issuesSort` | [`IssueSort`](#issuesort) | Sort order for issue lists. | +| `visibilityPipelineIdType` | [`VisibilityPipelineIdType`](#visibilitypipelineidtype) | Determines whether the pipeline list shows ID or IID. | ### `UserStatus` @@ -20947,6 +22507,29 @@ fields relate to interactions between the two entities. | `message` | [`String`](#string) | User status message. | | `messageHtml` | [`String`](#string) | HTML of the user status message. | +### `ValueStreamAnalyticsMetric` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `identifier` | [`String!`](#string) | Identifier for the metric. | +| `links` | [`[ValueStreamMetricLinkType!]!`](#valuestreammetriclinktype) | Optional links for drilling down. | +| `title` | [`String!`](#string) | Title for the metric. | +| `unit` | [`String`](#string) | Unit of measurement. | +| `value` | [`Float`](#float) | Value for the metric. | + +### `ValueStreamMetricLinkType` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `docsLink` | [`Boolean`](#boolean) | Link to the metric documentation. | +| `label` | [`String!`](#string) | Label for the link. | +| `name` | [`String!`](#string) | Name of the link group. | +| `url` | [`String!`](#string) | Drill-down URL. | + ### `VulnerabilitiesCountByDay` Represents the count of vulnerabilities by severity on a particular day. This data is retained for 365 days. @@ -21002,6 +22585,7 @@ Represents a vulnerability. | `severity` | [`VulnerabilitySeverity`](#vulnerabilityseverity) | Severity of the vulnerability (INFO, UNKNOWN, LOW, MEDIUM, HIGH, CRITICAL). | | `state` | [`VulnerabilityState`](#vulnerabilitystate) | State of the vulnerability (DETECTED, CONFIRMED, RESOLVED, DISMISSED). | | `stateComment` | [`String`](#string) | Comment given for the vulnerability state change. | +| `stateTransitions` | [`VulnerabilityStateTransitionTypeConnection`](#vulnerabilitystatetransitiontypeconnection) | List of state transitions related to the vulnerability. (see [Connections](#connections)) | | `title` | [`String`](#string) | Title of the vulnerability. | | `updatedAt` | [`Time`](#time) | Timestamp of when the vulnerability was last updated. | | `userNotesCount` | [`Int!`](#int) | Number of user notes attached to the vulnerability. | @@ -21171,6 +22755,19 @@ Represents the vulnerability details location within a file in the project. | `name` | [`String`](#string) | Name of the field. | | `offset` | [`Int!`](#int) | Offset of the module location. | +### `VulnerabilityDetailRow` + +Represents an individual row in a table. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `description` | [`String`](#string) | Description of the field. | +| `fieldName` | [`String`](#string) | Name of the field. | +| `name` | [`String`](#string) | Name of the field. | +| `row` | [`[VulnerabilityDetail!]!`](#vulnerabilitydetail) | Value of the field. | + ### `VulnerabilityDetailTable` Represents the vulnerability details table value. @@ -21183,7 +22780,7 @@ Represents the vulnerability details table value. | `fieldName` | [`String`](#string) | Name of the field. | | `headers` | [`[VulnerabilityDetail!]!`](#vulnerabilitydetail) | Table headers. | | `name` | [`String`](#string) | Name of the field. | -| `rows` | [`[VulnerabilityDetail!]!`](#vulnerabilitydetail) | Table rows. | +| `rows` | [`[VulnerabilityDetailRow!]!`](#vulnerabilitydetailrow) | Table rows. | ### `VulnerabilityDetailText` @@ -21417,10 +23014,10 @@ Check permissions for the current user on a vulnerability. | `adminVulnerability` | [`Boolean!`](#boolean) | Indicates the user can perform `admin_vulnerability` on this resource. | | `adminVulnerabilityExternalIssueLink` | [`Boolean!`](#boolean) | Indicates the user can perform `admin_vulnerability_external_issue_link` on this resource. | | `adminVulnerabilityIssueLink` | [`Boolean!`](#boolean) | Indicates the user can perform `admin_vulnerability_issue_link` on this resource. | -| `createVulnerability` | [`Boolean!`](#boolean) | Indicates the user can perform `create_vulnerability` on this resource. | | `createVulnerabilityExport` | [`Boolean!`](#boolean) | Indicates the user can perform `create_vulnerability_export` on this resource. | | `createVulnerabilityFeedback` | [`Boolean!`](#boolean) | Indicates the user can perform `create_vulnerability_feedback` on this resource. | | `destroyVulnerabilityFeedback` | [`Boolean!`](#boolean) | Indicates the user can perform `destroy_vulnerability_feedback` on this resource. | +| `readVulnerability` | [`Boolean!`](#boolean) | Indicates the user can perform `read_vulnerability` on this resource. | | `readVulnerabilityFeedback` | [`Boolean!`](#boolean) | Indicates the user can perform `read_vulnerability_feedback` on this resource. | | `updateVulnerabilityFeedback` | [`Boolean!`](#boolean) | Indicates the user can perform `update_vulnerability_feedback` on this resource. | @@ -21502,6 +23099,21 @@ Represents vulnerability counts by severity. | `medium` | [`Int`](#int) | Number of vulnerabilities of MEDIUM severity of the project. | | `unknown` | [`Int`](#int) | Number of vulnerabilities of UNKNOWN severity of the project. | +### `VulnerabilityStateTransitionType` + +Represents a state transition of a vulnerability. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `author` | [`UserCore!`](#usercore) | User who changed the state of the vulnerability. | +| `comment` | [`String`](#string) | Comment for the state change. | +| `createdAt` | [`Time!`](#time) | Time of the state change of the vulnerability. | +| `dismissalReason` | [`VulnerabilityDismissalReason`](#vulnerabilitydismissalreason) | Reason for the dismissal. | +| `fromState` | [`VulnerabilityState!`](#vulnerabilitystate) | State of the vulnerability before transition. | +| `toState` | [`VulnerabilityState!`](#vulnerabilitystate) | State of the vulnerability after transition. | + ### `VulnerableDependency` Represents a vulnerable dependency. Used in vulnerability location data. @@ -21556,16 +23168,18 @@ Represents vulnerability letter grades with associated projects. | Name | Type | Description | | ---- | ---- | ----------- | -| `author` **{warning-solid}** | [`UserCore`](#usercore) | **Introduced** in 15.9. This feature is in Alpha. It can be changed or removed at any time. User that created the work item. | +| `author` **{warning-solid}** | [`UserCore`](#usercore) | **Introduced** in 15.9. This feature is an Experiment. It can be changed or removed at any time. User that created the work item. | | `closedAt` | [`Time`](#time) | Timestamp of when the work item was closed. | | `confidential` | [`Boolean!`](#boolean) | Indicates the work item is confidential. | +| `createNoteEmail` | [`String`](#string) | User specific email address for the work item. | | `createdAt` | [`Time!`](#time) | Timestamp of when the work item was created. | | `description` | [`String`](#string) | Description of the work item. | | `descriptionHtml` | [`String`](#string) | GitLab Flavored Markdown rendering of `description`. | | `id` | [`WorkItemID!`](#workitemid) | Global ID of the work item. | | `iid` | [`ID!`](#id) | Internal ID of the work item. | | `lockVersion` | [`Int!`](#int) | Lock version of the work item. Incremented each time the work item is updated. | -| `project` **{warning-solid}** | [`Project!`](#project) | **Introduced** in 15.3. This feature is in Alpha. It can be changed or removed at any time. Project the work item belongs to. | +| `namespace` **{warning-solid}** | [`Namespace`](#namespace) | **Introduced** in 15.10. This feature is an Experiment. It can be changed or removed at any time. Namespace the work item belongs to. | +| `project` **{warning-solid}** | [`Project`](#project) | **Introduced** in 15.3. This feature is an Experiment. It can be changed or removed at any time. Project the work item belongs to. | | `state` | [`WorkItemState!`](#workitemstate) | State of the work item. | | `title` | [`String!`](#string) | Title of the work item. | | `titleHtml` | [`String`](#string) | GitLab Flavored Markdown rendering of `title`. | @@ -21575,6 +23189,20 @@ Represents vulnerability letter grades with associated projects. | `widgets` | [`[WorkItemWidget!]`](#workitemwidget) | Collection of widgets that belong to the work item. | | `workItemType` | [`WorkItemType!`](#workitemtype) | Type assigned to the work item. | +#### Fields with arguments + +##### `WorkItem.reference` + +Internal reference of the work item. Returned in shortened format by default. + +Returns [`String!`](#string). + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `full` | [`Boolean`](#boolean) | Boolean option specifying whether the reference should be returned in full. | + ### `WorkItemPermissions` Check permissions for the current user on a work item. @@ -21583,9 +23211,11 @@ Check permissions for the current user on a work item. | Name | Type | Description | | ---- | ---- | ----------- | +| `adminParentLink` | [`Boolean!`](#boolean) | Indicates the user can perform `admin_parent_link` on this resource. | | `adminWorkItem` | [`Boolean!`](#boolean) | Indicates the user can perform `admin_work_item` on this resource. | | `deleteWorkItem` | [`Boolean!`](#boolean) | Indicates the user can perform `delete_work_item` on this resource. | | `readWorkItem` | [`Boolean!`](#boolean) | Indicates the user can perform `read_work_item` on this resource. | +| `setWorkItemMetadata` | [`Boolean!`](#boolean) | Indicates the user can perform `set_work_item_metadata` on this resource. | | `updateWorkItem` | [`Boolean!`](#boolean) | Indicates the user can perform `update_work_item` on this resource. | ### `WorkItemType` @@ -21611,6 +23241,47 @@ Represents an assignees widget. | `canInviteMembers` | [`Boolean!`](#boolean) | Indicates whether the current user can invite members to the work item's project. | | `type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | +### `WorkItemWidgetAwardEmoji` + +Represents the award emoji widget. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `awardEmoji` | [`AwardEmojiConnection`](#awardemojiconnection) | Award emoji on the work item. (see [Connections](#connections)) | +| `downvotes` | [`Int!`](#int) | Number of downvotes the work item has received. | +| `type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | +| `upvotes` | [`Int!`](#int) | Number of upvotes the work item has received. | + +### `WorkItemWidgetCurrentUserTodos` + +Represents a todos widget. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | + +#### Fields with arguments + +##### `WorkItemWidgetCurrentUserTodos.currentUserTodos` + +To-do items for the current user. + +Returns [`TodoConnection!`](#todoconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `state` | [`TodoStateEnum`](#todostateenum) | State of the to-do items. | + ### `WorkItemWidgetDescription` Represents a description widget. @@ -21712,6 +23383,17 @@ four standard [pagination arguments](#connection-pagination-arguments): | ---- | ---- | ----------- | | `filter` | [`NotesFilterType`](#notesfiltertype) | Type of notes collection: ALL_NOTES, ONLY_COMMENTS, ONLY_ACTIVITY. | +### `WorkItemWidgetNotifications` + +Represents the notifications widget. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `subscribed` | [`Boolean!`](#boolean) | Whether the current user is subscribed to notifications on the work item. | +| `type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | + ### `WorkItemWidgetProgress` Represents a progress widget. @@ -21779,6 +23461,35 @@ Represents a weight widget. | `type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | | `weight` | [`Int`](#int) | Weight of the work item. | +### `Workspace` + +Represents a remote development workspace. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `actualState` | [`String!`](#string) | Actual state of the workspace. | +| `clusterAgent` | [`ClusterAgent!`](#clusteragent) | Kubernetes Agent associated with the workspace. | +| `createdAt` | [`Time!`](#time) | Timestamp of workspace creation. | +| `deploymentResourceVersion` | [`Int`](#int) | ResourceVersion of the Deployment resource for the workspace. | +| `desiredState` | [`String!`](#string) | Desired state of the workspace. | +| `desiredStateUpdatedAt` | [`Time!`](#time) | Timestamp of last update to desired state. | +| `devfile` | [`String!`](#string) | Source YAML of the devfile used to configure the workspace. | +| `devfilePath` | [`String!`](#string) | Project repo git path containing the devfile used to configure the workspace. | +| `devfileRef` | [`String!`](#string) | Project repo git ref containing the devfile used to configure the workspace. | +| `editor` | [`String!`](#string) | Editor used to configure the workspace. Must match a configured template. | +| `id` | [`RemoteDevelopmentWorkspaceID!`](#remotedevelopmentworkspaceid) | Global ID of the workspace. | +| `maxHoursBeforeTermination` | [`Int!`](#int) | Maximum hours the workspace can exist before it is automatically terminated. | +| `name` | [`String!`](#string) | Name of the workspace in Kubernetes. | +| `namespace` | [`String!`](#string) | Namespace of the workspace in Kubernetes. | +| `processedDevfile` | [`String!`](#string) | Processed YAML of the devfile used to configure the workspace. | +| `projectId` | [`ID!`](#id) | ID of the Project providing the Devfile for the workspace. | +| `respondedToAgentAt` | [`Time`](#time) | Timestamp of last response sent to GA4K for the workspace. | +| `updatedAt` | [`Time!`](#time) | Timestamp of last update to any mutable workspace property. | +| `url` | [`String!`](#string) | URL of the workspace. | +| `user` | [`UserCore!`](#usercore) | Owner of the workspace. | + ### `X509Certificate` Represents an X.509 certificate. @@ -21996,6 +23707,20 @@ User availability status. | `BUSY` | Busy. | | `NOT_SET` | Not Set. | +### `AvailableExportFields` + +Available fields to be exported as CSV. + +| Value | Description | +| ----- | ----------- | +| `AUTHOR` | Author name. | +| `AUTHOR_USERNAME` | Author username. | +| `CREATED_AT` | Date of creation. | +| `DESCRIPTION` | Description. | +| `ID` | Unique identifier. | +| `TITLE` | Title. | +| `TYPE` | Type of the work item. | + ### `BlobViewersType` Types of blob viewers. @@ -22012,6 +23737,7 @@ Include type. | Value | Description | | ----- | ----------- | +| `component` | Component include. | | `file` | Project file include. | | `local` | Local include. | | `remote` | Remote include. | @@ -22078,8 +23804,8 @@ Direction of access. | Value | Description | | ----- | ----------- | -| `IDLE` **{warning-solid}** | **Introduced** in 15.7. This feature is in Alpha. It can be changed or removed at any time. Runner is idle. | -| `RUNNING` **{warning-solid}** | **Introduced** in 15.7. This feature is in Alpha. It can be changed or removed at any time. Runner is executing jobs. | +| `IDLE` **{warning-solid}** | **Introduced** in 15.7. This feature is an Experiment. It can be changed or removed at any time. Runner is idle. | +| `RUNNING` **{warning-solid}** | **Introduced** in 15.7. This feature is an Experiment. It can be changed or removed at any time. Runner is executing jobs. | ### `CiRunnerMembershipFilter` @@ -22087,7 +23813,7 @@ Values for filtering runners in namespaces. The previous type name `RunnerMember | Value | Description | | ----- | ----------- | -| `ALL_AVAILABLE` **{warning-solid}** | **Introduced** in 15.5. This feature is in Alpha. It can be changed or removed at any time. Include all runners. This list includes runners for all projects in the group and subgroups, as well as for the parent groups and instance. | +| `ALL_AVAILABLE` **{warning-solid}** | **Introduced** in 15.5. This feature is an Experiment. It can be changed or removed at any time. Include all runners. This list includes runners for all projects in the group and subgroups, as well as for the parent groups and instance. | | `DESCENDANTS` | Include runners that have either a direct or inherited relationship. These runners can be specific to a project or a group. | | `DIRECT` | Include runners that have a direct relationship. | @@ -22178,6 +23904,15 @@ Mode of a commit action. | `BASE64` | Base64 encoding. | | `TEXT` | Text encoding. | +### `ComplianceFrameworkPresenceFilter` + +ComplianceFramework of a project for filtering. + +| Value | Description | +| ----- | ----------- | +| `ANY` | Any compliance framework is assigned. | +| `NONE` | No compliance framework is assigned. | + ### `ComplianceViolationReason` Reason for the compliance violation. @@ -22345,6 +24080,16 @@ Values for sorting tags. | `all` | All available organizations. | | `inactive` | Inactive organizations. | +### `DastPreScanVerificationCheckType` + +Check type of the pre scan verification step. + +| Value | Description | +| ----- | ----------- | +| `AUTHENTICATION` | Authentication check. | +| `CONNECTION` | Connection check. | +| `CRAWLING` | Crawling check. | + ### `DastPreScanVerificationStatus` Status of DAST pre scan verification. @@ -22459,6 +24204,17 @@ Weight of the data visualization palette. | `PENDING_DESTRUCTION` | Dependency proxy manifest has a status of pending_destruction. | | `PROCESSING` | Dependency proxy manifest has a status of processing. | +### `DependencySort` + +Values for sorting dependencies. + +| Value | Description | +| ----- | ----------- | +| `NAME_ASC` | Name by ascending order. | +| `NAME_DESC` | Name by descending order. | +| `PACKAGER_ASC` | Packager by ascending order. | +| `PACKAGER_DESC` | Packager by descending order. | + ### `DeploymentApprovalSummaryStatus` Status of the deployment approval summary. @@ -22543,6 +24299,7 @@ Detailed representation of whether a GitLab merge request can be merged. | `NOT_APPROVED` | Merge request must be approved before merging. | | `NOT_OPEN` | Merge request must be open before merging. | | `POLICIES_DENIED` | There are denied policies for the merge request. | +| `PREPARING` | Merge request diff is being created. | | `UNCHECKED` | Merge status has not been checked. | ### `DiffPositionType` @@ -22662,6 +24419,36 @@ Event action. | `REOPENED` | Reopened action. | | `UPDATED` | Updated action. | +### `GeoRegistryAction` + +Action to trigger on one or more Geo registries. + +| Value | Description | +| ----- | ----------- | +| `RESYNC` | Resync a registry. | +| `REVERIFY` | Reverify a registry. | + +### `GeoRegistryClass` + +Geo registry class. + +| Value | Description | +| ----- | ----------- | +| `CI_SECURE_FILE_REGISTRY` | Geo::CiSecureFileRegistry registry class. | +| `CONTAINER_REPOSITORY_REGISTRY` | Geo::ContainerRepositoryRegistry registry class. | +| `DEPENDENCY_PROXY_BLOB_REGISTRY` | Geo::DependencyProxyBlobRegistry registry class. | +| `DEPENDENCY_PROXY_MANIFEST_REGISTRY` | Geo::DependencyProxyManifestRegistry registry class. | +| `JOB_ARTIFACT_REGISTRY` | Geo::JobArtifactRegistry registry class. | +| `LFS_OBJECT_REGISTRY` | Geo::LfsObjectRegistry registry class. | +| `MERGE_REQUEST_DIFF_REGISTRY` | Geo::MergeRequestDiffRegistry registry class. | +| `PACKAGE_FILE_REGISTRY` | Geo::PackageFileRegistry registry class. | +| `PAGES_DEPLOYMENT_REGISTRY` | Geo::PagesDeploymentRegistry registry class. | +| `PIPELINE_ARTIFACT_REGISTRY` | Geo::PipelineArtifactRegistry registry class. | +| `PROJECT_WIKI_REPOSITORY_REGISTRY` | Geo::ProjectWikiRepositoryRegistry registry class. | +| `SNIPPET_REPOSITORY_REGISTRY` | Geo::SnippetRepositoryRegistry registry class. | +| `TERRAFORM_STATE_VERSION_REGISTRY` | Geo::TerraformStateVersionRegistry registry class. | +| `UPLOAD_REGISTRY` | Geo::UploadRegistry registry class. | + ### `GitlabSubscriptionsUserRole` Role of User. @@ -22692,6 +24479,7 @@ User permission on groups. | Value | Description | | ----- | ----------- | | `CREATE_PROJECTS` | Groups where the user can create projects. | +| `IMPORT_PROJECTS` | Groups where the user can import projects to. | | `TRANSFER_PROJECTS` | Groups where the user can transfer projects to. | ### `GroupReleaseSort` @@ -22732,6 +24520,7 @@ Issuable resource link type enum. | Value | Description | | ----- | ----------- | | `general` | General link type. | +| `pagerduty` | Pagerduty link type. | | `slack` | Slack link type. | | `zoom` | Zoom link type. | @@ -22767,6 +24556,15 @@ State of a GitLab issue or merge request. | `locked` | Discussion has been locked. | | `opened` | In open state. | +### `IssuableSubscriptionEvent` + +Values for subscribing and unsubscribing from issuables. + +| Value | Description | +| ----- | ----------- | +| `SUBSCRIBE` | Subscribe to an issuable. | +| `UNSUBSCRIBE` | Unsubscribe from an issuable. | + ### `IssueCreationIterationWildcardId` Iteration ID wildcard values for issue creation. @@ -22858,10 +24656,10 @@ Issue type. | ----- | ----------- | | `INCIDENT` | Incident issue type. | | `ISSUE` | Issue issue type. | -| `KEY_RESULT` **{warning-solid}** | **Introduced** in 15.7. This feature is in Alpha. It can be changed or removed at any time. Key Result issue type. Available only when feature flag `okrs_mvc` is enabled. | -| `OBJECTIVE` **{warning-solid}** | **Introduced** in 15.6. This feature is in Alpha. It can be changed or removed at any time. Objective issue type. Available only when feature flag `okrs_mvc` is enabled. | +| `KEY_RESULT` **{warning-solid}** | **Introduced** in 15.7. This feature is an Experiment. It can be changed or removed at any time. Key Result issue type. Available only when feature flag `okrs_mvc` is enabled. | +| `OBJECTIVE` **{warning-solid}** | **Introduced** in 15.6. This feature is an Experiment. It can be changed or removed at any time. Objective issue type. Available only when feature flag `okrs_mvc` is enabled. | | `REQUIREMENT` | Requirement issue type. | -| `TASK` **{warning-solid}** | **Introduced** in 15.2. This feature is in Alpha. It can be changed or removed at any time. Task issue type. | +| `TASK` **{warning-solid}** | **Introduced** in 15.2. This feature is an Experiment. It can be changed or removed at any time. Task issue type. | | `TEST_CASE` | Test Case issue type. | ### `IterationSearchableField` @@ -22892,7 +24690,6 @@ State of a GitLab iteration. | `closed` | Closed iteration. | | `current` | Current iteration. | | `opened` | Open iteration. | -| `started` **{warning-solid}** | **Deprecated** in 14.1. Use current instead. | | `upcoming` | Upcoming iteration. | ### `IterationWildcardId` @@ -22949,6 +24746,16 @@ List limit metric setting. | `issue_count` | Limit list by number of issues. | | `issue_weights` | Limit list by total weight of issues. | +### `MarkupFormat` + +List markup formats. + +| Value | Description | +| ----- | ----------- | +| `HTML` | HTML format. | +| `MARKDOWN` | Markdown format. | +| `RAW` | Raw format. | + ### `MeasurementIdentifier` Possible identifier types for a measurement. @@ -22979,6 +24786,18 @@ Access level of a group or project member. | `OWNER` | Owner access. | | `REPORTER` | Reporter access. | +### `MemberAccessLevelName` + +Name of access levels of a group or project member. + +| Value | Description | +| ----- | ----------- | +| `DEVELOPER` | Developer access. | +| `GUEST` | Guest access. | +| `MAINTAINER` | Maintainer access. | +| `OWNER` | Owner access. | +| `REPORTER` | Reporter access. | + ### `MemberSort` Values for sorting members. @@ -23139,9 +24958,11 @@ Values for sorting projects. | Value | Description | | ----- | ----------- | -| `ACTIVITY_DESC` | Sort by latest activity, in descending order. | +| `ACTIVITY_DESC` | Sort by latest activity, descending order. | | `SIMILARITY` | Most similar to the search query. | -| `STORAGE` | Sort by storage size. | +| `STORAGE` | Sort by excess repository storage size, descending order. | +| `STORAGE_SIZE_ASC` | Sort by total storage size, ascending order. | +| `STORAGE_SIZE_DESC` | Sort by total storage size, descending order. | ### `NegatedIterationWildcardId` @@ -23236,6 +25057,27 @@ Values for sorting group packages. | `VERSION_ASC` | Ordered by version in ascending order. | | `VERSION_DESC` | Ordered by version in descending order. | +### `PackageManager` + +Values for package manager. + +| Value | Description | +| ----- | ----------- | +| `BUNDLER` | Package manager: bundler. | +| `COMPOSER` | Package manager: composer. | +| `CONAN` | Package manager: conan. | +| `GO` | Package manager: go. | +| `GRADLE` | Package manager: gradle. | +| `MAVEN` | Package manager: maven. | +| `NPM` | Package manager: npm. | +| `NUGET` | Package manager: nuget. | +| `PIP` | Package manager: pip. | +| `PIPENV` | Package manager: pipenv. | +| `PNPM` | Package manager: pnpm. | +| `SBT` | Package manager: sbt. | +| `SETUPTOOLS` | Package manager: setuptools. | +| `YARN` | Package manager: yarn. | + ### `PackageSort` Values for sorting package. @@ -23348,6 +25190,17 @@ Event type of the pipeline associated with a merge request. | `SUCCESS` | Pipeline completed successfully. | | `WAITING_FOR_RESOURCE` | A resource (for example, a runner) that the pipeline requires to run is unavailable. | +### `ProductAnalyticsState` + +Current state of the product analytics stack. + +| Value | Description | +| ----- | ----------- | +| `COMPLETE` | Stack has been initialized and has data. | +| `CREATE_INSTANCE` | Stack has not been created yet. | +| `LOADING_INSTANCE` | Stack is currently initializing. | +| `WAITING_FOR_EVENTS` | Stack is waiting for events from users. | + ### `ProjectMemberRelation` Project member relation. @@ -23358,6 +25211,7 @@ Project member relation. | `DIRECT` | Direct members. | | `INHERITED` | Inherited members. | | `INVITED_GROUPS` | Invited Groups members. | +| `SHARED_INTO_ANCESTORS` | Shared Into Ancestors members. | ### `RegistryState` @@ -23370,6 +25224,15 @@ State of a Geo registry. | `STARTED` | Registry currently syncing. | | `SYNCED` | Registry that is synced. | +### `RelativePositionType` + +The position to which the object should be moved. + +| Value | Description | +| ----- | ----------- | +| `AFTER` | Object is moved after an adjacent object. | +| `BEFORE` | Object is moved before an adjacent object. | + ### `ReleaseAssetLinkType` Type of the link: `other`, `runbook`, `image`, `package`. @@ -23466,6 +25329,7 @@ The status of the security scan. | Value | Description | | ----- | ----------- | | `API_FUZZING` | API FUZZING scan report. | +| `BREACH_AND_ATTACK_SIMULATION` | BREACH AND ATTACK SIMULATION scan report. | | `CLUSTER_IMAGE_SCANNING` | CLUSTER IMAGE SCANNING scan report. | | `CONTAINER_SCANNING` | CONTAINER SCANNING scan report. | | `COVERAGE_FUZZING` | COVERAGE FUZZING scan report. | @@ -23482,6 +25346,7 @@ The type of the security scanner. | Value | Description | | ----- | ----------- | | `API_FUZZING` | API Fuzzing scanner. | +| `BREACH_AND_ATTACK_SIMULATION` | Breach And Attack Simulation scanner. | | `CLUSTER_IMAGE_SCANNING` | Cluster Image Scanning scanner. | | `CONTAINER_SCANNING` | Container Scanning scanner. | | `COVERAGE_FUZZING` | Coverage Fuzzing scanner. | @@ -23523,6 +25388,7 @@ State of a Sentry error. | `EXTERNAL_WIKI_SERVICE` | ExternalWikiService type. | | `GITHUB_SERVICE` | GithubService type. | | `GITLAB_SLACK_APPLICATION_SERVICE` | GitlabSlackApplicationService type (Gitlab.com only). | +| `GOOGLE_PLAY_SERVICE` | GooglePlayService type. | | `HANGOUTS_CHAT_SERVICE` | HangoutsChatService type. | | `HARBOR_SERVICE` | HarborService type. | | `IRKER_SERVICE` | IrkerService type. | @@ -23541,6 +25407,7 @@ State of a Sentry error. | `SHIMO_SERVICE` | ShimoService type. | | `SLACK_SERVICE` | SlackService type. | | `SLACK_SLASH_COMMANDS_SERVICE` | SlackSlashCommandsService type. | +| `SQUASH_TM_SERVICE` | SquashTmService type. | | `TEAMCITY_SERVICE` | TeamcityService type. | | `UNIFY_CIRCUIT_SERVICE` | UnifyCircuitService type. | | `WEBEX_TEAMS_SERVICE` | WebexTeamsService type. | @@ -23719,11 +25586,13 @@ Name of the feature that the callout is for. | ----- | ----------- | | `ACTIVE_USER_COUNT_THRESHOLD` | Callout feature name for active_user_count_threshold. | | `ARTIFACTS_MANAGEMENT_PAGE_FEEDBACK_BANNER` | Callout feature name for artifacts_management_page_feedback_banner. | +| `BRANCH_RULES_INFO_CALLOUT` | Callout feature name for branch_rules_info_callout. | | `BUY_PIPELINE_MINUTES_NOTIFICATION_DOT` | Callout feature name for buy_pipeline_minutes_notification_dot. | | `CANARY_DEPLOYMENT` | Callout feature name for canary_deployment. | | `CI_DEPRECATION_WARNING_FOR_TYPES_KEYWORD` | Callout feature name for ci_deprecation_warning_for_types_keyword. | | `CLOUD_LICENSING_SUBSCRIPTION_ACTIVATION_BANNER` | Callout feature name for cloud_licensing_subscription_activation_banner. | | `CLUSTER_SECURITY_WARNING` | Callout feature name for cluster_security_warning. | +| `CREATE_RUNNER_WORKFLOW_BANNER` | Callout feature name for create_runner_workflow_banner. | | `EOA_BRONZE_PLAN_BANNER` | Callout feature name for eoa_bronze_plan_banner. | | `FEATURE_FLAGS_NEW_VERSION` | Callout feature name for feature_flags_new_version. | | `GCP_SIGNUP_OFFER` | Callout feature name for gcp_signup_offer. | @@ -23737,6 +25606,7 @@ Name of the feature that the callout is for. | `NAMESPACE_STORAGE_LIMIT_BANNER_ERROR_THRESHOLD` | Callout feature name for namespace_storage_limit_banner_error_threshold. | | `NAMESPACE_STORAGE_LIMIT_BANNER_INFO_THRESHOLD` | Callout feature name for namespace_storage_limit_banner_info_threshold. | | `NAMESPACE_STORAGE_LIMIT_BANNER_WARNING_THRESHOLD` | Callout feature name for namespace_storage_limit_banner_warning_threshold. | +| `NAMESPACE_STORAGE_PRE_ENFORCEMENT_BANNER` | Callout feature name for namespace_storage_pre_enforcement_banner. | | `NEW_TOP_LEVEL_GROUP_ALERT` | Callout feature name for new_top_level_group_alert. | | `NEW_USER_SIGNUPS_CAP_REACHED` | Callout feature name for new_user_signups_cap_reached. | | `PERSONAL_ACCESS_TOKEN_EXPIRY` | Callout feature name for personal_access_token_expiry. | @@ -23751,10 +25621,6 @@ Name of the feature that the callout is for. | `SECURITY_CONFIGURATION_UPGRADE_BANNER` | Callout feature name for security_configuration_upgrade_banner. | | `SECURITY_NEWSLETTER_CALLOUT` | Callout feature name for security_newsletter_callout. | | `SECURITY_TRAINING_FEATURE_PROMOTION` | Callout feature name for security_training_feature_promotion. | -| `STORAGE_ENFORCEMENT_BANNER_FIRST_ENFORCEMENT_THRESHOLD` | Callout feature name for storage_enforcement_banner_first_enforcement_threshold. | -| `STORAGE_ENFORCEMENT_BANNER_FOURTH_ENFORCEMENT_THRESHOLD` | Callout feature name for storage_enforcement_banner_fourth_enforcement_threshold. | -| `STORAGE_ENFORCEMENT_BANNER_SECOND_ENFORCEMENT_THRESHOLD` | Callout feature name for storage_enforcement_banner_second_enforcement_threshold. | -| `STORAGE_ENFORCEMENT_BANNER_THIRD_ENFORCEMENT_THRESHOLD` | Callout feature name for storage_enforcement_banner_third_enforcement_threshold. | | `SUBMIT_LICENSE_USAGE_DATA_BANNER` | Callout feature name for submit_license_usage_data_banner. | | `SUGGEST_PIPELINE` | Callout feature name for suggest_pipeline. | | `SUGGEST_POPOVER_DISMISSED` | Callout feature name for suggest_popover_dismissed. | @@ -23768,8 +25634,6 @@ Name of the feature that the callout is for. | `UNFINISHED_TAG_CLEANUP_CALLOUT` | Callout feature name for unfinished_tag_cleanup_callout. | | `USER_REACHED_LIMIT_FREE_PLAN_ALERT` | Callout feature name for user_reached_limit_free_plan_alert. | | `VERIFICATION_REMINDER` | Callout feature name for verification_reminder. | -| `VSCODE_WEB_IDE` | Callout feature name for vscode_web_ide. | -| `VSCODE_WEB_IDE_CALLOUT` | Callout feature name for vscode_web_ide_callout. | | `WEB_IDE_ALERT_DISMISSED` | Callout feature name for web_ide_alert_dismissed. | | `WEB_IDE_CI_ENVIRONMENTS_GUIDANCE` | Callout feature name for web_ide_ci_environments_guidance. | @@ -23816,6 +25680,15 @@ Verification status of a GPG or X.509 signature for a commit. | `private` | Private visibility level. | | `public` | Public visibility level. | +### `VisibilityPipelineIdType` + +Determines whether the pipeline list shows ID or IID. + +| Value | Description | +| ----- | ----------- | +| `ID` | Display pipeline ID. | +| `IID` | Display pipeline IID. | + ### `VisibilityScopesEnum` | Value | Description | @@ -23947,6 +25820,15 @@ Weight ID wildcard values. | `ANY` | Weight is assigned. | | `NONE` | No weight is assigned. | +### `WorkItemAwardEmojiUpdateAction` + +Values for work item award emoji update enum. + +| Value | Description | +| ----- | ----------- | +| `ADD` | Adds the emoji. | +| `REMOVE` | Removes the emoji. | + ### `WorkItemSort` Values for sorting work items. @@ -23982,6 +25864,15 @@ Values for work item state events. | `CLOSE` | Closes the work item. | | `REOPEN` | Reopens the work item. | +### `WorkItemTodoUpdateAction` + +Values for work item to-do update enum. + +| Value | Description | +| ----- | ----------- | +| `ADD` | Adds the to-do. | +| `MARK_AS_DONE` | Marks the to-do as done. | + ### `WorkItemWidgetType` Type of a work item widget. @@ -23989,6 +25880,8 @@ Type of a work item widget. | Value | Description | | ----- | ----------- | | `ASSIGNEES` | Assignees widget. | +| `AWARD_EMOJI` | Award Emoji widget. | +| `CURRENT_USER_TODOS` | Current User Todos widget. | | `DESCRIPTION` | Description widget. | | `HEALTH_STATUS` | Health Status widget. | | `HIERARCHY` | Hierarchy widget. | @@ -23996,6 +25889,7 @@ Type of a work item widget. | `LABELS` | Labels widget. | | `MILESTONE` | Milestone widget. | | `NOTES` | Notes widget. | +| `NOTIFICATIONS` | Notifications widget. | | `PROGRESS` | Progress widget. | | `REQUIREMENT_LEGACY` | Requirement Legacy widget. | | `START_AND_DUE_DATE` | Start And Due Date widget. | @@ -24020,6 +25914,18 @@ A `AchievementsAchievementID` is a global ID. It is encoded as a string. An example `AchievementsAchievementID` is: `"gid://gitlab/Achievements::Achievement/1"`. +### `AchievementsUserAchievementID` + +A `AchievementsUserAchievementID` is a global ID. It is encoded as a string. + +An example `AchievementsUserAchievementID` is: `"gid://gitlab/Achievements::UserAchievement/1"`. + +### `AiModelID` + +A `AiModelID` is a global ID. It is encoded as a string. + +An example `AiModelID` is: `"gid://gitlab/Ai::Model/1"`. + ### `AlertManagementAlertID` A `AlertManagementAlertID` is a global ID. It is encoded as a string. @@ -24050,6 +25956,12 @@ A `AuditEventsExternalAuditEventDestinationID` is a global ID. It is encoded as An example `AuditEventsExternalAuditEventDestinationID` is: `"gid://gitlab/AuditEvents::ExternalAuditEventDestination/1"`. +### `AuditEventsInstanceExternalAuditEventDestinationID` + +A `AuditEventsInstanceExternalAuditEventDestinationID` is a global ID. It is encoded as a string. + +An example `AuditEventsInstanceExternalAuditEventDestinationID` is: `"gid://gitlab/AuditEvents::InstanceExternalAuditEventDestination/1"`. + ### `AuditEventsStreamingHeaderID` A `AuditEventsStreamingHeaderID` is a global ID. It is encoded as a string. @@ -24118,6 +26030,18 @@ A `CiRunnerID` is a global ID. It is encoded as a string. An example `CiRunnerID` is: `"gid://gitlab/Ci::Runner/1"`. +### `CiRunnerManagerID` + +A `CiRunnerManagerID` is a global ID. It is encoded as a string. + +An example `CiRunnerManagerID` is: `"gid://gitlab/Ci::RunnerManager/1"`. + +### `CiStageID` + +A `CiStageID` is a global ID. It is encoded as a string. + +An example `CiStageID` is: `"gid://gitlab/Ci::Stage/1"`. + ### `ClustersAgentID` A `ClustersAgentID` is a global ID. It is encoded as a string. @@ -24288,6 +26212,12 @@ An example `EpicTreeSortingID` is: `"gid://gitlab/EpicTreeSorting/1"`. Represents signed double-precision fractional values as specified by [IEEE 754](https://en.wikipedia.org/wiki/IEEE_floating_point). +### `GeoBaseRegistryID` + +A `GeoBaseRegistryID` is a global ID. It is encoded as a string. + +An example `GeoBaseRegistryID` is: `"gid://gitlab/Geo::BaseRegistry/1"`. + ### `GitlabErrorTrackingDetailedErrorID` A `GitlabErrorTrackingDetailedErrorID` is a global ID. It is encoded as a string. @@ -24557,6 +26487,12 @@ A `ReleasesLinkID` is a global ID. It is encoded as a string. An example `ReleasesLinkID` is: `"gid://gitlab/Releases::Link/1"`. +### `RemoteDevelopmentWorkspaceID` + +A `RemoteDevelopmentWorkspaceID` is a global ID. It is encoded as a string. + +An example `RemoteDevelopmentWorkspaceID` is: `"gid://gitlab/RemoteDevelopment::Workspace/1"`. + ### `SecurityTrainingProviderID` A `SecurityTrainingProviderID` is a global ID. It is encoded as a string. @@ -24641,12 +26577,6 @@ A `VulnerabilitiesExternalIssueLinkID` is a global ID. It is encoded as a string An example `VulnerabilitiesExternalIssueLinkID` is: `"gid://gitlab/Vulnerabilities::ExternalIssueLink/1"`. -### `VulnerabilitiesFindingID` - -A `VulnerabilitiesFindingID` is a global ID. It is encoded as a string. - -An example `VulnerabilitiesFindingID` is: `"gid://gitlab/Vulnerabilities::Finding/1"`. - ### `VulnerabilitiesScannerID` A `VulnerabilitiesScannerID` is a global ID. It is encoded as a string. @@ -24739,6 +26669,25 @@ One of: - [`NugetMetadata`](#nugetmetadata) - [`PypiMetadata`](#pypimetadata) +#### `Registrable` + +One of: + +- [`CiSecureFileRegistry`](#cisecurefileregistry) +- [`ContainerRepositoryRegistry`](#containerrepositoryregistry) +- [`DependencyProxyBlobRegistry`](#dependencyproxyblobregistry) +- [`DependencyProxyManifestRegistry`](#dependencyproxymanifestregistry) +- [`JobArtifactRegistry`](#jobartifactregistry) +- [`LfsObjectRegistry`](#lfsobjectregistry) +- [`MergeRequestDiffRegistry`](#mergerequestdiffregistry) +- [`PackageFileRegistry`](#packagefileregistry) +- [`PagesDeploymentRegistry`](#pagesdeploymentregistry) +- [`PipelineArtifactRegistry`](#pipelineartifactregistry) +- [`ProjectWikiRepositoryRegistry`](#projectwikirepositoryregistry) +- [`SnippetRepositoryRegistry`](#snippetrepositoryregistry) +- [`TerraformStateVersionRegistry`](#terraformstateversionregistry) +- [`UploadRegistry`](#uploadregistry) + #### `SecurityPolicySource` Represents a policy source. Its fields depend on the source type. @@ -24852,6 +26801,7 @@ Implementations: - [`EpicIssue`](#epicissue) - [`Issue`](#issue) - [`MergeRequest`](#mergerequest) +- [`WorkItemWidgetCurrentUserTodos`](#workitemwidgetcurrentusertodos) ##### Fields with arguments @@ -24925,6 +26875,21 @@ Implementations: | ---- | ---- | ----------- | | `events` | [`EventConnection`](#eventconnection) | List of events associated with the object. (see [Connections](#connections)) | +#### `ExternalAuditEventDestinationInterface` + +Implementations: + +- [`ExternalAuditEventDestination`](#externalauditeventdestination) +- [`InstanceExternalAuditEventDestination`](#instanceexternalauditeventdestination) + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `destinationUrl` | [`String!`](#string) | External destination to send audit events to. | +| `id` | [`ID!`](#id) | ID of the destination. | +| `verificationToken` | [`String!`](#string) | Verification token to validate source of event. | + #### `MemberInterface` Implementations: @@ -25122,6 +27087,7 @@ Implementations: | `savedReplies` | [`SavedReplyConnection`](#savedreplyconnection) | Saved replies authored by the user. Will not return saved replies if `saved_replies` feature flag is disabled. (see [Connections](#connections)) | | `state` | [`UserState!`](#userstate) | State of the user. | | `status` | [`UserStatus`](#userstatus) | User status. | +| `userAchievements` **{warning-solid}** | [`UserAchievementConnection`](#userachievementconnection) | **Introduced** in 15.10. This feature is an Experiment. It can be changed or removed at any time. Achievements for the user. Only returns for namespaces where the `achievements` feature flag is enabled. | | `userPermissions` | [`UserPermissions!`](#userpermissions) | Permissions for the current user on the resource. | | `username` | [`String!`](#string) | Username of the user. Unique within this instance of GitLab. | | `webPath` | [`String!`](#string) | Web path of the user. | @@ -25143,6 +27109,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `authorUsername` | [`String`](#string) | Username of the author. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | | `createdBefore` | [`Time`](#time) | Merge requests created before this timestamp. | @@ -25177,6 +27144,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `assigneeUsername` | [`String`](#string) | Username of the assignee. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | | `createdBefore` | [`Time`](#time) | Merge requests created before this timestamp. | @@ -25228,6 +27196,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `approved` | [`Boolean`](#boolean) | Limit results to approved merge requests. Available only when the feature flag `mr_approved_filter` is enabled. | | `assigneeUsername` | [`String`](#string) | Username of the assignee. | | `authorUsername` | [`String`](#string) | Username of the author. | | `createdAfter` | [`Time`](#time) | Merge requests created after this timestamp. | @@ -25338,11 +27307,29 @@ four standard [pagination arguments](#connection-pagination-arguments): | `state` | [`[TodoStateEnum!]`](#todostateenum) | State of the todo. | | `type` | [`[TodoTargetEnum!]`](#todotargetenum) | Type of the todo. | +###### `User.workspaces` + +Workspaces owned by the current user. + +Returns [`WorkspaceConnection`](#workspaceconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +####### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `ids` | [`[RemoteDevelopmentWorkspaceID!]`](#remotedevelopmentworkspaceid) | Array of global workspace IDs. For example, `["gid://gitlab/RemoteDevelopment::Workspace/1"]`. | + #### `WorkItemWidget` Implementations: - [`WorkItemWidgetAssignees`](#workitemwidgetassignees) +- [`WorkItemWidgetAwardEmoji`](#workitemwidgetawardemoji) +- [`WorkItemWidgetCurrentUserTodos`](#workitemwidgetcurrentusertodos) - [`WorkItemWidgetDescription`](#workitemwidgetdescription) - [`WorkItemWidgetHealthStatus`](#workitemwidgethealthstatus) - [`WorkItemWidgetHierarchy`](#workitemwidgethierarchy) @@ -25350,6 +27337,7 @@ Implementations: - [`WorkItemWidgetLabels`](#workitemwidgetlabels) - [`WorkItemWidgetMilestone`](#workitemwidgetmilestone) - [`WorkItemWidgetNotes`](#workitemwidgetnotes) +- [`WorkItemWidgetNotifications`](#workitemwidgetnotifications) - [`WorkItemWidgetProgress`](#workitemwidgetprogress) - [`WorkItemWidgetRequirementLegacy`](#workitemwidgetrequirementlegacy) - [`WorkItemWidgetStartAndDueDate`](#workitemwidgetstartandduedate) @@ -25371,6 +27359,59 @@ be used as arguments). Only general use input types are listed here. For mutation input types, see the associated mutation type above. +### `AiExplainCodeInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `messages` | [`[AiExplainCodeMessageInput!]!`](#aiexplaincodemessageinput) | Code messages that is passed to be explained by AI. | +| `resourceId` | [`AiModelID!`](#aimodelid) | Global ID of the resource to mutate. | + +### `AiExplainCodeMessageInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `content` | [`String!`](#string) | Content of the message. | +| `role` | [`String!`](#string) | Role of the message (system, user, assistant). | + +### `AiExplainVulnerabilityInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `resourceId` | [`AiModelID!`](#aimodelid) | Global ID of the resource to mutate. | + +### `AiGenerateDescriptionInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `content` | [`String!`](#string) | Content of the message. | +| `descriptionTemplateName` | [`String`](#string) | Name of the description template to use to generate message off of. | +| `resourceId` | [`AiModelID!`](#aimodelid) | Global ID of the resource to mutate. | + +### `AiSummarizeCommentsInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `resourceId` | [`AiModelID!`](#aimodelid) | Global ID of the resource to mutate. | + +### `AiTanukiBotInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `question` | [`String!`](#string) | GitLab documentation question for AI to answer. | +| `resourceId` | [`AiModelID!`](#aimodelid) | Global ID of the resource to mutate. | + ### `AlertManagementPayloadAlertFieldInput` Field that are available while modifying the custom mapping attributes for an HTTP integration. @@ -25439,6 +27480,16 @@ Attributes for defining a CI/CD variable. | `lastCommitId` | [`String`](#string) | Last known file commit ID. | | `previousPath` | [`String`](#string) | Original full path to the file being moved. | +### `ComplianceFrameworkFilters` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `id` | [`ComplianceManagementFrameworkID`](#compliancemanagementframeworkid) | ID of the compliance framework. | +| `not` | [`NegatedComplianceFrameworkFilters`](#negatedcomplianceframeworkfilters) | Negated compliance framework filter input. | +| `presenceFilter` | [`ComplianceFrameworkPresenceFilter`](#complianceframeworkpresencefilter) | Checks presence of compliance framework of the project, "none" and "any" values are supported. | + ### `ComplianceFrameworkInput` #### Arguments @@ -25460,6 +27511,7 @@ Attributes for defining a CI/CD variable. | `mergedAfter` | [`Date`](#date) | Merge requests merged after this date (inclusive). | | `mergedBefore` | [`Date`](#date) | Merge requests merged before this date (inclusive). | | `projectIds` | [`[ProjectID!]`](#projectid) | Filter compliance violations by project. | +| `targetBranch` | [`String`](#string) | Filter compliance violations by target branch. | ### `DastProfileCadenceInput` @@ -25589,6 +27641,15 @@ Represents an escalation rule. | `status` | [`EscalationRuleStatus!`](#escalationrulestatus) | Status required to prevent the rule from activating. | | `username` | [`String`](#string) | Username of the user to notify. | +### `GenerateTestFileInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `filePath` | [`String!`](#string) | File path to generate test files for. | +| `resourceId` | [`AiModelID!`](#aimodelid) | Global ID of the resource to mutate. | + ### `JiraUsersMappingInputType` #### Arguments @@ -25629,6 +27690,14 @@ Represents an escalation rule. | `types` | [`[IssueType!]`](#issuetype) | Filter by the given issue types. | | `weight` | [`String`](#string) | Filter by weight. | +### `NegatedComplianceFrameworkFilters` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `id` | [`ComplianceManagementFrameworkID`](#compliancemanagementframeworkid) | ID of the compliance framework. | + ### `NegatedEpicBoardIssueInput` #### Arguments @@ -25925,12 +27994,15 @@ A time-frame defined as a closed inclusive range of two dates. | Name | Type | Description | | ---- | ---- | ----------- | | `assigneesWidget` | [`WorkItemWidgetAssigneesInput`](#workitemwidgetassigneesinput) | Input for assignees widget. | +| `awardEmojiWidget` | [`WorkItemWidgetAwardEmojiUpdateInput`](#workitemwidgetawardemojiupdateinput) | Input for award emoji widget. | | `confidential` | [`Boolean`](#boolean) | Sets the work item confidentiality. | +| `currentUserTodosWidget` | [`WorkItemWidgetCurrentUserTodosInput`](#workitemwidgetcurrentusertodosinput) | Input for to-dos widget. | | `descriptionWidget` | [`WorkItemWidgetDescriptionInput`](#workitemwidgetdescriptioninput) | Input for description widget. | | `hierarchyWidget` | [`WorkItemWidgetHierarchyUpdateInput`](#workitemwidgethierarchyupdateinput) | Input for hierarchy widget. | | `id` | [`WorkItemID!`](#workitemid) | Global ID of the work item. | | `labelsWidget` | [`WorkItemWidgetLabelsUpdateInput`](#workitemwidgetlabelsupdateinput) | Input for labels widget. | | `milestoneWidget` | [`WorkItemWidgetMilestoneInput`](#workitemwidgetmilestoneinput) | Input for milestone widget. | +| `notificationsWidget` | [`WorkItemWidgetNotificationsUpdateInput`](#workitemwidgetnotificationsupdateinput) | Input for notifications widget. | | `startAndDueDateWidget` | [`WorkItemWidgetStartAndDueDateUpdateInput`](#workitemwidgetstartandduedateupdateinput) | Input for start and due date widget. | | `stateEvent` | [`WorkItemStateEvent`](#workitemstateevent) | Close or reopen a work item. | | `title` | [`String`](#string) | Title of the work item. | @@ -25943,6 +28015,24 @@ A time-frame defined as a closed inclusive range of two dates. | ---- | ---- | ----------- | | `assigneeIds` | [`[UserID!]!`](#userid) | Global IDs of assignees. | +### `WorkItemWidgetAwardEmojiUpdateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `action` | [`WorkItemAwardEmojiUpdateAction!`](#workitemawardemojiupdateaction) | Action for the update. | +| `name` | [`String!`](#string) | Emoji name. | + +### `WorkItemWidgetCurrentUserTodosInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `action` | [`WorkItemTodoUpdateAction!`](#workitemtodoupdateaction) | Action for the update. | +| `todoId` | [`TodoID`](#todoid) | Global ID of the to-do. If not present, all to-dos of the work item will be updated. | + ### `WorkItemWidgetDescriptionInput` #### Arguments @@ -25973,8 +28063,10 @@ A time-frame defined as a closed inclusive range of two dates. | Name | Type | Description | | ---- | ---- | ----------- | +| `adjacentWorkItemId` | [`WorkItemID`](#workitemid) | ID of the work item to be switched with. | | `childrenIds` | [`[WorkItemID!]`](#workitemid) | Global IDs of children work items. | | `parentId` | [`WorkItemID`](#workitemid) | Global ID of the parent work item. Use `null` to remove the association. | +| `relativePosition` | [`RelativePositionType`](#relativepositiontype) | Type of switch. Valid values are `BEFORE` or `AFTER`. | ### `WorkItemWidgetIterationInput` @@ -26001,6 +28093,14 @@ A time-frame defined as a closed inclusive range of two dates. | ---- | ---- | ----------- | | `milestoneId` | [`MilestoneID`](#milestoneid) | Milestone to assign to the work item. | +### `WorkItemWidgetNotificationsUpdateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `subscribed` | [`Boolean!`](#boolean) | Desired state of the subscription. | + ### `WorkItemWidgetProgressInput` #### Arguments diff --git a/doc/api/graphql/removed_items.md b/doc/api/graphql/removed_items.md index 57f1c49290f..4c506d93436 100644 --- a/doc/api/graphql/removed_items.md +++ b/doc/api/graphql/removed_items.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -10,6 +10,28 @@ GraphQL is a versionless API, unlike the REST API. Occasionally, items have to be updated or removed from the GraphQL API. According to our [process for removing items](index.md#deprecation-and-removal-process), here are the items that have been removed. +## GitLab 16.0 + +Fields removed in GitLab 16.0. + +### GraphQL Fields + +| Field name | GraphQL type | Deprecated in | Removal MR | Use instead | +|---|---|---|---|---| +| `name` | `PipelineSecurityReportFinding` | [15.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89571) | [!119055](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/119055) | `title` | +| `external` | `ReleaseAssetLink` | 15.9 | [!111750](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/111750) | None | +| `confidence` | `PipelineSecurityReportFinding` | 15.4 | [!118617](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118617) | None | +| `PAUSED` | `CiRunnerStatus` | 14.8 | [!118635](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118635) | `CiRunner.paused: true` | +| `ACTIVE` | `CiRunnerStatus` | 14.8 | [!118635](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118635) | `CiRunner.paused: false` | + +### GraphQL Mutations + +| Argument name | Mutation | Deprecated in | Use instead | +| -------------------- | -------------------- |---------------------------------------------------------------------|------------------------------------------------| +| - | `vulnerabilityFindingDismiss` | [15.5](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/99170) | `vulnerabilityDismiss` or `securityFindingDismiss` | +| - | `apiFuzzingCiConfigurationCreate` | [15.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87241) | `todos` | +| - | `CiCdSettingsUpdate` | [15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/361801) | `ProjectCiCdSettingsUpdate` | + ## GitLab 15.0 Fields removed in GitLab 15.0. diff --git a/doc/api/group_access_tokens.md b/doc/api/group_access_tokens.md index 82065590b33..446aa3668d8 100644 --- a/doc/api/group_access_tokens.md +++ b/doc/api/group_access_tokens.md @@ -123,6 +123,34 @@ curl --request POST --header "PRIVATE-TOKEN: " \ } ``` +## Rotate a group access token + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/403042) in GitLab 16.0 + +Rotate a group access token. Revokes the previous token and creates a new token that expires in one week. + +```plaintext +POST /groups/:id/access_tokens/:token_id/rotate +``` + +| Attribute | Type | required | Description | +|-----------|---------|----------|---------------------| +| `id` | integer or string | yes | ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding) | +| `token_id` | integer or string | yes | ID of the project access token | + +```shell +curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups//access_tokens//rotate" +``` + +### Responses + +- `200: OK` if existing token is successfully revoked and the new token is created. +- `400: Bad Request` if not rotated successfully. +- `401: Unauthorized` if either the: + - User does not have access to the token with the specified ID. + - Token with the specified ID does not exist. +- `404: Not Found` if the user is an administrator but the token with the specified ID does not exist. + ## Revoke a group access token > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77236) in GitLab 14.7. diff --git a/doc/api/group_badges.md b/doc/api/group_badges.md index ede7591c6d1..4c225e8aacb 100644 --- a/doc/api/group_badges.md +++ b/doc/api/group_badges.md @@ -1,6 +1,6 @@ --- -stage: Manage -group: Organization +stage: Data Stores +group: Tenant Scale info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -10,7 +10,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Placeholder tokens -Badges support placeholders that are replaced in real time in both the link and image URL. The allowed placeholders are: +[Badges](../user/project/badges.md) support placeholders that are replaced in real time in both the link and image URL. The allowed placeholders are: diff --git a/doc/api/group_boards.md b/doc/api/group_boards.md index f01c33607a7..9208f54c0bf 100644 --- a/doc/api/group_boards.md +++ b/doc/api/group_boards.md @@ -76,7 +76,7 @@ Example response: ] ``` -Users on [GitLab Premium or higher](https://about.gitlab.com/pricing/) see +Users on [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/) see different parameters, due to the ability to have multiple group boards. Example response: @@ -192,7 +192,7 @@ Example response: } ``` -Users on [GitLab Premium or higher](https://about.gitlab.com/pricing/) see +Users on [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/) see different parameters, due to the ability to have multiple group issue boards. Example response: diff --git a/doc/api/group_clusters.md b/doc/api/group_clusters.md index daffda8340c..a4369ecad5f 100644 --- a/doc/api/group_clusters.md +++ b/doc/api/group_clusters.md @@ -1,10 +1,10 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Group clusters API (certificate-based) (DEPRECATED) **(FREE)** +# Group clusters API (certificate-based) (deprecated) **(FREE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/30213) in GitLab 12.1. > - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. diff --git a/doc/api/group_import_export.md b/doc/api/group_import_export.md index c648b6bad37..c78f0ecb781 100644 --- a/doc/api/group_import_export.md +++ b/doc/api/group_import_export.md @@ -1,15 +1,13 @@ --- stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Group import/export API **(FREE)** +# Group import and export API **(FREE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20353) in GitLab 12.8. - -Group Import/Export allows you to export group structure and import it to a new location. -When used with [Project Import/Export](project_import_export.md), you can preserve connections with +Use the group import and export API to export a group structure and import it to a new location. +When you use the group import and export API with the [project import and export API](project_import_export.md), you can preserve connections with group-level relationships, such as connections between project issues and group epics. Group exports include the following: @@ -22,6 +20,18 @@ Group exports include the following: - Subgroups. Each subgroup includes all data above - Group wikis **(PREMIUM SELF)** +To preserve group-level relationships from imported projects, you should run group import and export first. This way, you can import project exports into the desired group structure. + +Imported groups have a `private` visibility level unless you import them into a parent group. +If you import groups into a parent group, the subgroups inherit by default a similar level of visibility. + +To preserve the member list and their respective permissions on imported groups, review the users in these groups. Make sure these users exist before importing the desired groups. + +## Prerequisites + +For information on prerequisites for group import and export API, see prerequisites for +[migrating groups by uploading an export file](../user/group/import/index.md#preparation). + ## Schedule new export Start a new group export. @@ -103,14 +113,3 @@ curl --request POST --header "PRIVATE-TOKEN: " \ NOTE: The maximum import file size can be set by the Administrator, default is `0` (unlimited). As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](settings.md#change-application-settings) or the [Admin Area](../user/admin_area/settings/account_and_limit_settings.md). Default [modified](https://gitlab.com/gitlab-org/gitlab/-/issues/251106) from 50 MB to 0 in GitLab 13.8. - -## Important notes - -Note the following: - -- To preserve group-level relationships from imported projects, run Group Import/Export first, - to allow project imports into the desired group structure. -- Imported groups are given a `private` visibility level, unless imported into a parent group. -- If imported into a parent group, subgroups inherit a similar level of visibility, unless otherwise restricted. -- To preserve the member list and their respective permissions on imported groups, - review the users in these groups. Make sure these users exist before importing the desired groups. diff --git a/doc/api/group_iterations.md b/doc/api/group_iterations.md index 3c445ee09dd..cae271e6365 100644 --- a/doc/api/group_iterations.md +++ b/doc/api/group_iterations.md @@ -22,13 +22,17 @@ GET /groups/:id/iterations?state=opened GET /groups/:id/iterations?state=closed GET /groups/:id/iterations?search=version GET /groups/:id/iterations?include_ancestors=false +GET /groups/:id/iterations?updated_before=2013-10-02T09%3A24%3A18Z +GET /groups/:id/iterations?updated_after=2013-10-02T09%3A24%3A18Z ``` | Attribute | Type | Required | Description | | ------------------- | ------- | -------- | ----------- | -| `state` | string | no | 'Return `opened`, `upcoming`, `current (previously started)`, `closed`, or `all` iterations. Filtering by `started` state is deprecated starting with 14.1, use `current` instead.' | +| `state` | string | no | 'Return `opened`, `upcoming`, `current`, `closed`, or `all` iterations.' | | `search` | string | no | Return only iterations with a title matching the provided string. | | `include_ancestors` | boolean | no | Include iterations from parent group and its ancestors. Defaults to `true`. | +| `updated_before` | datetime | no | Return only iterations updated before the given datetime. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/378662) in GitLab 15.10. | +| `updated_after` | datetime | no | Return only iterations updated after the given datetime. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/378662) in GitLab 15.10. | Example request: diff --git a/doc/api/group_level_variables.md b/doc/api/group_level_variables.md index 4037a778d7f..921a9922c9b 100644 --- a/doc/api/group_level_variables.md +++ b/doc/api/group_level_variables.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Authoring +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/api/group_milestones.md b/doc/api/group_milestones.md index fc95230cbba..52bce54119a 100644 --- a/doc/api/group_milestones.md +++ b/doc/api/group_milestones.md @@ -21,6 +21,8 @@ GET /groups/:id/milestones?state=active GET /groups/:id/milestones?state=closed GET /groups/:id/milestones?title=1.0 GET /groups/:id/milestones?search=version +GET /groups/:id/milestones?updated_before=2013-10-02T09%3A24%3A18Z +GET /groups/:id/milestones?updated_after=2013-10-02T09%3A24%3A18Z ``` Parameters: @@ -32,7 +34,9 @@ Parameters: | `state` | string | no | Return only `active` or `closed` milestones | | `title` | string | no | Return only the milestones having the given `title` | | `search` | string | no | Return only milestones with a title or description matching the provided string | -| `include_parent_milestones` | boolean | optional | Include milestones from parent group and its ancestors. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/196066) in GitLab 13.4 | +| `include_parent_milestones` | boolean | no | Include milestones from parent group and its ancestors. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/196066) in GitLab 13.4 | +| `updated_before` | datetime | no | Return only milestones updated before the given datetime. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). Introduced in GitLab 15.10 | +| `updated_after` | datetime | no | Return only milestones updated after the given datetime. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). Introduced in GitLab 15.10 | ```shell curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/milestones" diff --git a/doc/api/group_protected_branches.md b/doc/api/group_protected_branches.md new file mode 100644 index 00000000000..db42ca98f0e --- /dev/null +++ b/doc/api/group_protected_branches.md @@ -0,0 +1,469 @@ +--- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Group-level protected branches API **(PREMIUM SELF)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/110603) in GitLab 15.9 [with a flag](../administration/feature_flags.md) named `group_protected_branches`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `group_protected_branches`. +On GitLab.com, this feature is not available. + +## Valid access levels + +The access levels are defined in the `ProtectedRefAccess.allowed_access_levels` method. +These levels are recognized: + +```plaintext +0 => No access +30 => Developer access +40 => Maintainer access +60 => Admin access +``` + +## List protected branches + +Gets a list of protected branches from a group. If a wildcard is set, it is returned instead +of the exact name of the branches that match that wildcard. + +```plaintext +GET /groups/:id/protected_branches +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer or string | yes | The ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | +| `search` | string | no | Name or part of the name of protected branches to be searched for. | + +```shell +curl --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/groups/5/protected_branches" +``` + +Example response: + +```json +[ + { + "id": 1, + "name": "master", + "push_access_levels": [ + { + "id": 1, + "access_level": 40, + "user_id": null, + "group_id": 1234, + "access_level_description": "Maintainers" + } + ], + "merge_access_levels": [ + { + "id": 1, + "access_level": 40, + "user_id": null, + "group_id": 1234, + "access_level_description": "Maintainers" + } + ], + "allow_force_push":false, + "code_owner_approval_required": false + }, + { + "id": 1, + "name": "release/*", + "push_access_levels": [ + { + "id": 1, + "access_level": 40, + "user_id": null, + "group_id": null, + "access_level_description": "Maintainers" + } + ], + "merge_access_levels": [ + { + "id": 1, + "access_level": 40, + "user_id": null, + "group_id": 1234, + "access_level_description": "Maintainers" + } + ], + "allow_force_push":false, + "code_owner_approval_required": false + }, + ... +] +``` + +## Get a single protected branch or wildcard protected branch + +Gets a single protected branch or wildcard protected branch. + +```plaintext +GET /groups/:id/protected_branches/:name +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer or string | yes | The ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | +| `name` | string | yes | The name of the branch or wildcard. | + +```shell +curl --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/groups/5/protected_branches/master" +``` + +Example response: + +```json +{ + "id": 1, + "name": "master", + "push_access_levels": [ + { + "id": 1, + "access_level": 40, + "user_id": null, + "group_id": null, + "access_level_description": "Maintainers" + } + ], + "merge_access_levels": [ + { + "id": 1, + "access_level": null, + "user_id": null, + "group_id": 1234, + "access_level_description": "Example Merge Group" + } + ], + "allow_force_push":false, + "code_owner_approval_required": false +} +``` + +## Protect repository branches + +Protects a single repository branch using a wildcard protected branch. + +```plaintext +POST /groups/:id/protected_branches +``` + +```shell +curl --request POST --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/groups/5/protected_branches?name=*-stable&push_access_level=30&merge_access_level=30&unprotect_access_level=40" +``` + +| Attribute | Type | Required | Description | +| -------------------------------------------- | ---- | -------- | ----------- | +| `id` | integer or string | yes | The ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | +| `name` | string | yes | The name of the branch or wildcard. | +| `allow_force_push` | boolean | no | Allow all users with push access to force push. Default: `false`. | +| `allowed_to_merge` | array | no | Array of access levels allowed to merge, with each described by a hash of the form `{user_id: integer}`, `{group_id: integer}`, or `{access_level: integer}`. | +| `allowed_to_push` | array | no | Array of access levels allowed to push, with each described by a hash of the form `{user_id: integer}`, `{group_id: integer}`, or `{access_level: integer}`. | +| `allowed_to_unprotect` | array | no | Array of access levels allowed to unprotect, with each described by a hash of the form `{user_id: integer}`, `{group_id: integer}`, or `{access_level: integer}`. | +| `code_owner_approval_required` | boolean | no | Prevent pushes to this branch if it matches an item in the [`CODEOWNERS` file](../user/project/codeowners/index.md). Default: `false`. | +| `merge_access_level` | integer | no | Access levels allowed to merge. Defaults: `40`, Maintainer role. | +| `push_access_level` | integer | no | Access levels allowed to push. Defaults: `40`, Maintainer role. | +| `unprotect_access_level` | integer | no | Access levels allowed to unprotect. Defaults: `40`, Maintainer role. | + +Example response: + +```json +{ + "id": 1, + "name": "*-stable", + "push_access_levels": [ + { + "id": 1, + "access_level": 30, + "user_id": null, + "group_id": null, + "access_level_description": "Developers + Maintainers" + } + ], + "merge_access_levels": [ + { + "id": 1, + "access_level": 30, + "user_id": null, + "group_id": null, + "access_level_description": "Developers + Maintainers" + } + ], + "unprotect_access_levels": [ + { + "id": 1, + "access_level": 40, + "user_id": null, + "group_id": null, + "access_level_description": "Maintainers" + } + ], + "allow_force_push":false, + "code_owner_approval_required": false +} +``` + +### Example with user / group level access + +Elements in the `allowed_to_push` / `allowed_to_merge` / `allowed_to_unprotect` array should take the +form `{user_id: integer}`, `{group_id: integer}`, or `{access_level: integer}`. Each user must have +access to the project and each group must +[have this project shared](../user/project/members/share_project_with_groups.md). These access levels +allow [more granular control over protected branch access](../user/project/protected_branches.md). + +```shell +curl --request POST --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/groups/5/protected_branches?name=*-stable&allowed_to_push%5B%5D%5Buser_id%5D=1" +``` + +Example response: + +```json +{ + "id": 1, + "name": "*-stable", + "push_access_levels": [ + { + "id": 1, + "access_level": null, + "user_id": 1, + "group_id": null, + "access_level_description": "Administrator" + } + ], + "merge_access_levels": [ + { + "id": 1, + "access_level": 40, + "user_id": null, + "group_id": null, + "access_level_description": "Maintainers" + } + ], + "unprotect_access_levels": [ + { + "id": 1, + "access_level": 40, + "user_id": null, + "group_id": null, + "access_level_description": "Maintainers" + } + ], + "allow_force_push":false, + "code_owner_approval_required": false +} +``` + +### Example with allow to push and allow to merge access + +Example request: + +```shell +curl --request POST \ + --header "PRIVATE-TOKEN: " \ + --header "Content-Type: application/json" \ + --data '{ + "name": "master", + "allowed_to_push": [{"access_level": 30}], + "allowed_to_merge": [{ + "access_level": 30 + },{ + "access_level": 40 + } + ]}' + "https://gitlab.example.com/api/v4/groups/5/protected_branches" +``` + +Example response: + +```json +{ + "id": 5, + "name": "master", + "push_access_levels": [ + { + "id": 1, + "access_level": 30, + "access_level_description": "Developers + Maintainers", + "user_id": null, + "group_id": null + } + ], + "merge_access_levels": [ + { + "id": 1, + "access_level": 30, + "access_level_description": "Developers + Maintainers", + "user_id": null, + "group_id": null + }, + { + "id": 2, + "access_level": 40, + "access_level_description": "Maintainers", + "user_id": null, + "group_id": null + } + ], + "unprotect_access_levels": [ + { + "id": 1, + "access_level": 40, + "access_level_description": "Maintainers", + "user_id": null, + "group_id": null + } + ], + "allow_force_push":false, + "code_owner_approval_required": false +} +``` + +## Unprotect repository branches + +Unprotects the given protected branch or wildcard protected branch. + +```plaintext +DELETE /groups/:id/protected_branches/:name +``` + +```shell +curl --request DELETE --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/groups/5/protected_branches/*-stable" +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer or string | yes | The ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | +| `name` | string | yes | The name of the branch. | + +Example response: + +```json +{ + "name": "master", + "push_access_levels": [ + { + "id": 12, + "access_level": 40, + "access_level_description": "Maintainers", + "user_id": null, + "group_id": null + } + ] +} +``` + +## Update a protected branch + +Updates a protected branch. + +```plaintext +PATCH /groups/:id/protected_branches/:name +``` + +```shell +curl --request PATCH --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/groups/5/protected_branches/feature-branch?allow_force_push=true&code_owner_approval_required=true" +``` + +| Attribute | Type | Required | Description | +| -------------------------------------------- | ---- | -------- | ----------- | +| `id` | integer or string | yes | The ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | +| `name` | string | yes | The name of the branch. | +| `allow_force_push` | boolean | no | When enabled, members who can push to this branch can also force push. | +| `allowed_to_push` | array | no | Array of push access levels, with each described by a hash. | +| `allowed_to_merge` | array | no | Array of merge access levels, with each described by a hash. | +| `allowed_to_unprotect` | array | no | Array of unprotect access levels, with each described by a hash. | +| `code_owner_approval_required` | boolean | no | Prevent pushes to this branch if it matches an item in the [`CODEOWNERS` file](../user/project/codeowners/index.md). Default: `false`. | + +Elements in the `allowed_to_push`, `allowed_to_merge` and `allowed_to_unprotect` arrays should: + +- Be one of `user_id`, `group_id`, or `access_level`. +- Take the form `{user_id: integer}`, `{group_id: integer}`, or `{access_level: integer}`. + +To update: + +- `user_id`: Ensure the updated user has access to the project. You must also pass the + `id` of the `access_level` in the respective hash. +- `group_id`: Ensure the updated group [has this project shared](../user/project/members/share_project_with_groups.md). + You must also pass the `id` of the `access_level` in the respective hash. + +To delete: + +- You must pass `_destroy` set to `true`. See the following examples. + +### Example: create a `push_access_level` record + +```shell +curl --header 'Content-Type: application/json' --request PATCH \ + --data '{"allowed_to_push": [{access_level: 40}]}' \ + --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/groups/22034114/protected_branches/master" +``` + +Example response: + +```json +{ + "name": "master", + "push_access_levels": [ + { + "id": 12, + "access_level": 40, + "access_level_description": "Maintainers", + "user_id": null, + "group_id": null + } + ] +} +``` + +### Example: update a `push_access_level` record + +```shell +curl --header 'Content-Type: application/json' --request PATCH \ + --data '{"allowed_to_push": [{"id": 12, "access_level": 0}]' \ + --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/22034114/protected_branches/master" +``` + +Example response: + +```json +{ + "name": "master", + "push_access_levels": [ + { + "id": 12, + "access_level": 0, + "access_level_description": "No One", + "user_id": null, + "group_id": null + } + ] +} +``` + +### Example: delete a `push_access_level` record + +```shell +curl --header 'Content-Type: application/json' --request PATCH \ + --data '{"allowed_to_push": [{"id": 12, "_destroy": true}]}' \ + --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/22034114/protected_branches/master" +``` + +Example response: + +```json +{ + "name": "master", + "push_access_levels": [] +} +``` diff --git a/doc/api/group_protected_environments.md b/doc/api/group_protected_environments.md index 3003b6b840f..67b2b818ea9 100644 --- a/doc/api/group_protected_environments.md +++ b/doc/api/group_protected_environments.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: concepts, howto --- diff --git a/doc/api/group_relations_export.md b/doc/api/group_relations_export.md index 065ae03259a..5118b2f00f5 100644 --- a/doc/api/group_relations_export.md +++ b/doc/api/group_relations_export.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -11,7 +11,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w With the Group Relations Export API, you can partially export group structure. This API is similar to [group export](group_import_export.md), but it exports each top-level relation (for example, milestones/boards/labels) as a separate file -instead of one archive. The group relations export API is primarily used in [group migration](../user/group/index.md). +instead of one archive. The group relations export API is primarily used in [group migration](../user/group/import/index.md). ## Schedule new export diff --git a/doc/api/group_releases.md b/doc/api/group_releases.md index 9c395adbe04..a735f36eb82 100644 --- a/doc/api/group_releases.md +++ b/doc/api/group_releases.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/api/group_repository_storage_moves.md b/doc/api/group_repository_storage_moves.md index 8d685c75f60..95d261e79a9 100644 --- a/doc/api/group_repository_storage_moves.md +++ b/doc/api/group_repository_storage_moves.md @@ -1,6 +1,6 @@ --- stage: Create -group: Editor +group: IDE info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: reference --- diff --git a/doc/api/group_wikis.md b/doc/api/group_wikis.md index 6fb74ea00b7..c03224373de 100644 --- a/doc/api/group_wikis.md +++ b/doc/api/group_wikis.md @@ -1,6 +1,6 @@ --- -stage: Create -group: Editor +stage: Plan +group: Knowledge info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments" type: reference, api --- diff --git a/doc/api/groups.md b/doc/api/groups.md index 8b4850fa6de..6d295b50a01 100644 --- a/doc/api/groups.md +++ b/doc/api/groups.md @@ -1,11 +1,15 @@ --- -stage: Manage -group: Organization +stage: Data Stores +group: Tenant Scale info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Groups API **(FREE)** +Interact with [groups](../user/group/index.md) by using the REST API. + +The fields returned in responses vary based on the [permissions](../user/permissions.md) of the authenticated user. + ## List groups Get a list of visible groups for the authenticated user. When accessed without @@ -27,7 +31,7 @@ Parameters: | `search` | string | no | Return the list of authorized groups matching the search criteria | | `order_by` | string | no | Order groups by `name`, `path`, `id`, or `similarity` (if searching, [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332889) in GitLab 14.1). Default is `name` | | `sort` | string | no | Order groups in `asc` or `desc` order. Default is `asc` | -| `statistics` | boolean | no | Include group statistics (administrators only) | +| `statistics` | boolean | no | Include group statistics (administrators only).
*Note:* The REST API response does not provide the full `RootStorageStatistics` data that is shown in the UI. To match the data in the UI, use GraphQL instead of REST. For more information, see the [Group GraphQL reference](../api/graphql/reference/index.md#group).| | `with_custom_attributes` | boolean | no | Include [custom attributes](custom_attributes.md) in response (administrators only) | | `owned` | boolean | no | Limit to groups explicitly owned by the current user | | `min_access_level` | integer | no | Limit to groups where current user has at least this [role (`access_level`)](members.md#roles) | @@ -110,11 +114,14 @@ GET /groups?statistics=true "packages_size": 0, "snippets_size": 50, "uploads_size": 0 - } + }, + "wiki_access_level": "private" } ] ``` +Users of [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/) also see the `wiki_access_level` attribute. + You can search for groups by name or path, see below. You can filter by [custom attributes](custom_attributes.md) with: @@ -184,6 +191,8 @@ GET /groups/:id/subgroups ] ``` +Users of [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/) also see the `wiki_access_level` attribute. + ## List a group's descendant groups > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217115) in GitLab 13.5 @@ -267,6 +276,8 @@ GET /groups/:id/descendant_groups ] ``` +Users of [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/) also see the `wiki_access_level` attribute. + ## List a group's projects Get a list of projects in this group. When accessed without authentication, only public projects are returned. @@ -514,11 +525,6 @@ To get the details of all projects within a group, use either the [list a group' curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/4" ``` -NOTE: -There is [a known issue](https://gitlab.com/gitlab-org/gitlab/-/issues/345200) that can -prevent `runners_token` from being returned when the call has the `with_projects=false` -parameter. - This endpoint returns: - All projects and shared projects in GitLab 12.5 and earlier. @@ -695,10 +701,15 @@ Example response: The `prevent_sharing_groups_outside_hierarchy` attribute is present only on top-level groups. -Users of [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see -the `shared_runners_minutes_limit` and `extra_shared_runners_minutes_limit` parameters: +Users of [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/) also see the attributes: -Additional response parameters: +- `shared_runners_minutes_limit` +- `extra_shared_runners_minutes_limit` +- `marked_for_deletion_on` +- `membership_lock` +- `wiki_access_level` + +Additional response attributes: ```json { @@ -706,30 +717,9 @@ Additional response parameters: "description": "Aliquid qui quis dignissimos distinctio ut commodi voluptas est.", "shared_runners_minutes_limit": 133, "extra_shared_runners_minutes_limit": 133, - ... -} -``` - -Users of [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see -the `marked_for_deletion_on` attribute: - -```json -{ - "id": 4, - "description": "Aliquid qui quis dignissimos distinctio ut commodi voluptas est.", "marked_for_deletion_on": "2020-04-03", - ... -} -``` - -Users of [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see -the `membership_lock` attribute: - -```json -{ - "id": 4, - "description": "Aliquid qui quis dignissimos distinctio ut commodi voluptas est.", "membership_lock": false, + "wiki_access_level": "disabled", ... } ``` @@ -832,6 +822,7 @@ Parameters: | `membership_lock` **(PREMIUM)** | boolean | no | Users cannot be added to projects in this group. | | `extra_shared_runners_minutes_limit` **(PREMIUM SELF)** | integer | no | Can be set by administrators only. Additional CI/CD minutes for this group. | | `shared_runners_minutes_limit` **(PREMIUM SELF)** | integer | no | Can be set by administrators only. Maximum number of monthly CI/CD minutes for this group. Can be `nil` (default; inherit system default), `0` (unlimited), or `> 0`. | +| `wiki_access_level` **(PREMIUM)** | string | no | The wiki access level. Can be `disabled`, `private`, or `enabled`. | ### Options for `default_branch_protection` @@ -996,6 +987,7 @@ PUT /groups/:id | `unique_project_download_limit_alertlist` **(ULTIMATE)** | array of integers | no | List of user IDs that are emailed when the unique project download limit is exceeded. Available only on top-level groups. Default: `[]`, Maximum: 100 user IDs. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/110201) in GitLab 15.9. | | `auto_ban_user_on_excessive_projects_download` **(ULTIMATE)** | boolean | no | When enabled, users are automatically banned from the group when they download more than the maximum number of unique projects specified by `unique_project_download_limit` and `unique_project_download_limit_interval_in_seconds`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/94159) in GitLab 15.4. | | `ip_restriction_ranges` **(PREMIUM)** | string | no | Comma-separated list of IP addresses or subnet masks to restrict group access. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/351493) in GitLab 15.4. | +| `wiki_access_level` **(PREMIUM)** | string | no | The wiki access level. Can be `disabled`, `private`, or `enabled`. | NOTE: The `projects` and `shared_projects` attributes in the response are deprecated and [scheduled for removal in API v5](https://gitlab.com/gitlab-org/gitlab/-/issues/213797). @@ -1078,6 +1070,8 @@ Example response: The `prevent_sharing_groups_outside_hierarchy` attribute is present in the response only for top-level groups. +Users of [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/) also see the `wiki_access_level` attribute. + ### Disable the results limit **(FREE SELF)** The 100 results limit can break integrations developed using GitLab 12.4 and earlier. @@ -1142,7 +1136,7 @@ Only available to group owners and administrators. This endpoint: -- On Premium and higher tiers, marks the group for deletion. The deletion happens 7 days later by default, but you can change the retention period in the [instance settings](../user/admin_area/settings/visibility_and_access_controls.md#deletion-protection). +- On Premium and Ultimate tiers, marks the group for deletion. The deletion happens 7 days later by default, but you can change the retention period in the [instance settings](../user/admin_area/settings/visibility_and_access_controls.md#deletion-protection). - On Free tier, removes the group immediately and queues a background job to delete all projects in the group. - Deletes a subgroup immediately if the subgroup is marked for deletion (GitLab 15.4 and later). The endpoint does not immediately delete top-level groups. @@ -1740,7 +1734,7 @@ GET /groups/:id/push_rule } ``` -Users on [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see +Users on [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/) also see the `commit_committer_check` and `reject_unsigned_commits` parameters: ```json diff --git a/doc/api/import.md b/doc/api/import.md index 5e9fbf30226..be70868cca5 100644 --- a/doc/api/import.md +++ b/doc/api/import.md @@ -1,20 +1,26 @@ --- stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Import API **(FREE)** +Use the Import API to import repositories from GitHub or Bitbucket Server. + ## Import repository from GitHub -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/381902) in GitLab 15.8, GitLab no longer automatically creates namespaces or groups if the namespace or group name specified in `target_namespace` doesn't exist. GitLab also no longer falls back to using the user's personal namespace if the namespace or group name is taken or `target_namespace` is blank. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/381902) in GitLab 15.8, GitLab no longer automatically creates namespaces or groups if the namespace or group name specified in `target_namespace` doesn't exist. GitLab also no longer falls back to using the user's personal namespace if the namespace or group name is taken or `target_namespace` is blank. +> - Requirement for Maintainer role instead of Developer role introduced in GitLab 16.0 and backported to GitLab 15.11.1 and GitLab 15.10.5. +> - `collaborators_import` key in `optional_stages` was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/398154) in GitLab 16.0. Import your projects from GitHub to GitLab using the API. -The namespace set in `target_namespace` must exist. The namespace can be your user namespace or an existing group that -you have at least the Maintainer role for. Using the Developer role for this purpose was -[deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387891) in GitLab 15.8 and will be removed in GitLab 16.0. +Prerequisites: + +- [Prerequisites for GitHub importer](../user/project/import/github.md#prerequisites). +- The namespace set in `target_namespace` must exist. +- The namespace can be your user namespace or an existing group that you have at least the Maintainer role for. ```plaintext POST /import/github @@ -43,7 +49,8 @@ curl --request POST \ "optional_stages": { "single_endpoint_issue_events_import": true, "single_endpoint_notes_import": true, - "attachments_import": true + "attachments_import": true, + "collaborators_import": true } }' ``` @@ -53,6 +60,7 @@ The following keys are available for `optional_stages`: - `single_endpoint_issue_events_import`, for issue and pull request events import. - `single_endpoint_notes_import`, for an alternative and more thorough comments import. - `attachments_import`, for Markdown attachments import. +- `collaborators_import`, for importing direct repository collaborators who are not outside collaborators. For more information, see [Select additional items to import](../user/project/import/github.md#select-additional-items-to-import). @@ -77,7 +85,7 @@ token: - The GitLab project inherits the original project's visibility settings. As a result, the project is publicly accessible if the original project is public. - If the `path` or `target_namespace` does not exist, the project import fails. -## Cancel GitHub project import +### Cancel GitHub project import > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/364783) in GitLab 15.5. @@ -122,14 +130,11 @@ Returns the following status codes: - `400 Bad Request`: the project import cannot be canceled. - `404 Not Found`: the project associated with `project_id` does not exist. -## Import GitHub gists into GitLab snippets +### Import GitHub gists into GitLab snippets -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/371099) in GitLab 15.8 [with a flag](../administration/feature_flags.md) named `github_import_gists`. Disabled by default. Enabled on GitLab.com. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, -ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `github_import_gists`. -On GitLab.com, this feature is available. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/371099) in GitLab 15.8 [with a flag](../administration/feature_flags.md) named `github_import_gists`. Disabled by default. Enabled on GitLab.com. +> - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/386579) in GitLab 15.10. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/386579) in GitLab 15.11. Feature flag `github_import_gists` removed. You can use the GitLab API to import personal GitHub gists (with up to 10 files) into personal GitLab snippets. GitHub gists with more than 10 files are skipped. You should manually migrate these GitHub gists. @@ -163,13 +168,16 @@ Returns the following status codes: ## Import repository from Bitbucket Server -Import your projects from Bitbucket Server to GitLab via the API. +Import your projects from Bitbucket Server to GitLab using the API. -NOTE: The Bitbucket Project Key is only used for finding the repository in Bitbucket. You must specify a `target_namespace` if you want to import the repository to a GitLab group. If you do not specify `target_namespace`, the project imports to your personal user namespace. +Prerequisites: + +- For more information, see [prerequisites for Bitbucket Server importer](../user/project/import/bitbucket_server.md#import-your-bitbucket-repositories). + ```plaintext POST /import/bitbucket_server ``` @@ -202,3 +210,9 @@ curl --request POST \ For information on automating user, group, and project import API calls, see [Automate group and project import](../user/project/import/index.md#automate-group-and-project-import). + +## Related topics + +- [Group migration by direct transfer API](bulk_imports.md). +- [Group import and export API](group_import_export.md). +- [Project import and export API](project_import_export.md). diff --git a/doc/api/index.md b/doc/api/index.md index 110534ceced..8073cbec94a 100644 --- a/doc/api/index.md +++ b/doc/api/index.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -10,8 +10,9 @@ Automate and interact with GitLab, and integrate with external applications. - [Integrations](../integration/index.md) - [Webhooks](../user/project/integrations/webhooks.md) -- [Visual Studio Code extension](../user/project/repository/vscode.md) - [REST API](rest/index.md) - [GraphQL API](graphql/index.md) -- [Lint `.gitlab-ci.yml`](lint.md) -- [GitLab as an OAuth2 provider](oauth2.md) +- [OAuth 2.0 identity provider API](oauth2.md) +- [GitLab CLI (glab)](../integration/glab/index.md) +- [Visual Studio Code extension](../user/project/repository/vscode.md) +- [Code Suggestions](../user/project/repository/code_suggestions.md) diff --git a/doc/api/instance_clusters.md b/doc/api/instance_clusters.md index 45243003004..751e3203990 100644 --- a/doc/api/instance_clusters.md +++ b/doc/api/instance_clusters.md @@ -1,10 +1,10 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Instance clusters API (certificate-based) (DEPRECATED) **(FREE SELF)** +# Instance clusters API (certificate-based) (deprecated) **(FREE SELF)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36001) in GitLab 13.2. > - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. diff --git a/doc/api/instance_level_ci_variables.md b/doc/api/instance_level_ci_variables.md index e71cf777b2c..840744bcae1 100644 --- a/doc/api/instance_level_ci_variables.md +++ b/doc/api/instance_level_ci_variables.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Authoring +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/api/integrations.md b/doc/api/integrations.md index 24e0f189aad..5b6c4d17915 100644 --- a/doc/api/integrations.md +++ b/doc/api/integrations.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -96,7 +96,7 @@ Parameters: ### Disable Apple App Store integration -Disable the Apple App Store integration for a project. Integration settings are preserved. +Disable the Apple App Store integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/apple_app_store @@ -133,7 +133,7 @@ Parameters: ### Disable Asana integration -Disable the Asana integration for a project. Integration settings are preserved. +Disable the Asana integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/asana @@ -168,7 +168,7 @@ Parameters: ### Disable Assembla integration -Disable the Assembla integration for a project. Integration settings are preserved. +Disable the Assembla integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/assembla @@ -208,7 +208,7 @@ Parameters: ### Disable Atlassian Bamboo CI integration -Disable the Atlassian Bamboo CI integration for a project. Integration settings are preserved. +Disable the Atlassian Bamboo CI integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/bamboo @@ -244,7 +244,7 @@ Parameters: ### Disable Bugzilla integration -Disable the Bugzilla integration for a project. Integration settings are preserved. +Disable the Bugzilla integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/bugzilla @@ -283,7 +283,7 @@ Parameters: ### Disable Buildkite integration -Disable the Buildkite integration for a project. Integration settings are preserved. +Disable the Buildkite integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/buildkite @@ -320,7 +320,7 @@ Parameters: ### Disable Campfire integration -Disable the Campfire integration for a project. Integration settings are preserved. +Disable the Campfire integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/campfire @@ -360,7 +360,7 @@ Parameters: ### Disable Datadog integration -Disable the Datadog integration for a project. Integration settings are preserved. +Disable the Datadog integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/datadog @@ -405,7 +405,7 @@ Parameters: ### Disable Unify Circuit integration -Disable the Unify Circuit integration for a project. Integration settings are preserved. +Disable the Unify Circuit integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/unify-circuit @@ -450,7 +450,7 @@ Parameters: ### Disable Pumble integration -Disable the Pumble integration for a project. Integration settings are preserved. +Disable the Pumble integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/pumble @@ -495,7 +495,7 @@ Parameters: ### Disable Webex Teams integration -Disable the Webex Teams integration for a project. Integration settings are preserved. +Disable the Webex Teams integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/webex-teams @@ -531,7 +531,7 @@ Parameters: ### Disable Custom Issue Tracker integration -Disable the Custom Issue Tracker integration for a project. Integration settings are preserved. +Disable the Custom Issue Tracker integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/custom-issue-tracker @@ -565,7 +565,7 @@ Parameters: ### Disable Discord integration -Disable the Discord integration for a project. Integration settings are preserved. +Disable the Discord integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/discord @@ -604,7 +604,7 @@ Parameters: ### Disable Drone CI integration -Disable the Drone CI integration for a project. Integration settings are preserved. +Disable the Drone CI integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/drone-ci @@ -643,7 +643,7 @@ Parameters: ### Disable Emails on Push integration -Disable the Emails on Push integration for a project. Integration settings are preserved. +Disable the Emails on Push integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/emails-on-push @@ -679,7 +679,7 @@ Parameters: ### Disable EWM integration -Disable the EWM integration for a project. Integration settings are preserved. +Disable the EWM integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/ewm @@ -715,7 +715,7 @@ Parameters: ### Disable Confluence integration -Disable the Confluence integration for a project. Integration settings are preserved. +Disable the Confluence integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/confluence @@ -753,7 +753,7 @@ Parameters: ### Disable Shimo integration -Disable the Shimo integration for a project. Integration settings are preserved. +Disable the Shimo integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/shimo @@ -779,7 +779,7 @@ Parameters: ### Disable External wiki integration -Disable the External wiki integration for a project. Integration settings are preserved. +Disable the External wiki integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/external-wiki @@ -815,7 +815,7 @@ Parameters: ### Disable GitHub integration -Disable the GitHub integration for a project. Integration settings are preserved. +Disable the GitHub integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/github @@ -861,7 +861,7 @@ Parameters: ### Disable Hangouts Chat integration -Disable the Hangouts Chat integration for a project. Integration settings are preserved. +Disable the Hangouts Chat integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/hangouts-chat @@ -901,7 +901,7 @@ Parameters: ### Disable Irker (IRC gateway) integration -Disable the Irker (IRC gateway) integration for a project. Integration settings are preserved. +Disable the Irker (IRC gateway) integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/irker @@ -941,9 +941,12 @@ Parameters: | --------- | ---- | -------- | ----------- | | `url` | string | yes | The URL to the Jira project which is being linked to this GitLab project. For example, `https://jira.example.com`. | | `api_url` | string | no | The base URL to the Jira instance API. Web URL value is used if not set. For example, `https://jira-api.example.com`. | -| `username` | string | yes | The username of the user created to be used with GitLab/Jira. | -| `password` | string | yes | The password of the user created to be used with GitLab/Jira. | -| `active` | boolean | no | Activates or deactivates the integration. Defaults to false (deactivated). | +| `username` | string | no | The email or username to be used with Jira. For Jira Cloud use an email, for Jira Data Center and Jira Server use a username. Required when using Basic authentication (`jira_auth_type` is `0`) | +| `password` | string | yes | The Jira API token, password, or personal access token to be used with Jira. When your authentication method is Basic (`jira_auth_type` is `0`) use an API token for Jira Cloud, or a password for Jira Data Center or Jira Server. When your authentication method is Jira personal access token (`jira_auth_type` is `1`) use a personal access token. | +| `active` | boolean | no | Activates or deactivates the integration. Defaults to `false` (deactivated). | +| `jira_auth_type`| integer | no | The authentication method to be used with Jira. `0` means Basic Authentication. `1` means Jira personal access token. Defaults to `0`. | +| `jira_issue_prefix` | string | no | Prefix to match Jira issue keys. | +| `jira_issue_regex` | string | no | Regular expression to match Jira issue keys. | | `jira_issue_transition_automatic` | boolean | no | Enable [automatic issue transitions](../integration/jira/issues.md#automatic-issue-transitions). Takes precedence over `jira_issue_transition_id` if enabled. Defaults to `false` | | `jira_issue_transition_id` | string | no | The ID of one or more transitions for [custom issue transitions](../integration/jira/issues.md#custom-issue-transitions). Ignored if `jira_issue_transition_automatic` is enabled. Defaults to a blank string, which disables custom transitions. | | `commit_events` | boolean | false | Enable notifications for commit events | @@ -952,7 +955,7 @@ Parameters: ### Disable Jira integration -Disable the Jira integration for a project. Integration settings are preserved. +Disable the Jira integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/jira @@ -1011,7 +1014,7 @@ Parameters: ### Disable Slack Slash Command integration -Disable the Slack Slash Command integration for a project. Integration settings are preserved. +Disable the Slack Slash Command integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/slack-slash-commands @@ -1045,7 +1048,7 @@ Parameters: ### Disable Mattermost Slash Command integration -Disable the Mattermost Slash Command integration for a project. Integration settings are preserved. +Disable the Mattermost Slash Command integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/mattermost-slash-commands @@ -1076,7 +1079,7 @@ Parameters: ### Disable Packagist integration -Disable the Packagist integration for a project. Integration settings are preserved. +Disable the Packagist integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/packagist @@ -1114,7 +1117,7 @@ Parameters: ### Disable Pipeline-Emails integration -Disable the Pipeline-Emails integration for a project. Integration settings are preserved. +Disable the Pipeline-Emails integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/pipelines-email @@ -1151,7 +1154,7 @@ Parameters: ### Disable Pivotal Tracker integration -Disable the Pivotal Tracker integration for a project. Integration settings are preserved. +Disable the Pivotal Tracker integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/pivotaltracker @@ -1187,7 +1190,7 @@ Parameters: ### Disable Prometheus integration -Disable the Prometheus integration for a project. Integration settings are preserved. +Disable the Prometheus integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/prometheus @@ -1225,7 +1228,7 @@ Parameters: ### Disable Pushover integration -Disable the Pushover integration for a project. Integration settings are preserved. +Disable the Pushover integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/pushover @@ -1261,7 +1264,7 @@ Parameters: ### Disable Redmine integration -Disable the Redmine integration for a project. Integration settings are preserved. +Disable the Redmine integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/redmine @@ -1322,7 +1325,7 @@ Parameters: ### Disable Slack integration -Disable the Slack integration for a project. Integration settings are preserved. +Disable the Slack integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/slack @@ -1368,7 +1371,7 @@ Parameters: ### Disable Microsoft Teams integration -Disable the Microsoft Teams integration for a project. Integration settings are preserved. +Disable the Microsoft Teams integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/microsoft-teams @@ -1425,7 +1428,7 @@ Parameters: ### Disable Mattermost notifications integration -Disable the Mattermost notifications integration for a project. Integration settings are preserved. +Disable the Mattermost notifications integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/mattermost @@ -1467,7 +1470,7 @@ Parameters: ### Disable JetBrains TeamCity CI integration -Disable the JetBrains TeamCity CI integration for a project. Integration settings are preserved. +Disable the JetBrains TeamCity CI integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/teamcity @@ -1508,7 +1511,7 @@ Parameters: ### Disable Jenkins CI integration -Disable the Jenkins CI integration for a project. Integration settings are preserved. +Disable the Jenkins CI integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/jenkins @@ -1545,7 +1548,7 @@ Parameters: ### Disable Jenkins CI (Deprecated) integration -Disable the Jenkins CI (Deprecated) integration for a project. Integration settings are preserved. +Disable the Jenkins CI (Deprecated) integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/jenkins-deprecated @@ -1582,7 +1585,7 @@ Parameters: ### Disable MockCI integration -Disable the MockCI integration for a project. Integration settings are preserved. +Disable the MockCI integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/mock-ci @@ -1596,6 +1599,43 @@ Get MockCI integration settings for a project. GET /projects/:id/integrations/mock-ci ``` +## Squash TM + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/337855) in GitLab 15.10. + +Update [Squash TM](https://www.squashtest.com/product-squash-tm?lang=en) requirements when GitLab issues are modified. + +### Create/Edit Squash TM integration + +Set Squash TM integration settings for a project. + +```plaintext +PUT /projects/:id/integrations/squash-tm +``` + +Parameters: + +| Parameter | Type | Required | Description | +|-------------------------|--------|----------|-------------------------------| +| `url` | string | yes | URL of the Squash TM webhook. | +| `token` | string | no | Optional token | + +### Disable Squash TM integration + +Disable the Squash TM integration for a project. Integration settings are preserved. + +```plaintext +DELETE /projects/:id/integrations/squash-tm +``` + +### Get Squash TM integration settings + +Get Squash TM integration settings for a project. + +```plaintext +GET /projects/:id/integrations/squash-tm +``` + ## YouTrack YouTrack issue tracker @@ -1617,7 +1657,7 @@ Parameters: ### Disable YouTrack integration -Disable the YouTrack integration for a project. Integration settings are preserved. +Disable the YouTrack integration for a project. Integration settings are reset. ```plaintext DELETE /projects/:id/integrations/youtrack diff --git a/doc/api/issues.md b/doc/api/issues.md index e252655f781..03cc107415f 100644 --- a/doc/api/issues.md +++ b/doc/api/issues.md @@ -182,7 +182,7 @@ Example response: ] ``` -Issues created by users on GitLab Premium or higher include the `weight` property: +Issues created by users on GitLab Premium or Ultimate include the `weight` property: ```json [ @@ -195,7 +195,7 @@ Issues created by users on GitLab Premium or higher include the `weight` propert ] ``` -Issues created by users on GitLab Premium or higher include the `epic` property: +Issues created by users on GitLab Premium or Ultimate include the `epic` property: ```json { @@ -213,7 +213,7 @@ Issues created by users on GitLab Premium or higher include the `epic` property: } ``` -Issues created by users on GitLab Premium or higher include the `iteration` property: +Issues created by users on GitLab Premium or Ultimate include the `iteration` property: ```json { @@ -409,7 +409,7 @@ Example response: ] ``` -Issues created by users on GitLab Premium or higher include the `weight` property: +Issues created by users on GitLab Premium or Ultimate include the `weight` property: ```json [ @@ -422,7 +422,7 @@ Issues created by users on GitLab Premium or higher include the `weight` propert ] ``` -Issues created by users on GitLab Premium or higher include the `epic` property: +Issues created by users on GitLab Premium or Ultimate include the `epic` property: ```json { @@ -619,7 +619,7 @@ Example response: ] ``` -Issues created by users on GitLab Premium or higher include the `weight` property: +Issues created by users on GitLab Premium or Ultimate include the `weight` property: ```json [ @@ -632,7 +632,7 @@ Issues created by users on GitLab Premium or higher include the `weight` propert ] ``` -Issues created by users on GitLab Premium or higher include the `epic` property: +Issues created by users on GitLab Premium or Ultimate include the `epic` property: ```json { @@ -781,7 +781,7 @@ Example response: } ``` -Issues created by users on GitLab Premium or higher include the `weight` property: +Issues created by users on GitLab Premium or Ultimate include the `weight` property: ```json { @@ -792,7 +792,7 @@ Issues created by users on GitLab Premium or higher include the `weight` propert } ``` -Issues created by users on GitLab Premium or higher include the `epic` property: +Issues created by users on GitLab Premium or Ultimate include the `epic` property: ```json { @@ -941,7 +941,7 @@ Example response: } ``` -Issues created by users on GitLab Premium or higher include the `weight` property: +Issues created by users on GitLab Premium or Ultimate include the `weight` property: ```json { @@ -952,7 +952,7 @@ Issues created by users on GitLab Premium or higher include the `weight` propert } ``` -Issues created by users on GitLab Premium or higher include the `epic` property: +Issues created by users on GitLab Premium or Ultimate include the `epic` property: ```json { @@ -1090,7 +1090,7 @@ Example response: } ``` -Issues created by users on GitLab Premium or higher include the `weight` property: +Issues created by users on GitLab Premium or Ultimate include the `weight` property: ```json { @@ -1101,7 +1101,7 @@ Issues created by users on GitLab Premium or higher include the `weight` propert } ``` -Issues created by users on GitLab Premium or higher include the `epic` property: +Issues created by users on GitLab Premium or Ultimate include the `epic` property: ```json { @@ -1267,7 +1267,7 @@ Example response: } ``` -Issues created by users on GitLab Premium or higher include the `weight` property: +Issues created by users on GitLab Premium or Ultimate include the `weight` property: ```json { @@ -1278,7 +1278,7 @@ Issues created by users on GitLab Premium or higher include the `weight` propert } ``` -Issues created by users on GitLab Premium or higher include the `epic` property: +Issues created by users on GitLab Premium or Ultimate include the `epic` property: ```json { @@ -1452,7 +1452,7 @@ Example response: } ``` -Issues created by users on GitLab Premium or higher include the `weight` property: +Issues created by users on GitLab Premium or Ultimate include the `weight` property: ```json { @@ -1463,7 +1463,7 @@ Issues created by users on GitLab Premium or higher include the `weight` propert } ``` -Issues created by users on GitLab Premium or higher include the `epic` property: +Issues created by users on GitLab Premium or Ultimate include the `epic` property: ```json { @@ -1704,7 +1704,7 @@ Example response: } ``` -Issues created by users on GitLab Premium or higher include the `weight` property: +Issues created by users on GitLab Premium or Ultimate include the `weight` property: ```json { @@ -1715,7 +1715,7 @@ Issues created by users on GitLab Premium or higher include the `weight` propert } ``` -Issues created by users on GitLab Premium or higher include the `epic` property: +Issues created by users on GitLab Premium or Ultimate include the `epic` property: ```json { diff --git a/doc/api/iterations.md b/doc/api/iterations.md index 4997a917a5a..567a2def09f 100644 --- a/doc/api/iterations.md +++ b/doc/api/iterations.md @@ -24,13 +24,17 @@ GET /projects/:id/iterations?state=opened GET /projects/:id/iterations?state=closed GET /projects/:id/iterations?search=version GET /projects/:id/iterations?include_ancestors=false +GET /projects/:id/iterations?updated_before=2013-10-02T09%3A24%3A18Z +GET /projects/:id/iterations?updated_after=2013-10-02T09%3A24%3A18Z ``` | Attribute | Type | Required | Description | | ------------------- | ------- | -------- | ----------- | -| `state` | string | no | 'Return `opened`, `upcoming`, `current (previously started)`, `closed`, or `all` iterations. Filtering by `started` state is deprecated starting with 14.1, please use `current` instead.' | +| `state` | string | no | 'Return `opened`, `upcoming`, `current`, `closed`, or `all` iterations.' | | `search` | string | no | Return only iterations with a title matching the provided string. | | `include_ancestors` | boolean | no | Include iterations from parent group and its ancestors. Defaults to `true`. | +| `updated_before` | datetime | no | Return only iterations updated before the given datetime. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/378662) in GitLab 15.10. | +| `updated_after` | datetime | no | Return only iterations updated after the given datetime. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/378662) in GitLab 15.10. | Example request: diff --git a/doc/api/job_artifacts.md b/doc/api/job_artifacts.md index b73eafea3c4..3b6eaf9b670 100644 --- a/doc/api/job_artifacts.md +++ b/doc/api/job_artifacts.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Insights +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -64,7 +64,7 @@ Possible response status codes: > The use of `CI_JOB_TOKEN` in the artifacts download API was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2346) in [GitLab Premium](https://about.gitlab.com/pricing/) 9.5. -Download the artifacts zipped archive from the latest successful pipeline for +Download the artifacts zipped archive from the latest **successful** pipeline for the given reference name and job, provided the job finished successfully. This is the same as [getting the job's artifacts](#get-job-artifacts), but by defining the job's name instead of its ID. @@ -167,8 +167,8 @@ Possible response status codes: > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/23538) in GitLab 11.5. > - The use of `CI_JOB_TOKEN` in the artifacts download API was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55042) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.10. -Download a single artifact file for a specific job of the latest successful -pipeline for the given reference name from inside the job's artifacts archive. +Download a single artifact file for a specific job of the latest **successful** pipeline +for the given reference name from inside the job's artifacts archive. The file is extracted from the archive and streamed to the client. The artifact file provides more detail than what is available in the @@ -295,7 +295,7 @@ If the artifacts were deleted successfully, a response with status `204 No Conte Delete artifacts of a project that can be deleted. -By default, [artifacts from the most recent successful pipeline of each ref are kept](../ci/pipelines/job_artifacts.md#keep-artifacts-from-most-recent-successful-jobs). +By default, [artifacts from the most recent successful pipeline of each ref are kept](../ci/jobs/job_artifacts.md#keep-artifacts-from-most-recent-successful-jobs). ```plaintext DELETE /projects/:id/artifacts diff --git a/doc/api/jobs.md b/doc/api/jobs.md index 2d65ea82edd..f060d86c335 100644 --- a/doc/api/jobs.md +++ b/doc/api/jobs.md @@ -331,9 +331,9 @@ In earlier GitLab versions, jobs are sorted by ID in ascending order (oldest fir In GitLab 13.9 and later, this endpoint can include retried jobs in the response with `include_retried` set to `true`. -## List pipeline bridges +## List pipeline trigger jobs -Get a list of bridge jobs for a pipeline. +Get a list of trigger jobs for a pipeline. ```plaintext GET /projects/:id/pipelines/:pipeline_id/bridges @@ -867,8 +867,9 @@ POST /projects/:id/jobs/:job_id/play Example request: ```shell -curl --request POST "https://gitlab.example.com/api/v4/projects/1/jobs/1/play - --header "PRIVATE-TOKEN: " +curl --request POST "https://gitlab.example.com/api/v4/projects/1/jobs/1/play" \ + --header "Content-Type: application/json" \ + --header "PRIVATE-TOKEN: " \ --data @variables.json ``` diff --git a/doc/api/keys.md b/doc/api/keys.md index e7bdc70017c..337758e6c33 100644 --- a/doc/api/keys.md +++ b/doc/api/keys.md @@ -7,22 +7,25 @@ type: reference, api # Keys API **(FREE)** +If using a SHA256 fingerprint in an API call, you should URL-encode the fingerprint. + ## Get SSH key with user by ID of an SSH key -Get SSH key with user by ID of an SSH key. Note only administrators can lookup SSH key with user by ID of an SSH key. +Get SSH key with user by ID of an SSH key. Only available to administrators. ```plaintext GET /keys/:id ``` -| Attribute | Type | Required | Description | -|:----------|:--------|:---------|:---------------------| -| `id` | integer | yes | The ID of an SSH key | +| Attribute | Type | Required | Description | +|:----------|:--------|:---------|:----------------------| +| `id` | integer | yes | The ID of an SSH key. | Example request: ```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/keys/1" +curl --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/keys/1" ``` ```json @@ -75,22 +78,15 @@ You can search for a user that owns a specific SSH key. Note only administrators GET /keys ``` -| Attribute | Type | Required | Description | -|:--------------|:-------|:---------|:------------------------------| -| `fingerprint` | string | yes | The fingerprint of an SSH key | +| Attribute | Type | Required | Description | +|:--------------|:-------|:---------|:-------------------------------| +| `fingerprint` | string | yes | The fingerprint of an SSH key. | Example request: ```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/keys?fingerprint=ba:81:59:68:d7:6c:cd:02:02:bf:6a:9b:55:4e:af:d1" -``` - -If using sha256 fingerprint API calls, make sure that the fingerprint is URL-encoded. - -For example, `/` is represented by `%2F` and `:` is represented by`%3A`: - -```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/keys?fingerprint=SHA256%3AnUhzNyftwADy8AH3wFY31tAKs7HufskYTte2aXo%2FlCg" +curl --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/keys?fingerprint=ba:81:59:68:d7:6c:cd:02:02:bf:6a:9b:55:4e:af:d1" ``` Example response: @@ -142,15 +138,21 @@ Example response: ## Get user by deploy key fingerprint -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/119209) in GitLab 12.7. +Deploy keys are bound to the creating user. If you query with a deploy key +fingerprint, you get additional information about the projects using that key. -Deploy keys are bound to the creating user, so if you query with a deploy key -fingerprint you get additional information about the projects using that key. +Example request with an MD5 fingerprint: -Example request: +```shell +curl --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/keys?fingerprint=ba:81:59:68:d7:6c:cd:02:02:bf:6a:9b:55:4e:af:d1" +``` + +In this SHA256 example, `/` is represented by `%2F` and `:` is represented by`%3A`: ```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/keys?fingerprint=SHA256%3AnUhzNyftwADy8AH3wFY31tAKs7HufskYTte2aXo%2FlCg" +curl --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/keys?fingerprint=SHA256%3AnUhzNyftwADy8AH3wFY31tAKs7HufskYTte2aXo%2FlCg" ``` Example response: diff --git a/doc/api/license.md b/doc/api/license.md index ca9d9cf386d..39da6af30d4 100644 --- a/doc/api/license.md +++ b/doc/api/license.md @@ -233,3 +233,35 @@ Returns: - `204 No Content` if the license is successfully deleted. - `403 Forbidden` if the current user in not permitted to delete the license. - `404 Not Found` if the license to delete could not be found. + +## Trigger recalculation of billable users + +```plaintext +PUT /license/:id/refresh_billable_users +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer | yes | ID of the GitLab license. | + +```shell +curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/license/:id/refresh_billable_users" +``` + +Example response: + +```json +{ + "success": true +} +``` + +Returns: + +- `202 Accepted` if the request to refresh billable users is successfully initiated. +- `403 Forbidden` if the current user in not permitted to refresh billable users for the license. +- `404 Not Found` if the license could not be found. + +| Attribute | Type | Description | +|:-----------------------------|:--------------|:------------------------------------------| +| `success` | boolean | Whether the request succeeded or not. | diff --git a/doc/api/lint.md b/doc/api/lint.md index e50832a9528..cfd34f6a40c 100644 --- a/doc/api/lint.md +++ b/doc/api/lint.md @@ -6,7 +6,113 @@ info: To determine the technical writer assigned to the Stage/Group associated w # CI Lint API **(FREE)** -## Validate the CI YAML configuration +## Validate a CI YAML configuration with a namespace + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/231352) in GitLab 13.6. + +Checks if CI/CD YAML configuration is valid. This endpoint has namespace +specific context. + +```plaintext +POST /projects/:id/ci/lint +``` + +| Attribute | Type | Required | Description | +| ---------- | ------- | -------- | -------- | +| `content` | string | yes | The CI/CD configuration content. | +| `dry_run` | boolean | no | Run [pipeline creation simulation](../ci/lint.md#simulate-a-pipeline), or only do static check. This is false by default. | +| `include_jobs` | boolean | no | If the list of jobs that would exist in a static check or pipeline simulation should be included in the response. This is false by default. | +| `ref` | string | no | When `dry_run` is `true`, sets the branch or tag to use. Defaults to the project's default branch when not set. | + +Example request: + +```shell +curl --header "Content-Type: application/json" "https://gitlab.example.com/api/v4/projects/:id/ci/lint" --data '{"content": "{ \"image\": \"ruby:2.6\", \"services\": [\"postgres\"], \"before_script\": [\"bundle install\", \"bundle exec rake db:create\"], \"variables\": {\"DB_NAME\": \"postgres\"}, \"types\": [\"test\", \"deploy\", \"notify\"], \"rspec\": { \"script\": \"rake spec\", \"tags\": [\"ruby\", \"postgres\"], \"only\": [\"branches\"]}}"}' +``` + +Example responses: + +- Valid configuration: + + ```json + { + "valid": true, + "merged_yaml": "---\n:test_job:\n :script: echo 1\n", + "errors": [], + "warnings": [] + } + ``` + +- Invalid configuration: + + ```json + { + "valid": false, + "merged_yaml": "---\n:test_job:\n :script: echo 1\n", + "errors": [ + "jobs config should contain at least one visible job" + ], + "warnings": [] + } + ``` + +## Validate a project's CI configuration + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/231352) in GitLab 13.5. + +Checks if a project's latest (`HEAD` of the project's default branch) +`.gitlab-ci.yml` configuration is valid. This endpoint uses all namespace +specific data available, including variables and local includes. + +```plaintext +GET /projects/:id/ci/lint +``` + +| Attribute | Type | Required | Description | +| ---------- | ------- | -------- | -------- | +| `dry_run` | boolean | no | Run pipeline creation simulation, or only do static check. | +| `include_jobs` | boolean | no | If the list of jobs that would exist in a static check or pipeline simulation should be included in the response. This is false by default. | +| `ref` | string | no | When `dry_run` is `true`, sets the branch or tag to use. Defaults to the project's default branch when not set. | + +Example request: + +```shell +curl "https://gitlab.example.com/api/v4/projects/:id/ci/lint" +``` + +Example responses: + +- Valid configuration: + +```json +{ + "valid": true, + "merged_yaml": "---\n:test_job:\n :script: echo 1\n", + "errors": [], + "warnings": [] +} +``` + +- Invalid configuration: + +```json +{ + "valid": false, + "merged_yaml": "---\n:test_job:\n :script: echo 1\n", + "errors": [ + "jobs config should contain at least one visible job" + ], + "warnings": [] +} +``` + + + +## Validate the CI YAML configuration (deprecated) + +WARNING: +This endpoint was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/381669) in GitLab 15.7 +and was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/403256) in 16.0. Use [`POST /projects/:id/ci/lint`](#validate-a-ci-yaml-configuration-with-a-namespace) instead. Checks if CI/CD YAML configuration is valid. This endpoint validates basic CI/CD configuration syntax. It doesn't have any namespace-specific context. @@ -154,105 +260,7 @@ Example response: } ``` -## Validate a CI YAML configuration with a namespace - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/231352) in GitLab 13.6. - -Checks if CI/CD YAML configuration is valid. This endpoint has namespace -specific context. - -```plaintext -POST /projects/:id/ci/lint -``` - -| Attribute | Type | Required | Description | -| ---------- | ------- | -------- | -------- | -| `content` | string | yes | The CI/CD configuration content. | -| `dry_run` | boolean | no | Run [pipeline creation simulation](../ci/lint.md#simulate-a-pipeline), or only do static check. This is false by default. | -| `include_jobs` | boolean | no | If the list of jobs that would exist in a static check or pipeline simulation should be included in the response. This is false by default. | -| `ref` | string | no | When `dry_run` is `true`, sets the branch or tag to use. Defaults to the project's default branch when not set. | - -Example request: - -```shell -curl --header "Content-Type: application/json" "https://gitlab.example.com/api/v4/projects/:id/ci/lint" --data '{"content": "{ \"image\": \"ruby:2.6\", \"services\": [\"postgres\"], \"before_script\": [\"bundle install\", \"bundle exec rake db:create\"], \"variables\": {\"DB_NAME\": \"postgres\"}, \"types\": [\"test\", \"deploy\", \"notify\"], \"rspec\": { \"script\": \"rake spec\", \"tags\": [\"ruby\", \"postgres\"], \"only\": [\"branches\"]}}"}' -``` - -Example responses: - -- Valid configuration: - - ```json - { - "valid": true, - "merged_yaml": "---\n:test_job:\n :script: echo 1\n", - "errors": [], - "warnings": [] - } - ``` - -- Invalid configuration: - - ```json - { - "valid": false, - "merged_yaml": "---\n:test_job:\n :script: echo 1\n", - "errors": [ - "jobs config should contain at least one visible job" - ], - "warnings": [] - } - ``` - -## Validate a project's CI configuration - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/231352) in GitLab 13.5. - -Checks if a project's latest (`HEAD` of the project's default branch) -`.gitlab-ci.yml` configuration is valid. This endpoint uses all namespace -specific data available, including variables and local includes. - -```plaintext -GET /projects/:id/ci/lint -``` - -| Attribute | Type | Required | Description | -| ---------- | ------- | -------- | -------- | -| `dry_run` | boolean | no | Run pipeline creation simulation, or only do static check. | -| `include_jobs` | boolean | no | If the list of jobs that would exist in a static check or pipeline simulation should be included in the response. This is false by default. | -| `ref` | string | no | When `dry_run` is `true`, sets the branch or tag to use. Defaults to the project's default branch when not set. | - -Example request: - -```shell -curl "https://gitlab.example.com/api/v4/projects/:id/ci/lint" -``` - -Example responses: - -- Valid configuration: - -```json -{ - "valid": true, - "merged_yaml": "---\n:test_job:\n :script: echo 1\n", - "errors": [], - "warnings": [] -} -``` - -- Invalid configuration: - -```json -{ - "valid": false, - "merged_yaml": "---\n:test_job:\n :script: echo 1\n", - "errors": [ - "jobs config should contain at least one visible job" - ], - "warnings": [] -} -``` + ## Use jq to create and process YAML & JSON payloads diff --git a/doc/api/managed_licenses.md b/doc/api/managed_licenses.md index 6aee60c57e0..e7ac247ae4a 100644 --- a/doc/api/managed_licenses.md +++ b/doc/api/managed_licenses.md @@ -2,145 +2,16 @@ stage: Fulfillment group: Utilization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: 'index.md' --- -# Managed Licenses API **(ULTIMATE)** +# Managed Licenses API (removed) **(ULTIMATE)** -WARNING: -"approval" and "blacklisted" approval statuses are changed to "allowed" and "denied" in GitLab 15.0. +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/390417) in GitLab 15.9 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/397067) in 16.0. -## List managed licenses - -Get all managed licenses for a given project. - -```plaintext -GET /projects/:id/managed_licenses -``` - -| Attribute | Type | Required | Description | -| --------- | ------- | -------- | --------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | - -```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/managed_licenses" -``` - -Example response: - -```json -[ - { - "id": 1, - "name": "MIT", - "approval_status": "allowed" - }, - { - "id": 3, - "name": "ISC", - "approval_status": "denied" - } -] -``` - -## Show an existing managed license - -Shows an existing managed license. - -```plaintext -GET /projects/:id/managed_licenses/:managed_license_id -``` - -| Attribute | Type | Required | Description | -| --------------- | ------- | --------------------------------- | ------------------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | -| `managed_license_id` | integer/string | yes | The ID or URL-encoded name of the license belonging to the project | - -```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/managed_licenses/6" -``` - -Example response: - -```json -{ - "id": 1, - "name": "MIT", - "approval_status": "denied" -} -``` - -## Create a new managed license - -Creates a new managed license for the given project with the given name and approval status. - -```plaintext -POST /projects/:id/managed_licenses -``` - -| Attribute | Type | Required | Description | -| ------------- | ------- | -------- | ---------------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | -| `name` | string | yes | The name of the managed license | -| `approval_status` | string | yes | The approval status of the license. "allowed" or "denied". | - -```shell -curl --data "name=MIT&approval_status=denied" --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/managed_licenses" -``` - -Example response: - -```json -{ - "id": 1, - "name": "MIT", - "approval_status": "allowed" -} -``` - -## Delete a managed license - -Deletes a managed license with a given ID. - -```plaintext -DELETE /projects/:id/managed_licenses/:managed_license_id -``` - -| Attribute | Type | Required | Description | -| --------- | ------- | -------- | --------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | -| `managed_license_id` | integer/string | yes | The ID or URL-encoded name of the license belonging to the project | - -```shell -curl --request DELETE --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/managed_licenses/4" -``` - -When successful, it replies with an HTTP 204 response. - -## Edit an existing managed license - -Updates an existing managed license with a new approval status. - -```plaintext -PATCH /projects/:id/managed_licenses/:managed_license_id -``` - -| Attribute | Type | Required | Description | -| --------------- | ------- | --------------------------------- | ------------------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | -| `managed_license_id` | integer/string | yes | The ID or URL-encoded name of the license belonging to the project | -| `approval_status` | string | yes | The approval status of the license. "allowed" or "denied". | - -```shell -curl --request PATCH --data "approval_status=denied" \ - --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/managed_licenses/6" -``` - -Example response: - -```json -{ - "id": 1, - "name": "MIT", - "approval_status": "denied" -} -``` + + + + diff --git a/doc/api/member_roles.md b/doc/api/member_roles.md index a7fc93e0df5..3ef6e287418 100644 --- a/doc/api/member_roles.md +++ b/doc/api/member_roles.md @@ -63,6 +63,8 @@ Adds a member role to a group. POST /groups/:id/member_roles ``` +To add a member role to a group, the group must be at root-level (have no parent group). + | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | diff --git a/doc/api/members.md b/doc/api/members.md index 950289effd2..02fa4be3d64 100644 --- a/doc/api/members.md +++ b/doc/api/members.md @@ -316,9 +316,8 @@ This API endpoint works on top-level groups only. It does not work on subgroups. This function takes [pagination](rest/index.md#pagination) parameters `page` and `per_page` to restrict the list of users. -[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/262875) in GitLab 13.7, the `search` and -`sort` parameters allow you to search for billable group members by name and sort the results, -respectively. +[In GitLab 13.7 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/262875), use the `search` parameter +to search for billable group members by name and `sort` to sort the results. ```plaintext GET /groups/:id/billable_members diff --git a/doc/api/merge_request_approvals.md b/doc/api/merge_request_approvals.md index 0df2b90e64d..ccd79c697a0 100644 --- a/doc/api/merge_request_approvals.md +++ b/doc/api/merge_request_approvals.md @@ -6,7 +6,8 @@ info: "To determine the technical writer assigned to the Stage/Group associated # Merge request approvals API **(PREMIUM)** -> Changing approval configuration with the `/approvals` endpoint was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/11132) in GitLab 12.3. +> - Changing approval configuration with the `/approvals` endpoint was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/11132) in GitLab 12.3. +> - Endpoint `/approvals` [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/353097) in GitLab 16.0. Configuration for [approvals on all merge requests](../user/project/merge_requests/approvals/index.md) @@ -312,7 +313,7 @@ Supported attributes: | `applies_to_all_protected_branches` | boolean | **{dotted-circle}** No | Whether the rule is applied to all protected branches. If set to `true`, the value of `protected_branch_ids` is ignored. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. | | `group_ids` | Array | **{dotted-circle}** No | The IDs of groups as approvers. | | `protected_branch_ids` | Array | **{dotted-circle}** No | The IDs of protected branches to scope the rule by. To identify the ID, [use the API](protected_branches.md#list-protected-branches). | -| `report_type` | string | **{dotted-circle}** No | The report type required when the rule type is `report_approver`. The supported report types are `license_scanning` and `code_coverage`. | +| `report_type` | string | **{dotted-circle}** No | The report type required when the rule type is `report_approver`. The supported report types are `license_scanning` [(Deprecated in GitLab 15.9)](../update/deprecations.md#license-check-and-the-policies-tab-on-the-license-compliance-page) and `code_coverage`. | | `rule_type` | string | **{dotted-circle}** No | The type of rule. `any_approver` is a pre-configured default rule with `approvals_required` at `0`. Other rules are `regular`. | | `user_ids` | Array | **{dotted-circle}** No | The IDs of users as approvers. | | `usernames` | string array | **{dotted-circle}** No | The usernames for this rule. | @@ -595,49 +596,6 @@ Supported attributes: } ``` -### Change approval configuration (deprecated) - -> - Moved to GitLab Premium in GitLab 13.9. -> - Endpoint `/approvals` [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/11132) in GitLab 12.3. - -WARNING: -The `/approvals` endpoint was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/11132) in GitLab 12.3 -and is planned for removal in 16.0. To change the approvals required for a merge request, -use the `/approval_rules` endpoint described in [Create merge request level rule](#create-merge-request-level-rule). -on this page. This change is a breaking change. - -If you are allowed to, you can change `approvals_required` using the following -endpoint: - -```plaintext -POST /projects/:id/merge_requests/:merge_request_iid/approvals -``` - -Supported attributes: - -| Attribute | Type | Required | Description | -|----------------------|-------------------|------------------------|-------------------------------------------------------------------------------| -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](rest/index.md#namespaced-path-encoding). | -| `approvals_required` | integer | **{check-circle}** Yes | Approvals required before MR can be merged. | -| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of the merge request. | - -```json -{ - "id": 5, - "iid": 5, - "project_id": 1, - "title": "Approvals API", - "description": "Test", - "state": "opened", - "created_at": "2016-06-08T00:19:52.638Z", - "updated_at": "2016-06-08T21:20:42.470Z", - "merge_status": "cannot_be_merged", - "approvals_required": 2, - "approvals_left": 2, - "approved_by": [] -} -``` - ### Get the approval state of merge requests > Moved to GitLab Premium in 13.9. @@ -972,11 +930,11 @@ Supported attributes: | Attribute | Type | Required | Description | |------------------------|-------------------|------------------------|-------------------------------------------------------------------------------| | `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](rest/index.md#namespaced-path-encoding). | -| `approvals_required` | integer | **{check-circle}** Yes | The number of required approvals for this rule. | | `approval_rule_id` | integer | **{check-circle}** Yes | The ID of an approval rule. | | `merge_request_iid` | integer | **{check-circle}** Yes | The IID of a merge request. | -| `name` | string | **{check-circle}** Yes | The name of the approval rule. | +| `approvals_required` | integer | **{check-circle}** No | The number of required approvals for this rule. | | `group_ids` | Array | **{dotted-circle}** No | The IDs of groups as approvers. | +| `name` | string | **{check-circle}** No | The name of the approval rule. | | `remove_hidden_groups` | boolean | **{dotted-circle}** No | Whether hidden groups should be removed. | | `user_ids` | Array | **{dotted-circle}** No | The IDs of users as approvers. | | `usernames` | string array | **{dotted-circle}** No | The usernames for this rule. | @@ -1140,3 +1098,23 @@ Supported attributes: |---------------------|-------------------|------------------------|-------------------------------------------------------------------------------| | `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](rest/index.md#namespaced-path-encoding). | | `merge_request_iid` | integer | **{check-circle}** Yes | The IID of a merge request. | + +## Reset approvals of a merge request + +Clear all approvals of merge request. + +Available only for [bot users](../user/project/settings/project_access_tokens.md#bot-users-for-projects) +based on project or group tokens. Users without bot permissions receive a `401 Unauthorized` response. + +```plaintext +PUT /projects/:id/merge_requests/:merge_request_iid/reset_approvals +``` + +| Attribute | Type | Required | Description | +|---------------------|-------------------|----------|-----------------------------------------------------------------------------------------------------------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | +| `merge_request_iid` | integer | **{check-circle}** Yes | The internal ID of the merge request. | + +```shell +curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/76/merge_requests/1/reset_approvals" +``` diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index 024593b2c6b..1be5f6204a1 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -11,9 +11,16 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - `merged_by` was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/350534) in GitLab 14.7. > - `merge_user` was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/349031) as an eventual replacement for `merged_by` in GitLab 14.7. > - `merge_status` was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/3169#note_1162532204) in favor of `detailed_merge_status` in GitLab 15.6. +> - `with_merge_status_recheck` [changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/115948) in GitLab 15.11 [with a flag](../administration/feature_flags.md) named `restrict_merge_status_recheck` to be ignored for requests from users insufficient permissions. Disabled by default. +> - `approvals_before_merge` was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/119503) in GitLab 16.0. Every API call to merge requests must be authenticated. +## Removals in API v5 + +The `approvals_before_merge` attribute has been deprecated, and is scheduled to be removed +in API v5 in favor of the [Merge request approvals API](merge_request_approvals.md). + ## List merge requests Get all merge requests the authenticated user has access to. By @@ -45,6 +52,7 @@ Supported attributes: | ------------------------------- | -------------- | -------- | ----------- | | `approved_by_ids` **(PREMIUM)** | integer array | **{dotted-circle}** No | Returns merge requests which have been approved by all the users with the given `id`. Maximum of 5. `None` returns merge requests with no approvals. `Any` returns merge requests with an approval. | | `approver_ids` **(PREMIUM)** | integer array | **{dotted-circle}** No | Returns merge requests which have specified all the users with the given `id` as individual approvers. `None` returns merge requests without approvers. `Any` returns merge requests with an approver. | +| `approved` | string | **{dotted-circle}** No | Filters merge requests by their `approved` status. `yes` returns only approved merge requests. `no` returns only non-approved merge requests. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3159) in GitLab 15.11. Available only when the feature flag `mr_approved_filter` is enabled. | | `assignee_id` | integer | **{dotted-circle}** No | Returns merge requests assigned to the given user `id`. `None` returns unassigned merge requests. `Any` returns merge requests with an assignee. | | `author_id` | integer | **{dotted-circle}** No | Returns merge requests created by the given user `id`. Mutually exclusive with `author_username`. Combine with `scope=all` or `scope=assigned_to_me`. | | `author_username` | string | **{dotted-circle}** No | Returns merge requests created by the given `username`. Mutually exclusive with `author_id`. | @@ -71,7 +79,7 @@ Supported attributes: | `updated_before` | datetime | **{dotted-circle}** No | Returns merge requests updated on or before the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | | `view` | string | **{dotted-circle}** No | If `simple`, returns the `iid`, URL, title, description, and basic state of merge request. | | `with_labels_details` | boolean | **{dotted-circle}** No | If `true`, response returns more details for each label in labels field: `:name`, `:color`, `:description`, `:description_html`, `:text_color`. Default is `false`. | -| `with_merge_status_recheck` | boolean | **{dotted-circle}** No | If `true`, this projection requests (but does not guarantee) that the `merge_status` field be recalculated asynchronously. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31890) in GitLab 13.0. | +| `with_merge_status_recheck` | boolean | **{dotted-circle}** No | If `true`, this projection requests (but does not guarantee) that the `merge_status` field be recalculated asynchronously. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31890) in GitLab 13.0. In GitLab 15.11 and later, enable the `restrict_merge_status_recheck` feature [flag](../administration/feature_flags.md) for this attribute to be ignored when requested by users without at least the Developer role. | | `wip` | string | **{dotted-circle}** No | Filter merge requests against their `wip` status. `yes` to return *only* draft merge requests, `no` to return *non-draft* merge requests. | ```json @@ -194,20 +202,6 @@ Supported attributes: ] ``` -Users on [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see -the `approvals_before_merge` parameter: - -```json -[ - { - "id": 1, - "title": "test1", - "approvals_before_merge": null - ... - } -] -``` - ### Merge requests list response notes - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31890) in GitLab 13.0, listing merge requests may @@ -248,6 +242,7 @@ Supported attributes: | `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | | `approved_by_ids` **(PREMIUM)** | integer array | **{dotted-circle}** No | Returns merge requests which have been approved by all the users with the given `id`, with a maximum of 5. `None` returns merge requests with no approvals. `Any` returns merge requests with an approval. | | `approver_ids` **(PREMIUM)** | integer array | **{dotted-circle}** No | Returns merge requests which have specified all the users with the given `id` as individual approvers. `None` returns merge requests without approvers. `Any` returns merge requests with an approver. | +| `approved` | string | **{dotted-circle}** No | Filters merge requests by their `approved` status. `yes` returns only approved merge requests. `no` returns only non-approved merge requests. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3159) in GitLab 15.11. Available only when the feature flag `mr_approved_filter` is enabled. | | `assignee_id` | integer | **{dotted-circle}** No | Returns merge requests assigned to the given user `id`. `None` returns unassigned merge requests. `Any` returns merge requests with an assignee. | | `author_id` | integer | **{dotted-circle}** No | Returns merge requests created by the given user `id`. Mutually exclusive with `author_username`. | | `author_username` | string | **{dotted-circle}** No | Returns merge requests created by the given `username`. Mutually exclusive with `author_id`. | @@ -273,7 +268,7 @@ Supported attributes: | `view` | string | **{dotted-circle}** No | If `simple`, returns the `iid`, URL, title, description, and basic state of merge request. | | `wip` | string | **{dotted-circle}** No | Filter merge requests against their `wip` status. `yes` to return *only* draft merge requests, `no` to return *non-draft* merge requests. | | `with_labels_details` | boolean | **{dotted-circle}** No | If `true`, response returns more details for each label in labels field: `:name`, `:color`, `:description`, `:description_html`, `:text_color`. Default is `false`. | -| `with_merge_status_recheck` | boolean | **{dotted-circle}** No | If `true`, this projection requests (but does not guarantee) that the `merge_status` field be recalculated asynchronously. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31890) in GitLab 13.0. | +| `with_merge_status_recheck` | boolean | **{dotted-circle}** No | If `true`, this projection requests (but does not guarantee) that the `merge_status` field be recalculated asynchronously. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31890) in GitLab 13.0. In GitLab 15.11 and later, enable the `restrict_merge_status_recheck` feature [flag](../administration/feature_flags.md) for this attribute to be ignored when requested by users without at least the Developer role. | ```json [ @@ -397,20 +392,6 @@ Supported attributes: ] ``` -Users on [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see -the `approvals_before_merge` parameter: - -```json -[ - { - "id": 1, - "title": "test1", - "approvals_before_merge": null - ... - } -] -``` - For important notes on response data, read [Merge requests list response notes](#merge-requests-list-response-notes). ## List group merge requests @@ -438,6 +419,7 @@ Supported attributes: | `approved_by_ids` **(PREMIUM)** | integer array | **{dotted-circle}** No | Returns merge requests which have been approved by all the users with the given `id`, with a maximum of 5. `None` returns merge requests with no approvals. `Any` returns merge requests with an approval. | | `approved_by_usernames` **(PREMIUM)** | string array | **{dotted-circle}** No | Returns merge requests which have been approved by all the users with the given `username`, with a maximum of 5. `None` returns merge requests with no approvals. `Any` returns merge requests with an approval. | | `approver_ids` **(PREMIUM)** | integer array | **{dotted-circle}** No | Returns merge requests which have specified all the users with the given `id`s as individual approvers. `None` returns merge requests without approvers. `Any` returns merge requests with an approver. | +| `approved` | string | **{dotted-circle}** No | Filters merge requests by their `approved` status. `yes` returns only approved merge requests. `no` returns only non-approved merge requests. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3159) in GitLab 15.11. Available only when the feature flag `mr_approved_filter` is enabled. | | `assignee_id` | integer | **{dotted-circle}** No | Returns merge requests assigned to the given user `id`. `None` returns unassigned merge requests. `Any` returns merge requests with an assignee. | | `author_id` | integer | **{dotted-circle}** No | Returns merge requests created by the given user `id`. Mutually exclusive with `author_username`. | | `author_username` | string | **{dotted-circle}** No | Returns merge requests created by the given `username`. Mutually exclusive with `author_id`. | @@ -461,7 +443,7 @@ Supported attributes: | `updated_before` | datetime | **{dotted-circle}** No | Returns merge requests updated on or before the given time. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | | `view` | string | **{dotted-circle}** No | If `simple`, returns the `iid`, URL, title, description, and basic state of merge request. | | `with_labels_details` | boolean | **{dotted-circle}** No | If `true`, response returns more details for each label in labels field: `:name`, `:color`, `:description`, `:description_html`, `:text_color`. Default is `false`. | -| `with_merge_status_recheck` | boolean | **{dotted-circle}** No | If `true`, this projection requests (but does not guarantee) that the `merge_status` field be recalculated asynchronously. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31890) in GitLab 13.0. | +| `with_merge_status_recheck` | boolean | **{dotted-circle}** No | If `true`, this projection requests (but does not guarantee) that the `merge_status` field be recalculated asynchronously. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31890) in GitLab 13.0. In GitLab 15.11 and later, enable the `restrict_merge_status_recheck` feature [flag](../administration/feature_flags.md) for this attribute to be ignored when requested by users without at least the Developer role. | ```json [ @@ -583,20 +565,6 @@ Supported attributes: ] ``` -Users on [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see -the `approvals_before_merge` parameter: - -```json -[ - { - "id": 1, - "title": "test1", - "approvals_before_merge": null - ... - } -] -``` - For important notes on response data, read [Merge requests list response notes](#merge-requests-list-response-notes). ## Get single MR @@ -604,7 +572,7 @@ For important notes on response data, read [Merge requests list response notes]( Shows information about a single merge request. **Note**: the `changes_count` value in the response is a string, not an -integer. This is because when an merge request has too many changes to display and store, +integer. When an merge request has too many changes to display and store, it is capped at 1,000. In that case, the API returns the string `"1000+"` for the changes count. @@ -626,18 +594,18 @@ Supported attributes: | Attribute | Type | Description | |----------------------------------|------|-------------| -| `approvals_before_merge` | integer | **(PREMIUM)** Number of approvals required before this merge request can merge. | +| `approvals_before_merge`| integer | **(PREMIUM)** Number of approvals required before this merge request can merge. To configure approval rules, see [Merge request approvals API](merge_request_approvals.md). [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/353097) in GitLab 16.0. | | `assignee` | object | First assignee of the merge request. | | `assignees` | array | Assignees of the merge request. | | `author` | object | User who created this merge request. | | `blocking_discussions_resolved` | boolean | Indicates if all discussions are resolved only if all are required before merge request can be merged. | -| `changes_count` | string | Number of changes made on the merge request. | +| `changes_count` | string | Number of changes made on the merge request. Empty when the merge request is created, and populates asynchronously. See [Empty API Fields for new merge requests](#empty-api-fields-for-new-merge-requests).| | `closed_at` | datetime | Timestamp of when the merge request was closed. | | `closed_by` | object | User who closed this merge request. | | `created_at` | datetime | Timestamp of when the merge request was created. | | `description` | string | Description of the merge request. Contains Markdown rendered as HTML for caching. | | `detailed_merge_status` | string | Detailed merge status of the merge request. Read [merge status](#merge-status) for a list of potential values. | -| `diff_refs` | object | References of the base SHA, the head SHA, and the start SHA for this merge request. Corresponds to the latest diff version of the merge request. Empty when the merge request is created, and populates asynchronously. See [Empty `diff_refs` for new merge requests](#empty-diff_refs-for-new-merge-requests). | +| `diff_refs` | object | References of the base SHA, the head SHA, and the start SHA for this merge request. Corresponds to the latest diff version of the merge request. Empty when the merge request is created, and populates asynchronously. See [Empty API fields for new merge requests](#empty-api-fields-for-new-merge-requests). | | `discussion_locked` | boolean | Indicates if comments on the merge request are locked to members only. | | `downvotes` | integer | Number of downvotes for the merge request. | | `draft` | boolean | Indicates if the merge request is a draft. | @@ -749,7 +717,7 @@ Supported attributes: }, "has_conflicts": false, "blocking_discussions_resolved": true, - "approvals_before_merge": null, + "approvals_before_merge": null, // deprecated, use [Merge request approvals API](merge_request_approvals.md) "subscribed": true, "changes_count": "1", "latest_build_started_at": "2022-05-13T09:46:50.032Z", @@ -817,7 +785,7 @@ Supported attributes: "user": { "can_merge": true }, - "approvals_before_merge": { // Available for GitLab Premium and higher tiers only + "approvals_before_merge": { // Available for GitLab Premium and Ultimate tiers only "id": 1, "title": "test1", "approvals_before_merge": null @@ -842,7 +810,8 @@ Use `detailed_merge_status` instead of `merge_status` to account for all potenti - The `detailed_merge_status` field can contain one of the following values related to the merge request: - `blocked_status`: Blocked by another merge request. - `broken_status`: Can't merge into the target branch due to a potential conflict. - - `checking`: Mergeability checks are still in progress. + - `checking`: Git is testing if a valid merge is possible. + - `unchecked`: Git has not yet tested if a valid merge is possible. - `ci_must_pass`: A CI/CD pipeline must succeed before merge. - `ci_still_running`: A CI/CD pipeline is still running. - `discussions_not_resolved`: All discussions must be resolved before merge. @@ -852,7 +821,6 @@ Use `detailed_merge_status` instead of `merge_status` to account for all potenti - `not_approved`: Approval is required before merge. - `not_open`: The merge request must be open before merge. - `policies_denied`: The merge request contains denied policies. - - `unchecked`: The merge status has not been checked. ## Get single merge request participants @@ -1174,7 +1142,8 @@ Example response: ## List merge request pipelines -Get a list of merge request pipelines. +Get a list of merge request pipelines. The pagination parameters `page` and +`per_page` can be used to restrict the list of merge request pipelines. ```plaintext GET /projects/:id/merge_requests/:merge_request_iid/pipelines @@ -1275,8 +1244,8 @@ POST /projects/:id/merge_requests | `target_branch` | string | **{check-circle}** Yes | The target branch. | | `title` | string | **{check-circle}** Yes | Title of MR. | | `allow_collaboration` | boolean | **{dotted-circle}** No | Allow commits from members who can merge to the target branch. | +| `approvals_before_merge` **(PREMIUM)** | integer | **{dotted-circle}** No | Number of approvals required before this can be merged (see below). To configure approval rules, see [Merge request approvals API](merge_request_approvals.md). [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/353097) in GitLab 16.0. | | `allow_maintainer_to_push` | boolean | **{dotted-circle}** No | Alias of `allow_collaboration`. | -| `approvals_before_merge` **(PREMIUM)** | integer | **{dotted-circle}** No | Number of approvals required before this can be merged (see below). | | `assignee_id` | integer | **{dotted-circle}** No | Assignee user ID. | | `assignee_ids` | integer array | **{dotted-circle}** No | The ID of the users to assign the merge request to. Set to `0` or provide an empty value to unassign all assignees. | | `description` | string | **{dotted-circle}** No | Description of the merge request. Limited to 1,048,576 characters. | @@ -1288,13 +1257,6 @@ POST /projects/:id/merge_requests | `squash_on_merge` | boolean | no | Indicates if the merge request will be squashed when merged. | | `target_project_id` | integer | **{dotted-circle}** No | Numeric ID of the target project. | -If `approvals_before_merge` is not provided, it inherits the value from the target project. If provided, the following conditions must hold for it to take effect: - -- The target project's `approvals_before_merge` must be greater than zero. A - value of zero disables approvals for that project. -- The provided value of `approvals_before_merge` must be greater than the - target project's `approvals_before_merge`. - ```json { "id": 1, @@ -1416,18 +1378,6 @@ If `approvals_before_merge` is not provided, it inherits the value from the targ } ``` -Users of [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see -the `approvals_before_merge` parameter: - -```json -{ - "id": 1, - "title": "test1", - "approvals_before_merge": null - ... -} -``` - For important notes on response data, read [Single merge request response notes](#single-merge-request-response-notes). ## Update MR @@ -1599,18 +1549,6 @@ Must include at least one non-required attribute from above. } ``` -Users on [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see -the `approvals_before_merge` parameter: - -```json -{ - "id": 1, - "title": "test1", - "approvals_before_merge": null - ... -} -``` - For important notes on response data, read [Single merge request response notes](#single-merge-request-response-notes). ## Delete a merge request @@ -1788,18 +1726,6 @@ Supported attributes: } ``` -Users on [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see -the `approvals_before_merge` parameter: - -```json -{ - "id": 1, - "title": "test1", - "approvals_before_merge": null - ... -} -``` - This API returns specific HTTP status codes on failure: | HTTP Status | Message | Reason | @@ -1817,7 +1743,7 @@ Merge the changes between the merge request source and target branches into `ref ref, of the target project repository, if possible. This ref has the state the target branch would have if a regular merge action was taken. -This is not a regular merge action given it doesn't change the merge request target branch state in any manner. +This action isn't a regular merge action, because it doesn't change the merge request target branch state in any manner. This ref (`refs/merge-requests/:iid/merge`) isn't necessarily overwritten when submitting requests to this API, though it makes sure the ref has the latest possible state. @@ -2001,18 +1927,6 @@ Supported attributes: } ``` -Users on [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see -the `approvals_before_merge` parameter: - -```json -{ - "id": 1, - "title": "test1", - "approvals_before_merge": null - ... -} -``` - For important notes on response data, read [Single merge request response notes](#single-merge-request-response-notes). ## Rebase a merge request @@ -2037,7 +1951,7 @@ PUT /projects/:id/merge_requests/:merge_request_iid/rebase curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/76/merge_requests/1/rebase" ``` -This is an asynchronous request. The API returns a `HTTP 202 Accepted` response +This request is asynchronous. The API returns a `HTTP 202 Accepted` response if the request is enqueued successfully, with a response containing: ```json @@ -2078,26 +1992,6 @@ If the rebase operation fails, the response includes the following: } ``` -## Reset approvals of a merge request - -Clear all approvals of merge request. - -Available only for [bot users](../user/project/settings/project_access_tokens.md#bot-users-for-projects) -based on project or group tokens. Users without bot permissions receive a `401 Unauthorized` response. - -```plaintext -PUT /projects/:id/merge_requests/:merge_request_iid/reset_approvals -``` - -| Attribute | Type | Required | Description | -|---------------------|-------------------|----------|-----------------------------------------------------------------------------------------------------------------| -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | -| `merge_request_iid` | integer | **{check-circle}** Yes | The internal ID of the merge request. | - -```shell -curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/76/merge_requests/1/reset_approvals" -``` - ## Comments on merge requests Comments are done via the [notes](notes.md) resource. @@ -2333,18 +2227,6 @@ Example response: } ``` -Users on [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see -the `approvals_before_merge` parameter: - -```json -{ - "id": 1, - "title": "test1", - "approvals_before_merge": null - ... -} -``` - For important notes on response data, read [Single merge request response notes](#single-merge-request-response-notes). ## Unsubscribe from a merge request @@ -2504,18 +2386,6 @@ Example response: } ``` -Users on [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see -the `approvals_before_merge` parameter: - -```json -{ - "id": 1, - "title": "test1", - "approvals_before_merge": null - ... -} -``` - For important notes on response data, read [Single merge request response notes](#single-merge-request-response-notes). ## Create a to-do item @@ -2918,11 +2788,11 @@ To track which state was set, who did it, and when it happened, check out ## Troubleshooting -### Empty `diff_refs` for new merge requests +### Empty API fields for new merge requests -When a merge request is created, the `diff_refs` field is initially empty. This field -is populated asynchronously after the merge request is created. For more -information, see the issue -[`diff_refs` empty after merge request is created](https://gitlab.com/gitlab-org/gitlab/-/issues/386562), +When a merge request is created, the `diff_refs` and `changes_count` fields are +initially empty. These fields are populated asynchronously after the +merge request is created. For more information, see the issue +[Some merge request API fields (`diff_refs`, `changes_count`) empty after MR is created](https://gitlab.com/gitlab-org/gitlab/-/issues/386562), and the [related discussion](https://forum.gitlab.com/t/diff-refs-empty-after-mr-is-created/78975) in the GitLab forums. diff --git a/doc/api/merge_trains.md b/doc/api/merge_trains.md index 6d5d12a618c..c041b01da04 100644 --- a/doc/api/merge_trains.md +++ b/doc/api/merge_trains.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/36146) in GitLab 12.9. > - Using this API you can consume [Merge Train](../ci/pipelines/merge_trains.md) entries. -Every API call to merge trains must be authenticated with Developer or higher [permissions](../user/permissions.md). +Every API call to merge trains must be authenticated with at lease the Developer [role](../user/permissions.md). If a user is not a member of a project and the project is private, a `GET` request on that project returns a `404` status code. @@ -220,3 +220,74 @@ Example response: "duration":null } ``` + +## Add a merge request to a merge train + +Add a merge request to the merge train targeting the merge request's target branch. + +```plaintext +POST /projects/:id/merge_trains/merge_requests/:merge_request_iid +``` + +Supported attributes: + +| Attribute | Type | Required | Description | +| ------------------------------ | -------------- | -------- | ----------- | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `merge_request_iid` | integer | yes | The internal ID of the merge request. | +| `when_pipeline_succeeds` | boolean | no | If true, the merge request is added to the merge train when the pipeline succeeds. When false or unspecified, the merge request is added directly to the merge train. | +| `sha` | string | no | If present, the SHA must match the `HEAD` of the source branch, otherwise the merge fails. | +| `squash` | boolean | no | If true, the commits are squashed into a single commit on merge. | + +Example request: + +```shell +curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/597/merge_trains/merge_requests/1" +``` + +Example response: + +```json +[ + { + "id": 267, + "merge_request": { + "id": 273, + "iid": 1, + "project_id": 597, + "title": "My title 9", + "description": null, + "state": "opened", + "created_at": "2022-10-31T19:06:05.725Z", + "updated_at": "2022-10-31T19:06:05.725Z", + "web_url": "http://localhost/namespace18/project21/-/merge_requests/1" + }, + "user": { + "id": 933, + "username": "user12", + "name": "Sidney Jones31", + "state": "active", + "avatar_url": "https://www.gravatar.com/avatar/6c8365de387cb3db10ecc7b1880203c4?s=80\u0026d=identicon", + "web_url": "http://localhost/user12" + }, + "pipeline": { + "id": 273, + "iid": 1, + "project_id": 598, + "sha": "b83d6e391c22777fca1ed3012fce84f633d7fed0", + "ref": "main", + "status": "pending", + "source": "push", + "created_at": "2022-10-31T19:06:06.231Z", + "updated_at": "2022-10-31T19:06:06.231Z", + "web_url": "http://localhost/namespace19/project22/-/pipelines/273" + }, + "created_at": "2022-10-31T19:06:06.237Z", + "updated_at":"2022-10-31T19:06:06.237Z", + "target_branch":"main", + "status":"idle", + "merged_at":null, + "duration":null + } +] +``` diff --git a/doc/api/metadata.md b/doc/api/metadata.md index 92f7177482a..9a02ca2a5e4 100644 --- a/doc/api/metadata.md +++ b/doc/api/metadata.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/api/milestones.md b/doc/api/milestones.md index 998beeb9b3b..e1acf4c14bb 100644 --- a/doc/api/milestones.md +++ b/doc/api/milestones.md @@ -21,6 +21,8 @@ GET /projects/:id/milestones?state=active GET /projects/:id/milestones?state=closed GET /projects/:id/milestones?title=1.0 GET /projects/:id/milestones?search=version +GET /projects/:id/milestones?updated_before=2013-10-02T09%3A24%3A18Z +GET /projects/:id/milestones?updated_after=2013-10-02T09%3A24%3A18Z ``` Parameters: @@ -28,11 +30,13 @@ Parameters: | Attribute | Type | Required | Description | | ---------------------------- | ------ | -------- | ----------- | | `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | -| `iids[]` | integer array | optional | Return only the milestones having the given `iid` (Note: ignored if `include_parent_milestones` is set as `true`) | -| `state` | string | optional | Return only `active` or `closed` milestones | -| `title` | string | optional | Return only the milestones having the given `title` | -| `search` | string | optional | Return only milestones with a title or description matching the provided string | -| `include_parent_milestones` | boolean | optional | Include group milestones from parent group and its ancestors. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/196066) in GitLab 13.4 | +| `iids[]` | integer array | no | Return only the milestones having the given `iid` (Note: ignored if `include_parent_milestones` is set as `true`) | +| `state` | string | no | Return only `active` or `closed` milestones | +| `title` | string | no | Return only the milestones having the given `title` | +| `search` | string | no | Return only milestones with a title or description matching the provided string | +| `include_parent_milestones` | boolean | no | Include group milestones from parent group and its ancestors. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/196066) in GitLab 13.4 | +| `updated_before` | datetime | no | Return only milestones updated before the given datetime. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). Introduced in GitLab 15.10 | +| `updated_after` | datetime | no | Return only milestones updated after the given datetime. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). Introduced in GitLab 15.10 | ```shell curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/milestones" diff --git a/doc/api/namespaces.md b/doc/api/namespaces.md index cbf7ea079bc..15ce73fdbc3 100644 --- a/doc/api/namespaces.md +++ b/doc/api/namespaces.md @@ -9,8 +9,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w Usernames and group names fall under a special category called [namespaces](../user/namespace/index.md). -For users and groups supported API calls see the [users](users.md) and -[groups](groups.md) documentation respectively. +You might also want to view documentation for: + +- [Users](users.md) +- [Groups](groups.md) [Pagination](rest/index.md#pagination) is used. @@ -100,9 +102,9 @@ Owners also see the `plan` property associated with a namespace: ] ``` -Users on GitLab.com also see `max_seats_used` and `seats_in_use` parameters. +Users on GitLab.com also see `max_seats_used`, `seats_in_use` and `max_seats_used_changed_at` parameters. `max_seats_used` is the highest number of users the group had. `seats_in_use` is -the number of license seats currently being used. Both values are updated +the number of license seats currently being used. `max_seats_used_changed_at` shows the date when the `max_seats_used` value changed. All the values are updated once a day. `max_seats_used` and `seats_in_use` are non-zero only for namespaces on paid plans. @@ -114,6 +116,7 @@ once a day. "name": "user1", "billable_members_count": 2, "max_seats_used": 3, + "max_seats_used_changed_at":"2023-02-13T12:00:02.000Z", "seats_in_use": 2, ... } diff --git a/doc/api/notes.md b/doc/api/notes.md index 9c453c6390f..41b4ab7fd9a 100644 --- a/doc/api/notes.md +++ b/doc/api/notes.md @@ -78,6 +78,7 @@ GET /projects/:id/issues/:issue_iid/notes?sort=asc&order_by=updated_at "system": true, "noteable_id": 377, "noteable_type": "Issue", + "project_id": 5, "noteable_iid": 377, "resolvable": false, "confidential": false, @@ -100,6 +101,7 @@ GET /projects/:id/issues/:issue_iid/notes?sort=asc&order_by=updated_at "system": true, "noteable_id": 121, "noteable_type": "Issue", + "project_id": 5, "noteable_iid": 121, "resolvable": false, "confidential": true, @@ -147,7 +149,7 @@ Parameters: | `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | | `issue_iid` | integer | yes | The IID of an issue. | | `body` | string | yes | The content of a note. Limited to 1,000,000 characters. | -| `confidential` | boolean | no | **Deprecated:** will be removed in GitLab 16.0 and renamed to `internal`. The confidential flag of a note. Default is false. | +| `confidential` | boolean | no | **Deprecated:** Scheduled to be removed in GitLab 16.0 and renamed to `internal`. The confidential flag of a note. Default is false. | | `internal` | boolean | no | The internal flag of a note. Overrides `confidential` when both parameters are submitted. Default is false. | | `created_at` | string | no | Date time string, ISO 8601 formatted. It must be after 1970-01-01. Example: `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | @@ -171,7 +173,7 @@ Parameters: | `issue_iid` | integer | yes | The IID of an issue. | | `note_id` | integer | yes | The ID of a note. | | `body` | string | no | The content of a note. Limited to 1,000,000 characters. | -| `confidential` | boolean | no | **Deprecated:** will be removed in GitLab 16.0. The confidential flag of a note. Default is false. | +| `confidential` | boolean | no | **Deprecated:** Scheduled to be removed in GitLab 16.0. The confidential flag of a note. Default is false. | ```shell curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/issues/11/notes/636?body=note" @@ -239,9 +241,9 @@ Parameters: ```json { - "id": 52, - "title": "Snippet", - "file_name": "snippet.rb", + "id": 302, + "body": "closed", + "attachment": null, "author": { "id": 1, "username": "pipin", @@ -250,9 +252,16 @@ Parameters: "state": "active", "created_at": "2013-09-30T13:46:01Z" }, - "expires_at": null, - "updated_at": "2013-10-02T07:34:20Z", - "created_at": "2013-10-02T07:34:20Z" + "created_at": "2013-10-02T09:22:45Z", + "updated_at": "2013-10-02T10:22:45Z", + "system": true, + "noteable_id": 377, + "noteable_type": "Issue", + "project_id": 5, + "noteable_iid": 377, + "resolvable": false, + "confidential": false, + "internal": false } ``` @@ -263,7 +272,7 @@ curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/a ### Create new snippet note Creates a new note for a single snippet. Snippet notes are user comments on snippets. -If you create a note where the body only contains an Award Emoji, GitLab returns this object. +If you create a note where the body only contains an emoji reaction, GitLab returns this object. ```plaintext POST /projects/:id/snippets/:snippet_id/notes @@ -379,6 +388,7 @@ Parameters: "system": false, "noteable_id": 2, "noteable_type": "MergeRequest", + "project_id": 5, "noteable_iid": 2, "resolvable": false, "confidential": false, @@ -392,8 +402,13 @@ curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/a ### Create new merge request note -Creates a new note for a single merge request. -If you create a note where the body only contains an Award Emoji, GitLab returns this object. +Creates a new note for a single merge request. Notes are not attached to specific +lines in a merge request. For other approaches with more granular control, see +[Post comment to commit](commits.md#post-comment-to-commit) in the Commits API, +and [Create a new thread in the merge request diff](discussions.md#create-a-new-thread-in-the-merge-request-diff) +in the Discussions API. + +If you create a note where the body only contains an emoji reaction, GitLab returns this object. ```plaintext POST /projects/:id/merge_requests/:merge_request_iid/notes @@ -407,7 +422,7 @@ Parameters: | `merge_request_iid` | integer | yes | The IID of a project merge request | | `body` | string | yes | The content of a note. Limited to 1,000,000 characters. | | `created_at` | string | no | Date time string, ISO 8601 formatted. Example: `2016-03-11T03:45:40Z` (requires administrator or project/group owner rights) | -| `merge_request_diff_sha`| string | no | Required for the `/merge` [quick action](../user/project/quick_actions.md). The SHA of the head commit, which ensures the merge request wasn't updated after the API request was sent. | +| `merge_request_diff_head_sha`| string | no | Required for the `/merge` [quick action](../user/project/quick_actions.md). The SHA of the head commit, which ensures the merge request wasn't updated after the API request was sent. | ### Modify existing merge request note @@ -425,7 +440,7 @@ Parameters: | `merge_request_iid` | integer | yes | The IID of a project merge request | | `note_id` | integer | no | The ID of a note | | `body` | string | yes | The content of a note. Limited to 1,000,000 characters. | -| `confidential` | boolean | no | **Deprecated:** will be removed in GitLab 16.0. The confidential flag of a note. Default is false. | +| `confidential` | boolean | no | **Deprecated:** Scheduled to be removed in GitLab 16.0. The confidential flag of a note. Default is false. | ```shell curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/notes/1?body=note" @@ -496,9 +511,9 @@ Parameters: ```json { - "id": 52, - "title": "Epic", - "file_name": "epic.rb", + "id": 302, + "body": "Epic note", + "attachment": null, "author": { "id": 1, "username": "pipin", @@ -507,9 +522,14 @@ Parameters: "state": "active", "created_at": "2013-09-30T13:46:01Z" }, - "expires_at": null, - "updated_at": "2013-10-02T07:34:20Z", - "created_at": "2013-10-02T07:34:20Z", + "created_at": "2013-10-02T09:22:45Z", + "updated_at": "2013-10-02T10:22:45Z", + "system": true, + "noteable_id": 11, + "noteable_type": "Epic", + "project_id": 5, + "noteable_iid": 11, + "resolvable": false, "confidential": false, "internal": false } @@ -522,7 +542,7 @@ curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/a ### Create new epic note Creates a new note for a single epic. Epic notes are comments users can post to an epic. -If you create a note where the body only contains an Award Emoji, GitLab returns this object. +If you create a note where the body only contains an emoji reaction, GitLab returns this object. ```plaintext POST /groups/:id/epics/:epic_id/notes @@ -535,7 +555,7 @@ Parameters: | `body` | string | yes | The content of a note. Limited to 1,000,000 characters. | | `epic_id` | integer | yes | The ID of an epic | | `id` | integer or string | yes | The ID or [URL-encoded path of the group](rest/index.md#namespaced-path-encoding) | -| `confidential` | boolean | no | **Deprecated:** will be removed in GitLab 16.0 and is renamed to `internal`. The confidential flag of a note. Default is `false`. | +| `confidential` | boolean | no | **Deprecated:** Scheduled to be removed in GitLab 16.0 and is renamed to `internal`. The confidential flag of a note. Default is `false`. | | `internal` | boolean | no | The internal flag of a note. Overrides `confidential` when both parameters are submitted. Default is `false`. | ```shell @@ -558,7 +578,7 @@ Parameters: | `epic_id` | integer | yes | The ID of an epic | | `note_id` | integer | yes | The ID of a note | | `body` | string | yes | The content of a note. Limited to 1,000,000 characters. | -| `confidential` | boolean | no | **Deprecated:** will be removed in GitLab 16.0. The confidential flag of a note. Default is false. | +| `confidential` | boolean | no | **Deprecated:** Scheduled to be removed in GitLab 16.0. The confidential flag of a note. Default is false. | ```shell curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/epics/11/notes/1?body=note" diff --git a/doc/api/oauth2.md b/doc/api/oauth2.md index 3e470c5cb91..eb9d1d3bc8a 100644 --- a/doc/api/oauth2.md +++ b/doc/api/oauth2.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # OAuth 2.0 identity provider API **(FREE)** GitLab provides an API to allow third-party services to access GitLab resources on a user's behalf -with the [OAuth2](https://oauth.net/2/) protocol. +with the [OAuth 2.0](https://oauth.net/2/) protocol. To configure GitLab for this, see [Configure GitLab as an OAuth 2.0 authentication identity provider](../integration/oauth_provider.md). @@ -45,6 +45,8 @@ GitLab supports the following authorization flows: - **Resource owner password credentials:** To be used **only** for securely hosted, first-party services. GitLab recommends against use of this flow. +Device Authorization Grant is not supported. [Issue 332682](https://gitlab.com/gitlab-org/gitlab/-/issues/332682) proposes to add support. + The draft specification for [OAuth 2.1](https://oauth.net/2.1/) specifically omits both the Implicit grant and Resource Owner Password Credentials flows. diff --git a/doc/api/openapi/openapi.yaml b/doc/api/openapi/openapi.yaml index 9ee2b8119be..e090eab1e5d 100644 --- a/doc/api/openapi/openapi.yaml +++ b/doc/api/openapi/openapi.yaml @@ -20,7 +20,7 @@ info: The feature uses the current [GitLab session cookie](https://docs.gitlab.com/ee/api/index.html#session-cookie), so each request is made using your account. - Instructions for using this tool can be found in [Interactive API Documentation](openapi_interactive.md). + Instructions for using this tool can be found in [Interactive API Documentation](https://docs.gitlab.com/ee/api/openapi/openapi_interactive.html). version: v4 title: GitLab API diff --git a/doc/api/openapi/openapi_interactive.md b/doc/api/openapi/openapi_interactive.md index 1cf6ba6482c..c3e3f5f372b 100644 --- a/doc/api/openapi/openapi_interactive.md +++ b/doc/api/openapi/openapi_interactive.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/api/packages.md b/doc/api/packages.md index 32b21ce354d..86eaf3028cf 100644 --- a/doc/api/packages.md +++ b/doc/api/packages.md @@ -352,6 +352,9 @@ Can return the following status codes: - `204 No Content`, if the package was deleted successfully. - `404 Not Found`, if the package was not found. +If [request forwarding](../user/packages/package_registry/supported_functionality.md#forwarding-requests) is enabled, +deleting a package can introduce a [dependency confusion risk](../user/packages/package_registry/supported_functionality.md#deleting-packages). + ## Delete a package file > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32107) in GitLab 13.12. diff --git a/doc/api/packages/composer.md b/doc/api/packages/composer.md index e75dbfeb92f..857f87a751e 100644 --- a/doc/api/packages/composer.md +++ b/doc/api/packages/composer.md @@ -18,7 +18,7 @@ package registry, see the [Composer package registry documentation](../../user/p NOTE: These endpoints do not adhere to the standard API authentication methods. See the [Composer package registry documentation](../../user/packages/composer_repository/index.md) -for details on which headers and token types are supported. +for details on which headers and token types are supported. Undocumented authentication methods might be removed in the future. ## Base repository request diff --git a/doc/api/packages/conan.md b/doc/api/packages/conan.md index c37296ad664..a2c20756737 100644 --- a/doc/api/packages/conan.md +++ b/doc/api/packages/conan.md @@ -17,7 +17,7 @@ package registry, see the [Conan package registry documentation](../../user/pack NOTE: These endpoints do not adhere to the standard API authentication methods. -See each route for details on how credentials are expected to be passed. +See each route for details on how credentials are expected to be passed. Undocumented authentication methods might be removed in the future. NOTE: The Conan registry is not FIPS compliant and is disabled when [FIPS mode](../../development/fips_compliance.md) is enabled. @@ -123,7 +123,7 @@ GET /users/authenticate ``` ```shell -curl --user : "https://gitlab.example.com/packages/conan/v1/users/authenticate +curl --user : "https://gitlab.example.com/api/v4/packages/conan/v1/users/authenticate" ``` Example response: @@ -167,7 +167,7 @@ GET /conans/:package_name/:package_version/:package_username/:pack | --------- | ---- | -------- | ----------- | | `package_name` | string | yes | Name of a package. | | `package_version` | string | yes | Version of a package. | -| `package_username` | string | yes | Conan username of a package. This is the `+`-separated full path of your project. | +| `package_username` | string | yes | Conan username of a package. This attribute is the `+`-separated full path of your project. | | `package_channel` | string | yes | Channel of a package. | ```shell @@ -199,7 +199,7 @@ GET /conans/:package_name/:package_version/:package_username/:pack | --------- | ---- | -------- | ----------- | | `package_name` | string | yes | Name of a package. | | `package_version` | string | yes | Version of a package. | -| `package_username` | string | yes | Conan username of a package. This is the `+`-separated full path of your project. | +| `package_username` | string | yes | Conan username of a package. This attribute is the `+`-separated full path of your project. | | `package_channel` | string | yes | Channel of a package. | | `conan_package_reference` | string | yes | Reference hash of a Conan package. Conan generates this value. | @@ -231,7 +231,7 @@ GET /conans/:package_name/:package_version/:package_username/:pack | --------- | ---- | -------- | ----------- | | `package_name` | string | yes | Name of a package. | | `package_version` | string | yes | Version of a package. | -| `package_username` | string | yes | Conan username of a package. This is the `+`-separated full path of your project. | +| `package_username` | string | yes | Conan username of a package. This attribute is the `+`-separated full path of your project. | | `package_channel` | string | yes | Channel of a package. | ```shell @@ -265,7 +265,7 @@ GET /conans/:package_name/:package_version/:package_username/:pack | --------- | ---- | -------- | ----------- | | `package_name` | string | yes | Name of a package. | | `package_version` | string | yes | Version of a package. | -| `package_username` | string | yes | Conan username of a package. This is the `+`-separated full path of your project. | +| `package_username` | string | yes | Conan username of a package. This attribute is the `+`-separated full path of your project. | | `package_channel` | string | yes | Channel of a package. | | `conan_package_reference` | string | yes | Reference hash of a Conan package. Conan generates this value. | @@ -290,8 +290,8 @@ the project-level route, the returned URLs contain `/projects/:id`. > Introduced in GitLab 12.5. -Returns a list of recipe filenames with their associated download URLs. -This is the same payload as the [recipe manifest](#recipe-manifest) endpoint. +Recipe download URLs return a list of recipe filenames with their associated download URLs. +This attribute is the same payload as the [recipe manifest](#recipe-manifest) endpoint. ```plaintext GET /conans/:package_name/:package_version/:package_username/:package_channel/download_urls @@ -301,7 +301,7 @@ GET /conans/:package_name/:package_version/:package_username/:pack | --------- | ---- | -------- | ----------- | | `package_name` | string | yes | Name of a package. | | `package_version` | string | yes | Version of a package. | -| `package_username` | string | yes | Conan username of a package. This is the `+`-separated full path of your project. | +| `package_username` | string | yes | Conan username of a package. This attribute is the `+`-separated full path of your project. | | `package_channel` | string | yes | Channel of a package. | ```shell @@ -325,8 +325,8 @@ the project-level route, the returned URLs contain `/projects/:id`. > Introduced in GitLab 12.5. -Returns a list of package filenames with their associated download URLs. -This is the same payload as the [package manifest](#package-manifest) endpoint. +Package download URLs return a list of package filenames with their associated download URLs. +This URL is the same payload as the [package manifest](#package-manifest) endpoint. ```plaintext GET /conans/:package_name/:package_version/:package_username/:package_channel/packages/:conan_package_reference/download_urls @@ -336,7 +336,7 @@ GET /conans/:package_name/:package_version/:package_username/:pack | --------- | ---- | -------- | ----------- | | `package_name` | string | yes | Name of a package. | | `package_version` | string | yes | Version of a package. | -| `package_username` | string | yes | Conan username of a package. This is the `+`-separated full path of your project. | +| `package_username` | string | yes | Conan username of a package. This attribute is the `+`-separated full path of your project. | | `package_channel` | string | yes | Channel of a package. | | `conan_package_reference` | string | yes | Reference hash of a Conan package. Conan generates this value. | @@ -371,7 +371,7 @@ POST /conans/:package_name/:package_version/:package_username/:pac | --------- | ---- | -------- | ----------- | | `package_name` | string | yes | Name of a package. | | `package_version` | string | yes | Version of a package. | -| `package_username` | string | yes | Conan username of a package. This is the `+`-separated full path of your project. | +| `package_username` | string | yes | Conan username of a package. This attribute is the `+`-separated full path of your project. | | `package_channel` | string | yes | Channel of a package. | Example request JSON payload: @@ -417,7 +417,7 @@ POST /conans/:package_name/:package_version/:package_username/:pac | --------- | ---- | -------- | ----------- | | `package_name` | string | yes | Name of a package. | | `package_version` | string | yes | Version of a package. | -| `package_username` | string | yes | Conan username of a package. This is the `+`-separated full path of your project. | +| `package_username` | string | yes | Conan username of a package. This attribute is the `+`-separated full path of your project. | | `package_channel` | string | yes | Channel of a package. | | `conan_package_reference` | string | yes | Reference hash of a Conan package. Conan generates this value. | @@ -468,7 +468,7 @@ GET packages/conan/v1/files/:package_name/:package_version/:package_username/:pa | --------- | ---- | -------- | ----------- | | `package_name` | string | yes | Name of a package. | | `package_version` | string | yes | Version of a package. | -| `package_username` | string | yes | Conan username of a package. This is the `+`-separated full path of your project. | +| `package_username` | string | yes | Conan username of a package. This attribute is the `+`-separated full path of your project. | | `package_channel` | string | yes | Channel of a package. | | `recipe_revision` | string | yes | Revision of the recipe. GitLab does not yet support Conan revisions, so the default value of `0` is always used. | | `file_name` | string | yes | The name and file extension of the requested file. | @@ -501,7 +501,7 @@ PUT packages/conan/v1/files/:package_name/:package_version/:package_username/:pa | --------- | ---- | -------- | ----------- | | `package_name` | string | yes | Name of a package. | | `package_version` | string | yes | Version of a package. | -| `package_username` | string | yes | Conan username of a package. This is the `+`-separated full path of your project. | +| `package_username` | string | yes | Conan username of a package. This attribute is the `+`-separated full path of your project. | | `package_channel` | string | yes | Channel of a package. | | `recipe_revision` | string | yes | Revision of the recipe. GitLab does not yet support Conan revisions, so the default value of `0` is always used. | | `file_name` | string | yes | The name and file extension of the requested file. | @@ -531,7 +531,7 @@ GET packages/conan/v1/files/:package_name/:package_version/:package_username/:pa | --------- | ---- | -------- | ----------- | | `package_name` | string | yes | Name of a package. | | `package_version` | string | yes | Version of a package. | -| `package_username` | string | yes | Conan username of a package. This is the `+`-separated full path of your project. | +| `package_username` | string | yes | Conan username of a package. This attribute is the `+`-separated full path of your project. | | `package_channel` | string | yes | Channel of a package. | | `recipe_revision` | string | yes | Revision of the recipe. GitLab does not yet support Conan revisions, so the default value of `0` is always used. | | `conan_package_reference` | string | yes | Reference hash of a Conan package. Conan generates this value. | @@ -566,7 +566,7 @@ PUT packages/conan/v1/files/:package_name/:package_version/:package_username/:pa | --------- | ---- | -------- | ----------- | | `package_name` | string | yes | Name of a package. | | `package_version` | string | yes | Version of a package. | -| `package_username` | string | yes | Conan username of a package. This is the `+`-separated full path of your project. | +| `package_username` | string | yes | Conan username of a package. This attribute is the `+`-separated full path of your project. | | `package_channel` | string | yes | Channel of a package. | | `recipe_revision` | string | yes | Revision of the recipe. GitLab does not yet support Conan revisions, so the default value of `0` is always used. | | `conan_package_reference` | string | yes | Reference hash of a Conan package. Conan generates this value. | @@ -596,7 +596,7 @@ DELETE /conans/:package_name/:package_version/:package_username/:p | --------- | ---- | -------- | ----------- | | `package_name` | string | yes | Name of a package. | | `package_version` | string | yes | Version of a package. | -| `package_username` | string | yes | Conan username of a package. This is the `+`-separated full path of your project. | +| `package_username` | string | yes | Conan username of a package. This attribute is the `+`-separated full path of your project. | | `package_channel` | string | yes | Channel of a package. | ```shell diff --git a/doc/api/packages/debian.md b/doc/api/packages/debian.md index 87b583de5e6..267ebb59c19 100644 --- a/doc/api/packages/debian.md +++ b/doc/api/packages/debian.md @@ -24,7 +24,7 @@ package registry, see the [Debian registry documentation](../../user/packages/de NOTE: These endpoints do not adhere to the standard API authentication methods. See the [Debian registry documentation](../../user/packages/debian_repository/index.md) -for details on which headers and token types are supported. +for details on which headers and token types are supported. Undocumented authentication methods might be removed in the future. ## Enable the Debian API @@ -46,7 +46,8 @@ See [Authenticate to the Debian Package Repositories](../../user/packages/debian ## Upload a package file -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62028) in GitLab 14.0. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62028) in GitLab 14.0. +> - Upload with explicit distribution and component [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/101838) in GitLab 15.9. Upload a Debian package file: @@ -54,18 +55,29 @@ Upload a Debian package file: PUT projects/:id/packages/debian/:file_name ``` -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `id` | string | yes | The ID or full path of the project. | -| `file_name` | string | yes | The name of the Debian package file. | +| Attribute | Type | Required | Description | +| -------------- | ------ | -------- | ----------- | +| `id` | string | yes | The ID or full path of the project. | +| `file_name` | string | yes | The name of the Debian package file. | +| `distribution` | string | no | The distribution codename or suite. Used with `component` for upload with explicit distribution and component. | +| `component` | string | no | The package file component. Used with `distribution` for upload with explicit distribution and component. | ```shell curl --request PUT \ - --user : \ + --user ":" \ --upload-file path/to/mypkg.deb \ "https://gitlab.example.com/api/v4/projects/1/packages/debian/mypkg.deb" ``` +Upload with explicit distribution and component: + +```shell +curl --request PUT \ + --user ":" \ + --upload-file /path/to/myother.deb \ + "https://gitlab.example.com/api/v4/projects/1/packages/debian/myother.deb?distribution=sid&component=main" +``` + ## Download a package > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64923) in GitLab 14.2. diff --git a/doc/api/packages/debian_group_distributions.md b/doc/api/packages/debian_group_distributions.md index 23bc85bf0a0..0c7f4cdfeb8 100644 --- a/doc/api/packages/debian_group_distributions.md +++ b/doc/api/packages/debian_group_distributions.md @@ -45,7 +45,7 @@ GET /groups/:id/-/debian_distributions | `suite` | string | no | Filter with specific `suite`. | ```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/debian_distributions" +curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/-/debian_distributions" ``` Example response: @@ -86,7 +86,7 @@ GET /groups/:id/-/debian_distributions/:codename | `codename` | integer | yes | The `codename` of a distribution. | ```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/debian_distributions/unstable" +curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/-/debian_distributions/unstable" ``` Example response: @@ -125,7 +125,7 @@ GET /groups/:id/-/debian_distributions/:codename/key.asc | `codename` | integer | yes | The `codename` of a distribution. | ```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/debian_distributions/unstable/key.asc" +curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/-/debian_distributions/unstable/key.asc" ``` Example response: @@ -170,7 +170,7 @@ POST /groups/:id/-/debian_distributions | `architectures` | architectures | no | The new Debian distribution's list of architectures. | ```shell -curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/debian_distributions?codename=sid" +curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/-/debian_distributions?codename=sid" ``` Example response: @@ -217,7 +217,7 @@ PUT /groups/:id/-/debian_distributions/:codename | `architectures` | architectures | no | The Debian distribution's new list of architectures. | ```shell -curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/debian_distributions/unstable?suite=new-suite&valid_time_duration_seconds=604800" +curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/-/debian_distributions/unstable?suite=new-suite&valid_time_duration_seconds=604800" ``` Example response: @@ -256,5 +256,5 @@ DELETE /groups/:id/-/debian_distributions/:codename | `codename` | integer | yes | The codename of the Debian distribution. | ```shell -curl --request DELETE --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/debian_distributions/unstable" +curl --request DELETE --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/-/debian_distributions/unstable" ``` diff --git a/doc/api/packages/go_proxy.md b/doc/api/packages/go_proxy.md index 08a7138a82a..1cfc0f0b296 100644 --- a/doc/api/packages/go_proxy.md +++ b/doc/api/packages/go_proxy.md @@ -20,7 +20,7 @@ For instructions on how to work with the Go Proxy, see the [Go Proxy package doc NOTE: These endpoints do not adhere to the standard API authentication methods. See the [Go Proxy package documentation](../../user/packages/go_proxy/index.md) -for details on which headers and token types are supported. +for details on which headers and token types are supported. Undocumented authentication methods might be removed in the future. ## List diff --git a/doc/api/packages/helm.md b/doc/api/packages/helm.md index 0e9a72c2389..d69f524c47c 100644 --- a/doc/api/packages/helm.md +++ b/doc/api/packages/helm.md @@ -19,7 +19,7 @@ Package Registry, see the [Helm registry documentation](../../user/packages/helm NOTE: These endpoints do not adhere to the standard API authentication methods. See the [Helm registry documentation](../../user/packages/helm_repository/index.md) -for details on which headers and token types are supported. +for details on which headers and token types are supported. Undocumented authentication methods might be removed in the future. ## Download a chart index diff --git a/doc/api/packages/maven.md b/doc/api/packages/maven.md index 4086b68b750..733f4735ba2 100644 --- a/doc/api/packages/maven.md +++ b/doc/api/packages/maven.md @@ -18,7 +18,7 @@ package registry, see the [Maven package registry documentation](../../user/pack NOTE: These endpoints do not adhere to the standard API authentication methods. See [Maven package registry documentation](../../user/packages/maven_repository/index.md) -for details on which headers and token types are supported. +for details on which headers and token types are supported. Undocumented authentication methods might be removed in the future. ## Download a package file at the instance-level diff --git a/doc/api/packages/npm.md b/doc/api/packages/npm.md index 7e8732d9553..bf48fbc8f65 100644 --- a/doc/api/packages/npm.md +++ b/doc/api/packages/npm.md @@ -18,7 +18,7 @@ package registry, see the [npm package registry documentation](../../user/packag NOTE: These endpoints do not adhere to the standard API authentication methods. See the [npm package registry documentation](../../user/packages/npm_registry/index.md) -for details on which headers and token types are supported. +for details on which headers and token types are supported. Undocumented authentication methods might be removed in the future. ## Download a package diff --git a/doc/api/packages/nuget.md b/doc/api/packages/nuget.md index ebcb51be447..a0761b56645 100644 --- a/doc/api/packages/nuget.md +++ b/doc/api/packages/nuget.md @@ -18,7 +18,7 @@ package registry, see the [NuGet package registry documentation](../../user/pack NOTE: These endpoints do not adhere to the standard API authentication methods. See the [NuGet package registry documentation](../../user/packages/nuget_repository/index.md) -for details on which headers and token types are supported. +for details on which headers and token types are supported. Undocumented authentication methods might be removed in the future. ## Package index diff --git a/doc/api/packages/pypi.md b/doc/api/packages/pypi.md index e9546f50e36..4e7c59adf3a 100644 --- a/doc/api/packages/pypi.md +++ b/doc/api/packages/pypi.md @@ -18,7 +18,7 @@ package registry, see the [PyPI package registry documentation](../../user/packa NOTE: These endpoints do not adhere to the standard API authentication methods. See the [PyPI package registry documentation](../../user/packages/pypi_repository/index.md) -for details on which headers and token types are supported. +for details on which headers and token types are supported. Undocumented authentication methods might be removed in the future. NOTE: [Twine 3.4.2](https://twine.readthedocs.io/en/stable/changelog.html?highlight=FIPS#id28) or greater diff --git a/doc/api/packages/rubygems.md b/doc/api/packages/rubygems.md index c2d05f24085..17338161a7b 100644 --- a/doc/api/packages/rubygems.md +++ b/doc/api/packages/rubygems.md @@ -19,7 +19,7 @@ package registry, see the [Ruby gems registry documentation](../../user/packages NOTE: These endpoints do not adhere to the standard API authentication methods. See the [Ruby gems registry documentation](../../user/packages/rubygems_registry/index.md) -for details on which headers and token types are supported. +for details on which headers and token types are supported. Undocumented authentication methods might be removed in the future. ## Enable the Ruby gems API diff --git a/doc/api/packages/terraform-modules.md b/doc/api/packages/terraform-modules.md index 6b3d6b477b2..ff6ac24495e 100644 --- a/doc/api/packages/terraform-modules.md +++ b/doc/api/packages/terraform-modules.md @@ -4,16 +4,16 @@ group: Package Registry info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Terraform Registry API **(FREE)** +# Terraform Module Registry API **(FREE)** -This is the API documentation for [Terraform Modules](../../user/packages/terraform_module_registry/index.md). +This is the API documentation for the [Terraform Module Registry](../../user/packages/terraform_module_registry/index.md). WARNING: This API is used by the [Terraform CLI](https://www.terraform.io/) -and is generally not meant for manual consumption. +and is generally not meant for manual consumption. Undocumented authentication methods might be removed in the future. For instructions on how to upload and install Terraform modules from the GitLab -infrastructure registry, see the [Terraform modules registry documentation](../../user/packages/terraform_module_registry/index.md). +Terraform Module Registry, see the [Terraform Module Registry documentation](../../user/packages/terraform_module_registry/index.md). ## List available versions for a specific module @@ -35,7 +35,7 @@ curl --header "Authorization: Bearer " "https://gitlab.ex Example response: -```shell +```json { "modules": [ { @@ -93,9 +93,9 @@ curl --header "Authorization: Bearer " "https://gitlab.ex Example response: -```shell +```json { - "name": "hellow-world/local", + "name": "hello-world/local", "provider": "local", "providers": [ "local" @@ -132,9 +132,9 @@ curl --header "Authorization: Bearer " "https://gitlab.ex Example response: -```shell +```json { - "name": "hellow-world/local", + "name": "hello-world/local", "provider": "local", "providers": [ "local" @@ -171,7 +171,7 @@ curl --header "Authorization: Bearer " "https://gitlab.ex Example response: -```shell +```plaintext HTTP/1.1 204 No Content Content-Length: 0 X-Terraform-Get: /api/v4/packages/terraform/modules/v1/group/hello-world/local/1.0.0/file?token=&archive=tgz @@ -200,7 +200,7 @@ curl --header "Authorization: Bearer " "https://gitlab.ex Example response: -```shell +```plaintext HTTP/1.1 204 No Content Content-Length: 0 X-Terraform-Get: /api/v4/packages/terraform/modules/v1/group/hello-world/local/1.0.0/file?token=&archive=tgz diff --git a/doc/api/pages.md b/doc/api/pages.md index dbe7416b939..2821f5d510c 100644 --- a/doc/api/pages.md +++ b/doc/api/pages.md @@ -1,6 +1,6 @@ --- -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -10,9 +10,13 @@ Endpoints for managing [GitLab Pages](https://about.gitlab.com/stages-devops-lif The GitLab Pages feature must be enabled to use these endpoints. Find out more about [administering](../administration/pages/index.md) and [using](../user/project/pages/index.md) the feature. -## Unpublish pages +## Unpublish Pages -Remove pages. The user must have administrator access. +Prerequisite: + +- You must have administrator access to the instance. + +Remove Pages. ```plaintext DELETE /projects/:id/pages diff --git a/doc/api/pages_domains.md b/doc/api/pages_domains.md index 815cea7cc63..7d52c803c88 100644 --- a/doc/api/pages_domains.md +++ b/doc/api/pages_domains.md @@ -1,6 +1,6 @@ --- -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -12,7 +12,11 @@ The GitLab Pages feature must be enabled to use these endpoints. Find out more a ## List all Pages domains -Get a list of all Pages domains. The user must have administrator access. +Prerequisite: + +- You must have administrator access to the instance. + +Get a list of all Pages domains. ```plaintext GET /pages/domains diff --git a/doc/api/personal_access_tokens.md b/doc/api/personal_access_tokens.md index 717e995f510..691c094f9eb 100644 --- a/doc/api/personal_access_tokens.md +++ b/doc/api/personal_access_tokens.md @@ -177,7 +177,7 @@ curl --request GET --header "PRIVATE-TOKEN: " "https://gitlab > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/373999) in GitLab 15.5 -Get a single personal access token by using passing the token in a header. +Get a single personal access token and information about that token by passing the token in a header. ```plaintext GET /personal_access_tokens/self @@ -205,6 +205,36 @@ Example response: } ``` +## Rotate a personal access token + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/403042) in GitLab 16.0 + +Rotate a personal access token. Revokes the previous token and creates a new token that expires in one week. + +```plaintext +POST /personal_access_tokens/:id/rotate +``` + +| Attribute | Type | Required | Description | +|-----------|---------|----------|---------------------| +| `id` | integer/string | yes | ID of personal access token | + +NOTE: +Non-administrators can rotate their own tokens. Administrators can rotate tokens of any user. + +```shell +curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/personal_access_tokens//rotate" +``` + +### Responses + +- `200: OK` if the existing token is successfully revoked and the new token successfully created. +- `400: Bad Request` if not rotated successfully. +- `401: Unauthorized` if either the: + - User does not have access to the token with the specified ID. + - Token with the specified ID does not exist. +- `404: Not Found` if the user is an administrator but the token with the specified ID does not exist. + ## Revoke a personal access token Revoke a personal access token by either: diff --git a/doc/api/pipeline_triggers.md b/doc/api/pipeline_triggers.md index 1ffda1c0d79..50acac6bc2a 100644 --- a/doc/api/pipeline_triggers.md +++ b/doc/api/pipeline_triggers.md @@ -155,7 +155,7 @@ or a [CI/CD job token](../ci/jobs/ci_job_token.md) for authentication. With a CI/CD job token, the [triggered pipeline is a multi-project pipeline](../ci/pipelines/downstream_pipelines.md#trigger-a-multi-project-pipeline-by-using-the-api). The job that authenticates the request becomes associated with the upstream pipeline, -which is visible on the [pipeline graph](../ci/pipelines/downstream_pipelines.md#view-multi-project-pipelines-in-pipeline-graphs). +which is visible on the pipeline graph. If you use a trigger token in a job, the job is not associated with the upstream pipeline. diff --git a/doc/api/pipelines.md b/doc/api/pipelines.md index 7049d3c3927..fed0e553a9e 100644 --- a/doc/api/pipelines.md +++ b/doc/api/pipelines.md @@ -15,7 +15,14 @@ Read more on [pagination](rest/index.md#pagination). ## List project pipelines -> `iid` in response [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/342223) in GitLab 14.6. +> - `iid` in response [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/342223) in GitLab 14.6. +> - `name` in request and response [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/115310) in GitLab 15.11 [with a flag](../administration/feature_flags.md) named `pipeline_name_in_api`. Disabled by default. + +FLAG: +On self-managed GitLab, by default the `name` field is not available. +To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) +named `pipeline_name_in_api`. This feature is not ready for production use. +On GitLab.com, this feature is not available. List pipelines in a project. Child pipelines are not included in the results, but you can [get child pipeline](pipelines.md#get-a-single-pipeline) individually. @@ -36,6 +43,7 @@ GET /projects/:id/pipelines | `username`| string | no | The username of the user who triggered pipelines | | `updated_after` | datetime | no | Return pipelines updated after the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | | `updated_before` | datetime | no | Return pipelines updated before the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `name` | string | no | Return pipelines with the specified name. Introduced in GitLab 15.11, not available by default. | | `order_by`| string | no | Order pipelines by `id`, `status`, `ref`, `updated_at` or `user_id` (default: `id`) | | `sort` | string | no | Sort pipelines in `asc` or `desc` order (default: `desc`) | @@ -55,6 +63,7 @@ Example of response "source": "push", "ref": "new-pipeline", "sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", + "name": "Build pipeline", "web_url": "https://example.com/foo/bar/pipelines/47", "created_at": "2016-08-11T11:28:34.085Z", "updated_at": "2016-08-11T11:32:35.169Z" @@ -67,6 +76,7 @@ Example of response "source": "web", "ref": "new-pipeline", "sha": "eb94b618fb5865b26e80fdd8ae531b7a63ad851a", + "name": "Build pipeline", "web_url": "https://example.com/foo/bar/pipelines/48", "created_at": "2016-08-12T10:06:04.561Z", "updated_at": "2016-08-12T10:09:56.223Z" @@ -76,7 +86,14 @@ Example of response ## Get a single pipeline -> `iid` in response [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/342223) in GitLab 14.6. +> - `iid` in response [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/342223) in GitLab 14.6. +> - `name` in response [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/115310) in GitLab 15.11 [with a flag](../administration/feature_flags.md) named `pipeline_name_in_api`. Disabled by default. + +FLAG: +On self-managed GitLab, by default the `name` field is not available. +To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) +named `pipeline_name_in_api`. This feature is not ready for production use. +On GitLab.com, this feature is not available. Get one pipeline from a project. @@ -103,6 +120,7 @@ Example of response "id": 46, "iid": 11, "project_id": 1, + "name": "Build pipeline", "status": "success", "ref": "main", "sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a", @@ -271,6 +289,14 @@ Sample response: ## Get the latest pipeline +> `name` in response [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/115310) in GitLab 15.11 [with a flag](../administration/feature_flags.md) named `pipeline_name_in_api`. Disabled by default. + +FLAG: +On self-managed GitLab, by default the `name` field is not available. +To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) +named `pipeline_name_in_api`. This feature is not ready for production use. +On GitLab.com, this feature is not available. + Get the latest pipeline for a specific ref in a project. ```plaintext @@ -292,6 +318,7 @@ Example of response "id": 287, "iid": 144, "project_id": 21, + "name": "Build pipeline", "sha": "50f0acb76a40e34a4ff304f7347dcc6587da8a14", "ref": "main", "status": "success", diff --git a/doc/api/plan_limits.md b/doc/api/plan_limits.md index 8f6a7ae3e8d..f93f246c3f0 100644 --- a/doc/api/plan_limits.md +++ b/doc/api/plan_limits.md @@ -37,7 +37,6 @@ Example response: { "ci_pipeline_size": 0, "ci_active_jobs": 0, - "ci_active_pipelines": 0, "ci_project_subscriptions": 2, "ci_pipeline_schedules": 10, "ci_needs_size_limit": 50, @@ -68,7 +67,6 @@ PUT /application/plan_limits | `plan_name` | string | yes | Name of the plan to update. | | `ci_pipeline_size` | integer | no | Maximum number of jobs in a single pipeline. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85895) in GitLab 15.0. | | `ci_active_jobs` | integer | no | Total number of jobs in currently active pipelines. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85895) in GitLab 15.0. | -| `ci_active_pipelines` | integer | no | Maximum number of active pipelines per project. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85895) in GitLab 15.0. | | `ci_project_subscriptions` | integer | no | Maximum number of pipeline subscriptions to and from a project. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85895) in GitLab 15.0. | | `ci_pipeline_schedules` | integer | no | Maximum number of pipeline schedules. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85895) in GitLab 15.0. | | `ci_needs_size_limit` | integer | no | Maximum number of [DAG](../ci/directed_acyclic_graph/index.md) dependencies that a job can have. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85895) in GitLab 15.0. | @@ -94,7 +92,6 @@ Example response: { "ci_pipeline_size": 0, "ci_active_jobs": 0, - "ci_active_pipelines": 0, "ci_project_subscriptions": 2, "ci_pipeline_schedules": 10, "ci_needs_size_limit": 50, diff --git a/doc/api/product_analytics.md b/doc/api/product_analytics.md index c687acdb5db..c37fe223778 100644 --- a/doc/api/product_analytics.md +++ b/doc/api/product_analytics.md @@ -6,7 +6,9 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Product analytics API **(ULTIMATE)** -> Introduced in GitLab 15.4 [with a flag](../administration/feature_flags.md) named `cube_api_proxy`. Disabled by default. +> - Introduced in GitLab 15.4 [with a flag](../administration/feature_flags.md) named `cube_api_proxy`. Disabled by default. +> - `cube_api_proxy` removed and replaced with `product_analytics_internal_preview` in GitLab 15.10. +> - `product_analytics_internal_preview` replaced with `product_analytics_dashboards` in GitLab 15.11. FLAG: On self-managed GitLab, by default this feature is not available. To make it available per project or for your entire instance, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `cube_api_proxy`. diff --git a/doc/api/project_access_tokens.md b/doc/api/project_access_tokens.md index 6711d1b0261..437bdaa70f4 100644 --- a/doc/api/project_access_tokens.md +++ b/doc/api/project_access_tokens.md @@ -132,6 +132,34 @@ curl --request POST --header "PRIVATE-TOKEN: " \ } ``` +## Rotate a project access token + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/403042) in GitLab 16.0 + +Rotate a project access token. Revokes the previous token and creates a new token that expires in one week. + +```plaintext +POST /projects/:id/access_tokens/:token_id/rotate +``` + +| Attribute | Type | required | Description | +|-----------|---------|----------|---------------------| +| `id` | integer or string | yes | ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) | +| `token_id` | integer or string | yes | ID of the project access token | + +```shell +curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects//access_tokens//rotate" +``` + +### Responses + +- `200: OK` if the existing token is successfully revoked and the new token is successfully created. +- `400: Bad Request` if not rotated successfully. +- `401: Unauthorized` if either the: + - User does not have access to the token with the specified ID. + - Token with the specified ID does not exist. +- `404: Not Found` if the user is an administrator but the token with the specified ID does not exist. + ## Revoke a project access token > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/238991) in GitLab 13.9. diff --git a/doc/api/project_badges.md b/doc/api/project_badges.md index 3dad40a3f96..a8940a7875c 100644 --- a/doc/api/project_badges.md +++ b/doc/api/project_badges.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Placeholder tokens -Badges support placeholders that are replaced in real-time in both the link and image URL. The allowed placeholders are: +[Badges](../user/project/badges.md) support placeholders that are replaced in real-time in both the link and image URL. The allowed placeholders are: diff --git a/doc/api/project_clusters.md b/doc/api/project_clusters.md index 01081bdd6d9..65efcaf13b9 100644 --- a/doc/api/project_clusters.md +++ b/doc/api/project_clusters.md @@ -1,10 +1,10 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Project clusters API (certificate-based) (DEPRECATED) **(FREE)** +# Project clusters API (certificate-based) (deprecated) **(FREE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/23922) in GitLab 11.7. > - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. diff --git a/doc/api/project_import_export.md b/doc/api/project_import_export.md index 00e73d41b46..a162bc3e5af 100644 --- a/doc/api/project_import_export.md +++ b/doc/api/project_import_export.md @@ -4,12 +4,19 @@ group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Project import/export API **(FREE)** +# Project import and export API **(FREE)** -See also: +Use the project import and export API to import and export projects using file transfers. -- [Project import/export documentation](../user/project/settings/import_export.md). -- [Project import/export administration Rake tasks](../administration/raketasks/project_import_export.md). **(FREE SELF)** +Before using the project import and export API, you might want to use the +[group import and export API](group_import_export.md). + +## Prerequisites + +For information on prerequisites for project import and export API, see: + +- Prerequisites for [project export](../user/project/settings/import_export.md#export-a-project-and-its-data). +- Prerequisites for [project import](../user/project/settings/import_export.md#import-a-project-and-its-data). ## Schedule an export @@ -39,14 +46,15 @@ POST /projects/:id/export | Attribute | Type | Required | Description | | --------- | -------------- | -------- | ---------------------------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | -| `description` | string | no | Overrides the project description | -| `upload` | hash | no | Hash that contains the information to upload the exported project to a web server | -| `upload[url]` | string | yes | The URL to upload the project | -| `upload[http_method]` | string | no | The HTTP method to upload the exported project. Only `PUT` and `POST` methods allowed. Default is `PUT` | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. +| `upload[url]` | string | yes | The URL to upload the project. +| `description` | string | no | Overrides the project description. +| `upload` | hash | no | Hash that contains the information to upload the exported project to a web server. +| `upload[http_method]` | string | no | The HTTP method to upload the exported project. Only `PUT` and `POST` methods allowed. Default is `PUT`. ```shell -curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/export" \ +curl --request POST --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects/1/export" \ --data "upload[http_method]=PUT" \ --data-urlencode "upload[url]=https://example-bucket.s3.eu-west-3.amazonaws.com/backup?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIMBJHN2O62W8IELQ%2F20180312%2Feu-west-3%2Fs3%2Faws4_request&X-Amz-Date=20180312T110328Z&X-Amz-Expires=900&X-Amz-SignedHeaders=host&X-Amz-Signature=8413facb20ff33a49a147a0b4abcff4c8487cc33ee1f7e450c46e8f695569dbd" ``` @@ -67,10 +75,11 @@ GET /projects/:id/export | Attribute | Type | Required | Description | | --------- | -------------- | -------- | ---------------------------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. ```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/export" +curl --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects/1/export" ``` Status can be one of: @@ -113,9 +122,9 @@ Download the finished export. GET /projects/:id/export/download ``` -| Attribute | Type | Required | Description | -| --------- | -------------- | -------- | ---------------------------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | +| Attribute | Type | Required | Description | +| --------- | ----------------- | -------- | ---------------------------------------- | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. ```shell curl --header "PRIVATE-TOKEN: " --remote-header-name \ @@ -129,18 +138,20 @@ ls *export.tar.gz ## Import a file +> Requirement for Maintainer role instead of Developer role introduced in GitLab 16.0 and backported to GitLab 15.11.1 and GitLab 15.10.5. + ```plaintext POST /projects/import ``` -| Attribute | Type | Required | Description | -| --------- | -------------- | -------- | ---------------------------------------- | -| `namespace` | integer/string | no | The ID or path of the namespace to import the project to. Defaults to the current user's namespace.

Requires at least the Maintainer role on the destination group to import to. Using the Developer role for this purpose was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387891) in GitLab 15.8 and will be removed in GitLab 16.0. | -| `name` | string | no | The name of the project to be imported. Defaults to the path of the project if not provided | -| `file` | string | yes | The file to be uploaded | -| `path` | string | yes | Name and path for new project | -| `overwrite` | boolean | no | If there is a project with the same path the import overwrites it. Default to false | -| `override_params` | Hash | no | Supports all fields defined in the [Project API](projects.md) | +| Attribute | Type | Required | Description | +| ----------- | -------------- | -------- | ---------------------------------------- | +| `file` | string | yes | The file to be uploaded. | +| `path` | string | yes | Name and path for new project. | +| `name` | string | no | The name of the project to be imported. Defaults to the path of the project if not provided. | +| `namespace` | integer or string | no | The ID or path of the namespace to import the project to. Defaults to the current user's namespace.

Requires at least the Maintainer role on the destination group to import to. | +| `override_params` | Hash | no | Supports all fields defined in the [Project API](projects.md). | +| `overwrite` | boolean | no | If there is a project with the same path the import overwrites it. Defaults to `false`. | The override parameters passed take precedence over all values defined inside the export file. @@ -189,39 +200,29 @@ requests.post(url, headers=headers, data=data, files=files) ``` NOTE: -The maximum import file size can be set by the Administrator, default is `0` (unlimited).. +The maximum import file size can be set by the Administrator. It defaults to `0` (unlimited). As an administrator, you can modify the maximum import file size. To do so, use the `max_import_size` option in the [Application settings API](settings.md#change-application-settings) or the [Admin Area](../user/admin_area/settings/account_and_limit_settings.md). Default [modified](https://gitlab.com/gitlab-org/gitlab/-/issues/251106) from 50 MB to 0 in GitLab 13.8. -## Import a file from a remote object storage - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/282503) in GitLab 13.12 in [Beta](../policy/alpha-beta-support.md#beta-features). - -This endpoint is behind a feature flag that is enabled by default. +## Import a file from a remote object storage (Beta) -To enable this endpoint: +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/282503) in GitLab 13.12 in [Beta](../policy/alpha-beta-support.md#beta) [with a flag](../administration/feature_flags.md) named `import_project_from_remote_file`. Enabled by default. -```ruby -Feature.enable(:import_project_from_remote_file) -``` - -To disable this endpoint: - -```ruby -Feature.disable(:import_project_from_remote_file) -``` +FLAG: +On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](../administration/feature_flags.md) named `import_project_from_remote_file`. +On GitLab.com, this feature is available. ```plaintext POST /projects/remote-import ``` -| Attribute | Type | Required | Description | -| ----------------- | -------------- | -------- | ---------------------------------------- | -| `namespace` | integer/string | no | The ID or path of the namespace to import the project to. Defaults to the current user's namespace. | -| `name` | string | no | The name of the project to import. If not provided, defaults to the path of the project. | -| `url` | string | yes | URL for the file to import. | -| `path` | string | yes | Name and path for the new project. | -| `overwrite` | boolean | no | Whether to overwrite a project with the same path when importing. Defaults to `false`. | -| `override_params` | Hash | no | Supports all fields defined in the [Project API](projects.md). | +| Attribute | Type | Required | Description | +| ----------------- | ----------------- | -------- | ---------------------------------------- | +| `path` | string | yes | Name and path for the new project. +| `url` | string | yes | URL for the file to import. +| `name` | string | no | The name of the project to import. If not provided, defaults to the path of the project. +| `namespace` | integer or string | no | The ID or path of the namespace to import the project to. Defaults to the current user's namespace. +| `overwrite` | boolean | no | Whether to overwrite a project with the same path when importing. Defaults to `false`. +| `override_params` | Hash | no | Supports all fields defined in the [Project API](projects.md). The passed override parameters take precedence over all values defined in the export file. @@ -249,16 +250,14 @@ curl --request POST \ } ``` -The `Content-Length` header must return a valid number. The maximum file size is 10 gigabytes. +The `Content-Length` header must return a valid number. The maximum file size is 10 GB. The `Content-Type` header must be `application/gzip`. ## Import a file from AWS S3 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/348874) in GitLab 14.9 in [Beta](https://about.gitlab.com/handbook/product/gitlab-the-product/#beta), [with a flag](../administration/feature_flags.md) named `import_project_from_remote_file_s3`. Disabled by default. > - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/348874) in GitLab 14.10. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `import_project_from_remote_file_s3`. On GitLab.com, this feature is available. +> - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/350571) in GitLab 15.11. Feature flag `import_project_from_remote_file_s3` removed. ```plaintext POST /projects/remote-import-s3 @@ -266,14 +265,14 @@ POST /projects/remote-import-s3 | Attribute | Type | Required | Description | | ------------------- | -------------- | -------- | ---------------------------------------- | -| `namespace` | integer/string | no | The ID or path of the namespace to import the project to. Defaults to the current user's namespace. | -| `name` | string | no | The name of the project to import. If not provided, defaults to the path of the project. | -| `path` | string | yes | The full path of the new project. | -| `region` | string | yes | [AWS S3 region name where the file is stored.](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html#Regions) | -| `bucket_name` | string | yes | [AWS S3 bucket name where the file is stored.](https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html) | -| `file_key` | string | yes | [AWS S3 file key to identify the file.](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingObjects.html) | -| `access_key_id` | string | yes | [AWS S3 access key ID.](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys). | -| `secret_access_key` | string | yes | [AWS S3 secret access key.](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) | +| `access_key_id` | string | yes | [AWS S3 access key ID](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys). +| `bucket_name` | string | yes | [AWS S3 bucket name](https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html) where the file is stored. +| `file_key` | string | yes | [AWS S3 file key](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingObjects.html) to identify the file. +| `path` | string | yes | The full path of the new project. +| `region` | string | yes | [AWS S3 region name](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html#Regions) where the file is stored. +| `secret_access_key` | string | yes | [AWS S3 secret access key](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys). +| `name` | string | no | The name of the project to import. If not provided, defaults to the path of the project. +| `namespace` | integer or string | no | The ID or path of the namespace to import the project to. Defaults to the current user's namespace. The passed override parameters take precedence over all values defined in the export file. @@ -340,10 +339,11 @@ GET /projects/:id/import | Attribute | Type | Required | Description | | --------- | -------------- | -------- | ---------------------------------------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. ```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/import" +curl --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects/1/import" ``` Status can be one of: @@ -356,8 +356,10 @@ Status can be one of: If the status is `failed`, it includes the import error message under `import_error`. If the status is `failed`, `started` or `finished`, the `failed_relations` array might -be populated with any occurrences of relations that failed to import either due to -unrecoverable errors or because retries were exhausted (a typical example are query timeouts.) +be populated with any occurrences of relations that failed to import due to either: + +- Unrecoverable errors. +- Retries were exhausted. A typical example: query timeouts. NOTE: An element's `id` field in `failed_relations` references the failure record, not the relation. @@ -439,3 +441,8 @@ GitHub and how many were already imported: } } ``` + +## Related topics + +- [Migrating projects using file exports](../user/project/settings/import_export.md). +- [Project import and export Rake tasks](../administration/raketasks/project_import_export.md). diff --git a/doc/api/project_level_variables.md b/doc/api/project_level_variables.md index 92ca2849e8e..fa699c34a8a 100644 --- a/doc/api/project_level_variables.md +++ b/doc/api/project_level_variables.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Authoring +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: reference, api --- diff --git a/doc/api/project_relations_export.md b/doc/api/project_relations_export.md index fa6bdfa2900..835a53c7ecc 100644 --- a/doc/api/project_relations_export.md +++ b/doc/api/project_relations_export.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/api/project_snippets.md b/doc/api/project_snippets.md index 242edf7d768..e39836f2781 100644 --- a/doc/api/project_snippets.md +++ b/doc/api/project_snippets.md @@ -13,11 +13,9 @@ You can set it with the `visibility` field in the snippet. Constants for snippet visibility levels are: -| visibility | Description | -| ---------- | ----------- | -| `private` | The snippet is visible only to project members. | -| `internal` | The snippet is visible for any authenticated user except [external users](../user/admin_area/external_users.md). | -| `public` | The snippet can be accessed without any authentication. | +- **Private**: The snippet is visible only to project members. +- **Internal**: The snippet is visible for any authenticated user except [external users](../user/admin_area/external_users.md). +- **Public**: The snippet can be accessed without any authentication. NOTE: From July 2019, the `Internal` visibility setting is disabled for new projects, groups, @@ -35,9 +33,9 @@ GET /projects/:id/snippets Parameters: -| Attribute | Type | Required | Description | -|-----------|----------------|----------|-----------------------------------------------------------------------------------------------------------------| -| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | +| Attribute | Type | Required | Description | +|-----------|----------------|----------|-------------| +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. ## Single snippet @@ -49,10 +47,10 @@ GET /projects/:id/snippets/:snippet_id Parameters: -| Attribute | Type | Required | Description | -|--------------|----------------|----------|-----------------------------------------------------------------------------------------------------------------| -| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | -| `snippet_id` | integer | yes | The ID of a project's snippet. | +| Attribute | Type | Required | Description | +|--------------|----------------|----------|-------------| +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. +| `snippet_id` | integer | yes | The ID of a project's snippet. ```json { @@ -86,17 +84,17 @@ POST /projects/:id/snippets Parameters: -| Attribute | Type | Required | Description | -|:------------------|:----------------|:---------|:----------------------------------------------------------------------------------------------------------------| -| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | -| `title` | string | yes | Title of a snippet. | -| `file_name` | string | no | Deprecated: Use `files` instead. Name of a snippet file. | -| `content` | string | no | Deprecated: Use `files` instead. Content of a snippet. | -| `description` | string | no | Description of a snippet. | -| `visibility` | string | no | Snippet's [visibility](#snippet-visibility-level) | -| `files` | array of hashes | no | An array of snippet files. | -| `files:file_path` | string | yes | File path of the snippet file. | -| `files:content` | string | yes | Content of the snippet file. | +| Attribute | Type | Required | Description | +|:------------------|:----------------|:---------|:------------| +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. +| `files:content` | string | yes | Content of the snippet file. +| `files:file_path` | string | yes | File path of the snippet file. +| `title` | string | yes | Title of a snippet. +| `content` | string | no | Deprecated: Use `files` instead. Content of a snippet. +| `description` | string | no | Description of a snippet. +| `file_name` | string | no | Deprecated: Use `files` instead. Name of a snippet file. +| `files` | array of hashes | no | An array of snippet files. +| `visibility` | string | no | Snippet's [visibility](#snippet-visibility-level). Example request: @@ -127,7 +125,7 @@ curl --request POST "https://gitlab.com/api/v4/projects/:id/snippets" \ Updates an existing project snippet. The user must have permission to change an existing snippet. -Updates to snippets with multiple files *must* use the `files` attribute. +Updates to snippets with multiple files must use the `files` attribute. ```plaintext PUT /projects/:id/snippets/:snippet_id @@ -135,20 +133,20 @@ PUT /projects/:id/snippets/:snippet_id Parameters: -| Attribute | Type | Required | Description | -|:----------------------|:----------------|:---------|:----------------------------------------------------------------------------------------------------------------| -| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | -| `snippet_id` | integer | yes | The ID of a project's snippet. | -| `title` | string | no | Title of a snippet. | -| `file_name` | string | no | Deprecated: Use `files` instead. Name of a snippet file. | -| `content` | string | no | Deprecated: Use `files` instead. Content of a snippet. | -| `description` | string | no | Description of a snippet. | -| `visibility` | string | no | Snippet's [visibility](#snippet-visibility-level) | -| `files` | array of hashes | no | An array of snippet files. | -| `files:action` | string | yes | Type of action to perform on the file, one of: `create`, `update`, `delete`, `move` | -| `files:file_path` | string | no | File path of the snippet file. | -| `files:previous_path` | string | no | Previous path of the snippet file. | -| `files:content` | string | no | Content of the snippet file. | +| Attribute | Type | Required | Description | +|:----------------------|:----------------|:---------|:------------| +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. +| `files:action` | string | yes | Type of action to perform on the file. One of: `create`, `update`, `delete`, `move`. +| `snippet_id` | integer | yes | The ID of a project's snippet. +| `content` | string | no | Deprecated: Use `files` instead. Content of a snippet. +| `description` | string | no | Description of a snippet. +| `files` | array of hashes | no | An array of snippet files. +| `files:content` | string | no | Content of the snippet file. +| `files:file_path` | string | no | File path of the snippet file. +| `file_name` | string | no | Deprecated: Use `files` instead. Name of a snippet file. +| `files:previous_path` | string | no | Previous path of the snippet file. +| `title` | string | no | Title of a snippet. +| `visibility` | string | no | Snippet's [visibility](#snippet-visibility-level). Example request: @@ -186,10 +184,10 @@ DELETE /projects/:id/snippets/:snippet_id Parameters: -| Attribute | Type | Required | Description | -|:-------------|:---------------|:---------|:----------------------------------------------------------------------------------------------------------------| -| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | -| `snippet_id` | integer | yes | The ID of a project's snippet. | +| Attribute | Type | Required | Description | +|:-------------|:---------------|:---------|:------------| +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. +| `snippet_id` | integer | yes | The ID of a project's snippet. Example request: @@ -208,10 +206,10 @@ GET /projects/:id/snippets/:snippet_id/raw Parameters: -| Attribute | Type | Required | Description | +| Attribute | Type | Required | Description | |:-------------|:---------------|:---------|:----------------------------------------------------------------------------------------------------------------| -| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | -| `snippet_id` | integer | yes | The ID of a project's snippet. | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. +| `snippet_id` | integer | yes | The ID of a project's snippet. Example request: @@ -230,12 +228,12 @@ GET /projects/:id/snippets/:snippet_id/files/:ref/:file_path/raw Parameters: -| Attribute | Type | Required | Description | -|:-------------|:---------------|:---------|:----------------------------------------------------------------------------------------------------------------| -| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | -| `snippet_id` | integer | yes | The ID of a project's snippet. | -| `ref` | string | yes | The name of a branch, tag or commit, for example, main. | -| `file_path` | string | yes | The URL-encoded path to the file, for example, snippet%2Erb. | +| Attribute | Type | Required | Description | +|:-------------|:---------------|:---------|:------------| +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. +| `file_path` | string | yes | The URL-encoded path to the file, for example, `snippet%2Erb`. +| `ref` | string | yes | The name of a branch, tag or commit, for example, `main`. +| `snippet_id` | integer | yes | The ID of a project's snippet. Example request: @@ -252,10 +250,10 @@ Available only for users with administrator access. GET /projects/:id/snippets/:snippet_id/user_agent_detail ``` -| Attribute | Type | Required | Description | -|--------------|----------------|----------|-----------------------------------------------------------------------------------------------------------------| -| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | -| `snippet_id` | Integer | yes | The ID of a snippet. | +| Attribute | Type | Required | Description | +|--------------|----------------|----------|-------------| +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. +| `snippet_id` | Integer | yes | The ID of a snippet. Example request: diff --git a/doc/api/project_vulnerabilities.md b/doc/api/project_vulnerabilities.md index 9effe54b8e2..446c629c3bf 100644 --- a/doc/api/project_vulnerabilities.md +++ b/doc/api/project_vulnerabilities.md @@ -1,6 +1,6 @@ --- -stage: Create -group: Source Code +stage: Govern +group: Threat Insights info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments" type: reference, api --- diff --git a/doc/api/projects.md b/doc/api/projects.md index b502d547ddc..3105da44906 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -1,6 +1,6 @@ --- -stage: Manage -group: Organization +stage: Data Stores +group: Tenant Scale info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -15,12 +15,24 @@ The visibility level is determined by the `visibility` field in the project. For details, see [Project visibility](../user/public_access.md). +The fields returned in responses vary based on the [permissions](../user/permissions.md) of the authenticated user. + +## Removals in API v5 + +These attributes are deprecated, and are scheduled to be removed in v5 of the API: + +- `tag_list`: Use the `topics` attribute instead. +- `marked_for_deletion_at`: Use the `marked_for_deletion_on` attribute instead. + Available only to [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/). +- `approvals_before_merge`: Use the [Merge request approvals API](merge_request_approvals.md) instead. + Available only to [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/). + ## Project merge method -There are three options for `merge_method` to choose from: +The `merge_method` can use these options: - `merge`: a merge commit is created for every merge, and merging is allowed if - there are no conflicts. + no conflicts are present. - `rebase_merge`: a merge commit is created for every merge, but merging is only allowed if fast-forward merge is possible. You can make sure that the target branch would build after this merge request builds and merges. @@ -55,10 +67,10 @@ GET /projects | `repository_storage` | string | **{dotted-circle}** No | Limit results to projects stored on `repository_storage`. _(administrators only)_ | | `search_namespaces` | boolean | **{dotted-circle}** No | Include ancestor namespaces when matching search criteria. Default is `false`. | | `search` | string | **{dotted-circle}** No | Return list of projects matching the search criteria. | -| `simple` | boolean | **{dotted-circle}** No | Return only limited fields for each project. This is a no-op without authentication where only simple fields are returned. | +| `simple` | boolean | **{dotted-circle}** No | Return only limited fields for each project. This operation is a no-op without authentication where only simple fields are returned. | | `sort` | string | **{dotted-circle}** No | Return projects sorted in `asc` or `desc` order. Default is `desc`. | | `starred` | boolean | **{dotted-circle}** No | Limit by projects starred by the current user. | -| `statistics` | boolean | **{dotted-circle}** No | Include project statistics. Only available to Reporter or higher level role members. | +| `statistics` | boolean | **{dotted-circle}** No | Include project statistics. Available only to users with at least the Reporter role. | | `topic` | string | **{dotted-circle}** No | Comma-separated topic names. Limit results to projects that match all of given topics. See `topics` attribute. | | `topic_id` | integer | **{dotted-circle}** No | Limit results to projects with the assigned topic given by the topic ID. | | `visibility` | string | **{dotted-circle}** No | Limit by visibility `public`, `internal`, or `private`. | @@ -67,6 +79,8 @@ GET /projects | `with_issues_enabled` | boolean | **{dotted-circle}** No | Limit by enabled issues feature. | | `with_merge_requests_enabled` | boolean | **{dotted-circle}** No | Limit by enabled merge requests feature. | | `with_programming_language` | string | **{dotted-circle}** No | Limit by projects which use the given programming language. | +| `updated_before` | datetime | **{dotted-circle}** No | Limit results to projects last updated before the specified time. Format: ISO 8601 (`YYYY-MM-DDTHH:MM:SSZ`). [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/393979) in GitLab 15.10. For this filter to work, you must also provide `updated_at` as the `order_by` attribute. | +| `updated_after` | datetime | **{dotted-circle}** No | Limit results to projects last updated after the specified time. Format: ISO 8601 (`YYYY-MM-DDTHH:MM:SSZ`). [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/393979) in GitLab 15.10. For this filter to work, you must also provide `updated_at` as the `order_by` attribute. | This endpoint supports [keyset pagination](rest/index.md#keyset-based-pagination) for selected `order_by` options. @@ -128,12 +142,14 @@ When the user is authenticated and `simple` is not set this returns something li [ { "id": 4, - "description": null, + "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + "description_html": "

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

", "name": "Diaspora Client", "name_with_namespace": "Diaspora / Diaspora Client", "path": "diaspora-client", "path_with_namespace": "diaspora/diaspora-client", "created_at": "2013-09-30T13:46:02Z", + "updated_at": "2013-09-30T13:46:02Z", "default_branch": "main", "tag_list": [ //deprecated, use `topics` instead "example", @@ -260,29 +276,6 @@ When the user is authenticated and `simple` is not set this returns something li ] ``` -NOTE: -The `tag_list` attribute has been deprecated -and is removed in API v5 in favor of the `topics` attribute. - -NOTE: -For users of [GitLab Premium or higher](https://about.gitlab.com/pricing/), -the `marked_for_deletion_at` attribute has been deprecated, and is removed -in API v5 in favor of the `marked_for_deletion_on` attribute. - -Users of [GitLab Premium or higher](https://about.gitlab.com/pricing/) -can also see the `approvals_before_merge` parameter: - -```json -[ - { - "id": 4, - "description": null, - "approvals_before_merge": 0, - ... - } -] -``` - You can filter by [custom attributes](custom_attributes.md) with: ```plaintext @@ -332,21 +325,24 @@ GET /users/:user_id/projects | `order_by` | string | **{dotted-circle}** No | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at`. | | `owned` | boolean | **{dotted-circle}** No | Limit by projects explicitly owned by the current user. | | `search` | string | **{dotted-circle}** No | Return list of projects matching the search criteria. | -| `simple` | boolean | **{dotted-circle}** No | Return only limited fields for each project. This is a no-op without authentication where only simple fields are returned. | +| `simple` | boolean | **{dotted-circle}** No | Return only limited fields for each project. Without authentication, this operation is a no-op; only simple fields are returned. | | `sort` | string | **{dotted-circle}** No | Return projects sorted in `asc` or `desc` order. Default is `desc`. | | `starred` | boolean | **{dotted-circle}** No | Limit by projects starred by the current user. | -| `statistics` | boolean | **{dotted-circle}** No | Include project statistics. Only available to Reporter or higher level role members. | +| `statistics` | boolean | **{dotted-circle}** No | Include project statistics. Available only to users with at least the Reporter role. | | `visibility` | string | **{dotted-circle}** No | Limit by visibility `public`, `internal`, or `private`. | | `with_custom_attributes` | boolean | **{dotted-circle}** No | Include [custom attributes](custom_attributes.md) in response. _(administrator only)_ | | `with_issues_enabled` | boolean | **{dotted-circle}** No | Limit by enabled issues feature. | | `with_merge_requests_enabled` | boolean | **{dotted-circle}** No | Limit by enabled merge requests feature. | | `with_programming_language` | string | **{dotted-circle}** No | Limit by projects which use the given programming language. | +| `updated_before` | datetime | **{dotted-circle}** No | Limit results to projects last updated before the specified time. Format: ISO 8601 (`YYYY-MM-DDTHH:MM:SSZ`). [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/393979) in GitLab 15.10. | +| `updated_after` | datetime | **{dotted-circle}** No | Limit results to projects last updated after the specified time. Format: ISO 8601 (`YYYY-MM-DDTHH:MM:SSZ`). [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/393979) in GitLab 15.10. | ```json [ { "id": 4, - "description": null, + "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + "description_html": "

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

", "default_branch": "master", "visibility": "private", "ssh_url_to_repo": "git@example.com:diaspora/diaspora-client.git", @@ -382,6 +378,7 @@ GET /users/:user_id/projects "container_registry_access_level": "disabled", "security_and_compliance_access_level": "disabled", "created_at": "2013-09-30T13:46:02Z", + "updated_at": "2013-09-30T13:46:02Z", "last_activity_at": "2013-09-30T13:46:02Z", "creator_id": 3, "import_url": null, @@ -451,7 +448,8 @@ GET /users/:user_id/projects }, { "id": 6, - "description": null, + "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + "description_html": "

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

", "default_branch": "master", "visibility": "private", "ssh_url_to_repo": "git@example.com:brightbox/puppet.git", @@ -487,6 +485,7 @@ GET /users/:user_id/projects "container_registry_access_level": "disabled", "security_and_compliance_access_level": "disabled", "created_at": "2013-09-30T13:46:02Z", + "updated_at": "2013-09-30T13:46:02Z", "last_activity_at": "2013-09-30T13:46:02Z", "creator_id": 3, "import_url": null, @@ -536,7 +535,7 @@ GET /users/:user_id/projects "auto_devops_enabled": true, "auto_devops_deploy_strategy": "continuous", "repository_storage": "default", - "approvals_before_merge": 0, + "approvals_before_merge": 0, // Deprecated. Use merge request approvals API instead. "mirror": false, "mirror_user_id": 45, "mirror_trigger_builds": false, @@ -599,14 +598,16 @@ GET /users/:user_id/starred_projects | `order_by` | string | **{dotted-circle}** No | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at`. | | `owned` | boolean | **{dotted-circle}** No | Limit by projects explicitly owned by the current user. | | `search` | string | **{dotted-circle}** No | Return list of projects matching the search criteria. | -| `simple` | boolean | **{dotted-circle}** No | Return only limited fields for each project. This is a no-op without authentication where only simple fields are returned. | +| `simple` | boolean | **{dotted-circle}** No | Return only limited fields for each project. Without authentication, this operation is a no-op; only simple fields are returned. | | `sort` | string | **{dotted-circle}** No | Return projects sorted in `asc` or `desc` order. Default is `desc`. | | `starred` | boolean | **{dotted-circle}** No | Limit by projects starred by the current user. | -| `statistics` | boolean | **{dotted-circle}** No | Include project statistics. Only available to Reporter or higher level role members. | +| `statistics` | boolean | **{dotted-circle}** No | Include project statistics. Available only to users with at least the Reporter role. | | `visibility` | string | **{dotted-circle}** No | Limit by visibility `public`, `internal`, or `private`. | | `with_custom_attributes` | boolean | **{dotted-circle}** No | Include [custom attributes](custom_attributes.md) in response. _(administrator only)_ | | `with_issues_enabled` | boolean | **{dotted-circle}** No | Limit by enabled issues feature. | | `with_merge_requests_enabled` | boolean | **{dotted-circle}** No | Limit by enabled merge requests feature. | +| `updated_before` | datetime | **{dotted-circle}** No | Limit results to projects last updated before the specified time. Format: ISO 8601 (`YYYY-MM-DDTHH:MM:SSZ`). [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/393979) in GitLab 15.10. | +| `updated_after` | datetime | **{dotted-circle}** No | Limit results to projects last updated after the specified time. Format: ISO 8601 (`YYYY-MM-DDTHH:MM:SSZ`). [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/393979) in GitLab 15.10. | ```shell curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/users/5/starred_projects" @@ -618,7 +619,8 @@ Example response: [ { "id": 4, - "description": null, + "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + "description_html": "

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

", "default_branch": "master", "visibility": "private", "ssh_url_to_repo": "git@example.com:diaspora/diaspora-client.git", @@ -654,6 +656,7 @@ Example response: "container_registry_access_level": "disabled", "security_and_compliance_access_level": "disabled", "created_at": "2013-09-30T13:46:02Z", + "updated_at": "2013-09-30T13:46:02Z", "last_activity_at": "2013-09-30T13:46:02Z", "creator_id": 3, "namespace": { @@ -712,7 +715,8 @@ Example response: }, { "id": 6, - "description": null, + "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + "description_html": "

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

", "default_branch": "master", "visibility": "private", "ssh_url_to_repo": "git@example.com:brightbox/puppet.git", @@ -748,6 +752,7 @@ Example response: "container_registry_access_level": "disabled", "security_and_compliance_access_level": "disabled", "created_at": "2013-09-30T13:46:02Z", + "updated_at": "2013-09-30T13:46:02Z", "last_activity_at": "2013-09-30T13:46:02Z", "creator_id": 3, "namespace": { @@ -789,7 +794,7 @@ Example response: "auto_devops_enabled": true, "auto_devops_deploy_strategy": "continuous", "repository_storage": "default", - "approvals_before_merge": 0, + "approvals_before_merge": 0, // Deprecated. Use merge request approvals API instead. "mirror": false, "mirror_user_id": 45, "mirror_trigger_builds": false, @@ -846,13 +851,14 @@ GET /projects/:id |--------------------------|----------------|------------------------|-------------| | `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | | `license` | boolean | **{dotted-circle}** No | Include project license data. | -| `statistics` | boolean | **{dotted-circle}** No | Include project statistics. Only available to Reporter or higher level role members. | +| `statistics` | boolean | **{dotted-circle}** No | Include project statistics. Available only to users with at least the Reporter role. | | `with_custom_attributes` | boolean | **{dotted-circle}** No | Include [custom attributes](custom_attributes.md) in response. _(administrators only)_ | ```json { "id": 3, - "description": null, + "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + "description_html": "

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

", "default_branch": "master", "visibility": "private", "ssh_url_to_repo": "git@example.com:diaspora/diaspora-project-site.git", @@ -898,6 +904,7 @@ GET /projects/:id "next_run_at": "2020-01-07T21:42:58.658Z" }, "created_at": "2013-09-30T13:46:02Z", + "updated_at": "2013-09-30T13:46:02Z", "last_activity_at": "2013-09-30T13:46:02Z", "creator_id": 3, "namespace": { @@ -969,7 +976,7 @@ GET /projects/:id "squash_option": "default_on", "auto_devops_enabled": true, "auto_devops_deploy_strategy": "continuous", - "approvals_before_merge": 0, + "approvals_before_merge": 0, // Deprecated. Use merge request approvals API instead. "mirror": false, "mirror_user_id": 45, "mirror_trigger_builds": false, @@ -1014,22 +1021,6 @@ GET /projects/:id } ``` -NOTE: -The `tag_list` attribute has been deprecated -and is removed in API v5 in favor of the `topics` attribute. - -Users of [GitLab Premium or higher](https://about.gitlab.com/pricing/) -can also see the `approvals_before_merge` parameter: - -```json -{ - "id": 3, - "description": null, - "approvals_before_merge": 0, - ... -} -``` - Users of [GitLab Ultimate](https://about.gitlab.com/pricing/) can also see the `only_allow_merge_if_all_status_checks_passed` parameters using GitLab 15.5 and later: @@ -1046,7 +1037,7 @@ parameters using GitLab 15.5 and later: If the project is a fork, the `forked_from_project` field appears in the response. For this field, if the upstream project is private, a valid token for authentication must be provided. The field `mr_default_target_self` appears as well. If this value is `false`, then all merge requests -will target the upstream project by default. +target the upstream project by default. ```json { @@ -1100,7 +1091,7 @@ will target the upstream project by default. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55718) in GitLab 13.10. -Users of [GitLab Premium or higher](https://about.gitlab.com/pricing/) +Users of [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/) can also see the `issues_template` and `merge_requests_template` parameters for managing [issue and merge request description templates](../user/project/description_templates.md). @@ -1226,6 +1217,8 @@ Refer to the [Events API documentation](events.md#list-a-projects-visible-events ## Create project +> `operations_access_level` [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/385798) in GitLab 16.0. + Creates a new project owned by the authenticated user. If your HTTP repository isn't publicly accessible, add authentication information @@ -1253,8 +1246,8 @@ curl --request POST --header "PRIVATE-TOKEN: " \ | `allow_merge_on_skipped_pipeline` | boolean | **{dotted-circle}** No | Set whether or not merge requests can be merged with skipped jobs. | | `only_allow_merge_if_all_status_checks_passed` **(ULTIMATE)** | boolean | **{dotted-circle}** No | Indicates that merges of merge requests should be blocked unless all status checks have passed. Defaults to false. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/369859) in GitLab 15.5 with feature flag `only_allow_merge_if_all_status_checks_passed` disabled by default. | | `analytics_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private` or `enabled` | -| `approvals_before_merge` **(PREMIUM)** | integer | **{dotted-circle}** No | How many approvers should approve merge requests by default. To configure approval rules, see [Merge request approvals API](merge_request_approvals.md). | -| `auto_cancel_pending_pipelines` | string | **{dotted-circle}** No | Auto-cancel pending pipelines. This isn't a boolean, but enabled/disabled. | +| `approvals_before_merge` **(PREMIUM)** | integer | **{dotted-circle}** No | How many approvers should approve merge requests by default. To configure approval rules, see [Merge request approvals API](merge_request_approvals.md). [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/353097) in GitLab 16.0. | +| `auto_cancel_pending_pipelines` | string | **{dotted-circle}** No | Auto-cancel pending pipelines. This action toggles between an enabled state and a disabled state; it is not a boolean. | | `auto_devops_deploy_strategy` | string | **{dotted-circle}** No | Auto Deploy strategy (`continuous`, `manual` or `timed_incremental`). | | `auto_devops_enabled` | boolean | **{dotted-circle}** No | Enable Auto DevOps for this project. | | `autoclose_referenced_issues` | boolean | **{dotted-circle}** No | Set whether auto-closing referenced issues on default branch. | @@ -1272,8 +1265,8 @@ curl --request POST --header "PRIVATE-TOKEN: " \ | `external_authorization_classification_label` **(PREMIUM)** | string | **{dotted-circle}** No | The classification label for the project. | | `forking_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | | `group_with_project_templates_id` **(PREMIUM)** | integer | **{dotted-circle}** No | For group-level custom templates, specifies ID of group from which all the custom project templates are sourced. Leave empty for instance-level templates. Requires `use_custom_template` to be true. | -| `import_url` | string | **{dotted-circle}** No | URL to import repository from. When this isn't empty, you must not set `initialize_with_readme` to `true`. Doing so might result in the [following error](https://gitlab.com/gitlab-org/gitlab/-/issues/360266): `not a git repository`. | -| `initialize_with_readme` | boolean | **{dotted-circle}** No | Whether to create a Git repository with just a `README.md` file. Default is `false`. When this is true, you must not pass `import_url` or other attributes of this endpoint which specify alternative contents for the repository. Doing so might result in the [following error](https://gitlab.com/gitlab-org/gitlab/-/issues/360266): `not a git repository`. | +| `import_url` | string | **{dotted-circle}** No | URL to import repository from. When the URL value isn't empty, you must not set `initialize_with_readme` to `true`. Doing so might result in the [following error](https://gitlab.com/gitlab-org/gitlab/-/issues/360266): `not a git repository`. | +| `initialize_with_readme` | boolean | **{dotted-circle}** No | Whether to create a Git repository with just a `README.md` file. Default is `false`. When this boolean is true, you must not pass `import_url` or other attributes of this endpoint which specify alternative contents for the repository. Doing so might result in the [following error](https://gitlab.com/gitlab-org/gitlab/-/issues/360266): `not a git repository`. | | `issues_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | | `issues_enabled` | boolean | **{dotted-circle}** No | _(Deprecated)_ Enable issues for this project. Use `issues_access_level` instead. | | `jobs_enabled` | boolean | **{dotted-circle}** No | _(Deprecated)_ Enable jobs for this project. Use `builds_access_level` instead. | @@ -1288,7 +1281,6 @@ curl --request POST --header "PRIVATE-TOKEN: " \ | `namespace_id` | integer | **{dotted-circle}** No | Namespace for the new project (defaults to the current user's namespace). | | `only_allow_merge_if_all_discussions_are_resolved` | boolean | **{dotted-circle}** No | Set whether merge requests can only be merged when all the discussions are resolved. | | `only_allow_merge_if_pipeline_succeeds` | boolean | **{dotted-circle}** No | Set whether merge requests can only be merged with successful pipelines. This setting is named [**Pipelines must succeed**](../user/project/merge_requests/merge_when_pipeline_succeeds.md#require-a-successful-pipeline-for-merge) in the project settings. | -| `operations_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/385798) in GitLab 15.8. | | `packages_enabled` | boolean | **{dotted-circle}** No | Enable or disable packages repository feature. | | `pages_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, `enabled`, or `public`. | | `printing_merge_request_link_enabled` | boolean | **{dotted-circle}** No | Show link to create/view merge request when pushing from the command line. | @@ -1312,7 +1304,7 @@ curl --request POST --header "PRIVATE-TOKEN: " \ | `squash_option` | string | **{dotted-circle}** No | One of `never`, `always`, `default_on`, or `default_off`. | | `tag_list` | array | **{dotted-circle}** No | _([Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/328226) in GitLab 14.0)_ The list of tags for a project; put array of tags, that should be finally assigned to a project. Use `topics` instead. | | `template_name` | string | **{dotted-circle}** No | When used without `use_custom_template`, name of a [built-in project template](../user/project/index.md#create-a-project-from-a-built-in-template). When used with `use_custom_template`, name of a custom project template. | -| `template_project_id` **(PREMIUM)** | integer | **{dotted-circle}** No | When used with `use_custom_template`, project ID of a custom project template. This is preferable to using `template_name` since `template_name` may be ambiguous. | +| `template_project_id` **(PREMIUM)** | integer | **{dotted-circle}** No | When used with `use_custom_template`, project ID of a custom project template. Using a project ID is preferable to using `template_name` since `template_name` may be ambiguous. | | `topics` | array | **{dotted-circle}** No | The list of topics for a project; put array of topics, that should be finally assigned to a project. _([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/328226) in GitLab 14.0.)_ | | `use_custom_template` **(PREMIUM)** | boolean | **{dotted-circle}** No | Use either custom [instance](../user/admin_area/custom_project_templates.md) or [group](../user/group/custom_project_templates.md) (with `group_with_project_templates_id`) project template. | | `visibility` | string | **{dotted-circle}** No | See [project visibility level](#project-visibility-level). | @@ -1321,6 +1313,8 @@ curl --request POST --header "PRIVATE-TOKEN: " \ ## Create project for user +> `operations_access_level` [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/385798) in GitLab 16.0. + Creates a new project owned by the specified user. Available only for administrators. If your HTTP repository isn't publicly accessible, add authentication information @@ -1338,8 +1332,8 @@ POST /projects/user/:user_id | `allow_merge_on_skipped_pipeline` | boolean | **{dotted-circle}** No | Set whether or not merge requests can be merged with skipped jobs. | | `only_allow_merge_if_all_status_checks_passed` **(ULTIMATE)** | boolean | **{dotted-circle}** No | Indicates that merges of merge requests should be blocked unless all status checks have passed. Defaults to false. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/369859) in GitLab 15.5 with feature flag `only_allow_merge_if_all_status_checks_passed` disabled by default. | | `analytics_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private` or `enabled` | -| `approvals_before_merge` **(PREMIUM)** | integer | **{dotted-circle}** No | How many approvers should approve merge requests by default. To configure approval rules, see [Merge request approvals API](merge_request_approvals.md). | -| `auto_cancel_pending_pipelines` | string | **{dotted-circle}** No | Auto-cancel pending pipelines. This isn't a boolean, but enabled/disabled. | +| `approvals_before_merge` **(PREMIUM)** | integer | **{dotted-circle}** No | How many approvers should approve merge requests by default. [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/353097) in GitLab 16.0. To configure approval rules, see [Merge request approvals API](merge_request_approvals.md). | +| `auto_cancel_pending_pipelines` | string | **{dotted-circle}** No | Auto-cancel pending pipelines. This action toggles between an enabled state and a disabled state; it is not a boolean. | | `auto_devops_deploy_strategy` | string | **{dotted-circle}** No | Auto Deploy strategy (`continuous`, `manual` or `timed_incremental`). | | `auto_devops_enabled` | boolean | **{dotted-circle}** No | Enable Auto DevOps for this project. | | `autoclose_referenced_issues` | boolean | **{dotted-circle}** No | Set whether auto-closing referenced issues on default branch. | @@ -1372,7 +1366,6 @@ POST /projects/user/:user_id | `namespace_id` | integer | **{dotted-circle}** No | Namespace for the new project (defaults to the current user's namespace). | | `only_allow_merge_if_all_discussions_are_resolved` | boolean | **{dotted-circle}** No | Set whether merge requests can only be merged when all the discussions are resolved. | | `only_allow_merge_if_pipeline_succeeds` | boolean | **{dotted-circle}** No | Set whether merge requests can only be merged with successful jobs. | -| `operations_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/385798) in GitLab 15.8. | | `packages_enabled` | boolean | **{dotted-circle}** No | Enable or disable packages repository feature. | | `pages_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, `enabled`, or `public`. | | `path` | string | **{dotted-circle}** No | Custom repository name for new project. By default generated based on name. | @@ -1408,6 +1401,8 @@ POST /projects/user/:user_id ## Edit project +> `operations_access_level` [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/385798) in GitLab 16.0. + Updates an existing project. If your HTTP repository isn't publicly accessible, add authentication information @@ -1433,10 +1428,11 @@ Supported attributes: |-------------------------------------------------------------|----------------|------------------------|-------------| | `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | | `allow_merge_on_skipped_pipeline` | boolean | **{dotted-circle}** No | Set whether or not merge requests can be merged with skipped jobs. | +| `allow_pipeline_trigger_approve_deployment` **(PREMIUM)** | boolean | **{dotted-circle}** No | Set whether or not a pipeline triggerer is allowed to approve deployments. | | `only_allow_merge_if_all_status_checks_passed` **(ULTIMATE)** | boolean | **{dotted-circle}** No | Indicates that merges of merge requests should be blocked unless all status checks have passed. Defaults to false.

[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/369859) in GitLab 15.5 with feature flag `only_allow_merge_if_all_status_checks_passed` disabled by default. The feature flag was enabled by default in GitLab 15.9. | | `analytics_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private` or `enabled` | -| `approvals_before_merge` **(PREMIUM)** | integer | **{dotted-circle}** No | How many approvers should approve merge request by default. To configure approval rules, see [Merge request approvals API](merge_request_approvals.md). | -| `auto_cancel_pending_pipelines` | string | **{dotted-circle}** No | Auto-cancel pending pipelines. This isn't a boolean, but enabled/disabled. | +| `approvals_before_merge` **(PREMIUM)** | integer | **{dotted-circle}** No | How many approvers should approve merge requests by default. [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/353097) in GitLab 16.0. To configure approval rules, see [Merge request approvals API](merge_request_approvals.md). | +| `auto_cancel_pending_pipelines` | string | **{dotted-circle}** No | Auto-cancel pending pipelines. This action toggles between an enabled state and a disabled state; it is not a boolean. | | `auto_devops_deploy_strategy` | string | **{dotted-circle}** No | Auto Deploy strategy (`continuous`, `manual`, or `timed_incremental`). | | `auto_devops_enabled` | boolean | **{dotted-circle}** No | Enable Auto DevOps for this project. | | `autoclose_referenced_issues` | boolean | **{dotted-circle}** No | Set whether auto-closing referenced issues on default branch. | @@ -1476,12 +1472,11 @@ Supported attributes: | `mirror_trigger_builds` **(PREMIUM)** | boolean | **{dotted-circle}** No | Pull mirroring triggers builds. | | `mirror_user_id` **(PREMIUM)** | integer | **{dotted-circle}** No | User responsible for all the activity surrounding a pull mirror event. _(administrators only)_ | | `mirror` **(PREMIUM)** | boolean | **{dotted-circle}** No | Enables pull mirroring in a project. | -| `mr_default_target_self` | boolean | **{dotted-circle}** No | For forked projects, target merge requests to this project. If `false`, the target will be the upstream project. | +| `mr_default_target_self` | boolean | **{dotted-circle}** No | For forked projects, target merge requests to this project. If `false`, the target is the upstream project. | | `name` | string | **{dotted-circle}** No | The name of the project. | | `only_allow_merge_if_all_discussions_are_resolved` | boolean | **{dotted-circle}** No | Set whether merge requests can only be merged when all the discussions are resolved. | | `only_allow_merge_if_pipeline_succeeds` | boolean | **{dotted-circle}** No | Set whether merge requests can only be merged with successful jobs. | | `only_mirror_protected_branches` **(PREMIUM)** | boolean | **{dotted-circle}** No | Only mirror protected branches. | -| `operations_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/385798) in GitLab 15.8. | | `packages_enabled` | boolean | **{dotted-circle}** No | Enable or disable packages repository feature. | | `pages_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, `enabled`, or `public`. | | `path` | string | **{dotted-circle}** No | Custom repository name for the project. By default generated based on name. | @@ -1531,7 +1526,7 @@ POST /projects/:id/fork |------------------|----------------|------------------------|-------------| | `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | | `description` | string | **{dotted-circle}** No | The description assigned to the resultant project after forking. | -| `mr_default_target_self` | boolean | **{dotted-circle}** No | For forked projects, target merge requests to this project. If `false`, the target will be the upstream project. | +| `mr_default_target_self` | boolean | **{dotted-circle}** No | For forked projects, target merge requests to this project. If `false`, the target is the upstream project. | | `name` | string | **{dotted-circle}** No | The name assigned to the resultant project after forking. | | `namespace_id` | integer | **{dotted-circle}** No | The ID of the namespace that the project is forked to. | | `namespace_path` | string | **{dotted-circle}** No | The path of the namespace that the project is forked to. | @@ -1559,14 +1554,16 @@ GET /projects/:id/forks | `order_by` | string | **{dotted-circle}** No | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at`. | | `owned` | boolean | **{dotted-circle}** No | Limit by projects explicitly owned by the current user. | | `search` | string | **{dotted-circle}** No | Return list of projects matching the search criteria. | -| `simple` | boolean | **{dotted-circle}** No | Return only limited fields for each project. This is a no-op without authentication where only simple fields are returned. | +| `simple` | boolean | **{dotted-circle}** No | Return only limited fields for each project. Without authentication, this operation is a no-op; only simple fields are returned. | | `sort` | string | **{dotted-circle}** No | Return projects sorted in `asc` or `desc` order. Default is `desc`. | | `starred` | boolean | **{dotted-circle}** No | Limit by projects starred by the current user. | -| `statistics` | boolean | **{dotted-circle}** No | Include project statistics. Only available to Reporter or higher level role members. | +| `statistics` | boolean | **{dotted-circle}** No | Include project statistics. Available only to users with at least the Reporter role. | | `visibility` | string | **{dotted-circle}** No | Limit by visibility `public`, `internal`, or `private`. | | `with_custom_attributes` | boolean | **{dotted-circle}** No | Include [custom attributes](custom_attributes.md) in response. _(administrators only)_ | | `with_issues_enabled` | boolean | **{dotted-circle}** No | Limit by enabled issues feature. | | `with_merge_requests_enabled` | boolean | **{dotted-circle}** No | Limit by enabled merge requests feature. | +| `updated_before` | datetime | **{dotted-circle}** No | Limit results to projects last updated before the specified time. Format: ISO 8601 (`YYYY-MM-DDTHH:MM:SSZ`). [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/393979) in GitLab 15.10. | +| `updated_after` | datetime | **{dotted-circle}** No | Limit results to projects last updated after the specified time. Format: ISO 8601 (`YYYY-MM-DDTHH:MM:SSZ`). [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/393979) in GitLab 15.10. | ```shell curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/forks" @@ -1578,7 +1575,8 @@ Example responses: [ { "id": 3, - "description": null, + "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + "description_html": "

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

", "default_branch": "master", "visibility": "internal", "ssh_url_to_repo": "git@example.com:diaspora/diaspora-project-site.git", @@ -1609,6 +1607,7 @@ Example responses: "container_registry_access_level": "disabled", "security_and_compliance_access_level": "disabled", "created_at": "2013-09-30T13:46:02Z", + "updated_at": "2013-09-30T13:46:02Z", "last_activity_at": "2013-09-30T13:46:02Z", "creator_id": 3, "namespace": { @@ -1678,7 +1677,8 @@ Example response: ```json { "id": 3, - "description": null, + "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + "description_html": "

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

", "default_branch": "master", "visibility": "internal", "ssh_url_to_repo": "git@example.com:diaspora/diaspora-project-site.git", @@ -1709,6 +1709,7 @@ Example response: "container_registry_access_level": "disabled", "security_and_compliance_access_level": "disabled", "created_at": "2013-09-30T13:46:02Z", + "updated_at": "2013-09-30T13:46:02Z", "last_activity_at": "2013-09-30T13:46:02Z", "creator_id": 3, "namespace": { @@ -1784,7 +1785,8 @@ Example response: ```json { "id": 3, - "description": null, + "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + "description_html": "

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

", "default_branch": "master", "visibility": "internal", "ssh_url_to_repo": "git@example.com:diaspora/diaspora-project-site.git", @@ -1815,6 +1817,7 @@ Example response: "container_registry_access_level": "disabled", "security_and_compliance_access_level": "disabled", "created_at": "2013-09-30T13:46:02Z", + "updated_at": "2013-09-30T13:46:02Z", "last_activity_at": "2013-09-30T13:46:02Z", "creator_id": 3, "namespace": { @@ -1965,7 +1968,8 @@ Example response: ```json { "id": 3, - "description": null, + "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + "description_html": "

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

", "default_branch": "master", "visibility": "private", "ssh_url_to_repo": "git@example.com:diaspora/diaspora-project-site.git", @@ -2001,6 +2005,7 @@ Example response: "container_registry_access_level": "disabled", "security_and_compliance_access_level": "disabled", "created_at": "2013-09-30T13:46:02Z", + "updated_at": "2013-09-30T13:46:02Z", "last_activity_at": "2013-09-30T13:46:02Z", "creator_id": 3, "namespace": { @@ -2094,7 +2099,8 @@ Example response: ```json { "id": 3, - "description": null, + "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + "description_html": "

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

", "default_branch": "master", "visibility": "private", "ssh_url_to_repo": "git@example.com:diaspora/diaspora-project-site.git", @@ -2130,6 +2136,7 @@ Example response: "container_registry_access_level": "disabled", "security_and_compliance_access_level": "disabled", "created_at": "2013-09-30T13:46:02Z", + "updated_at": "2013-09-30T13:46:02Z", "last_activity_at": "2013-09-30T13:46:02Z", "creator_id": 3, "namespace": { @@ -2205,15 +2212,18 @@ This endpoint: - Deletes a project including all associated resources (including issues and merge requests). - In [GitLab 12.6](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) and later, on - [Premium or higher](https://about.gitlab.com/pricing/) tiers, + [Premium or Ultimate](https://about.gitlab.com/pricing/) tiers, [delayed project deletion](../user/project/settings/index.md#delayed-project-deletion) is applied if enabled. - From [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/220382) on - [Premium or higher](https://about.gitlab.com/pricing/) tiers, group + [Premium or Ultimate](https://about.gitlab.com/pricing/) tiers, group administrators can [configure](../user/group/manage.md#enable-delayed-project-deletion) projects within a group to be deleted after a delayed period. When enabled, actual deletion happens after the number of days specified in the [default deletion delay](../user/admin_area/settings/visibility_and_access_controls.md#deletion-protection). +- From [GitLab 15.11](https://gitlab.com/gitlab-org/gitlab/-/issues/396500) on + [Premium or Ultimate](https://about.gitlab.com/pricing/) tiers, deletes a project immediately if the project is already + marked for deletion, and the `permanently_remove` and `full_path` parameters are passed. WARNING: The default behavior of [Delayed Project deletion](https://gitlab.com/gitlab-org/gitlab/-/issues/32935) @@ -2224,9 +2234,11 @@ in GitLab 13.2, as discussed in [Enable delayed project deletion](../user/group/ DELETE /projects/:id ``` -| Attribute | Type | Required | Description | -|-----------|----------------|------------------------|-------------| -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| Attribute | Type | Required | Description | +|------------------------------------|-------------------|------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | +| `permanently_remove` **(PREMIUM)** | boolean/string | no | Immediately deletes a project if it is marked for deletion. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/396500) in GitLab 15.11 | +| `full_path` **(PREMIUM)** | string | no | Full path of project to use with `permanently_remove`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/396500) in GitLab 15.11. To find the project path, use `path_with_namespace` from [get single project](projects.md#get-single-project) | ## Restore project marked for deletion **(PREMIUM)** @@ -2244,9 +2256,12 @@ POST /projects/:id/restore ## Upload a file +> - Maximum attachment size enforcement [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/57250) in GitLab 13.11 [with a flag](../administration/feature_flags.md) named `enforce_max_attachment_size_upload_api`. Disabled by default. +> - Maximum attachment size [enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62542) in GitLab 13.11. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/112450) in GitLab 15.10. Feature flag `enforce_max_attachment_size_upload_api` removed. + Uploads a file to the specified project to be used in an issue or merge request -description, or a comment. GitLab versions 14.0 and later -[enforce](#max-attachment-size-enforcement) this limit. +description, or a comment. ```plaintext POST /projects/:id/uploads @@ -2282,42 +2297,6 @@ The returned `url` is relative to the project path. The returned `full_path` is the absolute path to the file. In Markdown contexts, the link is expanded when the format in `markdown` is used. -### Max attachment size enforcement - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/57250) in GitLab 13.11. - -GitLab 13.11 added enforcement of the [maximum attachment size limit](../user/admin_area/settings/account_and_limit_settings.md#max-attachment-size) behind the `enforce_max_attachment_size_upload_api` feature flag. GitLab 14.0 enables this by default. -To disable this enforcement: - -**In Omnibus installations:** - -1. Enter the Rails console: - - ```shell - sudo gitlab-rails console - ``` - -1. Disable the feature flag: - - ```ruby - Feature.disable(:enforce_max_attachment_size_upload_api) - ``` - -**In installations from source:** - -1. Enter the Rails console: - - ```shell - cd /home/git/gitlab - sudo -u git -H bundle exec rails console -e production - ``` - -1. Disable the feature flag: - - ```ruby - Feature.disable(:enforce_max_attachment_size_upload_api) - ``` - ## Upload a project avatar Uploads an avatar to the specified project. @@ -2501,7 +2480,7 @@ POST /projects/:id/hooks | `push_events` | boolean | **{dotted-circle}** No | Trigger hook on push events. | | `releases_events` | boolean | **{dotted-circle}** No | Trigger hook on release events. | | `tag_push_events` | boolean | **{dotted-circle}** No | Trigger hook on tag push events. | -| `token` | string | **{dotted-circle}** No | Secret token to validate received payloads; this isn't returned in the response. | +| `token` | string | **{dotted-circle}** No | Secret token to validate received payloads; the token isn't returned in the response. | | `wiki_page_events` | boolean | **{dotted-circle}** No | Trigger hook on wiki events. | ### Edit project hook @@ -2535,7 +2514,7 @@ PUT /projects/:id/hooks/:hook_id ### Delete project hook -Removes a hook from a project. This is an idempotent method and can be called +Removes a hook from a project. This method is idempotent, and can be called multiple times. Either the hook is available or not. ```plaintext @@ -2692,7 +2671,7 @@ PUT /projects/:id/push_rule > Moved to GitLab Premium in 13.9. -Removes a push rule from a project. This is an idempotent method and can be +Removes a push rule from a project. This method is idempotent and can be called multiple times. Either the push rule is available or not. ```plaintext @@ -2774,12 +2753,14 @@ Example response: ```json { "id": 7, - "description": "", + "description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + "description_html": "

Lorem ipsum dolor sit amet, consectetur adipiscing elit.

", "name": "hello-world", "name_with_namespace": "cute-cats / hello-world", "path": "hello-world", "path_with_namespace": "cute-cats/hello-world", "created_at": "2020-10-15T16:25:22.415Z", + "updated_at": "2020-10-15T16:25:22.415Z", "default_branch": "master", "tag_list": [], //deprecated, use `topics` instead "topics": [], @@ -2872,7 +2853,7 @@ Example response: "auto_devops_enabled": true, "auto_devops_deploy_strategy": "continuous", "autoclose_referenced_issues": true, - "approvals_before_merge": 0, + "approvals_before_merge": 0, // Deprecated. Use merge request approvals API instead. "mirror": false, "compliance_frameworks": [] } @@ -2933,13 +2914,14 @@ Example response: ## Configure pull mirroring for a project **(PREMIUM)** > - Moved to GitLab Premium in GitLab 13.9. -> - Field `mirror_branch_regex` [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/102608) in GitLab 15.8 [with a flag](../administration/feature_flags.md) named `mirror_only_branches_match_regex`. Disabled by default. +> - Field `mirror_branch_regex` [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/381667) in GitLab 15.8 [with a flag](../administration/feature_flags.md) named `mirror_only_branches_match_regex`. Disabled by default. +> - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/381667) in GitLab 16.0. FLAG: -On self-managed GitLab, by default the field `mirror_branch_regex` is not available. -To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) +On self-managed GitLab, by default the field `mirror_branch_regex` is available. +To hide the feature, ask an administrator to [disable the feature flag](../administration/feature_flags.md) named `mirror_only_branches_match_regex`. -On GitLab.com, this feature is not available. +On GitLab.com, this feature is available. Configure pull mirroring while [creating a new project](#create-project) or [updating an existing project](#edit-project) using the API diff --git a/doc/api/protected_branches.md b/doc/api/protected_branches.md index 5f6e03fde4d..d3091cb1e15 100644 --- a/doc/api/protected_branches.md +++ b/doc/api/protected_branches.md @@ -19,6 +19,8 @@ The access levels are defined in the `ProtectedRefAccess.allowed_access_levels` ## List protected branches +> Deploy key information [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/116846) in GitLab 16.0. + Gets a list of protected branches from a project as they are defined [in the UI](../user/project/protected_branches.md#configure-a-protected-branch). If a wildcard is set, it is returned instead of the exact name of the branches that match that wildcard. ```plaintext @@ -46,6 +48,12 @@ Example response: "id": 1, "access_level": 40, "access_level_description": "Maintainers" + }, + { + "id": 2, + "access_level": 40, + "access_level_description": "Deploy key", + "deploy_key_id": 1 } ], "merge_access_levels": [ @@ -82,7 +90,7 @@ Example response: ] ``` -Users on GitLab Premium or higher also see +Users on GitLab Premium or Ultimate also see the `user_id`, `group_id` and `inherited` parameters. If the `inherited` parameter exists, means the setting was inherited from the project's group. @@ -100,6 +108,14 @@ Example response: "user_id": null, "group_id": null, "access_level_description": "Maintainers" + }, + { + "id": 2, + "access_level": 40, + "access_level_description": "Deploy key", + "deploy_key_id": 1, + "user_id": null, + "group_id": null } ], "merge_access_levels": [ @@ -161,7 +177,7 @@ Example response: } ``` -Users on GitLab Premium or higher also see +Users on GitLab Premium or Ultimate also see the `user_id` and `group_id` parameters: Example response: @@ -208,16 +224,16 @@ curl --request POST --header "PRIVATE-TOKEN: " "https://gitla | Attribute | Type | Required | Description | | -------------------------------------------- | ---- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | -| `name` | string | yes | The name of the branch or wildcard | -| `push_access_level` | integer | no | Access levels allowed to push (defaults: `40`, Maintainer role) | -| `merge_access_level` | integer | no | Access levels allowed to merge (defaults: `40`, Maintainer role) | -| `unprotect_access_level` | integer | no | Access levels allowed to unprotect (defaults: `40`, Maintainer role) | -| `allow_force_push` | boolean | no | Allow all users with push access to force push. (default: `false`) | -| `allowed_to_push` **(PREMIUM)** | array | no | Array of access levels allowed to push, with each described by a hash of the form `{user_id: integer}`, `{group_id: integer}`, or `{access_level: integer}` | -| `allowed_to_merge` **(PREMIUM)** | array | no | Array of access levels allowed to merge, with each described by a hash of the form `{user_id: integer}`, `{group_id: integer}`, or `{access_level: integer}` | -| `allowed_to_unprotect` **(PREMIUM)** | array | no | Array of access levels allowed to unprotect, with each described by a hash of the form `{user_id: integer}`, `{group_id: integer}`, or `{access_level: integer}` | -| `code_owner_approval_required` **(PREMIUM)** | boolean | no | Prevent pushes to this branch if it matches an item in the [`CODEOWNERS` file](../user/project/code_owners.md). (defaults: false) | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. +| `name` | string | yes | The name of the branch or wildcard. +| `allow_force_push` | boolean | no | Allow all users with push access to force push. (default: `false`) +| `allowed_to_merge` **(PREMIUM)** | array | no | Array of access levels allowed to merge, with each described by a hash of the form `{user_id: integer}`, `{group_id: integer}`, or `{access_level: integer}`. +| `allowed_to_push` **(PREMIUM)** | array | no | Array of access levels allowed to push, with each described by a hash of the form `{user_id: integer}`, `{group_id: integer}`, or `{access_level: integer}`. +| `allowed_to_unprotect` **(PREMIUM)** | array | no | Array of access levels allowed to unprotect, with each described by a hash of the form `{user_id: integer}`, `{group_id: integer}`, or `{access_level: integer}`. The access level `No access` is not available for this field. | +| `code_owner_approval_required` **(PREMIUM)** | boolean | no | Prevent pushes to this branch if it matches an item in the [`CODEOWNERS` file](../user/project/codeowners/index.md). (defaults: false) +| `merge_access_level` | integer | no | Access levels allowed to merge. (defaults: `40`, Maintainer role) +| `push_access_level` | integer | no | Access levels allowed to push. (defaults: `40`, Maintainer role) +| `unprotect_access_level` | integer | no | Access levels allowed to unprotect. (defaults: `40`, Maintainer role) Example response: @@ -251,7 +267,7 @@ Example response: } ``` -Users on GitLab Premium or higher also see +Users on GitLab Premium or Ultimate also see the `user_id` and `group_id` parameters: Example response: @@ -369,7 +385,7 @@ Example response: "name": "master", "push_access_levels": [ { - "id": 1, + "id": 1, "access_level": 30, "access_level_description": "Developers + Maintainers", "user_id": null, @@ -378,14 +394,14 @@ Example response: ], "merge_access_levels": [ { - "id": 1, + "id": 1, "access_level": 30, "access_level_description": "Developers + Maintainers", "user_id": null, "group_id": null }, { - "id": 2, + "id": 2, "access_level": 40, "access_level_description": "Maintainers", "user_id": null, @@ -394,7 +410,7 @@ Example response: ], "unprotect_access_levels": [ { - "id": 1, + "id": 1, "access_level": 40, "access_level_description": "Maintainers", "user_id": null, @@ -437,15 +453,15 @@ PATCH /projects/:id/protected_branches/:name curl --request PATCH --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/protected_branches/feature-branch?allow_force_push=true&code_owner_approval_required=true" ``` -| Attribute | Type | Required | Description | +| Attribute | Type | Required | Description | | -------------------------------------------- | ---- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | -| `name` | string | yes | The name of the branch | -| `allow_force_push` | boolean | no | When enabled, members who can push to this branch can also force push. | -| `code_owner_approval_required` **(PREMIUM)** | boolean | no | Prevent pushes to this branch if it matches an item in the [`CODEOWNERS` file](../user/project/code_owners.md). Defaults to `false`. | -| `allowed_to_push` **(PREMIUM)** | array | no | Array of push access levels, with each described by a hash. | -| `allowed_to_merge` **(PREMIUM)** | array | no | Array of merge access levels, with each described by a hash. | -| `allowed_to_unprotect` **(PREMIUM)** | array | no | Array of unprotect access levels, with each described by a hash. | +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. +| `name` | string | yes | The name of the branch. +| `allow_force_push` | boolean | no | When enabled, members who can push to this branch can also force push. +| `allowed_to_push` **(PREMIUM)** | array | no | Array of push access levels, with each described by a hash. +| `allowed_to_merge` **(PREMIUM)** | array | no | Array of merge access levels, with each described by a hash. +| `allowed_to_unprotect` **(PREMIUM)** | array | no | Array of unprotect access levels, with each described by a hash. The access level `No access` is not available for this field. +| `code_owner_approval_required` **(PREMIUM)** | boolean | no | Prevent pushes to this branch if it matches an item in the [`CODEOWNERS` file](../user/project/codeowners/index.md). Defaults to `false`. | Elements in the `allowed_to_push`, `allowed_to_merge` and `allowed_to_unprotect` arrays should be one of `user_id`, `group_id` or `access_level`, and take the form `{user_id: integer}`, `{group_id: integer}` or @@ -529,3 +545,9 @@ Example response: "push_access_levels": [] } ``` + +## Related topics + +- [Protected branches](../user/project/protected_branches.md) +- [Branches](../user/project/repository/branches/index.md) +- [Branches API](branches.md) diff --git a/doc/api/protected_environments.md b/doc/api/protected_environments.md index 1ea3fc5adda..023046a4821 100644 --- a/doc/api/protected_environments.md +++ b/doc/api/protected_environments.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: concepts, howto --- diff --git a/doc/api/protected_tags.md b/doc/api/protected_tags.md index 633ca441fad..7aff87b54f0 100644 --- a/doc/api/protected_tags.md +++ b/doc/api/protected_tags.md @@ -8,17 +8,17 @@ info: To determine the technical writer assigned to the Stage/Group associated w **Valid access levels** -Currently, these levels are recognized: +These access levels are recognized: -```plaintext -0 => No access -30 => Developer access -40 => Maintainer access -``` +- `0`: No access +- `30`: Developer role +- `40`: Maintainer role ## List protected tags -Gets a list of protected tags from a project. +> Deploy key information [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/116846) in GitLab 16.0. + +Gets a list of [protected tags](../user/project/protected_tags.md) from a project. This function takes pagination parameters `page` and `per_page` to restrict the list of protected tags. ```plaintext @@ -27,10 +27,11 @@ GET /projects/:id/protected_tags | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | ```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/protected_tags" +curl --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects/5/protected_tags" ``` Example response: @@ -44,6 +45,12 @@ Example response: "id":1, "access_level": 40, "access_level_description": "Maintainers" + }, + { + "id": 2, + "access_level": 40, + "access_level_description": "Deploy key", + "deploy_key_id": 1 } ] }, @@ -54,7 +61,6 @@ Example response: ## Get a single protected tag or wildcard protected tag Gets a single protected tag or wildcard protected tag. -The pagination parameters `page` and `per_page` can be used to restrict the list of protected tags. ```plaintext GET /projects/:id/protected_tags/:name @@ -62,11 +68,12 @@ GET /projects/:id/protected_tags/:name | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | -| `name` | string | yes | The name of the tag or wildcard | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | +| `name` | string | yes | The name of the tag or wildcard. | ```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/protected_tags/release-1-0" +curl --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects/5/protected_tags/release-1-0" ``` Example response: @@ -86,23 +93,35 @@ Example response: ## Protect repository tags -Protects a single repository tag or several project repository -tags using a wildcard protected tag. +Protects a single repository tag, or several project repository +tags, using a wildcard protected tag. ```plaintext POST /projects/:id/protected_tags ``` ```shell -curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/protected_tags?name=*-stable&create_access_level=30" +curl --request POST --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects/5/protected_tags" -d '{ + "allowed_to_create" : [ + { + "user_id" : 1 + }, + { + "access_level" : 30 + } + ], + "create_access_level" : 30, + "name" : "*-stable" +}' ``` | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | -| `name` | string | yes | The name of the tag or wildcard | -| `create_access_level` | string | no | Access levels allowed to create (defaults: `40`, Maintainer role) | -| `allowed_to_create` | array | no | Array of access levels allowed to create tags, with each described by a hash of the form `{user_id: integer}`, `{group_id: integer}`, or `{access_level: integer}` | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | +| `name` | string | yes | The name of the tag or wildcard. | +| `allowed_to_create` | array | no | Array of access levels allowed to create tags, with each described by a hash of the form `{user_id: integer}`, `{group_id: integer}`, or `{access_level: integer}`. | +| `create_access_level` | string | no | Access levels allowed to create. Default: `40`, for Maintainer role. | Example response: @@ -128,10 +147,17 @@ DELETE /projects/:id/protected_tags/:name ``` ```shell -curl --request DELETE --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/protected_tags/*-stable" +curl --request DELETE --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects/5/protected_tags/*-stable" ``` | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `id` | integer/string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user | -| `name` | string | yes | The name of the tag | +| `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding) owned by the authenticated user. | +| `name` | string | yes | The name of the tag. | + +## Related topics + +- [Tags API](tags.md) for all tags +- [Tags](../user/project/repository/tags/index.md) user documentation +- [Protected tags](../user/project/protected_tags.md) user documentation diff --git a/doc/api/releases/index.md b/doc/api/releases/index.md index f23f2b36ed0..86c23283588 100644 --- a/doc/api/releases/index.md +++ b/doc/api/releases/index.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -26,7 +26,11 @@ For authentication, the Releases API accepts either: ## List Releases -> [Changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/72448) to allow for `JOB-TOKEN` in GitLab 14.5. +> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/72448) to allow for `JOB-TOKEN` in GitLab 14.5. + +> - The `external` field in Release Links was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/388975) in GitLab 15.9 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/112490) in 16.0. + Returns a paginated list of releases, sorted by `released_at`. @@ -158,14 +162,12 @@ Example response: "id":2, "name":"awesome-v0.2.msi", "url":"http://192.168.10.15:3000/msi", - "external":true, // deprecated in GitLab 15.9, will be removed in GitLab 16.0. "link_type":"other" }, { "id":1, "name":"awesome-v0.2.dmg", "url":"http://192.168.10.15:3000", - "external":true, // deprecated in GitLab 15.9, will be removed in GitLab 16.0. "link_type":"other" } ], @@ -256,7 +258,11 @@ Example response: ## Get a Release by a tag name -> [Changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/72448) to allow for `JOB-TOKEN` in GitLab 14.5. +> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/72448) to allow for `JOB-TOKEN` in GitLab 14.5. + +> - The `external` field in Release Links was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/388975) in GitLab 15.9 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/112490) in 16.0. + Gets a release for the given tag. @@ -386,7 +392,6 @@ Example response: "id":3, "name":"hoge", "url":"https://gitlab.example.com/root/awesome-app/-/tags/v0.11.1/binaries/linux-amd64", - "external":true, // deprecated in GitLab 15.9, will be removed in GitLab 16.0. "link_type":"other" } ] @@ -466,6 +471,13 @@ By default, GitLab fetches the release using `released_at` time. The use of the ## Create a release + + +> The `external` field in Release Links was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/388975) in GitLab 15.9 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/112490) in 16.0. + + + Creates a release. Developer level access to the project is required to create a release. ```plaintext @@ -594,7 +606,6 @@ Example response: "id":3, "name":"hoge", "url":"https://gitlab.example.com/root/awesome-app/-/tags/v0.11.1/binaries/linux-amd64", - "external":true, // deprecated in GitLab 15.9, will be removed in GitLab 16.0. "link_type":"other" } ], diff --git a/doc/api/releases/links.md b/doc/api/releases/links.md index 74f6b1c9efa..a1abded5ed4 100644 --- a/doc/api/releases/links.md +++ b/doc/api/releases/links.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -15,6 +15,13 @@ GitLab supports links to `http`, `https`, and `ftp` assets. ## List links of a release + + +> The `external` field in Release Links was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/388975) in GitLab 15.9 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/112490) in 16.0. + + + Get assets as links from a release. ```plaintext @@ -40,14 +47,12 @@ Example response: "id":2, "name":"awesome-v0.2.msi", "url":"http://192.168.10.15:3000/msi", - "external":true, // deprecated in GitLab 15.9, will be removed in GitLab 16.0. "link_type":"other" }, { "id":1, "name":"awesome-v0.2.dmg", "url":"http://192.168.10.15:3000", - "external":true, // deprecated in GitLab 15.9, will be removed in GitLab 16.0. "link_type":"other" } ] @@ -55,6 +60,13 @@ Example response: ## Get a release link + + +> The `external` field in Release Links was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/388975) in GitLab 15.9 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/112490) in 16.0. + + + Get an asset as a link from a release. ```plaintext @@ -80,13 +92,19 @@ Example response: "id":1, "name":"awesome-v0.2.dmg", "url":"http://192.168.10.15:3000", - "external":true, // deprecated in GitLab 15.9, will be removed in GitLab 16.0. "link_type":"other" } ``` ## Create a release link + + +> The `external` field in Release Links was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/388975) in GitLab 15.9 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/112490) in 16.0. + + + Creates an asset as a link from a release. ```plaintext @@ -122,13 +140,19 @@ Example response: "name":"hellodarwin-amd64", "url":"https://gitlab.example.com/mynamespace/hello/-/jobs/688/artifacts/raw/bin/hello-darwin-amd64", "direct_asset_url":"https://gitlab.example.com/mynamespace/hello/-/releases/v1.7.0/downloads/bin/hellodarwin-amd64", - "external":false, // deprecated in GitLab 15.9, will be removed in GitLab 16.0. "link_type":"other" } ``` ## Update a release link + + +> The `external` field in Release Links was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/388975) in GitLab 15.9 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/112490) in 16.0. + + + Updates an asset as a link from a release. ```plaintext @@ -164,13 +188,19 @@ Example response: "id":1, "name":"new name", "url":"http://192.168.10.15:3000", - "external":true, // deprecated in GitLab 15.9, will be removed in GitLab 16.0. "link_type":"runbook" } ``` ## Delete a release link + + +> The `external` field in Release Links was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/388975) in GitLab 15.9 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/112490) in 16.0. + + + Deletes an asset as a link from a release. ```plaintext @@ -196,7 +226,6 @@ Example response: "id":1, "name":"new name", "url":"http://192.168.10.15:3000", - "external":true, // deprecated in GitLab 15.9, will be removed in GitLab 16.0. "link_type":"other" } ``` diff --git a/doc/api/remote_mirrors.md b/doc/api/remote_mirrors.md index 93dffe69ef5..b59619b3477 100644 --- a/doc/api/remote_mirrors.md +++ b/doc/api/remote_mirrors.md @@ -91,13 +91,14 @@ Learn how to [configure a pull mirror](projects.md#configure-pull-mirroring-for- ## Create a push mirror -> Field `mirror_branch_regex` [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/102608) in GitLab 15.8 [with a flag](../administration/feature_flags.md) named `mirror_only_branches_match_regex`. Disabled by default. +> - Field `mirror_branch_regex` [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/381667) in GitLab 15.8 [with a flag](../administration/feature_flags.md) named `mirror_only_branches_match_regex`. Disabled by default. +> - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/381667) in GitLab 16.0. FLAG: -On self-managed GitLab, by default the field `mirror_branch_regex` is not available. -To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) +On self-managed GitLab, by default the field `mirror_branch_regex` is available. +To hide the feature, ask an administrator to [disable the feature flag](../administration/feature_flags.md) named `mirror_only_branches_match_regex`. -On GitLab.com, this feature is not available. +On GitLab.com, this feature is available. Push mirroring is disabled by default. To enable it, include the optional parameter `enabled` when you create the mirror: diff --git a/doc/api/repositories.md b/doc/api/repositories.md index a34fde54e90..d3bee8de2c5 100644 --- a/doc/api/repositories.md +++ b/doc/api/repositories.md @@ -164,7 +164,8 @@ Supported attributes: Example request: ```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.com/api/v4/projects//repository/archive?sha=&path=" +curl --header "PRIVATE-TOKEN: " \ + "https://gitlab.com/api/v4/projects//repository/archive?sha=&path=" ``` ## Compare branches, tags or commits @@ -278,10 +279,11 @@ GET /projects/:id/repository/merge_base | `id` | integer or string | yes | The ID or [URL-encoded path of the project](rest/index.md#namespaced-path-encoding). | | `refs` | array | yes | The refs to find the common ancestor of. Accepts multiple refs. | -Example request: +Example request, with the refs truncated for readability: ```shell -curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/repository/merge_base?refs[]=304d257dcb821665ab5110318fc58a007bd104ed&refs[]=0031876facac3f2b2702a0e53a26e89939a42209" +curl --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects/5/repository/merge_base?refs[]=304d257d&refs[]=0031876f" ``` Example response: @@ -385,26 +387,30 @@ If the last tag is `v0.9.0` and the default branch is `main`, the range of commi included in this example is `v0.9.0..main`: ```shell -curl --request POST --header "PRIVATE-TOKEN: token" --data "version=1.0.0" "https://gitlab.com/api/v4/projects/42/repository/changelog" +curl --request POST --header "PRIVATE-TOKEN: token" \ + --data "version=1.0.0" "https://gitlab.com/api/v4/projects/42/repository/changelog" ``` To generate the data on a different branch, specify the `branch` parameter. This command generates data from the `foo` branch: ```shell -curl --request POST --header "PRIVATE-TOKEN: token" --data "version=1.0.0&branch=foo" "https://gitlab.com/api/v4/projects/42/repository/changelog" +curl --request POST --header "PRIVATE-TOKEN: token" \ + --data "version=1.0.0&branch=foo" "https://gitlab.com/api/v4/projects/42/repository/changelog" ``` To use a different trailer, use the `trailer` parameter: ```shell -curl --request POST --header "PRIVATE-TOKEN: token" --data "version=1.0.0&trailer=Type" "https://gitlab.com/api/v4/projects/42/repository/changelog" +curl --request POST --header "PRIVATE-TOKEN: token" \ + --data "version=1.0.0&trailer=Type" "https://gitlab.com/api/v4/projects/42/repository/changelog" ``` To store the results in a different file, use the `file` parameter: ```shell -curl --request POST --header "PRIVATE-TOKEN: token" --data "version=1.0.0&file=NEWS" "https://gitlab.com/api/v4/projects/42/repository/changelog" +curl --request POST --header "PRIVATE-TOKEN: token" \ + --data "version=1.0.0&file=NEWS" "https://gitlab.com/api/v4/projects/42/repository/changelog" ``` ## Generate changelog data @@ -426,25 +432,30 @@ Supported attributes: | Attribute | Type | Required | Description | | :-------- | :------- | :--------- | :---------- | | `version` | string | yes | The version to generate the changelog for. The format must follow [semantic versioning](https://semver.org/). | -| `config_file` | string | no | The path of changelog configuration file in the project's Git repository, defaults to `.gitlab/changelog_config.yml`. | -| `date` | datetime | no | The date and time of the release, ISO 8601 formatted. Example: `2016-03-11T03:45:40Z`. Defaults to the current time. | +| `config_file` | string | no | The path of changelog configuration file in the project's Git repository. Defaults to `.gitlab/changelog_config.yml`. | +| `date` | datetime | no | The date and time of the release. Uses ISO 8601 format. Example: `2016-03-11T03:45:40Z`. Defaults to the current time. | | `from` | string | no | The start of the range of commits (as a SHA) to use for generating the changelog. This commit itself isn't included in the list. | | `to` | string | no | The end of the range of commits (as a SHA) to use for the changelog. This commit _is_ included in the list. Defaults to the HEAD of the default project branch. | -| `trailer` | string | no | The Git trailer to use for including commits, defaults to `Changelog`. | +| `trailer` | string | no | The Git trailer to use for including commits. Defaults to `Changelog`. | ```shell -curl --header "PRIVATE-TOKEN: token" "https://gitlab.com/api/v4/projects/42/repository/changelog?version=1.0.0" +curl --header "PRIVATE-TOKEN: token" \ + "https://gitlab.com/api/v4/projects/42/repository/changelog?version=1.0.0" ``` -Example Response: +Example response, with line breaks added for readability: ```json { - "notes": "## 1.0.0 (2021-11-17)\n\n### feature (2 changes)\n\n- [Title 2](namespace13/project13@ad608eb642124f5b3944ac0ac772fecaf570a6bf) ([merge request](namespace13/project13!2))\n- [Title 1](namespace13/project13@3c6b80ff7034fa0d585314e1571cc780596ce3c8) ([merge request](namespace13/project13!1))\n" + "notes": "## 1.0.0 (2021-11-17)\n\n### feature (2 changes)\n\n- + [Title 2](namespace13/project13@ad608eb642124f5b3944ac0ac772fecaf570a6bf) + ([merge request](namespace13/project13!2))\n- + [Title 1](namespace13/project13@3c6b80ff7034fa0d585314e1571cc780596ce3c8) + ([merge request](namespace13/project13!1))\n" } ``` ## Related topics - User documentation for [changelogs](../user/project/changelogs.md) -- Developer documentation for [changelog entries](../development/changelog.md) in GitLab. +- Developer documentation for [changelog entries](../development/changelog.md) in GitLab diff --git a/doc/api/resource_groups.md b/doc/api/resource_groups.md index cdba0c01f4f..23c982c9296 100644 --- a/doc/api/resource_groups.md +++ b/doc/api/resource_groups.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/api/rest/deprecations.md b/doc/api/rest/deprecations.md new file mode 100644 index 00000000000..db9b590606f --- /dev/null +++ b/doc/api/rest/deprecations.md @@ -0,0 +1,112 @@ +--- +stage: Manage +group: Import and Integrate +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# REST API deprecations and removals + +The following API changes will occur when the v4 API is removed. + +The date of this change is unknown. +For details, see [issue 216456](https://gitlab.com/gitlab-org/gitlab/-/issues/216456) +and [issue 387485](https://gitlab.com/gitlab-org/gitlab/-/issues/387485). + +## `geo_nodes` API endpoints + +Breaking change. [Related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/369140). + +The [`geo_nodes` API endpoints](../geo_nodes.md) are deprecated and are replaced by [`geo_sites`](../geo_sites.md). +It is a part of the global change on [how to refer to Geo deployments](../../administration/geo/glossary.md). +Nodes are renamed to sites across the application. The functionality of both endpoints remains the same. + +## `merged_by` API field + +Breaking change. [Related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/350534). + +The `merged_by` field in the [merge request API](../merge_requests.md#list-merge-requests) +has been deprecated in favor of the `merge_user` field which more correctly identifies who merged a merge request when +performing actions (merge when pipeline succeeds, add to merge train) other than a simple merge. + +API users are encouraged to use the new `merge_user` field instead. The `merged_by` field will be removed in v5 of the GitLab REST API. + +## `merge_status` API field + +Breaking change. [Related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/382032). + +The `merge_status` field in the [merge request API](../merge_requests.md#merge-status) +has been deprecated in favor of the `detailed_merge_status` field which more correctly identifies +all of the potential statuses that a merge request can be in. API users are encouraged to use the +new `detailed_merge_status` field instead. The `merge_status` field will be removed in v5 of the GitLab REST API. + +### Null value for `private_profile` attribute in User API + +Breaking change. [Related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/387005). + +When creating and updating users through the API, `null` was a valid value for the `private_profile` attribute, which would internally be converted to the default value. In v5 of the GitLab REST API, `null` will no longer be a valid value for this parameter, and the response will be a 400 if used. After this change, the only valid values will be `true` and `false`. + +## Single merge request changes API endpoint + +Breaking change. [Related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/322117). + +The endpoint to get +[changes from a single merge request](../merge_requests.md#get-single-merge-request-changes) +has been deprecated in favor the +[list merge request diffs](../merge_requests.md#list-merge-request-diffs) endpoint. +API users are encouraged to switch to the new diffs endpoint instead. + +The `changes from a single merge request` endpoint will be removed in v5 of the GitLab REST API. + +## Managed Licenses API endpoint + +Breaking change. [Related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/397067). + +The endpoint to get +[all managed licenses for a given project](../managed_licenses.md) +has been deprecated in favor the +[License Approval policy](../../user/compliance/license_approval_policies.md) feature. +Users who wish to continue to enforce approvals based on detected licenses are encouraged to create a new [License Approval policy](../../user/compliance/license_approval_policies.md) instead. + +The `managed licenses` endpoint will be removed in v5 of the GitLab REST API. + +## Approvers and Approver Group fields in Merge Request Approval API + +Breaking change. [Related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/353097). + +The endpoint to get the configuration of approvals for a project returns +empty arrays for `approvers` and `approval_groups`. +These fields were deprecated in favor of the endpoint to +[get project-level rules](../merge_request_approvals.md#get-project-level-rules) +for a merge request. API users are encouraged to switch to this endpoint instead. + +These fields will be removed from the `get configuration` endpoint in v5 of the GitLab REST API. + +## Runner usage of `active` replaced by `paused` + +Breaking change. [Related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/351109). + +Occurrences of the `active` identifier in the GitLab Runner GraphQL API endpoints will be +renamed to `paused` in GitLab 16.0. + +- In v4 of the REST API, starting in GitLab 14.8, you can use the `paused` property in place of `active` +- In v5 of the REST API, this change will affect endpoints taking or returning `active` property, such as: + - `GET /runners` + - `GET /runners/all` + - `GET /runners/:id` / `PUT /runners/:id` + - `PUT --form "active=false" /runners/:runner_id` + - `GET /projects/:id/runners` / `POST /projects/:id/runners` + - `GET /groups/:id/runners` + +The 16.0 release of GitLab Runner will start using the `paused` property when registering runners. + +## Runner status will not return `paused` + +Breaking change. [Related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/344648). + +In a future v5 of the REST API, the endpoints for GitLab Runner will not return `paused` or `active`. + +A runner's status will only relate to runner contact status, such as: +`online`, `offline`, or `not_connected`. Status `paused` or `active` will no longer appear. + +When checking if a runner is `paused`, API users are advised to check the boolean attribute +`paused` to be `true` instead. When checking if a runner is `active`, check if `paused` is `false`. diff --git a/doc/api/rest/index.md b/doc/api/rest/index.md index 57d13f2a54f..0bee7168cfb 100644 --- a/doc/api/rest/index.md +++ b/doc/api/rest/index.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -12,7 +12,7 @@ developers who are more comfortable with traditional API architecture. ## Compatibility guidelines -The HTTP API is versioned with a single number, which is currently `4`. This number +The HTTP API is versioned with a single number, which is `4`. This number symbolizes the major version number, as described by [SemVer](https://semver.org/). Because of this, backward-incompatible changes require this version number to change. @@ -83,9 +83,9 @@ authentication isn't provided. When authentication is not required, the document for each endpoint specifies this. For example, the [`/projects/:id` endpoint](../projects.md#get-single-project) does not require authentication. -There are several ways you can authenticate with the GitLab API: +You can authenticate with the GitLab API in several ways: -- [OAuth2 tokens](#oauth2-tokens) +- [OAuth 2.0 tokens](#oauth-20-tokens) - [Personal access tokens](../../user/profile/personal_access_tokens.md) - [Project access tokens](../../user/project/settings/project_access_tokens.md) - [Group access tokens](../../user/group/settings/group_access_tokens.md) @@ -94,8 +94,8 @@ There are several ways you can authenticate with the GitLab API: Project access tokens are supported by: -- Self-managed GitLab Free and higher. -- GitLab SaaS Premium and higher. +- Self-managed GitLab: Free, Premium, and Ultimate. +- GitLab SaaS: Premium and Ultimate. If you are an administrator, you or your application can authenticate as a specific user. To do so, use: @@ -116,30 +116,30 @@ NOTE: Deploy tokens can't be used with the GitLab public API. For details, see [Deploy Tokens](../../user/project/deploy_tokens/index.md). -### OAuth2 tokens +### OAuth 2.0 tokens -You can use an [OAuth2 token](../oauth2.md) to authenticate with the API by passing +You can use an [OAuth 2.0 token](../oauth2.md) to authenticate with the API by passing it in either the `access_token` parameter or the `Authorization` header. -Example of using the OAuth2 token in a parameter: +Example of using the OAuth 2.0 token in a parameter: ```shell curl "https://gitlab.example.com/api/v4/projects?access_token=OAUTH-TOKEN" ``` -Example of using the OAuth2 token in a header: +Example of using the OAuth 2.0 token in a header: ```shell curl --header "Authorization: Bearer OAUTH-TOKEN" "https://gitlab.example.com/api/v4/projects" ``` -Read more about [GitLab as an OAuth2 provider](../oauth2.md). +Read more about [GitLab as an OAuth 2.0 provider](../oauth2.md). NOTE: -We recommend OAuth access tokens have an expiration. You can use the `refresh_token` parameter +You should give OAuth access tokens an expiration. You can use the `refresh_token` parameter to refresh tokens. Integrations may need to be updated to use refresh tokens prior to expiration, which is based on the [`expires_in`](https://datatracker.ietf.org/doc/html/rfc6749#appendix-A.14) -property in the token endpoint response. See [OAuth2 token](../oauth2.md) documentation +property in the token endpoint response. See [OAuth 2.0 token](../oauth2.md) documentation for examples requesting a new access token using a refresh token. A default refresh setting of two hours is tracked in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/336598). @@ -299,52 +299,52 @@ insight into what went wrong. The following table gives an overview of how the API functions generally behave. -| Request type | Description | -|---------------|-------------| -| `GET` | Access one or more resources and return the result as JSON. | -| `POST` | Return `201 Created` if the resource is successfully created and return the newly created resource as JSON. | -| `GET` / `PUT` | Return `200 OK` if the resource is accessed or modified successfully. The (modified) result is returned as JSON. | +| Request type | Description | +|:--------------|:--------------------------------------------------------------------------------------------------------------------------------| +| `GET` | Access one or more resources and return the result as JSON. | +| `POST` | Return `201 Created` if the resource is successfully created and return the newly created resource as JSON. | +| `GET` / `PUT` | Return `200 OK` if the resource is accessed or modified successfully. The (modified) result is returned as JSON. | | `DELETE` | Returns `204 No Content` if the resource was deleted successfully or `202 Accepted` if the resource is scheduled to be deleted. | The following table shows the possible return codes for API requests. -| Return values | Description | -|--------------------------|-------------| -| `200 OK` | The `GET`, `PUT` or `DELETE` request was successful, and the resource itself is returned as JSON. | -| `202 Accepted` | The `GET`, `PUT` or `DELETE` request was successful, and the resource is scheduled for processing. | -| `204 No Content` | The server has successfully fulfilled the request, and there is no additional content to send in the response payload body. | -| `201 Created` | The `POST` request was successful, and the resource is returned as JSON. | -| `304 Not Modified` | The resource hasn't been modified since the last request. | -| `400 Bad Request` | A required attribute of the API request is missing. For example, the title of an issue is not given. | -| `401 Unauthorized` | The user isn't authenticated. A valid [user token](#authentication) is necessary. | -| `403 Forbidden` | The request isn't allowed. For example, the user isn't allowed to delete a project. | -| `404 Not Found` | A resource couldn't be accessed. For example, an ID for a resource couldn't be found. | -| `405 Method Not Allowed` | The request isn't supported. | -| `409 Conflict` | A conflicting resource already exists. For example, creating a project with a name that already exists. | -| `412` | The request was denied. This can happen if the `If-Unmodified-Since` header is provided when trying to delete a resource, which was modified in between. | -| `422 Unprocessable` | The entity couldn't be processed. | -| `429 Too Many Requests` | The user exceeded the [application rate limits](../../administration/instance_limits.md#rate-limits). | -| `500 Server Error` | While handling the request, something went wrong on the server. | +| Return values | Description | +|:--------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------| +| `200 OK` | The `GET`, `PUT` or `DELETE` request was successful, and the resource itself is returned as JSON. | +| `201 Created` | The `POST` request was successful, and the resource is returned as JSON. | +| `202 Accepted` | The `GET`, `PUT` or `DELETE` request was successful, and the resource is scheduled for processing. | +| `204 No Content` | The server has successfully fulfilled the request, and there is no additional content to send in the response payload body. | +| `304 Not Modified` | The resource hasn't been modified since the last request. | +| `400 Bad Request` | A required attribute of the API request is missing. For example, the title of an issue is not given. | +| `401 Unauthorized` | The user isn't authenticated. A valid [user token](#authentication) is necessary. | +| `403 Forbidden` | The request isn't allowed. For example, the user isn't allowed to delete a project. | +| `404 Not Found` | A resource couldn't be accessed. For example, an ID for a resource couldn't be found, or the user isn't authorized to access the resource. | +| `405 Method Not Allowed` | The request isn't supported. | +| `409 Conflict` | A conflicting resource already exists. For example, creating a project with a name that already exists. | +| `412 Precondition Failed` | The request was denied. This can happen if the `If-Unmodified-Since` header is provided when trying to delete a resource, which was modified in between. | +| `422 Unprocessable` | The entity couldn't be processed. | +| `429 Too Many Requests` | The user exceeded the [application rate limits](../../administration/instance_limits.md#rate-limits). | +| `500 Server Error` | While handling the request, something went wrong on the server. | ## Pagination GitLab supports the following pagination methods: -- Offset-based pagination. This is the default method and is available on all endpoints. +- Offset-based pagination. The default method and available on all endpoints. - Keyset-based pagination. Added to selected endpoints but being [progressively rolled out](https://gitlab.com/groups/gitlab-org/-/epics/2039). -For large collections, for performance reasons we recommend keyset pagination -(when available) instead of offset pagination. +For large collections, you should use keyset pagination +(when available) instead of offset pagination, for performance reasons. ### Offset-based pagination Sometimes, the returned result spans many pages. When listing resources, you can pass the following parameters: -| Parameter | Description | -|------------|-------------| -| `page` | Page number (default: `1`). | +| Parameter | Description | +|:-----------|:--------------------------------------------------------------| +| `page` | Page number (default: `1`). | | `per_page` | Number of items to list per page (default: `20`, max: `100`). | In the following example, we list 50 [namespaces](../namespaces.md) per page: @@ -397,14 +397,14 @@ x-total-pages: 3 GitLab also returns the following additional pagination headers: -| Header | Description | -|-----------------|-------------| -| `x-next-page` | The index of the next page. | +| Header | Description | +|:----------------|:-----------------------------------------------| +| `x-next-page` | The index of the next page. | | `x-page` | The index of the current page (starting at 1). | -| `x-per-page` | The number of items per page. | -| `x-prev-page` | The index of the previous page. | -| `x-total` | The total number of items. | -| `x-total-pages` | The total number of pages. | +| `x-per-page` | The number of items per page. | +| `x-prev-page` | The index of the previous page. | +| `x-total` | The total number of items. | +| `x-total-pages` | The total number of pages. | For GitLab.com users, [some pagination headers may not be returned](../../user/gitlab_com/index.md#pagination-response-headers). @@ -476,19 +476,23 @@ When the end of the collection is reached and there are no additional records to retrieve, the `Link` header is absent and the resulting array is empty. -We recommend using only the given link to retrieve the next page instead of +You should use only the given link to retrieve the next page instead of building your own URL. Apart from the headers shown, we don't expose additional pagination headers. +#### Supported resources + Keyset-based pagination is supported only for selected resources and ordering options: -| Resource | Options | Availability | -|:---------------------------------------------------------|:---------------------------------|:-------------------------------------------------------------------------------------------------------------| -| [Projects](../projects.md) | `order_by=id` only | Authenticated and unauthenticated users | -| [Groups](../groups.md) | `order_by=name`, `sort=asc` only | Unauthenticated users only | -| [Group audit events](../audit_events.md#group-audit-events) | `order_by=id`, `sort=desc` only | Authenticated users only ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/333968) in GitLab 15.2) | -| [Jobs](../jobs.md) | `order_by=id`, `sort=desc` only | Authenticated users only ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362172) in GitLab 15.9) | +| Resource | Options | Availability | +|:------------------------------------------------------------------|:---------------------------------|:--------------------------------------------------------------------------------------------------------------| +| [Group audit events](../audit_events.md#group-audit-events) | `order_by=id`, `sort=desc` only | Authenticated users only ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/333968) in GitLab 15.2) | +| [Groups](../groups.md) | `order_by=name`, `sort=asc` only | Unauthenticated users only | +| [Instance audit events](../audit_events.md#instance-audit-events) | `order_by=id`, `sort=desc` only | Authenticated users only ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/367528) in GitLab 15.11) | +| [Jobs](../jobs.md) | `order_by=id`, `sort=desc` only | Authenticated users only ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362172) in GitLab 15.9) | +| [Project audit events](../audit_events.md#project-audit-events) | `order_by=id`, `sort=desc` only | Authenticated users only ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/367528) in GitLab 15.10) | +| [Projects](../projects.md) | `order_by=id` only | Authenticated and unauthenticated users | ### Pagination response headers @@ -642,6 +646,14 @@ For example, suppose a project with `id: 42` has an issue with `id: 46` and Not all resources with the `iid` field are fetched by `iid`. For guidance regarding which field to use, see the documentation for the specific resource. +## `null` vs `false` + +In API responses, some boolean fields can have `null` values. +A `null` boolean has no default value and is neither `true` nor `false`. +GitLab treats `null` values in boolean fields the same as `false`. + +In boolean arguments, you should only set `true` or `false` values (not `null`). + ## Data validation and error reporting When working with the API you may encounter validation errors, in which case diff --git a/doc/api/runners.md b/doc/api/runners.md index be69b762555..a2614b95bb9 100644 --- a/doc/api/runners.md +++ b/doc/api/runners.md @@ -273,10 +273,10 @@ PUT /runners/:id | `id` | integer | yes | The ID of a runner | | `description` | string | no | The description of the runner | | `active` | boolean | no | Deprecated: Use `paused` instead. Flag indicating whether the runner is allowed to receive jobs | -| `paused` | boolean | no | Specifies whether the runner should ignore new jobs | +| `paused` | boolean | no | Specifies if the runner should ignore new jobs | | `tag_list` | array | no | The list of tags for the runner | -| `run_untagged` | boolean | no | Specifies whether the runner can execute untagged jobs | -| `locked` | boolean | no | Specifies whether the runner is locked | +| `run_untagged` | boolean | no | Specifies if the runner can execute untagged jobs | +| `locked` | boolean | no | Specifies if the runner is locked | | `access_level` | string | no | The access level of the runner; `not_protected` or `ref_protected` | | `maximum_timeout` | integer | no | Maximum timeout that limits the amount of time (in seconds) that runners can run jobs | @@ -656,20 +656,20 @@ Register a new runner for the instance. POST /runners ``` -| Attribute | Type | Required | Description | -|--------------------|--------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `token` | string | yes | [Registration token](#registration-and-authentication-tokens) | -| `description` | string | no | Runner's description | +| Attribute | Type | Required | Description | +|--------------------|--------------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `token` | string | yes | [Registration token](#registration-and-authentication-tokens) | +| `description` | string | no | Description of the runner | | `info` | hash | no | Runner's metadata. You can include `name`, `version`, `revision`, `platform`, and `architecture`, but only `version`, `platform`, and `architecture` are displayed in the Admin Area of the UI | -| `active` | boolean | no | Deprecated: Use `paused` instead. Specifies whether the runner is allowed to receive new jobs | -| `paused` | boolean | no | Specifies whether the runner should ignore new jobs | -| `locked` | boolean | no | Specifies whether the runner should be locked for the current project | -| `run_untagged` | boolean | no | Specifies whether the runner should handle untagged jobs | -| `tag_list` | string array | no | A list of runner tags | -| `access_level` | string | no | The access level of the runner; `not_protected` or `ref_protected` | -| `maximum_timeout` | integer | no | Maximum timeout that limits the amount of time (in seconds) that runners can run jobs | -| `maintainer_note` | string | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/350730), see `maintenance_note` | -| `maintenance_note` | string | no | Free-form maintenance notes for the runner (1024 characters) | +| `active` | boolean | no | Deprecated: Use `paused` instead. Specifies if the runner is allowed to receive new jobs | +| `paused` | boolean | no | Specifies if the runner should ignore new jobs | +| `locked` | boolean | no | Specifies if the runner should be locked for the current project | +| `run_untagged` | boolean | no | Specifies if the runner should handle untagged jobs | +| `tag_list` | string array | no | A list of runner tags | +| `access_level` | string | no | The access level of the runner; `not_protected` or `ref_protected` | +| `maximum_timeout` | integer | no | Maximum timeout that limits the amount of time (in seconds) that runners can run jobs | +| `maintainer_note` | string | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/350730), see `maintenance_note` | +| `maintenance_note` | string | no | Free-form maintenance notes for the runner (1024 characters) | ```shell curl --request POST "https://gitlab.example.com/api/v4/runners" \ @@ -679,9 +679,11 @@ curl --request POST "https://gitlab.example.com/api/v4/runners" \ Response: -| Status | Description | -|-----------|---------------------------------| -| 201 | Runner was created | +| Status | Description | +|-----------|-----------------------------------| +| 201 | Runner was created | +| 403 | Invalid runner registration token | +| 410 | Runner registration disabled | Example response: @@ -774,7 +776,7 @@ Example response: } ``` -## Reset instance's runner registration token +## Reset instance's runner registration token (deprecated) > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30942) in GitLab 14.3. > - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104691) in GitLab 15.7. @@ -793,7 +795,7 @@ curl --request POST --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/runners/reset_registration_token" ``` -## Reset project's runner registration token +## Reset project's runner registration token (deprecated) > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30942) in GitLab 14.3. > - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104691) in GitLab 15.7. @@ -812,7 +814,7 @@ curl --request POST --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects/9/runners/reset_registration_token" ``` -## Reset group's runner registration token +## Reset group's runner registration token (deprecated) > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30942) in GitLab 14.3. > - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104691) in GitLab 15.7. diff --git a/doc/api/search.md b/doc/api/search.md index 3c3ad3f6ab9..f7fa7babe71 100644 --- a/doc/api/search.md +++ b/doc/api/search.md @@ -31,8 +31,8 @@ GET /search | `scope` | string | Yes | The scope to search in. Values include `projects`, `issues`, `merge_requests`, `milestones`, `snippet_titles`, `users`. [Additional scopes](#additional-scopes): `blobs`, `commits`, `notes`, `wiki_blobs`. | | `search` | string | Yes | The search query. | | `confidential` | boolean | No | Filter by confidentiality. Supports `issues` scope; other scopes are ignored. | -| `order_by` | string | No | Allowed values are `created_at` only. If not set, results are sorted by `created_at` in descending order for basic search, or by the most relevant documents for Advanced Search.| -| `sort` | string | No | Allowed values are `asc` or `desc` only. If not set, results are sorted by `created_at` in descending order for basic search, or by the most relevant documents for Advanced Search.| +| `order_by` | string | No | Allowed values are `created_at` only. If not set, results are sorted by `created_at` in descending order for basic search, or by the most relevant documents for advanced search.| +| `sort` | string | No | Allowed values are `asc` or `desc` only. If not set, results are sorted by `created_at` in descending order for basic search, or by the most relevant documents for advanced search.| | `state` | string | No | Filter by state. Supports `issues` and `merge_requests` scopes; other scopes are ignored. | ### Scope: `projects` @@ -407,6 +407,7 @@ Example response: "system": false, "noteable_id": 22, "noteable_type": "Issue", + "project_id": 6, "noteable_iid": 2 } ] @@ -449,8 +450,8 @@ GET /groups/:id/search | `scope` | string | Yes | The scope to search in. Values include `issues`, `merge_requests`, `milestones`, `projects`, `users`. [Additional scopes](#additional-scopes): `blobs`, `commits`, `notes`, `wiki_blobs`. | | `search` | string | Yes | The search query. | | `confidential` | boolean | No | Filter by confidentiality. Supports only `issues` scope; other scopes are ignored. | -| `order_by` | string | No | Allowed values are `created_at` only. If not set, results are sorted by `created_at` in descending order for basic search, or by the most relevant documents for Advanced Search.| -| `sort` | string | No | Allowed values are `asc` or `desc` only. If not set, results are sorted by `created_at` in descending order for basic search, or by the most relevant documents for Advanced Search.| +| `order_by` | string | No | Allowed values are `created_at` only. If not set, results are sorted by `created_at` in descending order for basic search, or by the most relevant documents for advanced search.| +| `sort` | string | No | Allowed values are `asc` or `desc` only. If not set, results are sorted by `created_at` in descending order for basic search, or by the most relevant documents for advanced search.| | `state` | string | No | Filter by state. Supports `issues` and `merge_requests` only; other scopes are ignored. | The response depends on the requested scope. @@ -796,6 +797,7 @@ Example response: "system": false, "noteable_id": 22, "noteable_type": "Issue", + "project_id": 6, "noteable_iid": 2 } ] @@ -839,8 +841,8 @@ GET /projects/:id/search | `search` | string | Yes | The search query. | | `confidential` | boolean | No | Filter by confidentiality. Supports `issues` scope; other scopes are ignored. | | `ref` | string | No | The name of a repository branch or tag to search on. The project's default branch is used by default. Applicable only for scopes `blobs`, `commits`, and `wiki_blobs`. | -| `order_by` | string | No | Allowed values are `created_at` only. If not set, results are sorted by `created_at` in descending order for basic search, or by the most relevant documents for Advanced Search.| -| `sort` | string | No | Allowed values are `asc` or `desc` only. If not set, results are sorted by `created_at` in descending order for basic search, or by the most relevant documents for Advanced Search.| +| `order_by` | string | No | Allowed values are `created_at` only. If not set, results are sorted by `created_at` in descending order for basic search, or by the most relevant documents for advanced search.| +| `sort` | string | No | Allowed values are `asc` or `desc` only. If not set, results are sorted by `created_at` in descending order for basic search, or by the most relevant documents for advanced search.| | `state` | string | No | Filter by state. Supports the `issues` and `merge_requests` scopes; other scopes are ignored. | The response depends on the requested scope. @@ -1046,6 +1048,7 @@ Example response: "system": false, "noteable_id": 22, "noteable_type": "Issue", + "project_id": 6, "noteable_iid": 2 } ] diff --git a/doc/api/secure_files.md b/doc/api/secure_files.md index 11c718dde01..0b85bf05410 100644 --- a/doc/api/secure_files.md +++ b/doc/api/secure_files.md @@ -1,16 +1,23 @@ --- stage: Verify -group: Pipeline Authoring +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: reference, api --- # Project-level Secure Files API **(FREE)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78227) in GitLab 14.8. [Deployed behind the `ci_secure_files` flag](../administration/feature_flags.md), disabled by default. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/350748) in GitLab 15.7. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78227) in GitLab 14.8 [with a flag](../administration/feature_flags.md) named `ci_secure_files`. Disabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/350748) in GitLab 15.7. Feature flag `ci_secure_files` removed. -Limited to 100 secure files per project. Files must be smaller than 5 MB. Project-level Secure Files is an experimental feature developed by [GitLab Incubation Engineering](https://about.gitlab.com/handbook/engineering/incubation/). +This feature is part of [Mobile DevOps](../ci/mobile_devops.md) developed by [GitLab Incubation Engineering](https://about.gitlab.com/handbook/engineering/incubation/). +The feature is still in development, but you can: + +- [Request a feature](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=feature_request). +- [Report a bug](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=report_bug). +- [Share feedback](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=general_feedback). + +You can securely store up to 100 files for use in CI/CD pipelines as secure files. These files are stored securely outside of your project's repository and are not version controlled. It is safe to store sensitive information in these files. Secure files support both plain text and binary file types but must be 5 MB or less. ## List project secure files @@ -51,7 +58,7 @@ Example response: "checksum": "16630b189ab34b2e3504f4758e1054d2e478deda510b2b08cc0ef38d12e80aa2", "checksum_algorithm": "sha256", "created_at": "2022-02-22T22:22:22.222Z", - "expires_at": "2022-09-21T14:56:00.000Z", + "expires_at": "2023-09-21T14:55:59.000Z", "metadata": { "id":"75949910542696343243264405377658443914", "issuer": { @@ -67,7 +74,7 @@ Example response: "OU":"ABC123XYZ", "UID":"ABC123XYZ" }, - "expires_at":"2022-09-21T14:56:00.000Z" + "expires_at":"2023-09-21T14:55:59.000Z" } } ] diff --git a/doc/api/settings.md b/doc/api/settings.md index 94ed2b2e3db..c7ba93d310e 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -107,23 +107,29 @@ Example response: "external_pipeline_validation_service_token": null, "external_pipeline_validation_service_url": null, "jira_connect_application_key": null, - "jira_connect_proxy_url": null + "jira_connect_proxy_url": null, + "silent_mode_enabled": false } ``` Users on [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/) may also see the `group_owners_can_manage_default_branch_protection`, `file_template_project_id`, `delayed_project_deletion`, -`delayed_group_deletion`, `deletion_adjourned_period`, `disable_personal_access_tokens`, or the `geo_node_allowed_ips` parameters: +`delayed_group_deletion`, `default_project_deletion_protection`, `deletion_adjourned_period`, `disable_personal_access_tokens`, `geo_node_allowed_ips`, +or the `security_policy_global_group_approvers_enabled` parameters. + +From [GitLab 15.11](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/113332), with the `always_perform_delayed_deletion` feature flag enabled, +the `delayed_project_deletion` and `delayed_group_deletion` attributes will not be exposed. These attributes will be removed in GitLab 16.0. ```json { - "id" : 1, - "signup_enabled" : true, - "group_owners_can_manage_default_branch_protection" : true, + "id": 1, + "signup_enabled": true, + "group_owners_can_manage_default_branch_protection": true, "file_template_project_id": 1, "geo_node_allowed_ips": "0.0.0.0/0, ::/0", "delayed_project_deletion": false, "delayed_group_deletion": false, + "default_project_deletion_protection": false, "deletion_adjourned_period": 7, "disable_personal_access_tokens": false, ... @@ -227,7 +233,10 @@ Example response: "can_create_group": false, "jira_connect_application_key": "123", "jira_connect_proxy_url": "http://gitlab.example.com", - "user_defaults_to_private_profile": true + "user_defaults_to_private_profile": true, + "projects_api_rate_limit_unauthenticated": 400, + "silent_mode_enabled": false, + "security_policy_global_group_approvers_enabled": true } ``` @@ -240,8 +249,13 @@ these parameters: - `geo_status_timeout` - `delayed_project_deletion` - `delayed_group_deletion` +- `default_project_deletion_protection` - `deletion_adjourned_period` - `disable_personal_access_tokens` +- `security_policy_global_group_approvers_enabled` + +From [GitLab 15.11](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/113332), with the `always_perform_delayed_deletion` feature flag enabled, +the `delayed_project_deletion` and `delayed_group_deletion` attributes will not be exposed. These attributes will be removed in GitLab 16.0. Example responses: **(PREMIUM SELF)** @@ -268,9 +282,9 @@ listed in the descriptions of the relevant settings. | `akismet_api_key` | string | required by: `akismet_enabled` | API key for Akismet spam protection. | | `akismet_enabled` | boolean | no | (**If enabled, requires:** `akismet_api_key`) Enable or disable Akismet spam protection. | | `allow_group_owners_to_manage_ldap` **(PREMIUM)** | boolean | no | Set to `true` to allow group owners to manage LDAP. | -| `allow_local_requests_from_hooks_and_services` | boolean | no | (Deprecated: Use `allow_local_requests_from_web_hooks_and_services` instead) Allow requests to the local network from hooks and services. | +| `allow_local_requests_from_hooks_and_services` | boolean | no | (Deprecated: Use `allow_local_requests_from_web_hooks_and_services` instead) Allow requests to the local network from webhooks and integrations. | | `allow_local_requests_from_system_hooks` | boolean | no | Allow requests to the local network from system hooks. | -| `allow_local_requests_from_web_hooks_and_services` | boolean | no | Allow requests to the local network from web hooks and services. | +| `allow_local_requests_from_web_hooks_and_services` | boolean | no | Allow requests to the local network from webhooks and integrations. | | `allow_runner_registration_token` | boolean | no | Allow using a registration token to create a runner. Defaults to `true`. | | `archive_builds_in_human_readable` | string | no | Set the duration for which the jobs are considered as old and expired. After that time passes, the jobs are archived and no longer able to be retried. Make it empty to never expire jobs. It has to be no less than 1 day, for example: 15 days, 1 month, 2 years. | | `asset_proxy_enabled` | boolean | no | (**If enabled, requires:** `asset_proxy_url`) Enable proxying of assets. GitLab restart is required to apply changes. | @@ -282,9 +296,10 @@ listed in the descriptions of the relevant settings. | `auto_devops_domain` | string | no | Specify a domain to use by default for every project's Auto Review Apps and Auto Deploy stages. | | `auto_devops_enabled` | boolean | no | Enable Auto DevOps for projects by default. It automatically builds, tests, and deploys applications based on a predefined CI/CD configuration. | | `automatic_purchased_storage_allocation` | boolean | no | Enabling this permits automatic allocation of purchased storage in a namespace. | -| `bulk_import_enabled` | boolean | no | Enable migrating GitLab groups by direct transfer. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/383268) in GitLab 15.8. Requires [`bulk_import_projects` feature flag](../user/group/import/index.md#migrate-groups-by-direct-transfer-recommended) to also migrate projects. Setting also [available](../user/admin_area/settings/visibility_and_access_controls.md#enable-migration-of-groups-and-projects-by-direct-transfer) in the Admin Area. | +| `bulk_import_enabled` | boolean | no | Enable migrating GitLab groups by direct transfer. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/383268) in GitLab 15.8. Setting also [available](../user/admin_area/settings/visibility_and_access_controls.md#enable-migration-of-groups-and-projects-by-direct-transfer) in the Admin Area. | | `can_create_group` | boolean | no | Indicates whether users can create top-level groups. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/367754) in GitLab 15.5. Defaults to `true`. | | `check_namespace_plan` **(PREMIUM)** | boolean | no | Enabling this makes only licensed EE features available to projects if the project namespace's plan includes the feature or if the project is public. | +| `ci_max_includes` | integer | no | The maximum number of [includes](../ci/yaml/includes.md) per pipeline. Default is `150`. | | `commit_email_hostname` | string | no | Custom hostname (for private commit emails). | | `container_expiration_policies_enable_historic_entries` | boolean | no | Enable [cleanup policies](../user/packages/container_registry/reduce_container_registry_storage.md#enable-the-cleanup-policy) for all projects. | | `container_registry_cleanup_tags_service_max_list_size` | integer | no | The maximum number of tags that can be deleted in a single execution of [cleanup policies](../user/packages/container_registry/reduce_container_registry_storage.md#set-cleanup-limits-to-conserve-resources). | @@ -305,8 +320,10 @@ listed in the descriptions of the relevant settings. | `default_project_visibility` | string | no | What visibility level new projects receive. Can take `private`, `internal` and `public` as a parameter. Default is `private`. | | `default_projects_limit` | integer | no | Project limit per user. Default is `100000`. | | `default_snippet_visibility` | string | no | What visibility level new snippets receive. Can take `private`, `internal` and `public` as a parameter. Default is `private`. | -| `delayed_project_deletion` **(PREMIUM SELF)** | boolean | no | Enable delayed project deletion by default in new groups. Default is `false`. [From GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/issues/352960), can only be enabled when `delayed_group_deletion` is true. | -| `delayed_group_deletion` **(PREMIUM SELF)** | boolean | no | Enable delayed group deletion. Default is `true`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352959) in GitLab 15.0. [From GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/issues/352960), disables and locks the group-level setting for delayed protect deletion when set to `false`. | +| `default_syntax_highlighting_theme` | integer | no | Default syntax highlighting theme for new users and users who are not signed in. See [IDs of available themes](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/themes.rb#L16). +| `delayed_project_deletion` **(PREMIUM SELF)** | boolean | no | Enable delayed project deletion by default in new groups. Default is `false`. [From GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/issues/352960), can only be enabled when `delayed_group_deletion` is true. From [GitLab 15.11](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/113332), with the `always_perform_delayed_deletion` feature flag enabled, this attribute has been removed. This attribute will be completely removed in GitLab 16.0. | +| `delayed_group_deletion` **(PREMIUM SELF)** | boolean | no | Enable delayed group deletion. Default is `true`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352959) in GitLab 15.0. [From GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/issues/352960), disables and locks the group-level setting for delayed protect deletion when set to `false`. From [GitLab 15.11](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/113332), with the `always_perform_delayed_deletion` feature flag enabled, this attribute has been removed. This attribute will be completely removed in GitLab 16.0. | +| `default_project_deletion_protection` **(PREMIUM SELF)** | boolean | no | Enable default project deletion protection so only administrators can delete projects. Default is `false`. | | `deletion_adjourned_period` **(PREMIUM SELF)** | integer | no | The number of days to wait before deleting a project or group that is marked for deletion. Value must be between `1` and `90`. Defaults to `7`. [From GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/issues/352960), a hook on `deletion_adjourned_period` sets the period to `1` on every update, and sets both `delayed_project_deletion` and `delayed_group_deletion` to `false` if the period is `0`. | | `diff_max_patch_bytes` | integer | no | Maximum [diff patch size](../user/admin_area/diff_limits.md), in bytes. | | `diff_max_files` | integer | no | Maximum [files in a diff](../user/admin_area/diff_limits.md). | @@ -315,7 +332,7 @@ listed in the descriptions of the relevant settings. | `disable_feed_token` | boolean | no | Disable display of RSS/Atom and calendar feed tokens. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/231493) in GitLab 13.7. | | `disable_personal_access_token` **(PREMIUM SELF)** | boolean | no | Disable personal access tokens. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/384201) in GitLab 15.7. | | `disabled_oauth_sign_in_sources` | array of strings | no | Disabled OAuth sign-in sources. | -| `dns_rebinding_protection_enabled` | boolean | no | Enforce DNS rebinding attack protection. | +| `dns_rebinding_protection_enabled` | boolean | no | Enforce DNS-rebinding attack protection. | | `domain_denylist_enabled` | boolean | no | (**If enabled, requires:** `domain_denylist`) Allows blocking sign-ups from emails from specific domains. | | `domain_denylist` | array of strings | no | Users with email addresses that match these domains **cannot** sign up. Wildcards allowed. Use separate lines for multiple entries. For example: `domain.com`, `*.domain.com`. | | `domain_allowlist` | array of strings | no | Force people to use only corporate emails for sign-up. Default is `null`, meaning there is no restriction. | @@ -368,6 +385,7 @@ listed in the descriptions of the relevant settings. | `gitaly_timeout_default` | integer | no | Default Gitaly timeout, in seconds. This timeout is not enforced for Git fetch/push operations or Sidekiq jobs. Set to `0` to disable timeouts. | | `gitaly_timeout_fast` | integer | no | Gitaly fast operation timeout, in seconds. Some Gitaly operations are expected to be fast. If they exceed this threshold, there may be a problem with a storage shard and 'failing fast' can help maintain the stability of the GitLab instance. Set to `0` to disable timeouts. | | `gitaly_timeout_medium` | integer | no | Medium Gitaly timeout, in seconds. This should be a value between the Fast and the Default timeout. Set to `0` to disable timeouts. | +| `gitlab_dedicated_instance` | boolean | no | Indicates whether the instance was provisioned for GitLab Dedicated. | | `grafana_enabled` | boolean | no | Enable Grafana. | | `grafana_url` | string | no | Grafana URL. | | `gravatar_enabled` | boolean | no | Enable Gravatar. | @@ -386,7 +404,7 @@ listed in the descriptions of the relevant settings. | `housekeeping_incremental_repack_period` | integer | no | Deprecated. Number of Git pushes after which an incremental `git repack` is run. Use `housekeeping_optimize_repository_period` instead. For more information, see [Housekeeping fields](#housekeeping-fields).| | `housekeeping_optimize_repository_period`| integer | no | Number of Git pushes after which an incremental `git repack` is run. | | `html_emails_enabled` | boolean | no | Enable HTML emails. | -| `import_sources` | array of strings | no | Sources to allow project import from, possible values: `github`, `bitbucket`, `bitbucket_server`, `gitlab`, `fogbugz`, `git`, `gitlab_project`, `gitea`, `manifest`, and `phabricator`. | +| `import_sources` | array of strings | no | Sources to allow project import from, possible values: `github`, `bitbucket`, `bitbucket_server`, `fogbugz`, `git`, `gitlab_project`, `gitea`, and `manifest`. | | `in_product_marketing_emails_enabled` | boolean | no | Enable [in-product marketing emails](../user/profile/notifications.md#global-notification-settings). Enabled by default. | | `invisible_captcha_enabled` | boolean | no | Enable Invisible CAPTCHA spam detection during sign-up. Disabled by default. | | `issues_create_limit` | integer | no | Max number of issue creation requests per minute per user. Disabled by default.| @@ -417,7 +435,7 @@ listed in the descriptions of the relevant settings. | `maven_package_requests_forwarding` **(PREMIUM)** | boolean | no | Use repo.maven.apache.org as a default remote repository when the package is not found in the GitLab Package Registry for Maven. | | `npm_package_requests_forwarding` **(PREMIUM)** | boolean | no | Use npmjs.org as a default remote repository when the package is not found in the GitLab Package Registry for npm. | | `pypi_package_requests_forwarding` **(PREMIUM)** | boolean | no | Use pypi.org as a default remote repository when the package is not found in the GitLab Package Registry for PyPI. | -| `outbound_local_requests_whitelist` | array of strings | no | Define a list of trusted domains or IP addresses to which local requests are allowed when local requests for hooks and services are disabled. +| `outbound_local_requests_whitelist` | array of strings | no | Define a list of trusted domains or IP addresses to which local requests are allowed when local requests for webhooks and integrations are disabled. | `pages_domain_verification_enabled` | boolean | no | Require users to prove ownership of custom domains. Domain verification is an essential security measure for public GitLab sites. Users are required to demonstrate they control a domain before it is enabled. | | `password_authentication_enabled_for_git` | boolean | no | Enable authentication for Git over HTTP(S) via a GitLab account password. Default is `true`. | | `password_authentication_enabled_for_web` | boolean | no | Enable authentication for the web interface via a GitLab account password. Default is `true`. | @@ -434,6 +452,7 @@ listed in the descriptions of the relevant settings. | `plantuml_url` | string | required by: `plantuml_enabled` | The PlantUML instance URL for integration. | | `polling_interval_multiplier` | decimal | no | Interval multiplier used by endpoints that perform polling. Set to `0` to disable polling. | | `project_export_enabled` | boolean | no | Enable project export. | +| `projects_api_rate_limit_unauthenticated` | integer | no | [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/112283) in GitLab 15.10. Max number of requests per 10 minutes per IP address for unauthenticated requests to the [list all projects API](projects.md#list-all-projects). Default: 400. To disable throttling set to 0.| | `prometheus_metrics_enabled` | boolean | no | Enable Prometheus metrics. | | `protected_ci_variables` | boolean | no | CI/CD variables are protected by default. | | `push_event_activities_limit` | integer | no | Number of changes (branches or tags) in a single push to determine whether individual push events or bulk push events are created. [Bulk push events are created](../user/admin_area/settings/push_event_activities_limit.md) if it surpasses that value. | @@ -456,6 +475,7 @@ listed in the descriptions of the relevant settings. | `restricted_visibility_levels` | array of strings | no | Selected levels cannot be used by non-Administrator users for groups, projects or snippets. Can take `private`, `internal` and `public` as a parameter. Default is `null` which means there is no restriction. | | `rsa_key_restriction` | integer | no | The minimum allowed bit length of an uploaded RSA key. Default is `0` (no restriction). `-1` disables RSA keys. | | `session_expire_delay` | integer | no | Session duration in minutes. GitLab restart is required to apply changes. | +| `security_policy_global_group_approvers_enabled` | boolean | no | Whether to look up scan result policy approval groups globally or within project hierarchies. | | `shared_runners_enabled` | boolean | no | (**If enabled, requires:** `shared_runners_text` and `shared_runners_minutes`) Enable shared runners for new projects. | | `shared_runners_minutes` **(PREMIUM)** | integer | required by: `shared_runners_enabled` | Set the maximum number of CI/CD minutes that a group can use on shared runners per month. | | `shared_runners_text` | string | required by: `shared_runners_enabled` | Shared runners text. | @@ -465,11 +485,12 @@ listed in the descriptions of the relevant settings. | `sign_in_text` | string | no | Text on the login page. | | `signin_enabled` | string | no | (Deprecated: Use `password_authentication_enabled_for_web` instead) Flag indicating if password authentication is enabled for the web interface. | | `signup_enabled` | boolean | no | Enable registration. Default is `true`. | -| `slack_app_enabled` **(PREMIUM)** | boolean | no | (**If enabled, requires:** `slack_app_id`, `slack_app_secret` and `slack_app_secret`) Enable Slack app. | -| `slack_app_id` **(PREMIUM)** | string | required by: `slack_app_enabled` | The app ID of the Slack-app. | -| `slack_app_secret` **(PREMIUM)** | string | required by: `slack_app_enabled` | The app secret of the Slack-app. | -| `slack_app_signing_secret` **(PREMIUM)** | string | no | The signing secret of the Slack-app. | -| `slack_app_verification_token` **(PREMIUM)** | string | required by: `slack_app_enabled` | The verification token of the Slack-app. | +| `silent_mode_enabled` | boolean | no | Enable [Silent mode](../administration/silent_mode/index.md). Default is `false`. | +| `slack_app_enabled` | boolean | no | (**If enabled, requires:** `slack_app_id`, `slack_app_secret` and `slack_app_secret`) Enable Slack app. | +| `slack_app_id` | string | required by: `slack_app_enabled` | The app ID of the Slack-app. | +| `slack_app_secret` | string | required by: `slack_app_enabled` | The app secret of the Slack-app. | +| `slack_app_signing_secret` | string | no | The signing secret of the Slack-app. | +| `slack_app_verification_token` | string | required by: `slack_app_enabled` | The verification token of the Slack-app. | | `snippet_size_limit` | integer | no | Max snippet content size in **bytes**. Default: 52428800 Bytes (50 MB).| | `snowplow_app_id` | string | no | The Snowplow site name / application ID. (for example, `gitlab`) | | `snowplow_collector_hostname` | string | required by: `snowplow_enabled` | The Snowplow collector hostname. (for example, `snowplow.trx.gitlab.net`) | @@ -518,6 +539,7 @@ listed in the descriptions of the relevant settings. | `user_oauth_applications` | boolean | no | Allow users to register any application to use GitLab as an OAuth provider. | | `user_show_add_ssh_key_message` | boolean | no | When set to `false` disable the `You won't be able to pull or push project code via SSH` warning shown to users with no uploaded SSH key. | | `version_check_enabled` | boolean | no | Let GitLab inform you when an update is available. | +| `valid_runner_registrars` | array of strings | no | List of types which are allowed to register a GitLab Runner. Can be `[]`, `['group']`, `['project']` or `['group', 'project']`. | | `whats_new_variant` | string | no | What's new variant, possible values: `all_tiers`, `current_tier`, and `disabled`. | | `wiki_page_max_content_bytes` | integer | no | Maximum wiki page content size in **bytes**. Default: 52428800 Bytes (50 MB). The minimum value is 1024 bytes. | | `jira_connect_application_key` | String | no | Application ID of the OAuth application that should be used to authenticate with the GitLab for Jira Cloud app | diff --git a/doc/api/status_checks.md b/doc/api/status_checks.md index 7299e529bda..c024558b90c 100644 --- a/doc/api/status_checks.md +++ b/doc/api/status_checks.md @@ -10,9 +10,92 @@ type: reference, api > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3869) in GitLab 14.0, disabled behind the `:ff_external_status_checks` feature flag. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/320783) in GitLab 14.1. +## Get project external status check services + +You can request information about a project's external status check services using the following endpoint: + +```plaintext +GET /projects/:id/external_status_checks +``` + +**Parameters:** + +| Attribute | Type | Required | Description | +|---------------------|---------|----------|---------------------| +| `id` | integer | yes | ID of a project | + +```json +[ + { + "id": 1, + "name": "Compliance Tool", + "project_id": 6, + "external_url": "https://gitlab.com/example/compliance-tool", + "protected_branches": [ + { + "id": 14, + "project_id": 6, + "name": "master", + "created_at": "2020-10-12T14:04:50.787Z", + "updated_at": "2020-10-12T14:04:50.787Z", + "code_owner_approval_required": false + } + ] + } +] +``` + +## Create external status check service + +You can create a new external status check service for a project using the following endpoint: + +```plaintext +POST /projects/:id/external_status_checks +``` + +WARNING: +External status checks send information about all applicable merge requests to the +defined external service. This includes confidential merge requests. + +| Attribute | Type | Required | Description | +|------------------------|------------------|----------|------------------------------------------------| +| `id` | integer | yes | ID of a project | +| `name` | string | yes | Display name of external status check service | +| `external_url` | string | yes | URL of external status check service | +| `protected_branch_ids` | `array` | no | IDs of protected branches to scope the rule by | + +## Update external status check service + +You can update an existing external status check for a project using the following endpoint: + +```plaintext +PUT /projects/:id/external_status_checks/:check_id +``` + +| Attribute | Type | Required | Description | +|------------------------|------------------|----------|------------------------------------------------| +| `id` | integer | yes | ID of a project | +| `check_id` | integer | yes | ID of an external status check service | +| `name` | string | no | Display name of external status check service | +| `external_url` | string | no | URL of external status check service | +| `protected_branch_ids` | `array` | no | IDs of protected branches to scope the rule by | + +## Delete external status check service + +You can delete an external status check service for a project using the following endpoint: + +```plaintext +DELETE /projects/:id/external_status_checks/:check_id +``` + +| Attribute | Type | Required | Description | +|------------------------|----------------|----------|----------------------------------------| +| `check_id` | integer | yes | ID of an external status check service | +| `id` | integer | yes | ID of a project | + ## List status checks for a merge request -For a single merge request, list the external status checks that apply to it and their status. +For a single merge request, list the external status check services that apply to it and their status. ```plaintext GET /projects/:id/merge_requests/:merge_request_iid/status_checks @@ -29,13 +112,13 @@ GET /projects/:id/merge_requests/:merge_request_iid/status_checks [ { "id": 2, - "name": "Rule 1", + "name": "Service 1", "external_url": "https://gitlab.com/test-endpoint", "status": "passed" }, { "id": 1, - "name": "Rule 2", + "name": "Service 2", "external_url": "https://gitlab.com/test-endpoint-2", "status": "pending" } @@ -251,89 +334,6 @@ In case status check is already passed status code is 422 } ``` -## Get project external status checks - -You can request information about a project's external status checks using the following endpoint: - -```plaintext -GET /projects/:id/external_status_checks -``` - -**Parameters:** - -| Attribute | Type | Required | Description | -|---------------------|---------|----------|---------------------| -| `id` | integer | yes | ID of a project | - -```json -[ - { - "id": 1, - "name": "Compliance Check", - "project_id": 6, - "external_url": "https://gitlab.com/example/test.json", - "protected_branches": [ - { - "id": 14, - "project_id": 6, - "name": "master", - "created_at": "2020-10-12T14:04:50.787Z", - "updated_at": "2020-10-12T14:04:50.787Z", - "code_owner_approval_required": false - } - ] - } -] -``` - -## Create external status check - -You can create a new external status check for a project using the following endpoint: - -```plaintext -POST /projects/:id/external_status_checks -``` - -WARNING: -External status checks send information about all applicable merge requests to the -defined external service. This includes confidential merge requests. - -| Attribute | Type | Required | Description | -|------------------------|------------------|----------|------------------------------------------------| -| `id` | integer | yes | ID of a project | -| `name` | string | yes | Display name of external status check | -| `external_url` | string | yes | URL of external status check resource | -| `protected_branch_ids` | `array` | no | IDs of protected branches to scope the rule by | - -## Delete external status check - -You can delete an external status check for a project using the following endpoint: - -```plaintext -DELETE /projects/:id/external_status_checks/:check_id -``` - -| Attribute | Type | Required | Description | -|------------------------|----------------|----------|-----------------------| -| `check_id` | integer | yes | ID of an external status check | -| `id` | integer | yes | ID of a project | - -## Update external status check - -You can update an existing external status check for a project using the following endpoint: - -```plaintext -PUT /projects/:id/external_status_checks/:check_id -``` - -| Attribute | Type | Required | Description | -|------------------------|------------------|----------|------------------------------------------------| -| `id` | integer | yes | ID of a project | -| `check_id` | integer | yes | ID of an external status check | -| `name` | string | no | Display name of external status check | -| `external_url` | string | no | URL of external status check resource | -| `protected_branch_ids` | `array` | no | IDs of protected branches to scope the rule by | - ## Related topics -- [External status checks](../user/project/merge_requests/status_checks.md). +- [External status checks](../user/project/merge_requests/status_checks.md) diff --git a/doc/api/suggestions.md b/doc/api/suggestions.md index 1e1f226481c..bb1f3968cf3 100644 --- a/doc/api/suggestions.md +++ b/doc/api/suggestions.md @@ -11,7 +11,19 @@ This page describes the API for [suggesting changes](../user/project/merge_reque Every API call to suggestions must be authenticated. -## Applying a suggestion +## Create a suggestion + +To create a suggestion through the API, use the Discussions API to +[create a new thread in the merge request diff](discussions.md#create-new-merge-request-thread). +The format for suggestions is: + +````markdown +```suggestion:-3+0 +example text +``` +```` + +## Apply a suggestion Applies a suggested patch in a merge request. Users must have at least the Developer role to perform such action. @@ -43,7 +55,7 @@ Example response: } ``` -## Applying multiple suggestions +## Apply multiple suggestions ```plaintext PUT /suggestions/batch_apply diff --git a/doc/api/system_hooks.md b/doc/api/system_hooks.md index 1b72ef1da53..720ae457f37 100644 --- a/doc/api/system_hooks.md +++ b/doc/api/system_hooks.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -126,6 +126,8 @@ Example response: ## Test system hook +Executes the system hook with mock data. + ```plaintext POST /hooks/:id ``` @@ -140,7 +142,7 @@ Example request: curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/hooks/1" ``` -Example response: +The response is always the mock data: ```json { diff --git a/doc/api/templates/dockerfiles.md b/doc/api/templates/dockerfiles.md index 31b676558db..e9d57061510 100644 --- a/doc/api/templates/dockerfiles.md +++ b/doc/api/templates/dockerfiles.md @@ -14,7 +14,7 @@ in the GitLab repository. ## Override Dockerfile API templates **(PREMIUM SELF)** -In [GitLab Premium and higher](https://about.gitlab.com/pricing/) tiers, GitLab instance +In [GitLab Premium and Ultimate](https://about.gitlab.com/pricing/) tiers, GitLab instance administrators can override templates in the [Admin Area](../../user/admin_area/settings/instance_template_repository.md). diff --git a/doc/api/topics.md b/doc/api/topics.md index 8acf6bd645d..8f2aae9e070 100644 --- a/doc/api/topics.md +++ b/doc/api/topics.md @@ -1,6 +1,6 @@ --- -stage: Manage -group: Organization +stage: Data Stores +group: Tenant Scale info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/api/usage_data.md b/doc/api/usage_data.md index 33126521901..bf8924c1578 100644 --- a/doc/api/usage_data.md +++ b/doc/api/usage_data.md @@ -1,6 +1,6 @@ --- stage: Analytics -group: Product Intelligence +group: Analytics Instrumentation info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: reference, api --- @@ -35,7 +35,6 @@ Example response: product_section: enablement product_stage: enablement product_group: global_search - product_category: global_search value_type: number status: active time_frame: 28d diff --git a/doc/api/users.md b/doc/api/users.md index 4ecb3d48c0f..a2293090209 100644 --- a/doc/api/users.md +++ b/doc/api/users.md @@ -6,6 +6,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Users API **(FREE)** +This documentation has information on API calls, parameters and responses for the Users API. + +For information on user activities that update the user event timestamps, see [get user activities](#get-user-activities). + ## List users Get a list of users. @@ -41,7 +45,7 @@ GET /users You can also use `?search=` to search for users by name, username, or public email. For example, `/users?search=John`. When you search for a: -- Public email, you must use the full email address to get an exact match. +- Public email, you must use the full email address to get an exact match. A search might return a partial match. For example, if you search for the email `on@example.com`, the search can return both `on@example.com` and `jon@example.com`. - Name or username, you do not have to get an exact match because this is a fuzzy search. In addition, you can lookup users by username: @@ -209,7 +213,7 @@ GET /users ] ``` -Users on [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see the `shared_runners_minutes_limit`, `extra_shared_runners_minutes_limit`, `is_auditor`, and `using_license_seat` parameters. +Users on [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/) also see the `shared_runners_minutes_limit`, `extra_shared_runners_minutes_limit`, `is_auditor`, and `using_license_seat` parameters. ```json [ @@ -225,7 +229,7 @@ Users on [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see ] ``` -Users on [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see +Users on [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/) also see the `group_saml` provider option and `provisioned_by_group_id` parameter: ```json @@ -409,7 +413,7 @@ Example Responses: NOTE: The `plan` and `trial` parameters are only available on GitLab Enterprise Edition. -Users on [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see +Users on [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/) also see the `shared_runners_minutes_limit`, `is_auditor`, and `extra_shared_runners_minutes_limit` parameters. ```json @@ -423,7 +427,7 @@ the `shared_runners_minutes_limit`, `is_auditor`, and `extra_shared_runners_minu } ``` -Users on [GitLab.com Premium or higher](https://about.gitlab.com/pricing/) also +Users on [GitLab.com Premium or Ultimate](https://about.gitlab.com/pricing/) also see the `group_saml` option and `provisioned_by_group_id` parameter: ```json @@ -663,7 +667,7 @@ GET /user } ``` -Users on [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see the `shared_runners_minutes_limit`, `extra_shared_runners_minutes_limit` parameters. +Users on [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/) also see the `shared_runners_minutes_limit`, `extra_shared_runners_minutes_limit` parameters. ### For administrators **(FREE SELF)** @@ -727,7 +731,7 @@ Parameters: } ``` -Users on [GitLab Premium or higher](https://about.gitlab.com/pricing/) also see these +Users on [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/) also see these parameters: - `shared_runners_minutes_limit` @@ -838,7 +842,8 @@ Example response: "id": 1, "user_id": 1 "view_diffs_file_by_file": true, - "show_whitespace_in_diffs": false + "show_whitespace_in_diffs": false, + "pass_user_identities_to_ci_jwt": false } ``` @@ -859,16 +864,18 @@ PUT /user/preferences "id": 1, "user_id": 1 "view_diffs_file_by_file": true, - "show_whitespace_in_diffs": false + "show_whitespace_in_diffs": false, + "pass_user_identities_to_ci_jwt": false } ``` Parameters: -| Attribute | Required | Description | -| :--------------------------- | :------- | :---------------------------------------------------------- | -| `view_diffs_file_by_file` | Yes | Flag indicating the user sees only one file diff per page. | -| `show_whitespace_in_diffs` | Yes | Flag indicating the user sees whitespace changes in diffs. | +| Attribute | Required | Description | +| :------------------------------- | :------- | :--------------------------------------------------------------------------- | +| `view_diffs_file_by_file` | Yes | Flag indicating the user sees only one file diff per page. | +| `show_whitespace_in_diffs` | Yes | Flag indicating the user sees whitespace changes in diffs. | +| `pass_user_identities_to_ci_jwt` | Yes | Flag indicating the user passes their external identities as CI information. This attribute does not contain enough information to identify or authorize the user in an external system. The attribute is internal to GitLab, and must not be passed to third-party services. | ## User follow @@ -1301,7 +1308,25 @@ Parameters: | `key` | string | yes | New GPG key | ```shell -curl --data "key=-----BEGIN PGP PUBLIC KEY BLOCK-----\r\n\r\nxsBNBFV..." \ + +export KEY="$( gpg --armor --export )" + +curl --data-urlencode "key=-----BEGIN PGP PUBLIC KEY BLOCK----- +> xsBNBFVjnlIBCACibzXOLCiZiL2oyzYUaTOCkYnSUhymg3pdbfKtd4mpBa58xKBj +> t1pTHVpw3Sk03wmzhM/Ndlt1AV2YhLv++83WKr+gAHFYFiCV/tnY8bx3HqvVoy8O +> CfxWhw4QZK7+oYzVmJj8ZJm3ZjOC4pzuegNWlNLCUdZDx9OKlHVXLCX1iUbjdYWa +> qKV6tdV8hZolkbyjedQgrpvoWyeSHHpwHF7yk4gNJWMMI5rpcssL7i6mMXb/sDzO +> VaAtU5wiVducsOa01InRFf7QSTxoAm6Xy0PGv/k48M6xCALa9nY+BzlOv47jUT57 +> vilf4Szy9dKD0v9S0mQ+IHB+gNukWrnwtXx5ABEBAAHNFm5hbWUgKGNvbW1lbnQp +> IDxlbUBpbD7CwHUEEwECACkFAlVjnlIJEINgJNgv009/AhsDAhkBBgsJCAcDAgYV +> CAIJCgsEFgIDAQAAxqMIAFBHuBA8P1v8DtHonIK8Lx2qU23t8Mh68HBIkSjk2H7/ +> oO2cDWCw50jZ9D91PXOOyMPvBWV2IE3tARzCvnNGtzEFRtpIEtZ0cuctxeIF1id5 +> crfzdMDsmZyRHAOoZ9VtuD6mzj0ybQWMACb7eIHjZDCee3Slh3TVrLy06YRdq2I4 +> bjMOPePtK5xnIpHGpAXkB3IONxyITpSLKsA4hCeP7gVvm7r7TuQg1ygiUBlWbBYn +> iE5ROzqZjG1s7dQNZK/riiU2umGqGuwAb2IPvNiyuGR3cIgRE4llXH/rLuUlspAp +> o4nlxaz65VucmNbN1aMbDXLJVSqR1DuE00vEsL1AItI= +> =XQoy +> -----END PGP PUBLIC KEY BLOCK-----" \ --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/user/gpg_keys" ``` @@ -1311,7 +1336,7 @@ Example response: [ { "id": 1, - "key": "-----BEGIN PGP PUBLIC KEY BLOCK-----\r\n\r\nxsBNBFVjnlIBCACibzXOLCiZiL2oyzYUaTOCkYnSUhymg3pdbfKtd4mpBa58xKBj\r\nt1pTHVpw3Sk03wmzhM/Ndlt1AV2YhLv++83WKr+gAHFYFiCV/tnY8bx3HqvVoy8O\r\nCfxWhw4QZK7+oYzVmJj8ZJm3ZjOC4pzuegNWlNLCUdZDx9OKlHVXLCX1iUbjdYWa\r\nqKV6tdV8hZolkbyjedQgrpvoWyeSHHpwHF7yk4gNJWMMI5rpcssL7i6mMXb/sDzO\r\nVaAtU5wiVducsOa01InRFf7QSTxoAm6Xy0PGv/k48M6xCALa9nY+BzlOv47jUT57\r\nvilf4Szy9dKD0v9S0mQ+IHB+gNukWrnwtXx5ABEBAAHNFm5hbWUgKGNvbW1lbnQp\r\nIDxlbUBpbD7CwHUEEwECACkFAlVjnlIJEINgJNgv009/AhsDAhkBBgsJCAcDAgYV\r\nCAIJCgsEFgIDAQAAxqMIAFBHuBA8P1v8DtHonIK8Lx2qU23t8Mh68HBIkSjk2H7/\r\noO2cDWCw50jZ9D91PXOOyMPvBWV2IE3tARzCvnNGtzEFRtpIEtZ0cuctxeIF1id5\r\ncrfzdMDsmZyRHAOoZ9VtuD6mzj0ybQWMACb7eIHjZDCee3Slh3TVrLy06YRdq2I4\r\nbjMOPePtK5xnIpHGpAXkB3IONxyITpSLKsA4hCeP7gVvm7r7TuQg1ygiUBlWbBYn\r\niE5ROzqZjG1s7dQNZK/riiU2umGqGuwAb2IPvNiyuGR3cIgRE4llXH/rLuUlspAp\r\no4nlxaz65VucmNbN1aMbDXLJVSqR1DuE00vEsL1AItI=\r\n=XQoy\r\n-----END PGP PUBLIC KEY BLOCK-----", + "key": "-----BEGIN PGP PUBLIC KEY BLOCK-----\nxsBNBFVjnlIBCACibzXOLCiZiL2oyzYUaTOCkYnSUhymg3pdbfKtd4mpBa58xKBj\nt1pTHVpw3Sk03wmzhM/Ndlt1AV2YhLv++83WKr+gAHFYFiCV/tnY8bx3HqvVoy8O\nCfxWhw4QZK7+oYzVmJj8ZJm3ZjOC4pzuegNWlNLCUdZDx9OKlHVXLCX1iUbjdYWa\nqKV6tdV8hZolkbyjedQgrpvoWyeSHHpwHF7yk4gNJWMMI5rpcssL7i6mMXb/sDzO\nVaAtU5wiVducsOa01InRFf7QSTxoAm6Xy0PGv/k48M6xCALa9nY+BzlOv47jUT57\nvilf4Szy9dKD0v9S0mQ+IHB+gNukWrnwtXx5ABEBAAHNFm5hbWUgKGNvbW1lbnQp\nIDxlbUBpbD7CwHUEEwECACkFAlVjnlIJEINgJNgv009/AhsDAhkBBgsJCAcDAgYV\nCAIJCgsEFgIDAQAAxqMIAFBHuBA8P1v8DtHonIK8Lx2qU23t8Mh68HBIkSjk2H7/\noO2cDWCw50jZ9D91PXOOyMPvBWV2IE3tARzCvnNGtzEFRtpIEtZ0cuctxeIF1id5\ncrfzdMDsmZyRHAOoZ9VtuD6mzj0ybQWMACb7eIHjZDCee3Slh3TVrLy06YRdq2I4\nbjMOPePtK5xnIpHGpAXkB3IONxyITpSLKsA4hCeP7gVvm7r7TuQg1ygiUBlWbBYn\niE5ROzqZjG1s7dQNZK/riiU2umGqGuwAb2IPvNiyuGR3cIgRE4llXH/rLuUlspAp\no4nlxaz65VucmNbN1aMbDXLJVSqR1DuE00vEsL1AItI=\n=XQoy\n-----END PGP PUBLIC KEY BLOCK-----", "created_at": "2017-09-05T09:17:46.264Z" } ] @@ -1413,7 +1438,22 @@ Parameters: | `key_id` | integer | yes | ID of the GPG key | ```shell -curl --data "key=-----BEGIN PGP PUBLIC KEY BLOCK-----\r\n\r\nxsBNBFV..." \ +curl --data-urlencode "key=-----BEGIN PGP PUBLIC KEY BLOCK----- +> xsBNBFVjnlIBCACibzXOLCiZiL2oyzYUaTOCkYnSUhymg3pdbfKtd4mpBa58xKBj +> t1pTHVpw3Sk03wmzhM/Ndlt1AV2YhLv++83WKr+gAHFYFiCV/tnY8bx3HqvVoy8O +> CfxWhw4QZK7+oYzVmJj8ZJm3ZjOC4pzuegNWlNLCUdZDx9OKlHVXLCX1iUbjdYWa +> qKV6tdV8hZolkbyjedQgrpvoWyeSHHpwHF7yk4gNJWMMI5rpcssL7i6mMXb/sDzO +> VaAtU5wiVducsOa01InRFf7QSTxoAm6Xy0PGv/k48M6xCALa9nY+BzlOv47jUT57 +> vilf4Szy9dKD0v9S0mQ+IHB+gNukWrnwtXx5ABEBAAHNFm5hbWUgKGNvbW1lbnQp +> IDxlbUBpbD7CwHUEEwECACkFAlVjnlIJEINgJNgv009/AhsDAhkBBgsJCAcDAgYV +> CAIJCgsEFgIDAQAAxqMIAFBHuBA8P1v8DtHonIK8Lx2qU23t8Mh68HBIkSjk2H7/ +> oO2cDWCw50jZ9D91PXOOyMPvBWV2IE3tARzCvnNGtzEFRtpIEtZ0cuctxeIF1id5 +> crfzdMDsmZyRHAOoZ9VtuD6mzj0ybQWMACb7eIHjZDCee3Slh3TVrLy06YRdq2I4 +> bjMOPePtK5xnIpHGpAXkB3IONxyITpSLKsA4hCeP7gVvm7r7TuQg1ygiUBlWbBYn +> iE5ROzqZjG1s7dQNZK/riiU2umGqGuwAb2IPvNiyuGR3cIgRE4llXH/rLuUlspAp +> o4nlxaz65VucmNbN1aMbDXLJVSqR1DuE00vEsL1AItI= +> =XQoy +> -----END PGP PUBLIC KEY BLOCK-----" \ --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/users/2/gpg_keys" ``` @@ -1423,7 +1463,7 @@ Example response: [ { "id": 1, - "key": "-----BEGIN PGP PUBLIC KEY BLOCK-----\r\n\r\nxsBNBFVjnlIBCACibzXOLCiZiL2oyzYUaTOCkYnSUhymg3pdbfKtd4mpBa58xKBj\r\nt1pTHVpw3Sk03wmzhM/Ndlt1AV2YhLv++83WKr+gAHFYFiCV/tnY8bx3HqvVoy8O\r\nCfxWhw4QZK7+oYzVmJj8ZJm3ZjOC4pzuegNWlNLCUdZDx9OKlHVXLCX1iUbjdYWa\r\nqKV6tdV8hZolkbyjedQgrpvoWyeSHHpwHF7yk4gNJWMMI5rpcssL7i6mMXb/sDzO\r\nVaAtU5wiVducsOa01InRFf7QSTxoAm6Xy0PGv/k48M6xCALa9nY+BzlOv47jUT57\r\nvilf4Szy9dKD0v9S0mQ+IHB+gNukWrnwtXx5ABEBAAHNFm5hbWUgKGNvbW1lbnQp\r\nIDxlbUBpbD7CwHUEEwECACkFAlVjnlIJEINgJNgv009/AhsDAhkBBgsJCAcDAgYV\r\nCAIJCgsEFgIDAQAAxqMIAFBHuBA8P1v8DtHonIK8Lx2qU23t8Mh68HBIkSjk2H7/\r\noO2cDWCw50jZ9D91PXOOyMPvBWV2IE3tARzCvnNGtzEFRtpIEtZ0cuctxeIF1id5\r\ncrfzdMDsmZyRHAOoZ9VtuD6mzj0ybQWMACb7eIHjZDCee3Slh3TVrLy06YRdq2I4\r\nbjMOPePtK5xnIpHGpAXkB3IONxyITpSLKsA4hCeP7gVvm7r7TuQg1ygiUBlWbBYn\r\niE5ROzqZjG1s7dQNZK/riiU2umGqGuwAb2IPvNiyuGR3cIgRE4llXH/rLuUlspAp\r\no4nlxaz65VucmNbN1aMbDXLJVSqR1DuE00vEsL1AItI=\r\n=XQoy\r\n-----END PGP PUBLIC KEY BLOCK-----", + "key": "-----BEGIN PGP PUBLIC KEY BLOCK-----\nxsBNBFVjnlIBCACibzXOLCiZiL2oyzYUaTOCkYnSUhymg3pdbfKtd4mpBa58xKBj\nt1pTHVpw3Sk03wmzhM/Ndlt1AV2YhLv++83WKr+gAHFYFiCV/tnY8bx3HqvVoy8O\nCfxWhw4QZK7+oYzVmJj8ZJm3ZjOC4pzuegNWlNLCUdZDx9OKlHVXLCX1iUbjdYWa\nqKV6tdV8hZolkbyjedQgrpvoWyeSHHpwHF7yk4gNJWMMI5rpcssL7i6mMXb/sDzO\nVaAtU5wiVducsOa01InRFf7QSTxoAm6Xy0PGv/k48M6xCALa9nY+BzlOv47jUT57\nvilf4Szy9dKD0v9S0mQ+IHB+gNukWrnwtXx5ABEBAAHNFm5hbWUgKGNvbW1lbnQp\nIDxlbUBpbD7CwHUEEwECACkFAlVjnlIJEINgJNgv009/AhsDAhkBBgsJCAcDAgYV\nCAIJCgsEFgIDAQAAxqMIAFBHuBA8P1v8DtHonIK8Lx2qU23t8Mh68HBIkSjk2H7/\noO2cDWCw50jZ9D91PXOOyMPvBWV2IE3tARzCvnNGtzEFRtpIEtZ0cuctxeIF1id5\ncrfzdMDsmZyRHAOoZ9VtuD6mzj0ybQWMACb7eIHjZDCee3Slh3TVrLy06YRdq2I4\nbjMOPePtK5xnIpHGpAXkB3IONxyITpSLKsA4hCeP7gVvm7r7TuQg1ygiUBlWbBYn\niE5ROzqZjG1s7dQNZK/riiU2umGqGuwAb2IPvNiyuGR3cIgRE4llXH/rLuUlspAp\no4nlxaz65VucmNbN1aMbDXLJVSqR1DuE00vEsL1AItI=\n=XQoy\n-----END PGP PUBLIC KEY BLOCK-----", "created_at": "2017-09-05T09:17:46.264Z" } ] @@ -2027,7 +2067,7 @@ Pre-requisite: Get the last activity date for all users, sorted from oldest to newest. -The activities that update the timestamp are: +The activities that update the user event timestamps (`last_activity_on` and `current_sign_in_at`) are: - Git HTTP/SSH activities (such as clone, push) - User logging in to GitLab @@ -2163,3 +2203,43 @@ Returns: - `400 Bad request` if two factor authentication is not enabled for the specified user. - `403 Forbidden` if not authenticated as an administrator. - `404 User Not Found` if user cannot be found. + +## Create a CI runner **(FREE SELF)** + +It creates a new runner, linked to the current user. + +Requires administrator access or ownership of the target namespace or project. Token values are returned once. Make sure you save it because you can't access +it again. + +```plaintext +POST /user/runners +``` + +| Attribute | Type | Required | Description | +|--------------------|--------------|----------|---------------------------------------------------------------------------------------------------| +| `runner_type` | string | yes | Specifies the scope of the runner; `instance_type`, `group_type`, or `project_type`. | +| `group_id` | integer | no | The ID of the group that the runner is created in. Required if `runner_type` is `group_type`. | +| `project_id` | integer | no | The ID of the project that the runner is created in. Required if `runner_type` is `project_type`. | +| `description` | string | no | Description of the runner. | +| `paused` | boolean | no | Specifies if the runner should ignore new jobs. | +| `locked` | boolean | no | Specifies if the runner should be locked for the current project. | +| `run_untagged` | boolean | no | Specifies if the runner should handle untagged jobs. | +| `tag_list` | string array | no | A list of runner tags. | +| `access_level` | string | no | The access level of the runner; `not_protected` or `ref_protected`. | +| `maximum_timeout` | integer | no | Maximum timeout that limits the amount of time (in seconds) that runners can run jobs. | +| `maintenance_note` | string | no | Free-form maintenance notes for the runner (1024 characters). | + +```shell +curl --request POST --header "PRIVATE-TOKEN: " --data "runner_type=instance_type" \ + "https://gitlab.example.com/api/v4/user/runners" +``` + +Example response: + +```json +{ + "id": 9171, + "token": "glrt-kyahzxLaj4Dc1jQf4xjX", + "token_expires_at": null +} +``` diff --git a/doc/api/version.md b/doc/api/version.md index 0ceab92b9d6..d7a13d78c14 100644 --- a/doc/api/version.md +++ b/doc/api/version.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/api/visual_review_discussions.md b/doc/api/visual_review_discussions.md index f966af82b10..496c732b337 100644 --- a/doc/api/visual_review_discussions.md +++ b/doc/api/visual_review_discussions.md @@ -1,16 +1,21 @@ --- stage: Verify -group: Pipeline Insights +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Visual Review discussions API **(PREMIUM)** + +# Visual Review discussions API (deprecated) **(PREMIUM)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18710) in GitLab 12.5. > - [Moved](https://about.gitlab.com/blog/2021/01/26/new-gitlab-product-subscription-model/) to GitLab Premium in 13.9. +WARNING: +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387751) in GitLab 15.8 +and is planned for removal in 17.0. This change is a breaking change. + Visual Review discussions are notes on merge requests sent as -feedback from [Visual Reviews](../ci/review_apps/index.md#visual-reviews). +feedback from [Visual Reviews](../ci/review_apps/index.md#visual-reviews-deprecated). ## Create new merge request thread @@ -45,3 +50,4 @@ Parameters: ```shell curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/visual_review_discussions?body=comment" ``` + diff --git a/doc/api/vulnerability_exports.md b/doc/api/vulnerability_exports.md index e1473ba68b6..c72e4a36929 100644 --- a/doc/api/vulnerability_exports.md +++ b/doc/api/vulnerability_exports.md @@ -8,11 +8,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/197494) in GitLab 12.10. [Updated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30397) in GitLab 13.0. -WARNING: -This API is in an [Alpha](../policy/alpha-beta-support.md#alpha-features) stage and considered unstable. -The response payload may be subject to change or breakage -across GitLab releases. - Every API call to vulnerability exports must be [authenticated](rest/index.md#authentication). ## Create a project-level vulnerability export diff --git a/doc/api/wikis.md b/doc/api/wikis.md index 046901af56b..17317b7c594 100644 --- a/doc/api/wikis.md +++ b/doc/api/wikis.md @@ -1,6 +1,6 @@ --- -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/architecture/blueprints/_template.md b/doc/architecture/blueprints/_template.md index f7dea60e9b7..e22cc2e6857 100644 --- a/doc/architecture/blueprints/_template.md +++ b/doc/architecture/blueprints/_template.md @@ -50,10 +50,14 @@ Blueprint statuses you can use: - "accepted" - "ongoing" - "implemented" +- "postponed" - "rejected" --> + + + # {+ Title of Blueprint +} ## Design and implementation details @@ -153,3 +160,12 @@ Diagrams authored in GitLab flavored markdown are preferred. In cases where that is not feasible, images should be placed under `images/` in the same directory as the `index.md` for the proposal. --> + +## Alternative Solutions + + diff --git a/doc/architecture/blueprints/cells/cells-feature-admin-area.md b/doc/architecture/blueprints/cells/cells-feature-admin-area.md new file mode 100644 index 00000000000..31d5388d40b --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-admin-area.md @@ -0,0 +1,59 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Admin Area' +--- + + + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: Admin Area + +In our Cells architecture proposal we plan to share all admin related tables in +GitLab. This allows simpler management of all Cells in one interface and reduces +the risk of settings diverging in different Cells. This introduces challenges +with admin pages that allow you to manage data that will be spread across all +Cells. + +## 1. Definition + +There are consequences for admin pages that contain data that spans "the whole +instance" as the Admin pages may be served by any Cell or possibly just 1 cell. +There are already many parts of the Admin interface that will have data that +spans many cells. For example lists of all Groups, Projects, Topics, Jobs, +Analytics, Applications and more. There are also administrative monitoring +capabilities in the Admin page that will span many cells such as the "Background +Jobs" and "Background Migrations" pages. + +## 2. Data flow + +## 3. Proposal + +We will need to decide how to handle these exceptions with a few possible +options: + +1. Move all these pages out into a dedicated per-cell Admin section. Probably + the URL will need to be routable to a single Cell like `/cells//admin`, + then we can display this data per Cell. These pages will be distinct from + other Admin pages which control settings that are shared across all Cells. We + will also need to consider how this impacts self-managed customers and + whether, or not, this should be visible for single-cell instances of GitLab. +1. Build some aggregation interfaces for this data so that it can be fetched + from all Cells and presented in a single UI. This may be beneficial to an + administrator that needs to see and filter all data at a glance, especially + when they don't know which Cell the data is on. The downside, however, is + that building this kind of aggregation is very tricky when all the Cells are + designed to be totally independent, and it does also enforce more strict + requirements on compatibility between Cells. + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/cells-feature-agent-for-kubernetes.md b/doc/architecture/blueprints/cells/cells-feature-agent-for-kubernetes.md new file mode 100644 index 00000000000..37347cf836d --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-agent-for-kubernetes.md @@ -0,0 +1,30 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Agent for Kubernetes' +--- + + + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: Agent for Kubernetes + +> TL;DR + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/cells-feature-backups.md b/doc/architecture/blueprints/cells/cells-feature-backups.md new file mode 100644 index 00000000000..d596bdd2078 --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-backups.md @@ -0,0 +1,62 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Backups' +--- + + + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: Backups + +Each cells will take its own backups, and consequently have its own isolated +backup / restore procedure. + +## 1. Definition + +GitLab Backup takes a backup of the PostgreSQL database used by the application, +and also Git repository data. + +## 2. Data flow + +Each cell has a number of application databases to back up (e.g. `main`, and `ci`). + +Additionally, there may be cluster-wide metadata tables (e.g. `users` table) +which is directly accesible via PostgreSQL. + +## 3. Proposal + +### 3.1. Cluster-wide metadata + +It is currently unknown how cluster-wide metadata tables will be accessible. We +may choose to have cluster-wide metadata tables backed up separately, or have +each cell back up its copy of cluster-wide metdata tables. + +### 3.2 Consistency + +#### 3.2.1 Take backups independently + +As each cell will communicate with each other via API, and there will be no joins +to the users table, it should be acceptable for each cell to take a backup +independently of each other. + +#### 3.2.2 Enforce snapshots + +We can require that each cell take a snapshot for the PostgreSQL databases at +around the same time to allow for a consistent-enough backup. + +## 4. Evaluation + +As the number of cells increases, it will likely not be feasible to take a +snapshot at the same time for all cells. Hence taking backups independently is +the better option. + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/cells-feature-ci-runners.md b/doc/architecture/blueprints/cells/cells-feature-ci-runners.md new file mode 100644 index 00000000000..e352be17dd3 --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-ci-runners.md @@ -0,0 +1,170 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: CI Runners' +--- + + + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: CI Runners + +GitLab in order to execute CI jobs [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner/), +very often managed by customer in their infrastructure. + +All CI jobs created as part of CI pipeline are run in a context of project +it poses a challenge how to manage GitLab Runners. + +## 1. Definition + +There are 3 different types of runners: + +- instance-wide: runners that are registered globally with specific tags (selection criteria) +- group runners: runners that execute jobs from a given top-level group or subprojects of that group +- project runners: runners that execute jobs from projects or many projects: some runners might + have projects assigned from projects in different top-level groups. + +This alongside with existing data structure where `ci_runners` is a table describing +all types of runners poses a challenge how the `ci_runners` should be managed in a Cells environment. + +## 2. Data flow + +GitLab Runners use a set of globally scoped endpoints to: + +- registration of a new runner via registration token `https://gitlab.com/api/v4/runners` + ([subject for removal](../runner_tokens/index.md)) (`registration token`) +- requests jobs via an authenticated `https://gitlab.com/api/v4/jobs/request` endpoint (`runner token`) +- upload job status via `https://gitlab.com/api/v4/jobs/:job_id` (`build token`) +- upload trace via `https://gitlab.com/api/v4/jobs/:job_id/trace` (`build token`) +- download and upload artifacts via `https://gitlab.com/api/v4/jobs/:job_id/artifacts` (`build token`) + +Currently three types of authentication tokens are used: + +- runner registration token ([subject for removal](../runner_tokens/index.md)) +- runner token representing an registered runner in a system with specific configuration (`tags`, `locked`, etc.) +- build token representing an ephemeral token giving a limited access to updating a specific + job, uploading artifacts, downloading dependent artifacts, downloading and uploading + container registry images + +Each of those endpoints do receive an authentication token via header (`JOB-TOKEN` for `/trace`) +or body parameter (`token` all other endpoints). + +Since the CI pipeline would be created in a context of a specific Cell it would be required +that pick of a build would have to be processed by that particular Cell. This requires +that build picking depending on a solution would have to be either: + +- routed to correct Cell for a first time +- be made to be two phase: request build from global pool, claim build on a specific Cell using a Cell specific URL + +## 3. Proposal + +This section describes various proposals. Reader should consider that those +proposals do describe solutions for different problems. Many or some aspects +of those proposals might be the solution to the stated problem. + +### 3.1. Authentication tokens + +Even though the paths for CI Runners are not routable they can be made routable with +those two possible solutions: + +- The `https://gitlab.com/api/v4/jobs/request` uses a long polling mechanism with + a ticketing mechanism (based on `X-GitLab-Last-Update` header). Runner when first + starts sends a request to GitLab to which GitLab responds with either a build to pick + by runner. This value is completely controlled by GitLab. This allows GitLab + to use JWT or any other means to encode `cell` identifier that could be easily + decodable by Router. +- The majority of communication (in terms of volume) is using `build token` making it + the easiest target to change since GitLab is sole owner of the token that Runner later + uses for specific job. There were prior discussions about not storing `build token` + but rather using `JWT` token with defined scopes. Such token could encode the `cell` + to which router could easily route all requests. + +### 3.2. Request body + +- The most of used endpoints pass authentication token in request body. It might be desired + to use HTTP Headers as an easier way to access this information by Router without + a need to proxy requests. + +### 3.3. Instance-wide are Cell local + +We can pick a design where all runners are always registered and local to a given Cell: + +- Each Cell has it's own set of instance-wide runners that are updated at it's own pace +- The project runners can only be linked to projects from the same organization + creating strong isolation. +- In this model the `ci_runners` table is local to the Cell. +- In this model we would require the above endpoints to be scoped to a Cell in some way + or made routable. It might be via prefixing them, adding additional Cell parameter, + or providing much more robust way to decode runner token and match it to Cell. +- If routable token is used, we could move away from cryptographic random stored in + database to rather prefer to use JWT tokens that would encode +- The Admin Area showing registered Runners would have to be scoped to a Cell + +This model might be desired since it provides strong isolation guarantees. +This model does significantly increase maintenance overhead since each Cell is managed +separately. + +This model may require adjustments to runner tags feature so that projects have consistent runner experience across cells. + +### 3.4. Instance-wide are cluster-wide + +Contrary to proposal where all runners are Cell local, we can consider that runners +are global, or just instance-wide runners are global. + +However, this requires significant overhaul of system and to change the following aspects: + +- `ci_runners` table would likely have to be split decomposed into `ci_instance_runners`, ... +- all interfaces would have to be adopted to use correct table +- build queuing would have to be reworked to be two phase where each Cell would know of all pending + and running builds, but the actual claim of a build would happen against a Cell containing data +- likely `ci_pending_builds` and `ci_running_builds` would have to be made `cluster-wide` tables + increasing likelihood of creating hotspots in a system related to CI queueing + +This model makes it complex to implement from engineering side. Does make some data being shared +between Cells. Creates hotspots / scalability issues in a system (ex. during abuse) that +might impact experience of organizations on other Cells. + +### 3.5. GitLab CI Daemon + +Another potential solution to explore is to have a dedicated service responsible for builds queueing +owning it's database and working in a model of either sharded or celled service. There were prior +discussions about [CI/CD Daemon](https://gitlab.com/gitlab-org/gitlab/-/issues/19435). + +If the service would be sharded: + +- depending on a model if runners are cluster-wide or cell-local this service would have to fetch + data from all Cells +- if the sharded service would be used we could adapt a model of either sharing database containing + `ci_pending_builds/ci_running_builds` with the service +- if the sharded service would be used we could consider a push model where each Cell pushes to CI/CD Daemon + builds that should be picked by Runner +- the sharded service would be aware which Cell is responsible for processing the given build and could + route processing requests to designated Cell + +If the service would be celled: + +- all expectations of routable endpoints are still valid + +In general usage of CI Daemon does not help significantly with the stated problem. However, this offers +a few upsides related to more efficient processing and decoupling model: push model and it opens a way +to offer stateful communication with GitLab Runners (ex. gRPC or Websockets). + +## 4. Evaluation + +Considering all solutions it appears that solution giving the most promise is: + +- use "instance-wide are Cell local" +- refine endpoints to have routable identities (either via specific paths, or better tokens) + +Other potential upsides is to get rid of `ci_builds.token` and rather use a `JWT token` +that can much better and easier encode wider set of scopes allowed by CI runner. + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/cells-feature-container-registry.md b/doc/architecture/blueprints/cells/cells-feature-container-registry.md new file mode 100644 index 00000000000..a5761808941 --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-container-registry.md @@ -0,0 +1,132 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Container Registry' +--- + + + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: Container Registry + +GitLab Container Registry is a feature allowing to store Docker Container Images +in GitLab. You can read about GitLab integration [here](../../../user/packages/container_registry/index.md). + +## 1. Definition + +GitLab Container Registry is a complex service requiring usage of PostgreSQL, Redis +and Object Storage dependencies. Right now there's undergoing work to introduce +[Container Registry Metadata](../container_registry_metadata_database/index.md) +to optimize data storage and image retention policies of Container Registry. + +GitLab Container Registry is serving as a container for stored data, +but on it's own does not authenticate `docker login`. The `docker login` +is executed with user credentials (can be `personal access token`) +or CI build credentials (ephemeral `ci_builds.token`). + +Container Registry uses data deduplication. It means that the same blob +(image layer) that is shared between many projects is stored only once. +Each layer is hashed by `sha256`. + +The `docker login` does request JWT time-limited authentication token that +is signed by GitLab, but validated by Container Registry service. The JWT +token does store all authorized scopes (`container repository images`) +and operation types (`push` or `pull`). A single JWT authentication token +can be have many authorized scopes. This allows container registry and client +to mount existing blobs from another scopes. GitLab responds only with +authorized scopes. Then it is up to GitLab Container Registry to validate +if the given operation can be performed. + +The GitLab.com pages are always scoped to project. Each project can have many +container registry images attached. + +Currently in case of GitLab.com the actual registry service is served +via `https://registry.gitlab.com`. + +The main identifiable problems are: + +- the authentication request (`https://gitlab.com/jwt/auth`) that is processed by GitLab.com +- the `https://registry.gitlab.com` that is run by external service and uses it's own data store +- the data deduplication, the Cells architecture with registry run in a Cell would reduce + efficiency of data storage + +## 2. Data flow + +### 2.1. Authorization request that is send by `docker login` + +```shell +curl \ + --user "username:password" \ + "https://gitlab/jwt/auth?client_id=docker&offline_token=true&service=container_registry&scope=repository:gitlab-org/gitlab-build-images:push,pull" +``` + +Result is encoded and signed JWT token. Second base64 encoded string (split by `.`) contains JSON with authorized scopes. + +```json +{"auth_type":"none","access":[{"type":"repository","name":"gitlab-org/gitlab-build-images","actions":["pull"]}],"jti":"61ca2459-091c-4496-a3cf-01bac51d4dc8","aud":"container_registry","iss":"omnibus-gitlab-issuer","iat":1669309469,"nbf":166} +``` + +### 2.2. Docker client fetching tags + +```shell +curl \ + -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ + -H "Authorization: Bearer token" \ + https://registry.gitlab.com/v2/gitlab-org/gitlab-build-images/tags/list + +curl \ + -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ + -H "Authorization: Bearer token" \ + https://registry.gitlab.com/v2/gitlab-org/gitlab-build-images/manifests/danger-ruby-2.6.6 +``` + +### 2.3. Docker client fetching blobs and manifests + +```shell +curl \ + -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ + -H "Authorization: Bearer token" \ + https://registry.gitlab.com/v2/gitlab-org/gitlab-build-images/blobs/sha256:a3f2e1afa377d20897e08a85cae089393daa0ec019feab3851d592248674b416 +``` + +## 3. Proposal + +### 3.1. Shard Container Registry separately to Cells architecture + +Due to it's architecture it extensive architecture and in general highly scalable +horizontal architecture it should be evaluated if the GitLab Container Registry +should be run not in Cell, but in a Cluster and be scaled independently. + +This might be easier, but would definitely not offer the same amount of data isolation. + +### 3.2. Run Container Registry within a Cell + +It appears that except `/jwt/auth` which would likely have to be processed by Router +(to decode `scope`) the container registry could be run as a local service of a Cell. + +The actual data at least in case of GitLab.com is not forwarded via registry, +but rather served directly from Object Storage / CDN. + +Its design encodes container repository image in a URL that is easily routable. +It appears that we could re-use the same stateless Router service in front of Container Registry +to serve manifests and blobs redirect. + +The only downside is increased complexity of managing standalone registry for each Cell, +but this might be desired approach. + +## 4. Evaluation + +There do not seem any theoretical problems with running GitLab Container Registry in a Cell. +Service seems that can be easily made routable to work well. + +The practical complexities are around managing complex service from infrastructure side. + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/cells-feature-contributions-forks.md b/doc/architecture/blueprints/cells/cells-feature-contributions-forks.md new file mode 100644 index 00000000000..3e498c24144 --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-contributions-forks.md @@ -0,0 +1,121 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Contributions: Forks' +--- + + + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: Contributions: Forks + +[Forking workflow](../../../user/project/repository/forking_workflow.md) allows users +to copy existing project sources into their own namespace of choice (personal or group). + +## 1. Definition + +[Forking workflow](../../../user/project/repository/forking_workflow.md) is common workflow +with various usage patterns: + +- allows users to contribute back to upstream project +- persist repositories into their personal namespace +- copy to make changes and release as modified project + +Forks allow users not having write access to parent project to make changes. The forking workflow +is especially important for the Open Source community which is able to contribute back +to public projects. However, it is equally important in some companies which prefer the strong split +of responsibilites and tighter access control. The access to project is restricted +to designated list of developers. + +Forks enable: + +- tigther control of who can modify the upstream project +- split of the responsibilites: parent project might use CI configuration connecting to production systems +- run CI pipelines in context of fork in much more restrictive environment +- consider all forks to be unveted which reduces risks of leaking secrets, or any other information + tied with the project + +The forking model is problematic in Cells architecture for following reasons: + +- Forks are clones of existing repositories, forks could be created across different organizations, Cells and Gitaly shards. +- User can create merge request and contribute back to upstream project, this upstream project might in a different organization and Cell. +- The merge request CI pipeline is to executed in a context of source project, but presented in a context of target project. + +## 2. Data flow + +## 3. Proposals + +### 3.1. Intra-Cluster forks + +This proposal makes us to implement forks as a intra-ClusterCell forks where communication is done via API +between all trusted Cells of a cluster: + +- Forks when created, they are created always in context of user choice of group. +- Forks are isolated from Organization. +- Organization or group owner could disable forking across organizations or forking in general. +- When a Merge Request is created it is created in context of target project, referencing + external project on another Cell. +- To target project the merge reference is transfered that is used for presenting information + in context of target project. +- CI pipeline is fetched in context of source project as it-is today, the result is fetched into + Merge Request of target project. +- The Cell holding target project internally uses GraphQL to fetch status of source project + and include in context of the information for merge request. + +Upsides: + +- All existing forks continue to work as-is, as they are treated as intra-Cluster forks. + +Downsides: + +- The purpose of Organizations is to provide strong isolation between organizations + allowing to fork across does break security boundaries. +- However, this is no different to ability of users today to clone repository to local computer + and push it to any repository of choice. +- Access control of source project can be lower than those of target project. System today + requires that in order to contribute back the access level needs to be the same for fork and upstream. + +### 3.2. Forks are created in a personal namespace of the current organization + +Instead of creating projects across organizations, the forks are created in a user personal namespace +tied with the organization. Example: + +- Each user that is part of organization receives their personal namespace. For example for `GitLab Inc.` + it could be `gitlab.com/organization/gitlab-inc/@ayufan`. +- The user has to fork into it's own personal namespace of the organization. +- The user has that many personal namespaces as many organizations it belongs to. +- The personal namespace behaves similar to currently offered personal namespace. +- The user can manage and create projects within a personal namespace. +- The organization can prevent or disable usage of personal namespaces disallowing forks. +- All current forks are migrated into personal namespace of user in Organization. +- All forks are part of to the organization. +- The forks are not federated features. +- The personal namespace and forked project do not share configuration with parent project. + +### 3.3. Forks are created as internal projects under current project + +Instead of creating projects across organizations, the forks are attachments to existing projects. +Each user forking a project receives their unique project. Example: + +- For project: `gitlab.com/gitlab-org/gitlab`, forks would be created in `gitlab.com/gitlab-org/gitlab/@kamil-gitlab`. +- Forks are created in a context of current organization, they do not cross organization boundaries + and are managed by the organization. +- Tied to the user (or any other user-provided name of the fork). +- The forks are not federated features. + +Downsides: + +- Does not answer how to handle and migrate all exisiting forks. +- Might share current group / project settings - breaking some security boundaries. + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/cells-feature-dashboard.md b/doc/architecture/blueprints/cells/cells-feature-dashboard.md new file mode 100644 index 00000000000..135f69c6ed3 --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-dashboard.md @@ -0,0 +1,30 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Dashboard' +--- + + + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: Dashboard + +> TL;DR + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/cells-feature-data-migration.md b/doc/architecture/blueprints/cells/cells-feature-data-migration.md new file mode 100644 index 00000000000..ef0865b4081 --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-data-migration.md @@ -0,0 +1,131 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Data migration' +--- + + + +DISCLAIMER: +This page may contain information related to upcoming products, features and +functionality. It is important to note that the information presented is for +informational purposes only, so please do not rely on the information for +purchasing or planning purposes. Just like with all projects, the items +mentioned on the page are subject to change or delay, and the development, +release, and timing of any products, features, or functionality remain at the +sole discretion of GitLab Inc. + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: Data migration + +It is essential for Cells architecture to provide a way to migrate data out of big Cells +into smaller ones. This describes various approaches to provide this type of split. + +We also need to handle for cases where data is already violating the expected +isolation constraints of Cells (ie. references cannot span multiple +organizations). We know that existing features like linked issues allowed users +to link issues across any projects regardless of their hierarchy. There are many +similar features. All of this data will need to be migrated in some way before +it can be split across different cells. This may mean some data needs to be +deleted, or the feature changed and modelled slightly differently before we can +properly split or migrate the organizations between cells. + +Having schema deviations across different Cells, which is a necessary +consequence of different databases, will also impact our ability to migrate +data between cells. Different schemas impact our ability to reliably replicate +data across cells and especially impact our ability to validate that the data is +correctly replicated. It might force us to only be able to move data between +cells when the schemas are all in sync (slowing down deployments and the +rebalancing process) or possibly only migrate from newer to older schemas which +would be complex. + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +### 3.1. Split large Cells + +A single Cell can only be divided into many Cells. This is based on principle +that it is easier to create exact clone of an existing Cell in many replicas +out of which some will be made authoritative once migrated. Keeping those +replicas up-to date with Cell 0 is also much easier due to pre-existing +replication solutions that can replicate the whole systems: Geo, PostgreSQL +physical replication, etc. + +1. All data of an organization needs to not be divided across many Cells. +1. Split should be doable online. +1. New Cells cannot contain pre-existing data. +1. N Cells contain exact replica of Cell 0. +1. The data of Cell 0 is live replicated to as many Cells it needs to be split. +1. Once consensus is achieved between Cell 0 and N-Cells the organizations to be migrated away + are marked as read-only cluster-wide. +1. The `routes` is updated on for all organizations to be split to indicate an authoritative + Cell holding the most recent data, like `gitlab-org` on `cell-100`. +1. The data for `gitlab-org` on Cell 0, and on other non-authoritative N-Cells are dormant + and will be removed in the future. +1. All accesses to `gitlab-org` on a given Cell are validated about `cell_id` of `routes` + to ensure that given Cell is authoritative to handle the data. + +#### More challenges of this proposal + +1. There is no streaming replication capability for Elasticsearch, but you could + snapshot the whole Elasticsearch index and recreate, but this takes hours. + It could be handled by pausing Elasticsearch indexing on the initial cell during + the migration as indexing downtime is not a big issue, but this still needs + to be coordinated with the migration process +1. Syncing Redis, Gitaly, CI Postgres, Main Postgres, registry Postgres, other + new data stores snapshots in an online system would likely lead to gaps + without a long downtime. You need to choose a sync point and at the sync + point you need to stop writes to perform the migration. The more data stores + there are to migrate at the same time the longer the write downtime for the + failover. We would also need to find a reliable place in the application to + actually block updates to all these systems with a high degree of + confidence. In the past we've only been confident by shutting down all rails + services because any rails process could write directly to any of these at + any time due to async workloads or other surprising code paths. +1. How to efficiently delete all the orphaned data. Locating all `ci_builds` + associated with half the organizations would be very expensive if we have to + do joins. We haven't yet determined if we'd want to store an `organization_id` + column on every table, but this is the kind of thing it would be helpful for. + +### 3.2. Migrate organization from an existing Cell + +This is different to split, as we intend to perform logical and selective replication +of data belonging to a single organization. + +Today this type of selective replication is only implemented by Gitaly where we can migrate +Git repository from a single Gitaly node to another with minimal downtime. + +In this model we would require identifying all resources belonging to a given organization: +database rows, object storage files, Git repositories, etc. and selectively copy them over +to another (likely) existing Cell importing data into it. Ideally ensuring that we can +perform logical replication live of all changed data, but change similarly to split +which Cell is authoritative for this organization. + +1. It is hard to identify all resources belonging to organization. +1. It requires either downtime for organization or a robust system to identify + live changes made. +1. It likely will require a full database structure analysis (more robust than project import/export) + to perform selective PostgreSQL logical replication. + +#### More challenges of this proposal + +1. Logical replication is still not performant enough to keep up with our + scale. Even if we could use logical replication we still don't have an + efficient way to filter data related to a single organization without + joining all the way to the `organizations` table which will slow down + logical replication dramatically. + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/cells-feature-database-sequences.md b/doc/architecture/blueprints/cells/cells-feature-database-sequences.md new file mode 100644 index 00000000000..d94dc3be864 --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-database-sequences.md @@ -0,0 +1,95 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Database Sequences' +--- + + + +DISCLAIMER: +This page may contain information related to upcoming products, features and +functionality. It is important to note that the information presented is for +informational purposes only, so please do not rely on the information for +purchasing or planning purposes. Just like with all projects, the items +mentioned on the page are subject to change or delay, and the development, +release, and timing of any products, features, or functionality remain at the +sole discretion of GitLab Inc. + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: Database Sequences + +GitLab today ensures that every database row create has unique ID, allowing +to access Merge Request, CI Job or Project by a known global ID. + +Cells will use many distinct and not connected databases, each of them having +a separate IDs for most of entities. + +It might be desirable to retain globally unique IDs for all database rows +to allow migrating resources between Cells in the future. + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +This are some preliminary ideas how we can retain unique IDs across the system. + +### 3.1. UUID + +Instead of using incremental sequences use UUID (128 bit) that is stored in database. + +- This might break existing IDs and requires adding UUID column for all existing tables. +- This makes all indexes larger as it requires storing 128 bit instead of 32/64 bit in index. + +### 3.2. Use Cell index encoded in ID + +Since significant number of tables already use 64 bit ID numbers we could use MSB to encode +Cell ID effectively enabling + +- This might limit amount of Cells that can be enabled in system, as we might decide to only + allocate 1024 possible Cell numbers. +- This might make IDs to be migratable between Cells, since even if entity from Cell 1 is migrated to Cell 100 + this ID would still be unique. +- If resources are migrated the ID itself will not be enough to decode Cell number and we would need + lookup table. +- This requires updating all IDs to 32 bits. + +### 3.3. Allocate sequence ranges from central place + +Each Cell might receive its own range of the sequences as they are consumed from a centrally managed place. +Once Cell consumes all IDs assigned for a given table it would be replenished and a next range would be allocated. +Ranges would be tracked to provide a faster lookup table if a random access pattern is required. + +- This might make IDs to be migratable between Cells, since even if entity from Cell 1 is migrated to Cell 100 + this ID would still be unique. +- If resources are migrated the ID itself will not be enough to decode Cell number and we would need + much more robust lookup table as we could be breaking previously assigned sequence ranges. +- This does not require updating all IDs to 64 bits. +- This adds some performance penalty to all `INSERT` statements in Postgres or at least from Rails as we need to check for the sequence number and potentially wait for our range to be refreshed from the ID server +- The available range will need to be stored and incremented in a centralized place so that concurrent transactions cannot possibly get the same value. + +### 3.4. Define only some tables to require unique IDs + +Maybe this is acceptable only for some tables to have a globally unique IDs. It could be projects, groups +and other top-level entities. All other tables like `merge_requests` would only offer Cell-local ID, +but when referenced outside it would rather use IID (an ID that is monotonic in context of a given resource, like project). + +- This makes the ID 10000 for `merge_requests` be present on all Cells, which might be sometimes confusing + as for uniqueness of the resource. +- This might make random access by ID (if ever needed) be impossible without using composite key, like: `project_id+merge_request_id`. +- This would require us to implement a transformation/generation of new ID if we need to migrate records to another cell. This can lead to very difficult migration processes when these IDs are also used as foreign keys for other records being migrated. +- If IDs need to change when moving between cells this means that any links to records by ID would no longer work even if those links included the `project_id`. +- If we plan to allow these ids to not be unique and change the unique constraint to be based on a composite key then we'd need to update all foreign key references to be based on the composite key + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/cells-feature-git-access.md b/doc/architecture/blueprints/cells/cells-feature-git-access.md new file mode 100644 index 00000000000..70b3f136904 --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-git-access.md @@ -0,0 +1,164 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Git Access' +--- + + + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: Git Access + +This document describes impact of Cells architecture on all Git access (over HTTPS and SSH) +patterns providing explanation of how potentially those features should be changed +to work well with Cells. + +## 1. Definition + +Git access is done through out the application. It can be an operation performed by the system +(read Git repository) or by user (create a new file via Web IDE, `git clone` or `git push` via command line). + +The Cells architecture defines that all Git repositories will be local to the Cell, +so no repository could be shared with another Cell. + +The Cells architecture will require that any Git operation done can only be handled by a Cell holding +the data. It means that any operation either via Web interface, API, or GraphQL needs to be routed +to the correct Cell. It means that any `git clone` or `git push` operation can only be performed +in a context of a Cell. + +## 2. Data flow + +The are various operations performed today by the GitLab on a Git repository. This describes +the data flow how they behave today to better represent the impact. + +It appears that Git access does require changes only to a few endpoints that are scoped to project. +There appear to be different types of repositories: + +- Project: assigned to Group +- Wiki: additional repository assigned to Project +- Design: similar to Wiki, additional repository assigned to Project +- Snippet: creates a virtual project to hold repository, likely tied to the User + +### 2.1. Git clone over HTTPS + +Execution of: `git clone` over HTTPS + +```mermaid +sequenceDiagram + User ->> Workhorse: GET /gitlab-org/gitlab.git/info/refs?service=git-upload-pack + Workhorse ->> Rails: GET /gitlab-org/gitlab.git/info/refs?service=git-upload-pack + Rails ->> Workhorse: 200 OK + Workhorse ->> Gitaly: RPC InfoRefsUploadPack + Gitaly ->> User: Response + User ->> Workhorse: POST /gitlab-org/gitlab.git/git-upload-pack + Workhorse ->> Gitaly: RPC PostUploadPackWithSidechannel + Gitaly ->> User: Response +``` + +### 2.2. Git clone over SSH + +Execution of: `git clone` over SSH + +```mermaid +sequenceDiagram + User ->> Git SSHD: ssh git@gitlab.com + Git SSHD ->> Rails: GET /api/v4/internal/authorized_keys + Rails ->> Git SSHD: 200 OK (list of accepted SSH keys) + Git SSHD ->> User: Accept SSH + User ->> Git SSHD: git clone over SSH + Git SSHD ->> Rails: POST /api/v4/internal/allowed?project=/gitlab-org/gitlab.git&service=git-upload-pack + Rails ->> Git SSHD: 200 OK + Git SSHD ->> Gitaly: RPC SSHUploadPackWithSidechannel + Gitaly ->> User: Response +``` + +### 2.3. Git push over HTTPS + +Execution of: `git push` over HTTPS + +```mermaid +sequenceDiagram + User ->> Workhorse: GET /gitlab-org/gitlab.git/info/refs?service=git-receive-pack + Workhorse ->> Rails: GET /gitlab-org/gitlab.git/info/refs?service=git-receive-pack + Rails ->> Workhorse: 200 OK + Workhorse ->> Gitaly: RPC PostReceivePack + Gitaly ->> Rails: POST /api/v4/internal/allowed?gl_repository=project-111&service=git-receive-pack + Gitaly ->> Rails: POST /api/v4/internal/pre_receive?gl_repository=project-111 + Gitaly ->> Rails: POST /api/v4/internal/post_receive?gl_repository=project-111 + Gitaly ->> User: Response +``` + +### 2.4. Git push over SSHD + +Execution of: `git clone` over SSH + +```mermaid +sequenceDiagram + User ->> Git SSHD: ssh git@gitlab.com + Git SSHD ->> Rails: GET /api/v4/internal/authorized_keys + Rails ->> Git SSHD: 200 OK (list of accepted SSH keys) + Git SSHD ->> User: Accept SSH + User ->> Git SSHD: git clone over SSH + Git SSHD ->> Rails: POST /api/v4/internal/allowed?project=/gitlab-org/gitlab.git&service=git-receive-pack + Rails ->> Git SSHD: 200 OK + Git SSHD ->> Gitaly: RPC ReceivePack + Gitaly ->> Rails: POST /api/v4/internal/allowed?gl_repository=project-111 + Gitaly ->> Rails: POST /api/v4/internal/pre_receive?gl_repository=project-111 + Gitaly ->> Rails: POST /api/v4/internal/post_receive?gl_repository=project-111 + Gitaly ->> User: Response +``` + +### 2.5. Create commit via Web + +Execution of `Add CHANGELOG` to repository: + +```mermaid +sequenceDiagram + Web ->> Puma: POST /gitlab-org/gitlab/-/create/main + Puma ->> Gitaly: RPC TreeEntry + Gitaly ->> Rails: POST /api/v4/internal/allowed?gl_repository=project-111 + Gitaly ->> Rails: POST /api/v4/internal/pre_receive?gl_repository=project-111 + Gitaly ->> Rails: POST /api/v4/internal/post_receive?gl_repository=project-111 + Gitaly ->> Puma: Response + Puma ->> Web: See CHANGELOG +``` + +## 3. Proposal + +The Cells stateless router proposal requires that any ambiguous path (that is not routable) +will be made to be routable. It means that at least the following paths will have to be updated +do introduce a routable entity (project, group, or organization). + +Change: + +- `/api/v4/internal/allowed` => `/api/v4/internal/projects//allowed` +- `/api/v4/internal/pre_receive` => `/api/v4/internal/projects//pre_receive` +- `/api/v4/internal/post_receive` => `/api/v4/internal/projects//post_receive` +- `/api/v4/internal/lfs_authenticate` => `/api/v4/internal/projects//lfs_authenticate` + +Where: + +- `gl_repository` can be `project-1111` (`Gitlab::GlRepository`) +- `gl_repository` in some cases might be a full path to repository as executed by GitLab Shell (`/gitlab-org/gitlab.git`) + +## 4. Evaluation + +Supporting Git repositories if a Cell can access only its own repositories does not appear to be complex. + +The one major complication is supporting snippets, but this likely falls in the same category as for the approach +to support user's personal namespaces. + +## 4.1. Pros + +1. The API used for supporting HTTPS/SSH and Hooks are well defined and can easily be made routable. + +## 4.2. Cons + +1. The sharing of repositories objects is limited to the given Cell and Gitaly node. +1. The across-Cells forks are likely impossible to be supported (discover: how this work today across different Gitaly node). diff --git a/doc/architecture/blueprints/cells/cells-feature-gitlab-pages.md b/doc/architecture/blueprints/cells/cells-feature-gitlab-pages.md new file mode 100644 index 00000000000..7e4ab785095 --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-gitlab-pages.md @@ -0,0 +1,30 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: GitLab Pages' +--- + + + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: GitLab Pages + +> TL;DR + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/cells-feature-global-search.md b/doc/architecture/blueprints/cells/cells-feature-global-search.md new file mode 100644 index 00000000000..c1e2b93bc2d --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-global-search.md @@ -0,0 +1,48 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Global search' +--- + + + +DISCLAIMER: +This page may contain information related to upcoming products, features and +functionality. It is important to note that the information presented is for +informational purposes only, so please do not rely on the information for +purchasing or planning purposes. Just like with all projects, the items +mentioned on the page are subject to change or delay, and the development, +release, and timing of any products, features, or functionality remain at the +sole discretion of GitLab Inc. + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: Global search + +When we introduce multiple Cells we intend to isolate all services related to +those Cells. This will include Elasticsearch which means our current global +search functionality will not work. It may be possible to implement aggregated +search across all cells, but it is unlikely to be performant to do fan-out +searches across all cells especially once you start to do pagination which +requires setting the correct offset and page number for each search. + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +Likely first versions of Cells will simply not support global searches and then +we may later consider if building global searches to support popular use cases +is worthwhile. + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/cells-feature-graphql.md b/doc/architecture/blueprints/cells/cells-feature-graphql.md new file mode 100644 index 00000000000..d936a1b81ba --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-graphql.md @@ -0,0 +1,95 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: GraphQL' +--- + + + +DISCLAIMER: +This page may contain information related to upcoming products, features and +functionality. It is important to note that the information presented is for +informational purposes only, so please do not rely on the information for +purchasing or planning purposes. Just like with all projects, the items +mentioned on the page are subject to change or delay, and the development, +release, and timing of any products, features, or functionality remain at the +sole discretion of GitLab Inc. + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: GraphQL + +GitLab extensively uses GraphQL to perform efficient data query operations. +GraphQL due to it's nature is not directly routable. The way how GitLab uses +it calls the `/api/graphql` endpoint, and only query or mutation of body request +might define where the data can be accessed. + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +There are at least two main ways to implement GraphQL in Cells architecture. + +### 3.1. GraphQL routable by endpoint + +Change `/api/graphql` to `/api/organization//graphql`. + +- This breaks all existing usages of `/api/graphql` endpoint + since the API URI is changed. + +### 3.2. GraphQL routable by body + +As part of router parse GraphQL body to find a routable entity, like `project`. + +- This still makes the GraphQL query be executed only in context of a given Cell + and not allowing the data to be merged. + +```json +# Good example +{ + project(fullPath:"gitlab-org/gitlab") { + id + description + } +} + +# Bad example, since Merge Request is not routable +{ + mergeRequest(id: 1111) { + iid + description + } +} +``` + +### 3.3. Merging GraphQL Proxy + +Implement as part of router GraphQL Proxy which can parse body +and merge results from many Cells. + +- This might make pagination hard to achieve, or we might assume that + we execute many queries of which results are merged across all Cells. + +```json +{ + project(fullPath:"gitlab-org/gitlab"){ + id, description + } + group(fullPath:"gitlab-com") { + id, description + } +} +``` + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/cells-feature-organizations.md b/doc/architecture/blueprints/cells/cells-feature-organizations.md new file mode 100644 index 00000000000..03178d9e6ce --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-organizations.md @@ -0,0 +1,59 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Organizations' +--- + + + +DISCLAIMER: +This page may contain information related to upcoming products, features and +functionality. It is important to note that the information presented is for +informational purposes only, so please do not rely on the information for +purchasing or planning purposes. Just like with all projects, the items +mentioned on the page are subject to change or delay, and the development, +release, and timing of any products, features, or functionality remain at the +sole discretion of GitLab Inc. + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: Organizations + +One of the major designs of Cells architecture is strong isolation between Groups. +Organizations as described by this blueprint provides a way to have plausible UX +for joining together many Groups that are isolated from the rest of systems. + +## 1. Definition + +Cells do require that all groups and projects of a single organization can +only be stored on a single Cell since a Cell can only access data that it holds locally +and has very limited capabilities to read information from other Cells. + +Cells with Organizations do require strong isolation between organizations. + +It will have significant implications on various user-facing features, +like Todos, dropdowns allowing to select projects, references to other issues +or projects, or any other social functions present at GitLab. Today those functions +were able to reference anything in the whole system. With the introduction of +organizations such will be forbidden. + +This problem definition aims to answer effort and implications required to add +strong isolation between organizations to the system. Including features affected +and their data processing flow. The purpose is to ensure that our solution when +implemented consistently avoids data leakage between organizations residing on +a single Cell. + +## 2. Data flow + +## 3. Proposal + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/cells-feature-personal-namespaces.md b/doc/architecture/blueprints/cells/cells-feature-personal-namespaces.md new file mode 100644 index 00000000000..e8f5c250a8e --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-personal-namespaces.md @@ -0,0 +1,30 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Personal Namespaces' +--- + + + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: Personal Namespaces + +> TL;DR + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/cells-feature-router-endpoints-classification.md b/doc/architecture/blueprints/cells/cells-feature-router-endpoints-classification.md new file mode 100644 index 00000000000..7c2974ca258 --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-router-endpoints-classification.md @@ -0,0 +1,47 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Router Endpoints Classification' +--- + + + +DISCLAIMER: +This page may contain information related to upcoming products, features and +functionality. It is important to note that the information presented is for +informational purposes only, so please do not rely on the information for +purchasing or planning purposes. Just like with all projects, the items +mentioned on the page are subject to change or delay, and the development, +release, and timing of any products, features, or functionality remain at the +sole discretion of GitLab Inc. + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: Router Endpoints Classification + +Classification of all endpoints is essential to properly route request +hitting load balancer of a GitLab installation to a Cell that can serve it. + +Each Cell should be able to decode each request and classify for which Cell +it belongs to. + +GitLab currently implements hundreds of endpoints. This document tries +to describe various techniques that can be implemented to allow the Rails +to provide this information efficiently. + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/cells-feature-schema-changes.md b/doc/architecture/blueprints/cells/cells-feature-schema-changes.md new file mode 100644 index 00000000000..d712b24a8a0 --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-schema-changes.md @@ -0,0 +1,56 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Schema changes' +--- + + + +DISCLAIMER: +This page may contain information related to upcoming products, features and +functionality. It is important to note that the information presented is for +informational purposes only, so please do not rely on the information for +purchasing or planning purposes. Just like with all projects, the items +mentioned on the page are subject to change or delay, and the development, +release, and timing of any products, features, or functionality remain at the +sole discretion of GitLab Inc. + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: Schema changes + +When we introduce multiple Cells that own their own databases this will +complicate the process of making schema changes to Postgres and Elasticsearch. +Today we already need to be careful to make changes comply with our zero +downtime deployments. For example, +[when removing a column we need to make changes over 3 separate deployments](../../../development/database/avoiding_downtime_in_migrations.md#dropping-columns). +We have tooling like `post_migrate` that helps with these kinds of changes to +reduce the number of merge requests needed, but these will be complicated when +we are dealing with deploying multiple rails applications that will be at +different versions at any one time. This problem will be particularly tricky to +solve for shared databases like our plan to share the `users` related tables +among all Cells. + +A key benefit of Cells may be that it allows us to run different +customers on different versions of GitLab. We may choose to update our own cell +before all our customers giving us even more flexibility than our current +canary architecture. But doing this means that schema changes need to have even +more versions of backward compatibility support which could slow down +development as we need extra steps to make schema changes. + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/cells-feature-secrets.md b/doc/architecture/blueprints/cells/cells-feature-secrets.md new file mode 100644 index 00000000000..20260c89ccd --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-secrets.md @@ -0,0 +1,49 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Secrets' +--- + + + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: Secrets + +Where possible, each cell should have its own distinct set of secrets. +However, there will be some secrets that will be required to be the same for all +cells in the cluster + +## 1. Definition + +GitLab has a lot of +[secrets](https://docs.gitlab.com/charts/installation/secrets.html) that needs +to be configured. + +Some secrets are for inter-component communication, e.g. `GitLab Shell secret`, +and used only within a cell. + +Some secrets are used for features, e.g. `ci_jwt_signing_key`. + +## 2. Data flow + +## 3. Proposal + +1. Secrets used for features will need to be consistent across all cells, so that the UX is consistent. + 1. This is especially true for the `db_key_base` secret which is used for + encrypting data at rest in the database - so that projects that are + transferred to another cell will continue to work. We do not want to have + to re-encrypt such rows when we move projects/groups between cells. +1. Secrets which are used for intra-cell communication only should be uniquely generated + per-cell. + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/cells-feature-snippets.md b/doc/architecture/blueprints/cells/cells-feature-snippets.md new file mode 100644 index 00000000000..f5e72c0e3a0 --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-snippets.md @@ -0,0 +1,30 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Snippets' +--- + + + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: Snippets + +> TL;DR + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/cells-feature-template.md b/doc/architecture/blueprints/cells/cells-feature-template.md new file mode 100644 index 00000000000..3cece3dc99e --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-template.md @@ -0,0 +1,30 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Problem A' +--- + + + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: A + +> TL;DR + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/cells-feature-uploads.md b/doc/architecture/blueprints/cells/cells-feature-uploads.md new file mode 100644 index 00000000000..fdac3a9977c --- /dev/null +++ b/doc/architecture/blueprints/cells/cells-feature-uploads.md @@ -0,0 +1,30 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Uploads' +--- + + + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Cells: Uploads + +> TL;DR + +## 1. Definition + +## 2. Data flow + +## 3. Proposal + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/cells/glossary.md b/doc/architecture/blueprints/cells/glossary.md new file mode 100644 index 00000000000..c3ec5fd12e4 --- /dev/null +++ b/doc/architecture/blueprints/cells/glossary.md @@ -0,0 +1,106 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Glossary' +--- + +# Cells: Glossary + +We use the following terms to describe components and properties of the Cells architecture. + +## Cell + +> Pod was renamed to Cell in + +A Cell is a set of infrastructure components that contains multiple top-level groups that belong to different organizations. The components include both datastores (PostgreSQL, Redis etc.) and stateless services (web etc.). The infrastructure components provided within a Cell are shared among organizations and their top-level groups but not shared with other Cells. This isolation of infrastructure components means that Cells are independent from each other. + + + +### Cell properties + +- Each cell is independent from the others +- Infrastructure components are shared by organizations and their top-level groups within a Cell +- More Cells can be provisioned to provide horizontal scalability +- A failing Cell does not lead to failure of other Cells +- Noisy neighbor effects are limited to within a Cell +- Cells are not visible to organizations; it is an implementation detail +- Cells may be located in different geographical regions (for example, EU, US, JP, UK) + +Discouraged synonyms: GitLab instance, cluster, shard + +## Cluster + +A cluster is a collection of Cells. + + + +### Cluster properties + +- A cluster holds cluster-wide metadata, for example Users, Routes, Settings. + +Discouraged synonyms: whale + +## Organizations + +GitLab references [Organizations in the initial set up](../../../topics/set_up_organization.md) and users can add a (free text) organization to their profile. There is no Organization entity established in the GitLab codebase. + +As part of delivering Cells, we propose the introduction of an `organization` entity. Organizations would represent billable entities or customers. + +Organizations are a known concept, present for example in [AWS](https://docs.aws.amazon.com/whitepapers/latest/organizing-your-aws-environment/core-concepts.html) and [GCP](https://cloud.google.com/resource-manager/docs/cloud-platform-resource-hierarchy#organizations). + +Organizations work under the following assumptions: + +1. Users care about what happens within their organizations. +1. Features need to work within an organization. +1. Only few features need to work across organizations. +1. Users understand that the majority of pages they view are only scoped to a single organization at a time. +1. Organizations are located on a single cell. + +![Term Organization](images/term-organization.png) + +### Organization properties + +- Top-level groups belong to organizations +- Organizations are isolated from each other by default meaning that cross-group features will only work for group that exist within a single organization +- User namespaces must not belong to an organization + +Discouraged synonyms: Billable entities, customers + +## Top-Level group + +Top-level group is the name given to the top most group of all other groups. Groups and projects are nested underneath the top-level group. + +Example: + +`https://gitlab.com/gitlab-org/gitlab/`: + +- `gitlab-org` is a `top-level group`; the root for all groups and projects of an organization +- `gitlab` is a `project`; a project of the organization. + +The top-level group has served as the defacto Organization entity. With the creation of Organization, top-level groups will be [nested underneath Organizations](https://gitlab.com/gitlab-org/gitlab/-/issues/394796). + +Over time there won't be a distinction between a top-level group and a group. All features that make Top-level groups different from groups will move to Organization. + +Discouraged synonyms: Root-level namespace + +![Term Top-level Group](images/term-top-level-group.png) + +### Top-level group properties + +- Top-level groups belonging to an organization are located on the same Cell +- Top-level groups can interact with other top-level groups that belong to the same organization + +## Users + +Users are available globally and not restricted to a single Cell. Users belong to a single organization, but can participate in many organizations through group and project membership with varying permissions. Inside organizations, users can create multiple top-level groups. User activity is not limited to a single organization but their contributions (for example TODOs) are only aggregated within an organization. This avoids the need for aggregating across cells. + +### User properties + +- Users are shared globally across all Cells +- Users can create multiple top-level groups +- Users can be a member of multiple top-level groups +- Users belong to one organization. See [!395736](https://gitlab.com/gitlab-org/gitlab/-/issues/395736) +- Users can be members of groups and projects in different organizations +- Users can administer organizations +- User activity is aggregated in an organization +- Every user has one personal namespace diff --git a/doc/architecture/blueprints/cells/goals.md b/doc/architecture/blueprints/cells/goals.md new file mode 100644 index 00000000000..67dc25625c7 --- /dev/null +++ b/doc/architecture/blueprints/cells/goals.md @@ -0,0 +1,59 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Goals' +--- + +# Cells: Goals + +## Scalability + +The main goal of this new shared-infrastructure architecture is to provide additional scalability for our SaaS Platform. GitLab.com is largely monolithic and we have estimated (internal) that the current architecture has scalability limitations, even when database partitioning and decomposition are taken into account. + +Cells provide a horizontally scalable solution because additional Cells can be created based on demand. Cells can be provisioned and tuned as needed for optimal scalability. + +## Increased availability + +A major challenge for shared-infrastructure architectures is a lack of isolation between top-level groups. This can lead to noisy neighbor effects. A organization's behavior inside a top-level group can impact all other organizations. This is highly undesirable. Cells provide isolation at the cell level. A group of organizations is fully isolated from other organizations located on a different Cell. This minimizes noisy neighbor effects while still benefiting from the cost-efficiency of shared infrastructure. + +Additionally, Cells provide a way to implement disaster recovery capabilities. Entire Cells may be replicated to read-only standbys with automatic failover capabilities. + +## A consistent experience + +Organizations should have the same user experience on our SaaS platform as they do on a self-managed GitLab instance. + +## Regions + +GitLab.com is only hosted within the United States of America. Organizations located in other regions have voiced demand for local SaaS offerings. Cells provide a path towards [GitLab Regions](https://gitlab.com/groups/gitlab-org/-/epics/6037) because Cells may be deployed within different geographies. Depending on which of the organization's data is located outside a Cell, this may solve data residency and compliance problems. + +## Market segment + +Cells would provide a solution for organizations in the small to medium business (up to 100 users) and the mid-market segment (up to 2000 users). +(See [segmentation definitions](https://about.gitlab.com/handbook/sales/field-operations/gtm-resources/#segmentation).) +Larger organizations may benefit substantially from [GitLab Dedicated](../../../subscriptions/gitlab_dedicated/index.md). + +At this moment, GitLab.com has "social-network"-like capabilities that may not fit well into a more isolated organization model. Removing those features, however, possesses some challenges: + +1. How will existing `gitlab-org` contributors contribute to the namespace?? +1. How do we move existing top-level groups into the new model (effectively breaking their social features)? + +We should evaluate if the SMB and mid market segment is interested in these features, or if not having them is acceptable in most cases. + +## Self-managed + +For reasons of consistency, it is expected that self-managed instances will +adopt the cells architecture as well. To expand, self-managed instances can +continue with just a single Cell while supporting the option of adding additional +Cells. Organizations, and possible User decomposition will also be adopted for +self-managed instances. + +## High-level architecture problems to solve + +A number of technical issues need to be resolved to implement Cells (in no particular order). This section will be expanded. + +1. How are Cells provisioned? - [Design discussion](https://gitlab.com/gitlab-org/gitlab/-/issues/396641) +1. What is a Cells topology? - [Design discussion](https://gitlab.com/gitlab-org/gitlab/-/issues/396641) +1. How are users of an organization routed to the correct Cell? - +1. How do users authenticate with Cells and Organizations? - [Design discussion](https://gitlab.com/gitlab-org/gitlab/-/issues/395736) +1. How are Cells rebalanced? +1. How can Cells implement disaster recovery capabilities? diff --git a/doc/architecture/blueprints/cells/images/pods-and-fulfillment.png b/doc/architecture/blueprints/cells/images/pods-and-fulfillment.png new file mode 100644 index 00000000000..fea32d1800e Binary files /dev/null and b/doc/architecture/blueprints/cells/images/pods-and-fulfillment.png differ diff --git a/doc/architecture/blueprints/cells/images/term-cell.png b/doc/architecture/blueprints/cells/images/term-cell.png new file mode 100644 index 00000000000..799b2eccd95 Binary files /dev/null and b/doc/architecture/blueprints/cells/images/term-cell.png differ diff --git a/doc/architecture/blueprints/cells/images/term-cluster.png b/doc/architecture/blueprints/cells/images/term-cluster.png new file mode 100644 index 00000000000..03c92850b64 Binary files /dev/null and b/doc/architecture/blueprints/cells/images/term-cluster.png differ diff --git a/doc/architecture/blueprints/cells/images/term-organization.png b/doc/architecture/blueprints/cells/images/term-organization.png new file mode 100644 index 00000000000..dd6367ad84a Binary files /dev/null and b/doc/architecture/blueprints/cells/images/term-organization.png differ diff --git a/doc/architecture/blueprints/cells/images/term-top-level-group.png b/doc/architecture/blueprints/cells/images/term-top-level-group.png new file mode 100644 index 00000000000..4af2468f50d Binary files /dev/null and b/doc/architecture/blueprints/cells/images/term-top-level-group.png differ diff --git a/doc/architecture/blueprints/cells/impact.md b/doc/architecture/blueprints/cells/impact.md new file mode 100644 index 00000000000..878af4d1a5e --- /dev/null +++ b/doc/architecture/blueprints/cells/impact.md @@ -0,0 +1,58 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells: Cross-section impact' +--- + +# Cells: Cross-section impact + +Cells is a fundamental architecture change that impacts other sections and stages. This section summarizes and links to other groups that may be impacted and highlights potential conflicts that need to be resolved. The Tenant Scale group is not responsible for achieving the goals of other groups but we want to ensure that dependencies are resolved. + +## Summary + +Based on discussions with other groups the net impact of introducing Cells and a new entity called organizations is mostly neutral. It may slow down development in some areas. We did not discover major blockers for other teams. + +1. We need to resolve naming conflicts (proposal is TBD) +1. Cells requires introducing Organizations. Organizations are a new entity **above** top-level groups. Because this is a new entity, it may impact the ability to consolidate settings for Group::Organization and influence their decision on [how to approach introducing a an organization](https://gitlab.com/gitlab-org/gitlab/-/issues/376285#approach-2-organization-is-built-on-top-of-top-level-groups) +1. Organizations may make it slightly easier for Fulfillment to realize their billing plans. + +## Impact on Group::Organization + +We synced with the Organization PM and Designer ([recording](https://youtu.be/b5Opn9cFWFk)) and discussed the similarities and differences between the Cells and Organization proposal ([presentation](https://docs.google.com/presentation/d/1FsUi22Up15b_tu6p2m-yLML3hCZ3rgrZrmzJAxUsNmU/edit?usp=sharing)). + +### Goals of Group::Organization + +As defined in the [organization documentation](../../../user/organization/index.md): + +1. Create an entity to manage everything you do as a GitLab administrator, including: + 1. Defining and applying settings to all of your groups, subgroups, and projects. + 1. Aggregating data from all your groups, subgroups, and projects. +1. Reach feature parity between SaaS and self-managed installations, with all Admin Area settings moving to groups (?). Hardware controls remain on the instance level. + +The [organization roadmap outlines](https://gitlab.com/gitlab-org/gitlab/-/issues/368237#high-level-goals) the current goals in detail. + +### Potential conflicts with Cells + +- Organization defines a new entity as the primary organizational object for groups and projects. +- We will only introduce one entity +- Group::Organization highlighted the need to further validate the key assumption that users only care about what happens within their organization. + +## Impact on Fulfillment + +We synced with Fulfillment ([recording](https://youtu.be/FkQF3uF7vTY)) to discuss how Cells would impact them. Fulfillment is supportive of an entity above top-level groups. Their perspective is outline in [!5639](https://gitlab.com/gitlab-org/customers-gitlab-com/-/merge_requests/5639/diffs). + +### Goals of Fulfillment + +- Fulfillment has a longstanding plan to move billing from the top-level group to a level above. This would mean that a license applies for an organization and all its top-level groups. +- Fulfillment uses Zuora for billing and would like to have a 1-to-1 relationship between an organization and their Zuora entity called BillingAccount. They want to move away from tying a license to a single user. +- If a customer needs multiple organizations, the corresponding BillingAccounts can be rolled up into a consolidated billing account (similar to [AWS consolidated billing](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/consolidated-billing.html)) +- Ideally, a self-managed instance has a single Organization by default, which should be enough for most customers. +- Fulfillment prefers only one additional entity. + +A rough representation of this is: + +![Cells and Fulfillment](images/pods-and-fulfillment.png) + +### Potential conflicts with Cells + +- There are no known conflicts between Fulfillment's plans and Cells diff --git a/doc/architecture/blueprints/cells/index.md b/doc/architecture/blueprints/cells/index.md new file mode 100644 index 00000000000..9938875adb6 --- /dev/null +++ b/doc/architecture/blueprints/cells/index.md @@ -0,0 +1,360 @@ +--- +status: accepted +creation-date: "2022-09-07" +authors: [ "@ayufan", "@fzimmer", "@DylanGriffith" ] +coach: "@ayufan" +approvers: [ "@fzimmer" ] +owning-stage: "~devops::enablement" +participating-stages: [] +--- + + + +# Cells + +This document is a work-in-progress and represents a very early state of the Cells design. Significant aspects are not documented, though we expect to add them in the future. + +Cells is a new architecture for our Software as a Service platform. This architecture is horizontally-scalable, resilient, and provides a more consistent user experience. It may also provide additional features in the future, such as data residency control (regions) and federated features. + +For more information about Cells, see also: + +- [Glossary](glossary.md) +- [Goals](goals.md) +- [Cross-section impact](impact.md) + +## Work streams + +We can't ship the entire Cells architecture in one go - it is too large. +Instead, we are defining key work streams required by the project. + +Not all objectives need to be fulfilled to reach production readiness. +It is expected that some objectives will not be completed for General Availability (GA), +but will be enough to run Cells in production. + +### 1. Data access layer + +Before Cells can be run in production we need to prepare the codebase to accept the Cells architecture. +This preparation involves: + +- Allowing data sharing between Cells. +- Updating the tooling for discovering cross-Cell data traversal. +- Defining code practices for cross-Cell data traversal. +- Analyzing the data model to define the data affinity. + +Under this objective the following steps are expected: + +1. **Allow to share cluster-wide data with database-level data access layer.** + + Cells can connect to a database containing shared data. For example: + application settings, users, or routing information. + +1. **Evaluate the efficiency of database-level access vs. API-oriented access layer.** + + Reconsider the consequences of database-level data access for data migration, resiliency of updates and of interconnected systems when we share only a subset of data. + +1. **Cluster-unique identifiers** + + Every object has a unique identifier that can be used to access data across the cluster. The IDs for allocated projects, issues and any other objects are cluster-unique. + +1. **Cluster-wide deletions** + + If entities deleted in Cell 2 are cross-referenced, they are properly deleted or nullified across clusters. We will likely re-use existing [loose foreign keys](../../../development/database/loose_foreign_keys.md) to extend it with cross-Cells data removal. + +1. **Data access layer** + + Ensure that a stable data-access (versioned) layer that allows to share cluster-wide data is implemented. + +1. **Database migration** + + Ensure that migrations can be run independently between Cells, and we safely handle migrations of shared data in a way that does not impact other Cells. + +### 2. Essential workflows + +To make Cells viable we require to define and support +essential workflows before we can consider the Cells +to be of Beta quality. Essential workflows are meant +to cover the majority of application functionality +that makes the product mostly useable, but with some caveats. + +The current approach is to define workflows from top to bottom. +The order defines the presumed priority of the items. +This list is not exhaustive as we would be expecting +other teams to help and fix their workflows after +the initial phase, in which we fix the fundamental ones. + +To consider a project ready for the Beta phase, it is expected +that all features defined below are supported by Cells. +In the cases listed below, the workflows define a set of tables +to be properly attributed to the feature. In some cases, +a table with an ambiguous usage has to be broken down. +For example: `uploads` are used to store user avatars, +as well as uploaded attachments for comments. It would be expected +that `uploads` is split into `uploads` (describing group/project-level attachments) +and `global_uploads` (describing, for example, user avatars). + +Except for initial 2-3 quarters this work is highly parallel. +It would be expected that **group::tenant scale** would help other +teams to fix their feature set to work with Cells. The first 2-3 quarters +would be required to define a general split of data and build required tooling. + +1. **Instance-wide settings are shared across cluster.** + + The Admin Area section for most part is shared across a cluster. + +1. **User accounts are shared across cluster.** + + The purpose is to make `users` cluster-wide. + +1. **User can create group.** + + The purpose is to perform a targeted decomposition of `users` and `namespaces`, because the `namespaces` will be stored locally in the Cell. + +1. **User can create project.** + + The purpose is to perform a targeted decomposition of `users` and `projects`, because the `projects` will be stored locally in the Cell. + +1. **User can change profile avatar that is shared in cluster.** + + The purpose is to fix global uploads that are shared in cluster. + +1. **User can push to Git repository.** + + The purpose is to ensure that essential joins from the projects table are properly attributed to be + Cell-local, and as a result the essential Git workflow is supported. + +1. **User can run CI pipeline.** + + The purpose is that `ci_pipelines` (like `ci_stages`, `ci_builds`, `ci_job_artifacts`) and adjacent tables are properly attributed to be Cell-local. + +1. **User can create issue, merge request, and merge it after it is green.** + + The purpose is to ensure that `issues` and `merge requests` are properly attributed to be `Cell-local`. + +1. **User can manage group and project members.** + + The `members` table is properly attributed to be either `Cell-local` or `cluster-wide`. + +1. **User can manage instance-wide runners.** + + The purpose is to scope all CI Runners to be Cell-local. Instance-wide runners in fact become Cell-local runners. The expectation is to provide a user interface view and manage all runners per Cell, instead of per cluster. + +1. **User is part of organization and can only see information from the organization.** + + The purpose is to have many organizations per Cell, but never have a single organization spanning across many Cells. This is required to ensure that information shown within an organization is isolated, and does not require fetching information from other Cells. + +### 3. Additional workflows + +Some of these additional workflows might need to be supported, depending on the group decision. +This list is not exhaustive of work needed to be done. + +1. **User can use all group-level features.** +1. **User can use all project-level features.** +1. **User can share groups with other groups in an organization.** +1. **User can create system webhook.** +1. **User can upload and manage packages.** +1. **User can manage security detection features.** +1. **User can manage Kubernetes integration.** +1. TBD + +### 4. Routing layer + +The routing layer is meant to offer a consistent user experience where all Cells are presented +under a single domain (for example, `gitlab.com`), instead of +having to navigate to separate domains. + +The user will able to use `https://gitlab.com` to access Cell-enabled GitLab. Depending +on the URL access, it will be transparently proxied to the correct Cell that can serve this particular +information. For example: + +- All requests going to `https://gitlab.com/users/sign_in` are randomly distributed to all Cells. +- All requests going to `https://gitlab.com/gitlab-org/gitlab/-/tree/master` are always directed to Cell 5, for example. +- All requests going to `https://gitlab.com/my-username/my-project` are always directed to Cell 1. + +1. **Technology.** + + We decide what technology the routing service is written in. + The choice is dependent on the best performing language, and the expected way + and place of deployment of the routing layer. If it is required to make + the service multi-cloud it might be required to deploy it to the CDN provider. + Then the service needs to be written using a technology compatible with the CDN provider. + +1. **Cell discovery.** + + The routing service needs to be able to discover and monitor the health of all Cells. + +1. **Router endpoints classification.** + + The stateless routing service will fetch and cache information about endpoints + from one of the Cells. We need to implement a protocol that will allow us to + accurately describe the incoming request (its fingerprint), so it can be classified + by one of the Cells, and the results of that can be cached. We also need to implement + a mechanism for negative cache and cache eviction. + +1. **GraphQL and other ambigious endpoints.** + + Most endpoints have a unique sharding key: the organization, which directly + or indirectly (via a group or project) can be used to classify endpoints. + Some endpoints are ambiguous in their usage (they don't encode the sharding key), + or the sharding key is stored deep in the payload. In these cases, we need to decide how to handle endpoints like `/api/graphql`. + +### 5. Cell deployment + +We will run many Cells. To manage them easier, we need to have consistent +deployment procedures for Cells, including a way to deploy, manage, migrate, +and monitor. + +We are very likely to use tooling made for [GitLab Dedicated](https://about.gitlab.com/dedicated/) +with its control planes. + +1. **Extend GitLab Dedicated to support GCP.** +1. TBD + +### 6. Migration + +When we reach production and are able to store new organizations on new Cells, we need +to be able to divide big Cells into many smaller ones. + +1. **Use GitLab Geo to clone Cells.** + + The purpose is to use GitLab Geo to clone Cells. + +1. **Split Cells by cloning them.** + + Once Cell is cloned we change routing information for organizations. + Organization will encode `cell_id`. When we update `cell_id` it will automatically + make the given Cell to be authoritative to handle the traffic for the given organization. + +1. **Delete redundant data from previous Cells.** + + Since the organization is now stored on many Cells, once we change `cell_id` + we will have to remove data from all other Cells based on `organization_id`. + +## Availability of the feature + +We are following the [Support for Experiment, Beta, and Generally Available features](../../../policy/alpha-beta-support.md). + +### 1. Experiment + +Expectations: + +- We can deploy a Cell on staging or another testing environment by using a separate domain (ex. `cell2.staging.gitlab.com`) + using [Cell deployment](#5-cell-deployment) tooling. +- User can create organization, group and project, and run some of the [essential workflows](#2-essential-workflows). +- It is not expected to be able to run a router to serve all requests under a single domain. +- We expect data-loss of data stored on additional Cells. +- We expect to tear down and create many new Cells to validate tooling. + +### 2. Beta + +Expectations: + +- We can run many Cells under a single domain (ex. `staging.gitlab.com`). +- All features defined in [essential workflows](#2-essential-workflows) are supported. +- Not all aspects of [Routing layer](#4-routing-layer) are finalized. +- We expect additional Cells to be stable with minimal data loss. + +### 3. GA + +Expectations: + +- We can run many Cells under a single domain (for example, `staging.gitlab.com`). +- All features defined in [essential workflows](#2-essential-workflows) are supported. +- All features of [routing layer](#4-routing-layer) are supported. +- Most of [additional workflows](#3-additional-workflows) are supported. +- We don't expect to support any of [migration](#6-migration) aspects. + +### 4. Post GA + +Expectations: + +- We support all [additional workflows](#3-additional-workflows). +- We can [migrate](#6-migration) existing organizations onto new Cells. + +## Iteration plan + +The delivered iterations will focus on solving particular steps of a given +key work stream. + +It is expected that initial iterations will rather +be slow, because they require substantially more +changes to prepare the codebase for data split. + +One iteration describes one quarter's worth of work. + +1. Iteration 1 - FY24Q1 + + - Data access layer: Initial Admin Area settings are shared across cluster. + - Essential workflows: Allow to share cluster-wide data with database-level data access layer + +1. Iteration 2 - FY24Q2 + + - Essential workflows: User accounts are shared across cluster. + - Essential workflows: User can create group. + +1. Iteration 3 - FY24Q3 + + - Essential workflows: User can create project. + - Essential workflows: User can push to Git repository. + - Cell deployment: Extend GitLab Dedicated to support GCP + - Routing: Technology. + +1. Iteration 4 - FY24Q4 + + - Essential workflows: User can run CI pipeline. + - Essential workflows: User can create issue, merge request, and merge it after it is green. + - Data access layer: Evaluate the efficiency of database-level access vs. API-oriented access layer + - Data access layer: Cluster-unique identifiers. + - Routing: Cell discovery. + - Routing: Router endpoints classification. + +1. Iteration 5 - FY25Q1 + + - TBD + +## Technical Proposals + +The Cells architecture do have long lasting implications to data processing, location, scalability and the GitLab architecture. +This section links all different technical proposals that are being evaluated. + +- [Stateless Router That Uses a Cache to Pick Cell and Is Redirected When Wrong Cell Is Reached](proposal-stateless-router-with-buffering-requests.md) + +- [Stateless Router That Uses a Cache to Pick Cell and pre-flight `/api/v4/cells/learn`](proposal-stateless-router-with-routes-learning.md) + +## Impacted features + +The Cells architecture will impact many features requiring some of them to be rewritten, or changed significantly. +This is the list of known affected features with the proposed solutions. + +- [Cells: Git Access](cells-feature-git-access.md) +- [Cells: Data Migration](cells-feature-data-migration.md) +- [Cells: Database Sequences](cells-feature-database-sequences.md) +- [Cells: GraphQL](cells-feature-graphql.md) +- [Cells: Organizations](cells-feature-organizations.md) +- [Cells: Router Endpoints Classification](cells-feature-router-endpoints-classification.md) +- [Cells: Schema changes (Postgres and Elasticsearch migrations)](cells-feature-schema-changes.md) +- [Cells: Backups](cells-feature-backups.md) +- [Cells: Global Search](cells-feature-global-search.md) +- [Cells: CI Runners](cells-feature-ci-runners.md) +- [Cells: Admin Area](cells-feature-admin-area.md) +- [Cells: Secrets](cells-feature-secrets.md) +- [Cells: Container Registry](cells-feature-container-registry.md) +- [Cells: Contributions: Forks](cells-feature-contributions-forks.md) +- [Cells: Personal Namespaces](cells-feature-personal-namespaces.md) +- [Cells: Dashboard: Projects, Todos, Issues, Merge Requests, Activity, ...](cells-feature-dashboard.md) +- [Cells: Snippets](cells-feature-snippets.md) +- [Cells: Uploads](cells-feature-uploads.md) +- [Cells: GitLab Pages](cells-feature-gitlab-pages.md) +- [Cells: Agent for Kubernetes](cells-feature-agent-for-kubernetes.md) + +## Decision log + +- 2022-03-15: Google Cloud as the cloud service. For details, see [issue 396641](https://gitlab.com/gitlab-org/gitlab/-/issues/396641#note_1314932272). + +## Links + +- [Internal Pods presentation](https://docs.google.com/presentation/d/1x1uIiN8FR9fhL7pzFh9juHOVcSxEY7d2_q4uiKKGD44/edit#slide=id.ge7acbdc97a_0_155) +- [Internal link to all diagrams](https://drive.google.com/file/d/13NHzbTrmhUM-z_Bf0RjatUEGw5jWHSLt/view?usp=sharing) +- [Cells Epic](https://gitlab.com/groups/gitlab-org/-/epics/7582) +- [Database Group investigation](https://about.gitlab.com/handbook/engineering/development/enablement/data_stores/database/doc/root-namespace-sharding.html) +- [Shopify Pods architecture](https://shopify.engineering/a-pods-architecture-to-allow-shopify-to-scale) +- [Opstrace architecture](https://gitlab.com/gitlab-org/opstrace/opstrace/-/blob/main/docs/architecture/overview.md) diff --git a/doc/architecture/blueprints/cells/proposal-stateless-router-with-buffering-requests.md b/doc/architecture/blueprints/cells/proposal-stateless-router-with-buffering-requests.md new file mode 100644 index 00000000000..f352fea84b1 --- /dev/null +++ b/doc/architecture/blueprints/cells/proposal-stateless-router-with-buffering-requests.md @@ -0,0 +1,649 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells Stateless Router Proposal' +--- + + + +This document is a work-in-progress and represents a very early state of the +Pods design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Pods, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Proposal: Stateless Router + +We will decompose `gitlab_users`, `gitlab_routes` and `gitlab_admin` related +tables so that they can be shared between all cells and allow any cell to +authenticate a user and route requests to the correct cell. Cells may receive +requests for the resources they don't own, but they know how to redirect back +to the correct cell. + +The router is stateless and does not read from the `routes` database which +means that all interactions with the database still happen from the Rails +monolith. This architecture also supports regions by allowing for low traffic +databases to be replicated across regions. + +Users are not directly exposed to the concept of Cells but instead they see +different data dependent on their chosen "organization". +[Organizations](glossary.md#organizations) will be a new model introduced to enforce isolation in the +application and allow us to decide which request route to which cell, since an +organization can only be on a single cell. + +## Differences + +The main difference between this proposal and the one [with learning routes](proposal-stateless-router-with-routes-learning.md) +is that this proposal always sends requests to any of the Cells. If the requests cannot be processed, +the requests will be bounced back with relevant headers. This requires that request to be buffered. +It allows that request decoding can be either via URI or Body of request by Rails. +This means that each request might be sent more than once and be processed more than once as result. + +The [with learning routes proposal](proposal-stateless-router-with-routes-learning.md) requires that +routable information is always encoded in URI, and the router sends a pre-flight request. + +## Summary in diagrams + +This shows how a user request routes via DNS to the nearest router and the router chooses a cell to send the request to. + +```mermaid +graph TD; + user((User)); + dns[DNS]; + router_us(Router); + router_eu(Router); + cell_us0{Cell US0}; + cell_us1{Cell US1}; + cell_eu0{Cell EU0}; + cell_eu1{Cell EU1}; + user-->dns; + dns-->router_us; + dns-->router_eu; + subgraph Europe + router_eu-->cell_eu0; + router_eu-->cell_eu1; + end + subgraph United States + router_us-->cell_us0; + router_us-->cell_us1; + end +``` + +
More detail + +This shows that the router can actually send requests to any cell. The user will +get the closest router to them geographically. + +```mermaid +graph TD; + user((User)); + dns[DNS]; + router_us(Router); + router_eu(Router); + cell_us0{Cell US0}; + cell_us1{Cell US1}; + cell_eu0{Cell EU0}; + cell_eu1{Cell EU1}; + user-->dns; + dns-->router_us; + dns-->router_eu; + subgraph Europe + router_eu-->cell_eu0; + router_eu-->cell_eu1; + end + subgraph United States + router_us-->cell_us0; + router_us-->cell_us1; + end + router_eu-.->cell_us0; + router_eu-.->cell_us1; + router_us-.->cell_eu0; + router_us-.->cell_eu1; +``` + +
+ +
Even more detail + +This shows the databases. `gitlab_users` and `gitlab_routes` exist only in the +US region but are replicated to other regions. Replication does not have an +arrow because it's too hard to read the diagram. + +```mermaid +graph TD; + user((User)); + dns[DNS]; + router_us(Router); + router_eu(Router); + cell_us0{Cell US0}; + cell_us1{Cell US1}; + cell_eu0{Cell EU0}; + cell_eu1{Cell EU1}; + db_gitlab_users[(gitlab_users Primary)]; + db_gitlab_routes[(gitlab_routes Primary)]; + db_gitlab_users_replica[(gitlab_users Replica)]; + db_gitlab_routes_replica[(gitlab_routes Replica)]; + db_cell_us0[(gitlab_main/gitlab_ci Cell US0)]; + db_cell_us1[(gitlab_main/gitlab_ci Cell US1)]; + db_cell_eu0[(gitlab_main/gitlab_ci Cell EU0)]; + db_cell_eu1[(gitlab_main/gitlab_ci Cell EU1)]; + user-->dns; + dns-->router_us; + dns-->router_eu; + subgraph Europe + router_eu-->cell_eu0; + router_eu-->cell_eu1; + cell_eu0-->db_cell_eu0; + cell_eu0-->db_gitlab_users_replica; + cell_eu0-->db_gitlab_routes_replica; + cell_eu1-->db_gitlab_users_replica; + cell_eu1-->db_gitlab_routes_replica; + cell_eu1-->db_cell_eu1; + end + subgraph United States + router_us-->cell_us0; + router_us-->cell_us1; + cell_us0-->db_cell_us0; + cell_us0-->db_gitlab_users; + cell_us0-->db_gitlab_routes; + cell_us1-->db_gitlab_users; + cell_us1-->db_gitlab_routes; + cell_us1-->db_cell_us1; + end + router_eu-.->cell_us0; + router_eu-.->cell_us1; + router_us-.->cell_eu0; + router_us-.->cell_eu1; +``` + +
+ +## Summary of changes + +1. Tables related to User data (including profile settings, authentication credentials, personal access tokens) are decomposed into a `gitlab_users` schema +1. The `routes` table is decomposed into `gitlab_routes` schema +1. The `application_settings` (and probably a few other instance level tables) are decomposed into `gitlab_admin` schema +1. A new column `routes.cell_id` is added to `routes` table +1. A new Router service exists to choose which cell to route a request to. +1. A new concept will be introduced in GitLab called an organization and a user can select a "default organization" and this will be a user level setting. The default organization is used to redirect users away from ambiguous routes like `/dashboard` to organization scoped routes like `/organizations/my-organization/-/dashboard`. Legacy users will have a special default organization that allows them to keep using global resources on `Cell US0`. All existing namespaces will initially move to this public organization. +1. If a cell receives a request for a `routes.cell_id` that it does not own it returns a `302` with `X-Gitlab-Cell-Redirect` header so that the router can send the request to the correct cell. The correct cell can also set a header `X-Gitlab-Cell-Cache` which contains information about how this request should be cached to remember the cell. For example if the request was `/gitlab-org/gitlab` then the header would encode `/gitlab-org/* => Cell US0` (for example, any requests starting with `/gitlab-org/` can always be routed to `Cell US0` +1. When the cell does not know (from the cache) which cell to send a request to it just picks a random cell within it's region +1. Writes to `gitlab_users` and `gitlab_routes` are sent to a primary PostgreSQL server in our `US` region but reads can come from replicas in the same region. This will add latency for these writes but we expect they are infrequent relative to the rest of GitLab. + +## Detailed explanation of default organization in the first iteration + +All users will get a new column `users.default_organization` which they can +control in user settings. We will introduce a concept of the +`GitLab.com Public` organization. This will be set as the default organization for all existing +users. This organization will allow the user to see data from all namespaces in +`Cell US0` (for example, our original GitLab.com instance). This behavior can be invisible to +existing users such that they don't even get told when they are viewing a +global page like `/dashboard` that it's even scoped to an organization. + +Any new users with a default organization other than `GitLab.com Public` will have +a distinct user experience and will be fully aware that every page they load is +only ever scoped to a single organization. These users can never +load any global pages like `/dashboard` and will end up being redirected to +`/organizations//-/dashboard`. This may also be the case +for legacy APIs and such users may only ever be able to use APIs scoped to a +organization. + +## Detailed explanation of Admin Area settings + +We believe that maintaining and synchronizing Admin Area settings will be +frustrating and painful so to avoid this we will decompose and share all Admin Area +settings in the `gitlab_admin` schema. This should be safe (similar to other +shared schemas) because these receive very little write traffic. + +In cases where different cells need different settings (for example, the +Elasticsearch URL), we will either decide to use a templated +format in the relevant `application_settings` row which allows it to be dynamic +per cell. Alternatively if that proves difficult we'll introduce a new table +called `per_cell_application_settings` and this will have 1 row per cell to allow +setting different settings per cell. It will still be part of the `gitlab_admin` +schema and shared which will allow us to centrally manage it and simplify +keeping settings in sync for all cells. + +## Pros + +1. Router is stateless and can live in many regions. We use Anycast DNS to resolve to nearest region for the user. +1. Cells can receive requests for namespaces in the wrong cell and the user + still gets the right response as well as caching at the router that + ensures the next request is sent to the correct cell so the next request + will go to the correct cell +1. The majority of the code still lives in `gitlab` rails codebase. The Router doesn't actually need to understand how GitLab URLs are composed. +1. Since the responsibility to read and write `gitlab_users`, + `gitlab_routes` and `gitlab_admin` still lives in Rails it means minimal + changes will be needed to the Rails application compared to extracting + services that need to isolate the domain models and build new interfaces. +1. Compared to a separate routing service this allows the Rails application + to encode more complex rules around how to map URLs to the correct cell + and may work for some existing API endpoints. +1. All the new infrastructure (just a router) is optional and a single-cell + self-managed installation does not even need to run the Router and there are + no other new services. + +## Cons + +1. `gitlab_users`, `gitlab_routes` and `gitlab_admin` databases may need to be + replicated across regions and writes need to go across regions. We need to + do an analysis on write TPS for the relevant tables to determine if this is + feasible. +1. Sharing access to the database from many different Cells means that they are + all coupled at the Postgres schema level and this means changes to the + database schema need to be done carefully in sync with the deployment of all + Cells. This limits us to ensure that Cells are kept in closely similar + versions compared to an architecture with shared services that have an API + we control. +1. Although most data is stored in the right region there can be requests + proxied from another region which may be an issue for certain types + of compliance. +1. Data in `gitlab_users` and `gitlab_routes` databases must be replicated in + all regions which may be an issue for certain types of compliance. +1. The router cache may need to be very large if we get a wide variety of URLs + (for example, long tail). In such a case we may need to implement a 2nd level of + caching in user cookies so their frequently accessed pages always go to the + right cell the first time. +1. Having shared database access for `gitlab_users` and `gitlab_routes` + from multiple cells is an unusual architecture decision compared to + extracting services that are called from multiple cells. +1. It is very likely we won't be able to find cacheable elements of a + GraphQL URL and often existing GraphQL endpoints are heavily dependent on + ids that won't be in the `routes` table so cells won't necessarily know + what cell has the data. As such we'll probably have to update our GraphQL + calls to include an organization context in the path like + `/api/organizations//graphql`. +1. This architecture implies that implemented endpoints can only access data + that are readily accessible on a given Cell, but are unlikely + to aggregate information from many Cells. +1. All unknown routes are sent to the latest deployment which we assume to be `Cell US0`. + This is required as newly added endpoints will be only decodable by latest cell. + This Cell could later redirect to correct one that can serve the given request. + Since request processing might be heavy some Cells might receive significant amount + of traffic due to that. + +## Example database configuration + +Handling shared `gitlab_users`, `gitlab_routes` and `gitlab_admin` databases, while having dedicated `gitlab_main` and `gitlab_ci` databases should already be handled by the way we use `config/database.yml`. We should also, already be able to handle the dedicated EU replicas while having a single US primary for `gitlab_users` and `gitlab_routes`. Below is a snippet of part of the database configuration for the Cell architecture described above. + +
Cell US0 + +```yaml +# config/database.yml +production: + main: + host: postgres-main.cell-us0.primary.consul + load_balancing: + discovery: postgres-main.cell-us0.replicas.consul + ci: + host: postgres-ci.cell-us0.primary.consul + load_balancing: + discovery: postgres-ci.cell-us0.replicas.consul + users: + host: postgres-users-primary.consul + load_balancing: + discovery: postgres-users-replicas.us.consul + routes: + host: postgres-routes-primary.consul + load_balancing: + discovery: postgres-routes-replicas.us.consul + admin: + host: postgres-admin-primary.consul + load_balancing: + discovery: postgres-admin-replicas.us.consul +``` + +
+ +
Cell EU0 + +```yaml +# config/database.yml +production: + main: + host: postgres-main.cell-eu0.primary.consul + load_balancing: + discovery: postgres-main.cell-eu0.replicas.consul + ci: + host: postgres-ci.cell-eu0.primary.consul + load_balancing: + discovery: postgres-ci.cell-eu0.replicas.consul + users: + host: postgres-users-primary.consul + load_balancing: + discovery: postgres-users-replicas.eu.consul + routes: + host: postgres-routes-primary.consul + load_balancing: + discovery: postgres-routes-replicas.eu.consul + admin: + host: postgres-admin-primary.consul + load_balancing: + discovery: postgres-admin-replicas.eu.consul +``` + +
+ +## Request flows + +1. `gitlab-org` is a top level namespace and lives in `Cell US0` in the `GitLab.com Public` organization +1. `my-company` is a top level namespace and lives in `Cell EU0` in the `my-organization` organization + +### Experience for paying user that is part of `my-organization` + +Such a user will have a default organization set to `/my-organization` and will be +unable to load any global routes outside of this organization. They may load other +projects/namespaces but their MR/Todo/Issue counts at the top of the page will +not be correctly populated in the first iteration. The user will be aware of +this limitation. + +#### Navigates to `/my-company/my-project` while logged in + +1. User is in Europe so DNS resolves to the router in Europe +1. They request `/my-company/my-project` without the router cache, so the router chooses randomly `Cell EU1` +1. `Cell EU1` does not have `/my-company`, but it knows that it lives in `Cell EU0` so it redirects the router to `Cell EU0` +1. `Cell EU0` returns the correct response as well as setting the cache headers for the router `/my-company/* => Cell EU0` +1. The router now caches and remembers any request paths matching `/my-company/*` should go to `Cell EU0` + +```mermaid +sequenceDiagram + participant user as User + participant router_eu as Router EU + participant cell_eu0 as Cell EU0 + participant cell_eu1 as Cell EU1 + user->>router_eu: GET /my-company/my-project + router_eu->>cell_eu1: GET /my-company/my-project + cell_eu1->>router_eu: 302 /my-company/my-project X-Gitlab-Cell-Redirect={cell:Cell EU0} + router_eu->>cell_eu0: GET /my-company/my-project + cell_eu0->>user:

My Project... X-Gitlab-Cell-Cache={path_prefix:/my-company/} +``` + +#### Navigates to `/my-company/my-project` while not logged in + +1. User is in Europe so DNS resolves to the router in Europe +1. The router does not have `/my-company/*` cached yet so it chooses randomly `Cell EU1` +1. `Cell EU1` redirects them through a login flow +1. Still they request `/my-company/my-project` without the router cache, so the router chooses a random cell `Cell EU1` +1. `Cell EU1` does not have `/my-company`, but it knows that it lives in `Cell EU0` so it redirects the router to `Cell EU0` +1. `Cell EU0` returns the correct response as well as setting the cache headers for the router `/my-company/* => Cell EU0` +1. The router now caches and remembers any request paths matching `/my-company/*` should go to `Cell EU0` + +```mermaid +sequenceDiagram + participant user as User + participant router_eu as Router EU + participant cell_eu0 as Cell EU0 + participant cell_eu1 as Cell EU1 + user->>router_eu: GET /my-company/my-project + router_eu->>cell_eu1: GET /my-company/my-project + cell_eu1->>user: 302 /users/sign_in?redirect=/my-company/my-project + user->>router_eu: GET /users/sign_in?redirect=/my-company/my-project + router_eu->>cell_eu1: GET /users/sign_in?redirect=/my-company/my-project + cell_eu1->>user:

Sign in... + user->>router_eu: POST /users/sign_in?redirect=/my-company/my-project + router_eu->>cell_eu1: POST /users/sign_in?redirect=/my-company/my-project + cell_eu1->>user: 302 /my-company/my-project + user->>router_eu: GET /my-company/my-project + router_eu->>cell_eu1: GET /my-company/my-project + cell_eu1->>router_eu: 302 /my-company/my-project X-Gitlab-Cell-Redirect={cell:Cell EU0} + router_eu->>cell_eu0: GET /my-company/my-project + cell_eu0->>user:

My Project... X-Gitlab-Cell-Cache={path_prefix:/my-company/} +``` + +#### Navigates to `/my-company/my-other-project` after last step + +1. User is in Europe so DNS resolves to the router in Europe +1. The router cache now has `/my-company/* => Cell EU0`, so the router chooses `Cell EU0` +1. `Cell EU0` returns the correct response as well as the cache header again + +```mermaid +sequenceDiagram + participant user as User + participant router_eu as Router EU + participant cell_eu0 as Cell EU0 + participant cell_eu1 as Cell EU1 + user->>router_eu: GET /my-company/my-project + router_eu->>cell_eu0: GET /my-company/my-project + cell_eu0->>user:

My Project... X-Gitlab-Cell-Cache={path_prefix:/my-company/} +``` + +#### Navigates to `/gitlab-org/gitlab` after last step + +1. User is in Europe so DNS resolves to the router in Europe +1. The router has no cached value for this URL so randomly chooses `Cell EU0` +1. `Cell EU0` redirects the router to `Cell US0` +1. `Cell US0` returns the correct response as well as the cache header again + +```mermaid +sequenceDiagram + participant user as User + participant router_eu as Router EU + participant cell_eu0 as Cell EU0 + participant cell_us0 as Cell US0 + user->>router_eu: GET /gitlab-org/gitlab + router_eu->>cell_eu0: GET /gitlab-org/gitlab + cell_eu0->>router_eu: 302 /gitlab-org/gitlab X-Gitlab-Cell-Redirect={cell:Cell US0} + router_eu->>cell_us0: GET /gitlab-org/gitlab + cell_us0->>user:

GitLab.org... X-Gitlab-Cell-Cache={path_prefix:/gitlab-org/} +``` + +In this case the user is not on their "default organization" so their TODO +counter will not include their normal todos. We may choose to highlight this in +the UI somewhere. A future iteration may be able to fetch that for them from +their default organization. + +#### Navigates to `/` + +1. User is in Europe so DNS resolves to the router in Europe +1. Router does not have a cache for `/` route (specifically rails never tells it to cache this route) +1. The Router choose `Cell EU0` randomly +1. The Rails application knows the users default organization is `/my-organization`, so + it redirects the user to `/organizations/my-organization/-/dashboard` +1. The Router has a cached value for `/organizations/my-organization/*` so it then sends the + request to `POD EU0` +1. `Cell EU0` serves up a new page `/organizations/my-organization/-/dashboard` which is the same + dashboard view we have today but scoped to an organization clearly in the UI +1. The user is (optionally) presented with a message saying that data on this page is only + from their default organization and that they can change their default + organization if it's not right. + +```mermaid +sequenceDiagram + participant user as User + participant router_eu as Router EU + participant cell_eu0 as Cell EU0 + user->>router_eu: GET / + router_eu->>cell_eu0: GET / + cell_eu0->>user: 302 /organizations/my-organization/-/dashboard + user->>router: GET /organizations/my-organization/-/dashboard + router->>cell_eu0: GET /organizations/my-organization/-/dashboard + cell_eu0->>user:

My Company Dashboard... X-Gitlab-Cell-Cache={path_prefix:/organizations/my-organization/} +``` + +#### Navigates to `/dashboard` + +As above, they will end up on `/organizations/my-organization/-/dashboard` as +the rails application will already redirect `/` to the dashboard page. + +### Navigates to `/not-my-company/not-my-project` while logged in (but they don't have access since this project/group is private) + +1. User is in Europe so DNS resolves to the router in Europe +1. The router knows that `/not-my-company` lives in `Cell US1` so sends the request to this +1. The user does not have access so `Cell US1` returns 404 + +```mermaid +sequenceDiagram + participant user as User + participant router_eu as Router EU + participant cell_us1 as Cell US1 + user->>router_eu: GET /not-my-company/not-my-project + router_eu->>cell_us1: GET /not-my-company/not-my-project + cell_us1->>user: 404 +``` + +#### Creates a new top level namespace + +The user will be asked which organization they want the namespace to belong to. +If they select `my-organization` then it will end up on the same cell as all +other namespaces in `my-organization`. If they select nothing we default to +`GitLab.com Public` and it is clear to the user that this is isolated from +their existing organization such that they won't be able to see data from both +on a single page. + +### Experience for GitLab team member that is part of `/gitlab-org` + +Such a user is considered a legacy user and has their default organization set to +`GitLab.com Public`. This is a "meta" organization that does not really exist but +the Rails application knows to interpret this organization to mean that they are +allowed to use legacy global functionality like `/dashboard` to see data across +namespaces located on `Cell US0`. The rails backend also knows that the default cell to render any ambiguous +routes like `/dashboard` is `Cell US0`. Lastly the user will be allowed to +navigate to organizations on another cell like `/my-organization` but when they do the +user will see a message indicating that some data may be missing (for example, the +MRs/Issues/Todos) counts. + +#### Navigates to `/gitlab-org/gitlab` while not logged in + +1. User is in the US so DNS resolves to the US router +1. The router knows that `/gitlab-org` lives in `Cell US0` so sends the request + to this cell +1. `Cell US0` serves up the response + +```mermaid +sequenceDiagram + participant user as User + participant router_us as Router US + participant cell_us0 as Cell US0 + user->>router_us: GET /gitlab-org/gitlab + router_us->>cell_us0: GET /gitlab-org/gitlab + cell_us0->>user:

GitLab.org... X-Gitlab-Cell-Cache={path_prefix:/gitlab-org/} +``` + +#### Navigates to `/` + +1. User is in US so DNS resolves to the router in US +1. Router does not have a cache for `/` route (specifically rails never tells it to cache this route) +1. The Router chooses `Cell US1` randomly +1. The Rails application knows the users default organization is `GitLab.com Public`, so + it redirects the user to `/dashboards` (only legacy users can see + `/dashboard` global view) +1. Router does not have a cache for `/dashboard` route (specifically rails never tells it to cache this route) +1. The Router chooses `Cell US1` randomly +1. The Rails application knows the users default organization is `GitLab.com Public`, so + it allows the user to load `/dashboards` (only legacy users can see + `/dashboard` global view) and redirects to router the legacy cell which is `Cell US0` +1. `Cell US0` serves up the global view dashboard page `/dashboard` which is the same + dashboard view we have today + +```mermaid +sequenceDiagram + participant user as User + participant router_us as Router US + participant cell_us0 as Cell US0 + participant cell_us1 as Cell US1 + user->>router_us: GET / + router_us->>cell_us1: GET / + cell_us1->>user: 302 /dashboard + user->>router_us: GET /dashboard + router_us->>cell_us1: GET /dashboard + cell_us1->>router_us: 302 /dashboard X-Gitlab-Cell-Redirect={cell:Cell US0} + router_us->>cell_us0: GET /dashboard + cell_us0->>user:

Dashboard... +``` + +#### Navigates to `/my-company/my-other-project` while logged in (but they don't have access since this project is private) + +They get a 404. + +### Experience for non-authenticated users + +Flow is similar to authenticated users except global routes like `/dashboard` will +redirect to the login page as there is no default organization to choose from. + +### A new customers signs up + +They will be asked if they are already part of an organization or if they'd +like to create one. If they choose neither they end up no the default +`GitLab.com Public` organization. + +### An organization is moved from 1 cell to another + +TODO + +### GraphQL/API requests which don't include the namespace in the URL + +TODO + +### The autocomplete suggestion functionality in the search bar which remembers recent issues/MRs + +TODO + +### Global search + +TODO + +## Administrator + +### Loads `/admin` page + +1. Router picks a random cell `Cell US0` +1. Cell US0 redirects user to `/admin/cells/cellus0` +1. Cell US0 renders an Admin Area page and also returns a cache header to cache `/admin/cellss/cellus0/* => Cell US0`. The Admin Area page contains a dropdown list showing other cells they could select and it changes the query parameter. + +Admin Area settings in Postgres are all shared across all cells to avoid +divergence but we still make it clear in the URL and UI which cell is serving +the Admin Area page as there is dynamic data being generated from these pages and +the operator may want to view a specific cell. + +## More Technical Problems To Solve + +### Replicating User Sessions Between All Cells + +Today user sessions live in Redis but each cell will have their own Redis instance. We already use a dedicated Redis instance for sessions so we could consider sharing this with all cells like we do with `gitlab_users` PostgreSQL database. But an important consideration will be latency as we would still want to mostly fetch sessions from the same region. + +An alternative might be that user sessions get moved to a JWT payload that encodes all the session data but this has downsides. For example, it is difficult to expire a user session, when their password changes or for other reasons, if the session lives in a JWT controlled by the user. + +### How do we migrate between Cells + +Migrating data between cells will need to factor all data stores: + +1. PostgreSQL +1. Redis Shared State +1. Gitaly +1. Elasticsearch + +### Is it still possible to leak the existence of private groups via a timing attack? + +If you have router in EU, and you know that EU router by default redirects +to EU located Cells, you know their latency (lets assume 10 ms). Now, if your +request is bounced back and redirected to US which has different latency +(lets assume that roundtrip will be around 60 ms) you can deduce that 404 was +returned by US Cell and know that your 404 is in fact 403. + +We may defer this until we actually implement a cell in a different region. Such timing attacks are already theoretically possible with the way we do permission checks today but the timing difference is probably too small to be able to detect. + +One technique to mitigate this risk might be to have the router add a random +delay to any request that returns 404 from a cell. + +## Should runners be shared across all cells? + +We have 2 options and we should decide which is easier: + +1. Decompose runner registration and queuing tables and share them across all + cells. This may have implications for scalability, and we'd need to consider + if this would include group/project runners as this may have scalability + concerns as these are high traffic tables that would need to be shared. +1. Runners are registered per-cell and, we probably have a separate fleet of + runners for every cell or just register the same runners to many cells which + may have implications for queueing + +## How do we guarantee unique ids across all cells for things that cannot conflict? + +This project assumes at least namespaces and projects have unique ids across +all cells as many requests need to be routed based on their ID. Since those +tables are across different databases then guaranteeing a unique ID will +require a new solution. There are likely other tables where unique IDs are +necessary and depending on how we resolve routing for GraphQL and other APIs +and other design goals it may be determined that we want the primary key to be +unique for all tables. diff --git a/doc/architecture/blueprints/cells/proposal-stateless-router-with-routes-learning.md b/doc/architecture/blueprints/cells/proposal-stateless-router-with-routes-learning.md new file mode 100644 index 00000000000..aadc08016e3 --- /dev/null +++ b/doc/architecture/blueprints/cells/proposal-stateless-router-with-routes-learning.md @@ -0,0 +1,673 @@ +--- +stage: enablement +group: Tenant Scale +description: 'Cells Stateless Router Proposal' +--- + + + +This document is a work-in-progress and represents a very early state of the +Cells design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Cells, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Proposal: Stateless Router + +We will decompose `gitlab_users`, `gitlab_routes` and `gitlab_admin` related +tables so that they can be shared between all cells and allow any cell to +authenticate a user and route requests to the correct cell. Cells may receive +requests for the resources they don't own, but they know how to redirect back +to the correct cell. + +The router is stateless and does not read from the `routes` database which +means that all interactions with the database still happen from the Rails +monolith. This architecture also supports regions by allowing for low traffic +databases to be replicated across regions. + +Users are not directly exposed to the concept of Cells but instead they see +different data dependent on their chosen "organization". +[Organizations](glossary.md#organizations) will be a new model introduced to enforce isolation in the +application and allow us to decide which request route to which cell, since an +organization can only be on a single cell. + +## Differences + +The main difference between this proposal and one [with buffering requests](proposal-stateless-router-with-buffering-requests.md) +is that this proposal uses a pre-flight API request (`/api/v4/cells/learn`) to redirect the request body to the correct Cell. +This means that each request is sent exactly once to be processed, but the URI is used to decode which Cell it should be directed. + +## Summary in diagrams + +This shows how a user request routes via DNS to the nearest router and the router chooses a cell to send the request to. + +```mermaid +graph TD; + user((User)); + dns[DNS]; + router_us(Router); + router_eu(Router); + cell_us0{Cell US0}; + cell_us1{Cell US1}; + cell_eu0{Cell EU0}; + cell_eu1{Cell EU1}; + user-->dns; + dns-->router_us; + dns-->router_eu; + subgraph Europe + router_eu-->cell_eu0; + router_eu-->cell_eu1; + end + subgraph United States + router_us-->cell_us0; + router_us-->cell_us1; + end +``` + +### More detail + +This shows that the router can actually send requests to any cell. The user will +get the closest router to them geographically. + +```mermaid +graph TD; + user((User)); + dns[DNS]; + router_us(Router); + router_eu(Router); + cell_us0{Cell US0}; + cell_us1{Cell US1}; + cell_eu0{Cell EU0}; + cell_eu1{Cell EU1}; + user-->dns; + dns-->router_us; + dns-->router_eu; + subgraph Europe + router_eu-->cell_eu0; + router_eu-->cell_eu1; + end + subgraph United States + router_us-->cell_us0; + router_us-->cell_us1; + end + router_eu-.->cell_us0; + router_eu-.->cell_us1; + router_us-.->cell_eu0; + router_us-.->cell_eu1; +``` + +### Even more detail + +This shows the databases. `gitlab_users` and `gitlab_routes` exist only in the +US region but are replicated to other regions. Replication does not have an +arrow because it's too hard to read the diagram. + +```mermaid +graph TD; + user((User)); + dns[DNS]; + router_us(Router); + router_eu(Router); + cell_us0{Cell US0}; + cell_us1{Cell US1}; + cell_eu0{Cell EU0}; + cell_eu1{Cell EU1}; + db_gitlab_users[(gitlab_users Primary)]; + db_gitlab_routes[(gitlab_routes Primary)]; + db_gitlab_users_replica[(gitlab_users Replica)]; + db_gitlab_routes_replica[(gitlab_routes Replica)]; + db_cell_us0[(gitlab_main/gitlab_ci Cell US0)]; + db_cell_us1[(gitlab_main/gitlab_ci Cell US1)]; + db_cell_eu0[(gitlab_main/gitlab_ci Cell EU0)]; + db_cell_eu1[(gitlab_main/gitlab_ci Cell EU1)]; + user-->dns; + dns-->router_us; + dns-->router_eu; + subgraph Europe + router_eu-->cell_eu0; + router_eu-->cell_eu1; + cell_eu0-->db_cell_eu0; + cell_eu0-->db_gitlab_users_replica; + cell_eu0-->db_gitlab_routes_replica; + cell_eu1-->db_gitlab_users_replica; + cell_eu1-->db_gitlab_routes_replica; + cell_eu1-->db_cell_eu1; + end + subgraph United States + router_us-->cell_us0; + router_us-->cell_us1; + cell_us0-->db_cell_us0; + cell_us0-->db_gitlab_users; + cell_us0-->db_gitlab_routes; + cell_us1-->db_gitlab_users; + cell_us1-->db_gitlab_routes; + cell_us1-->db_cell_us1; + end + router_eu-.->cell_us0; + router_eu-.->cell_us1; + router_us-.->cell_eu0; + router_us-.->cell_eu1; +``` + +## Summary of changes + +1. Tables related to User data (including profile settings, authentication credentials, personal access tokens) are decomposed into a `gitlab_users` schema +1. The `routes` table is decomposed into `gitlab_routes` schema +1. The `application_settings` (and probably a few other instance level tables) are decomposed into `gitlab_admin` schema +1. A new column `routes.cell_id` is added to `routes` table +1. A new Router service exists to choose which cell to route a request to. +1. If a router receives a new request it will send `/api/v4/cells/learn?method=GET&path_info=/group-org/project` to learn which Cell can process it +1. A new concept will be introduced in GitLab called an organization +1. We require all existing endpoints to be routable by URI, or be fixed to a specific Cell for processing. This requires changing ambiguous endpoints like `/dashboard` to be scoped like `/organizations/my-organization/-/dashboard` +1. Endpoints like `/admin` would be routed always to the specific Cell, like `cell_0` +1. Each Cell can respond to `/api/v4/cells/learn` and classify each endpoint +1. Writes to `gitlab_users` and `gitlab_routes` are sent to a primary PostgreSQL server in our `US` region but reads can come from replicas in the same region. This will add latency for these writes but we expect they are infrequent relative to the rest of GitLab. + +## Pre-flight request learning + +While processing a request the URI will be decoded and a pre-flight request +will be sent for each non-cached endpoint. + +When asking for the endpoint GitLab Rails will return information about +the routable path. GitLab Rails will decode `path_info` and match it to +an existing endpoint and find a routable entity (like project). The router will +treat this as short-lived cache information. + +1. Prefix match: `/api/v4/cells/learn?method=GET&path_info=/gitlab-org/gitlab-test/-/issues` + + ```json + { + "path": "/gitlab-org/gitlab-test", + "cell": "cell_0", + "source": "routable" + } + ``` + +1. Some endpoints might require an exact match: `/api/v4/cells/learn?method=GET&path_info=/-/profile` + + ```json + { + "path": "/-/profile", + "cell": "cell_0", + "source": "fixed", + "exact": true + } + ``` + +## Detailed explanation of default organization in the first iteration + +All users will get a new column `users.default_organization` which they can +control in user settings. We will introduce a concept of the +`GitLab.com Public` organization. This will be set as the default organization for all existing +users. This organization will allow the user to see data from all namespaces in +`Cell US0` (ie. our original GitLab.com instance). This behavior can be invisible to +existing users such that they don't even get told when they are viewing a +global page like `/dashboard` that it's even scoped to an organization. + +Any new users with a default organization other than `GitLab.com Public` will have +a distinct user experience and will be fully aware that every page they load is +only ever scoped to a single organization. These users can never +load any global pages like `/dashboard` and will end up being redirected to +`/organizations//-/dashboard`. This may also be the case +for legacy APIs and such users may only ever be able to use APIs scoped to a +organization. + +## Detailed explanation of Admin Area settings + +We believe that maintaining and synchronizing Admin Area settings will be +frustrating and painful so to avoid this we will decompose and share all Admin Area +settings in the `gitlab_admin` schema. This should be safe (similar to other +shared schemas) because these receive very little write traffic. + +In cases where different cells need different settings (eg. the +Elasticsearch URL), we will either decide to use a templated +format in the relevant `application_settings` row which allows it to be dynamic +per cell. Alternatively if that proves difficult we'll introduce a new table +called `per_cell_application_settings` and this will have 1 row per cell to allow +setting different settings per cell. It will still be part of the `gitlab_admin` +schema and shared which will allow us to centrally manage it and simplify +keeping settings in sync for all cells. + +## Pros + +1. Router is stateless and can live in many regions. We use Anycast DNS to resolve to nearest region for the user. +1. Cells can receive requests for namespaces in the wrong cell and the user + still gets the right response as well as caching at the router that + ensures the next request is sent to the correct cell so the next request + will go to the correct cell +1. The majority of the code still lives in `gitlab` rails codebase. The Router doesn't actually need to understand how GitLab URLs are composed. +1. Since the responsibility to read and write `gitlab_users`, + `gitlab_routes` and `gitlab_admin` still lives in Rails it means minimal + changes will be needed to the Rails application compared to extracting + services that need to isolate the domain models and build new interfaces. +1. Compared to a separate routing service this allows the Rails application + to encode more complex rules around how to map URLs to the correct cell + and may work for some existing API endpoints. +1. All the new infrastructure (just a router) is optional and a single-cell + self-managed installation does not even need to run the Router and there are + no other new services. + +## Cons + +1. `gitlab_users`, `gitlab_routes` and `gitlab_admin` databases may need to be + replicated across regions and writes need to go across regions. We need to + do an analysis on write TPS for the relevant tables to determine if this is + feasible. +1. Sharing access to the database from many different Cells means that they are + all coupled at the Postgres schema level and this means changes to the + database schema need to be done carefully in sync with the deployment of all + Cells. This limits us to ensure that Cells are kept in closely similar + versions compared to an architecture with shared services that have an API + we control. +1. Although most data is stored in the right region there can be requests + proxied from another region which may be an issue for certain types + of compliance. +1. Data in `gitlab_users` and `gitlab_routes` databases must be replicated in + all regions which may be an issue for certain types of compliance. +1. The router cache may need to be very large if we get a wide variety of URLs + (ie. long tail). In such a case we may need to implement a 2nd level of + caching in user cookies so their frequently accessed pages always go to the + right cell the first time. +1. Having shared database access for `gitlab_users` and `gitlab_routes` + from multiple cells is an unusual architecture decision compared to + extracting services that are called from multiple cells. +1. It is very likely we won't be able to find cacheable elements of a + GraphQL URL and often existing GraphQL endpoints are heavily dependent on + ids that won't be in the `routes` table so cells won't necessarily know + what cell has the data. As such we'll probably have to update our GraphQL + calls to include an organization context in the path like + `/api/organizations//graphql`. +1. This architecture implies that implemented endpoints can only access data + that are readily accessible on a given Cell, but are unlikely + to aggregate information from many Cells. +1. All unknown routes are sent to the latest deployment which we assume to be `Cell US0`. + This is required as newly added endpoints will be only decodable by latest cell. + Likely this is not a problem for the `/cells/learn` is it is lightweight + to process and this should not cause a performance impact. + +## Example database configuration + +Handling shared `gitlab_users`, `gitlab_routes` and `gitlab_admin` databases, while having dedicated `gitlab_main` and `gitlab_ci` databases should already be handled by the way we use `config/database.yml`. We should also, already be able to handle the dedicated EU replicas while having a single US primary for `gitlab_users` and `gitlab_routes`. Below is a snippet of part of the database configuration for the Cell architecture described above. + +**Cell US0**: + +```yaml +# config/database.yml +production: + main: + host: postgres-main.cell-us0.primary.consul + load_balancing: + discovery: postgres-main.cell-us0.replicas.consul + ci: + host: postgres-ci.cell-us0.primary.consul + load_balancing: + discovery: postgres-ci.cell-us0.replicas.consul + users: + host: postgres-users-primary.consul + load_balancing: + discovery: postgres-users-replicas.us.consul + routes: + host: postgres-routes-primary.consul + load_balancing: + discovery: postgres-routes-replicas.us.consul + admin: + host: postgres-admin-primary.consul + load_balancing: + discovery: postgres-admin-replicas.us.consul +``` + +**Cell EU0**: + +```yaml +# config/database.yml +production: + main: + host: postgres-main.cell-eu0.primary.consul + load_balancing: + discovery: postgres-main.cell-eu0.replicas.consul + ci: + host: postgres-ci.cell-eu0.primary.consul + load_balancing: + discovery: postgres-ci.cell-eu0.replicas.consul + users: + host: postgres-users-primary.consul + load_balancing: + discovery: postgres-users-replicas.eu.consul + routes: + host: postgres-routes-primary.consul + load_balancing: + discovery: postgres-routes-replicas.eu.consul + admin: + host: postgres-admin-primary.consul + load_balancing: + discovery: postgres-admin-replicas.eu.consul +``` + +## Request flows + +1. `gitlab-org` is a top level namespace and lives in `Cell US0` in the `GitLab.com Public` organization +1. `my-company` is a top level namespace and lives in `Cell EU0` in the `my-organization` organization + +### Experience for paying user that is part of `my-organization` + +Such a user will have a default organization set to `/my-organization` and will be +unable to load any global routes outside of this organization. They may load other +projects/namespaces but their MR/Todo/Issue counts at the top of the page will +not be correctly populated in the first iteration. The user will be aware of +this limitation. + +#### Navigates to `/my-company/my-project` while logged in + +1. User is in Europe so DNS resolves to the router in Europe +1. They request `/my-company/my-project` without the router cache, so the router chooses randomly `Cell EU1` +1. The `/cells/learn` is sent to `Cell EU1`, which responds that resource lives on `Cell EU0` +1. `Cell EU0` returns the correct response +1. The router now caches and remembers any request paths matching `/my-company/*` should go to `Cell EU0` + +```mermaid +sequenceDiagram + participant user as User + participant router_eu as Router EU + participant cell_eu0 as Cell EU0 + participant cell_eu1 as Cell EU1 + user->>router_eu: GET /my-company/my-project + router_eu->>cell_eu1: /api/v4/cells/learn?method=GET&path_info=/my-company/my-project + cell_eu1->>router_eu: {path: "/my-company", cell: "cell_eu0", source: "routable"} + router_eu->>cell_eu0: GET /my-company/my-project + cell_eu0->>user:

My Project... +``` + +#### Navigates to `/my-company/my-project` while not logged in + +1. User is in Europe so DNS resolves to the router in Europe +1. The router does not have `/my-company/*` cached yet so it chooses randomly `Cell EU1` +1. The `/cells/learn` is sent to `Cell EU1`, which responds that resource lives on `Cell EU0` +1. `Cell EU0` redirects them through a login flow +1. User requests `/users/sign_in`, uses random Cell to run `/cells/learn` +1. The `Cell EU1` responds with `cell_0` as a fixed route +1. User after login requests `/my-company/my-project` which is cached and stored in `Cell EU0` +1. `Cell EU0` returns the correct response + +```mermaid +sequenceDiagram + participant user as User + participant router_eu as Router EU + participant cell_eu0 as Cell EU0 + participant cell_eu1 as Cell EU1 + user->>router_eu: GET /my-company/my-project + router_eu->>cell_eu1: /api/v4/cells/learn?method=GET&path_info=/my-company/my-project + cell_eu1->>router_eu: {path: "/my-company", cell: "cell_eu0", source: "routable"} + router_eu->>cell_eu0: GET /my-company/my-project + cell_eu0->>user: 302 /users/sign_in?redirect=/my-company/my-project + user->>router_eu: GET /users/sign_in?redirect=/my-company/my-project + router_eu->>cell_eu1: /api/v4/cells/learn?method=GET&path_info=/users/sign_in + cell_eu1->>router_eu: {path: "/users", cell: "cell_eu0", source: "fixed"} + router_eu->>cell_eu0: GET /users/sign_in?redirect=/my-company/my-project + cell_eu0-->>user:

Sign in... + user->>router_eu: POST /users/sign_in?redirect=/my-company/my-project + router_eu->>cell_eu0: POST /users/sign_in?redirect=/my-company/my-project + cell_eu0->>user: 302 /my-company/my-project + user->>router_eu: GET /my-company/my-project + router_eu->>cell_eu0: GET /my-company/my-project + router_eu->>cell_eu0: GET /my-company/my-project + cell_eu0->>user:

My Project... +``` + +#### Navigates to `/my-company/my-other-project` after last step + +1. User is in Europe so DNS resolves to the router in Europe +1. The router cache now has `/my-company/* => Cell EU0`, so the router chooses `Cell EU0` +1. `Cell EU0` returns the correct response as well as the cache header again + +```mermaid +sequenceDiagram + participant user as User + participant router_eu as Router EU + participant cell_eu0 as Cell EU0 + participant cell_eu1 as Cell EU1 + user->>router_eu: GET /my-company/my-project + router_eu->>cell_eu0: GET /my-company/my-project + cell_eu0->>user:

My Project... +``` + +#### Navigates to `/gitlab-org/gitlab` after last step + +1. User is in Europe so DNS resolves to the router in Europe +1. The router has no cached value for this URL so randomly chooses `Cell EU0` +1. `Cell EU0` redirects the router to `Cell US0` +1. `Cell US0` returns the correct response as well as the cache header again + +```mermaid +sequenceDiagram + participant user as User + participant router_eu as Router EU + participant cell_eu0 as Cell EU0 + participant cell_us0 as Cell US0 + user->>router_eu: GET /gitlab-org/gitlab + router_eu->>cell_eu0: /api/v4/cells/learn?method=GET&path_info=/gitlab-org/gitlab + cell_eu0->>router_eu: {path: "/gitlab-org", cell: "cell_us0", source: "routable"} + router_eu->>cell_us0: GET /gitlab-org/gitlab + cell_us0->>user:

GitLab.org... +``` + +In this case the user is not on their "default organization" so their TODO +counter will not include their normal todos. We may choose to highlight this in +the UI somewhere. A future iteration may be able to fetch that for them from +their default organization. + +#### Navigates to `/` + +1. User is in Europe so DNS resolves to the router in Europe +1. Router does not have a cache for `/` route (specifically rails never tells it to cache this route) +1. The Router choose `Cell EU0` randomly +1. The Rails application knows the users default organization is `/my-organization`, so + it redirects the user to `/organizations/my-organization/-/dashboard` +1. The Router has a cached value for `/organizations/my-organization/*` so it then sends the + request to `POD EU0` +1. `Cell EU0` serves up a new page `/organizations/my-organization/-/dashboard` which is the same + dashboard view we have today but scoped to an organization clearly in the UI +1. The user is (optionally) presented with a message saying that data on this page is only + from their default organization and that they can change their default + organization if it's not right. + +```mermaid +sequenceDiagram + participant user as User + participant router_eu as Router EU + participant cell_eu0 as Cell EU0 + user->>router_eu: GET / + router_eu->>cell_eu0: GET / + cell_eu0->>user: 302 /organizations/my-organization/-/dashboard + user->>router: GET /organizations/my-organization/-/dashboard + router->>cell_eu0: GET /organizations/my-organization/-/dashboard + cell_eu0->>user:

My Company Dashboard... X-Gitlab-Cell-Cache={path_prefix:/organizations/my-organization/} +``` + +#### Navigates to `/dashboard` + +As above, they will end up on `/organizations/my-organization/-/dashboard` as +the rails application will already redirect `/` to the dashboard page. + +### Navigates to `/not-my-company/not-my-project` while logged in (but they don't have access since this project/group is private) + +1. User is in Europe so DNS resolves to the router in Europe +1. The router knows that `/not-my-company` lives in `Cell US1` so sends the request to this +1. The user does not have access so `Cell US1` returns 404 + +```mermaid +sequenceDiagram + participant user as User + participant router_eu as Router EU + participant cell_us1 as Cell US1 + user->>router_eu: GET /not-my-company/not-my-project + router_eu->>cell_us1: GET /not-my-company/not-my-project + cell_us1->>user: 404 +``` + +#### Creates a new top level namespace + +The user will be asked which organization they want the namespace to belong to. +If they select `my-organization` then it will end up on the same cell as all +other namespaces in `my-organization`. If they select nothing we default to +`GitLab.com Public` and it is clear to the user that this is isolated from +their existing organization such that they won't be able to see data from both +on a single page. + +### Experience for GitLab team member that is part of `/gitlab-org` + +Such a user is considered a legacy user and has their default organization set to +`GitLab.com Public`. This is a "meta" organization that does not really exist but +the Rails application knows to interpret this organization to mean that they are +allowed to use legacy global functionality like `/dashboard` to see data across +namespaces located on `Cell US0`. The rails backend also knows that the default cell to render any ambiguous +routes like `/dashboard` is `Cell US0`. Lastly the user will be allowed to +navigate to organizations on another cell like `/my-organization` but when they do the +user will see a message indicating that some data may be missing (eg. the +MRs/Issues/Todos) counts. + +#### Navigates to `/gitlab-org/gitlab` while not logged in + +1. User is in the US so DNS resolves to the US router +1. The router knows that `/gitlab-org` lives in `Cell US0` so sends the request + to this cell +1. `Cell US0` serves up the response + +```mermaid +sequenceDiagram + participant user as User + participant router_us as Router US + participant cell_us0 as Cell US0 + user->>router_us: GET /gitlab-org/gitlab + router_us->>cell_us0: GET /gitlab-org/gitlab + cell_us0->>user:

GitLab.org... +``` + +#### Navigates to `/` + +1. User is in US so DNS resolves to the router in US +1. Router does not have a cache for `/` route (specifically rails never tells it to cache this route) +1. The Router chooses `Cell US1` randomly +1. The Rails application knows the users default organization is `GitLab.com Public`, so + it redirects the user to `/dashboards` (only legacy users can see + `/dashboard` global view) +1. Router does not have a cache for `/dashboard` route (specifically rails never tells it to cache this route) +1. The Router chooses `Cell US1` randomly +1. The Rails application knows the users default organization is `GitLab.com Public`, so + it allows the user to load `/dashboards` (only legacy users can see + `/dashboard` global view) and redirects to router the legacy cell which is `Cell US0` +1. `Cell US0` serves up the global view dashboard page `/dashboard` which is the same + dashboard view we have today + +```mermaid +sequenceDiagram + participant user as User + participant router_us as Router US + participant cell_us0 as Cell US0 + participant cell_us1 as Cell US1 + user->>router_us: GET / + router_us->>cell_us1: GET / + cell_us1->>user: 302 /dashboard + user->>router_us: GET /dashboard + router_us->>cell_us1: /api/v4/cells/learn?method=GET&path_info=/dashboard + cell_us1->>router_us: {path: "/dashboard", cell: "cell_us0", source: "routable"} + router_us->>cell_us0: GET /dashboard + cell_us0->>user:

Dashboard... +``` + +#### Navigates to `/my-company/my-other-project` while logged in (but they don't have access since this project is private) + +They get a 404. + +### Experience for non-authenticated users + +Flow is similar to logged in users except global routes like `/dashboard` will +redirect to the login page as there is no default organization to choose from. + +### A new customers signs up + +They will be asked if they are already part of an organization or if they'd +like to create one. If they choose neither they end up no the default +`GitLab.com Public` organization. + +### An organization is moved from 1 cell to another + +TODO + +### GraphQL/API requests which don't include the namespace in the URL + +TODO + +### The autocomplete suggestion functionality in the search bar which remembers recent issues/MRs + +TODO + +### Global search + +TODO + +## Administrator + +### Loads `/admin` page + +1. The `/admin` is locked to `Cell US0` +1. Some endpoints of `/admin`, like Projects in Admin are scoped to a Cell + and users needs to choose the correct one in a dropdown, which results in endpoint + like `/admin/cells/cell_0/projects`. + +Admin Area settings in Postgres are all shared across all cells to avoid +divergence but we still make it clear in the URL and UI which cell is serving +the Admin Area page as there is dynamic data being generated from these pages and +the operator may want to view a specific cell. + +## More Technical Problems To Solve + +### Replicating User Sessions Between All Cells + +Today user sessions live in Redis but each cell will have their own Redis instance. We already use a dedicated Redis instance for sessions so we could consider sharing this with all cells like we do with `gitlab_users` PostgreSQL database. But an important consideration will be latency as we would still want to mostly fetch sessions from the same region. + +An alternative might be that user sessions get moved to a JWT payload that encodes all the session data but this has downsides. For example, it is difficult to expire a user session, when their password changes or for other reasons, if the session lives in a JWT controlled by the user. + +### How do we migrate between Cells + +Migrating data between cells will need to factor all data stores: + +1. PostgreSQL +1. Redis Shared State +1. Gitaly +1. Elasticsearch + +### Is it still possible to leak the existence of private groups via a timing attack? + +If you have router in EU, and you know that EU router by default redirects +to EU located Cells, you know their latency (lets assume 10 ms). Now, if your +request is bounced back and redirected to US which has different latency +(lets assume that roundtrip will be around 60 ms) you can deduce that 404 was +returned by US Cell and know that your 404 is in fact 403. + +We may defer this until we actually implement a cell in a different region. Such timing attacks are already theoretically possible with the way we do permission checks today but the timing difference is probably too small to be able to detect. + +One technique to mitigate this risk might be to have the router add a random +delay to any request that returns 404 from a cell. + +## Should runners be shared across all cells? + +We have 2 options and we should decide which is easier: + +1. Decompose runner registration and queuing tables and share them across all + cells. This may have implications for scalability, and we'd need to consider + if this would include group/project runners as this may have scalability + concerns as these are high traffic tables that would need to be shared. +1. Runners are registered per-cell and, we probably have a separate fleet of + runners for every cell or just register the same runners to many cells which + may have implications for queueing + +## How do we guarantee unique ids across all cells for things that cannot conflict? + +This project assumes at least namespaces and projects have unique ids across +all cells as many requests need to be routed based on their ID. Since those +tables are across different databases then guaranteeing a unique ID will +require a new solution. There are likely other tables where unique IDs are +necessary and depending on how we resolve routing for GraphQL and other APIs +and other design goals it may be determined that we want the primary key to be +unique for all tables. diff --git a/doc/architecture/blueprints/ci_data_decay/index.md b/doc/architecture/blueprints/ci_data_decay/index.md index e26e7d5dbd3..2eac27def18 100644 --- a/doc/architecture/blueprints/ci_data_decay/index.md +++ b/doc/architecture/blueprints/ci_data_decay/index.md @@ -8,12 +8,14 @@ owning-stage: "~devops::verify" participating-stages: [] --- + + # CI/CD data time decay ## Summary GitLab CI/CD is one of the most data and compute intensive components of GitLab. -Since its [initial release in November 2012](https://about.gitlab.com/blog/2012/11/13/continuous-integration-server-from-gitlab/), +Since its initial release in 2012, the CI/CD subsystem has evolved significantly. It was [integrated into GitLab in September 2015](https://about.gitlab.com/releases/2015/09/22/gitlab-8-0-released/) and has become [one of the most beloved CI/CD solutions](https://about.gitlab.com/blog/2017/09/27/gitlab-leader-continuous-integration-forrester-wave/). @@ -231,7 +233,7 @@ In progress. - 2022-02-08: Pipeline partitioning PoC [merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80186) started. - 2022-02-23: Pipeline partitioning PoC [successful](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80186#note_852704724) - 2022-03-07: A way to attach an existing table as a partition [found and proven](https://gitlab.com/gitlab-org/gitlab/-/issues/353380#note_865237214). -- 2022-03-23: Pipeline partitioning design [Google Doc](https://docs.google.com/document/d/1ARdoTZDy4qLGf6Z1GIHh83-stG_ZLpqsibjKr_OXMgc) started. +- 2022-03-23: Pipeline partitioning design Google Doc (GitLab internal) started: `https://docs.google.com/document/d/1ARdoTZDy4qLGf6Z1GIHh83-stG_ZLpqsibjKr_OXMgc`. - 2022-03-29: Pipeline partitioning PoC [concluded](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80186#note_892674358). - 2022-04-15: Partitioned pipeline data associations PoC [shipped](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84071). - 2022-04-30: Additional [benchmarking started](https://gitlab.com/gitlab-org/gitlab/-/issues/361019) to evaluate impact. diff --git a/doc/architecture/blueprints/ci_data_decay/pipeline_partitioning.md b/doc/architecture/blueprints/ci_data_decay/pipeline_partitioning.md index ebe3c72adfc..5dea1090507 100644 --- a/doc/architecture/blueprints/ci_data_decay/pipeline_partitioning.md +++ b/doc/architecture/blueprints/ci_data_decay/pipeline_partitioning.md @@ -1,10 +1,15 @@ --- -stage: none -group: unassigned -comments: false +status: ongoing +creation-date: "2022-05-31" +authors: [ "@grzesiek" ] +coach: [ "@ayufan", "@grzesiek" ] +approvers: [ "@jreporter", "@cheryl.li" ] +owning-stage: "~devops::verify" description: 'Pipeline data partitioning design' --- + + # Pipeline data partitioning design ## What problem are we trying to solve? @@ -803,9 +808,11 @@ DRIs: | Role | Who | |---------------------|------------------------------------------------| | Author | Grzegorz Bizon, Principal Engineer | -| Recommender | Kamil Trzciński, Senior Distingiushed Engineer | -| Product Manager | James Heimbuck, Senior Product Manager | -| Engineering Manager | Scott Hampton, Engineering Manager | +| Recommender | Kamil Trzciński, Senior Distinguished Engineer | +| Product Leadership | Jackie Porter, Director of Product Management | +| Engineering Leadership | Caroline Simpson, Engineering Manager / Cheryl Li, Senior Engineering Manager | | Lead Engineer | Marius Bobin, Senior Backend Engineer | +| Senior Engineer | Maxime Orefice, Senior Backend Engineer | +| Senior Engineer | Tianwen Chen, Senior Backend Engineer | diff --git a/doc/architecture/blueprints/ci_pipeline_components/dev_workflow.md b/doc/architecture/blueprints/ci_pipeline_components/dev_workflow.md new file mode 100644 index 00000000000..fd897781cf5 --- /dev/null +++ b/doc/architecture/blueprints/ci_pipeline_components/dev_workflow.md @@ -0,0 +1,154 @@ +--- +stage: verify +group: pipeline authoring +description: 'Development workflow for a components repository' +--- + +# Development workflow for a components repository + +## Summary + +This page describes the process of creating a components repository. +It describes all the necessary steps, from the creation of the project to having new releases displayed in the +catalog page. + +## 1. Create a new project + +First, create a new project and add a `README.md` file, which is a planned future +requirement for a repository to become a catalog resource. + +## 2. Create a component inside the repository + +If you intend to have only one component in the repository, you can define it in the root directory. +Otherwise, create a directory for the component. +For more information, see the [directory structure of a components repository](index.md#structure-of-a-components-repository). + +This example defines a single component in the root directory. + +Create a `template.yml` file that contains the configuration we want to provide as a component: + +```yaml +spec: + inputs: + stage: + default: test +--- +.component-default-job: + image: busybox + stage: $[[ inputs.stage ]] + +component-job-1: + extends: .component-default-job + script: echo job 1 + +component-job-2: + extends: .component-default-job + script: echo job 2 +``` + +The example component configuration above adds two jobs, `component-job-1` and `component-job-2`, to a pipeline. + +## 3. Test changes in CI + +To test any changes pushed to our component, we create a `.gitlab-ci.yml` in the root directory: + +```yaml +## +# This configuration expects an access token with read-only access to the API +# to be saved as in a masked CI/CD variable named 'API_TOKEN' + +include: + # Leverage predefined variables to refer to the current project and SHA + - component: gitlab.com/$CI_PROJECT_PATH@$CI_COMMIT_SHA + +stages: [test, release] + +# Expect all `component-job-*` jobs are added +ensure-jobs-added: + image: badouralix/curl-jq + script: + - | + route="https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/pipelines/$CI_PIPELINE_ID/jobs" + count=`curl --silent --header "PRIVATE-TOKEN: $API_TOKEN" $route | jq 'map(select(.name | contains("component-job-"))) | length'` + if [ "$count" != "2" ]; then + exit 1 + fi + +# Ensure that a project description exists, because it will be important to display +# the resource in the catalog. +check-description: + image: badouralix/curl-jq + script: + - | + route="https://gitlab.com/api/v4/projects/$CI_PROJECT_ID" + desc=`curl --silent --header "PRIVATE-TOKEN: $API_TOKEN" $route | jq '.description'` + if [ "$desc" = "null" ]; then + echo "Description not set. Please set a projet description" + exit 1 + else + echo "Description set" + fi + +# Ensure that a `README.md` exists in the root directory as it represents the +# documentation for the whole components repository. +check-readme: + image: busybox + script: ls README.md || (echo "Please add a README.md file" && exit 1) + +# If we are tagging a release with a specific convention ("v" + number) and all +# previous checks succeeded, we proceed with creating a release automatically. +create-release: + stage: release + image: registry.gitlab.com/gitlab-org/release-cli:latest + rules: + - if: $CI_COMMIT_TAG =~ /^v\d+/ + script: echo "Creating release $CI_COMMIT_TAG" + release: + tag_name: $CI_COMMIT_TAG + description: "Release $CI_COMMIT_TAG of components repository $CI_PROJECT_PATH" +``` + +This pipeline contains examples of several tasks: + +- Use the component to ensure that the final configuration uses valid syntax. + This also ensures that the minimal requirements for the component to work are in place, + like inputs and secrets. +- Test that the created pipeline has the expected characteristics. + For example, ensure the `component-job-*` jobs are added to the pipeline. + - We call the [pipeline API endpoint](../../../api/pipelines.md#get-a-single-pipeline) with `curl` + and parse the data via `jq`. + - With this technique users could check things like ensuring certain jobs were included, + the job has the right properties set, or the log contains the expected output. +- Ensure that the project description is set. +- Ensure that the repository contains a `README.md` file. +- Create a [release automatically](../../../ci/yaml/index.md#release). When a tag is created and follows specific regex, create a release + after all previous checks pass. + +## 4. Run a pipeline + +Now run a new pipeline for the `main` branch, by pushing a change or manually running a pipeline: + +![Pipeline on main branch](img/pipeline_main.png) + +## 5. Create a tag + +As the pipeline for `main` is green, we can now [create our first tag](../../../user/project/repository/tags/index.md#create-a-tag): `v1.0`. + +As soon as the `v1.0` tag is created, we see a tag pipeline start. +This time the pipeline also has a `create-release` job in the `release` stage: + +![Pipeline on tag](img/pipeline_tag.png) + +When the `create-release` job finishes we should see the new release available in the **Releases** menu: + +![New components repository release](img/new_release.png) + +## 6. Publish the repository to the catalog + +To ensure that both the components repository and the new release is visible in the CI Catalog, +we need to publish it. + +Publishing a components repository makes it a catalog resource. + +The API endpoint for this action is under development. +For more details read the [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/387065). diff --git a/doc/architecture/blueprints/ci_pipeline_components/img/new_release.png b/doc/architecture/blueprints/ci_pipeline_components/img/new_release.png new file mode 100644 index 00000000000..eed5c55d5e3 Binary files /dev/null and b/doc/architecture/blueprints/ci_pipeline_components/img/new_release.png differ diff --git a/doc/architecture/blueprints/ci_pipeline_components/img/pipeline_main.png b/doc/architecture/blueprints/ci_pipeline_components/img/pipeline_main.png new file mode 100644 index 00000000000..8b03b96ba7e Binary files /dev/null and b/doc/architecture/blueprints/ci_pipeline_components/img/pipeline_main.png differ diff --git a/doc/architecture/blueprints/ci_pipeline_components/img/pipeline_tag.png b/doc/architecture/blueprints/ci_pipeline_components/img/pipeline_tag.png new file mode 100644 index 00000000000..d0814a479ae Binary files /dev/null and b/doc/architecture/blueprints/ci_pipeline_components/img/pipeline_tag.png differ diff --git a/doc/architecture/blueprints/ci_pipeline_components/index.md b/doc/architecture/blueprints/ci_pipeline_components/index.md index b1aee7c4217..ff4604b61bf 100644 --- a/doc/architecture/blueprints/ci_pipeline_components/index.md +++ b/doc/architecture/blueprints/ci_pipeline_components/index.md @@ -8,6 +8,8 @@ owning-stage: "~devops::verify" participating-stages: [] --- + + # CI/CD Catalog ## Summary @@ -92,9 +94,13 @@ This section defines some terms that are used throughout this document. With the identifying abstract concepts and are subject to changes as we refine the design by discovering new insights. - **Component** Is the reusable unit of pipeline configuration. -- **Project** Is the GitLab project attached to a repository. A project can contain multiple components. -- **Catalog** is the collection of projects that are set to contain components. -- **Version** is the release name of a tag in the project, which allows components to be pinned to a specific revision. +- **Components repository** represents a collection of CI components stored in the same project. +- **Project** is the GitLab project attached to a single components repository. +- **Catalog** is a collection of resources like components repositories. +- **Catalog resource** is the single item displayed in the catalog. A components repository is a catalog resource. +- **Version** is a specific revision of catalog resource. It maps to the released tag in the project, + which allows components to be pinned to a specific revision. +- **Steps** is a collection of instructions for how jobs can be executed. ## Definition of pipeline component @@ -128,7 +134,7 @@ Eventually, we want to make CI Catalog Components predictable. Including a component by its path, using a fixed `@` version, should always return the same configuration, regardless of a context from which it is getting included from. The resulting configuration should be the same for a given component version -and the set of inputs passed using `with:` keyword, hence it should be +and the set of inputs passed using `include:inputs` keyword, therefore it should be [deterministic](https://en.wikipedia.org/wiki/Deterministic_algorithm). A component should not produce side effects by being included and should be @@ -202,7 +208,6 @@ A component YAML file: - Should be **validated statically** (for example: using JSON schema validators). ```yaml ---- spec: inputs: website: @@ -217,18 +222,14 @@ spec: # content of the component ``` -Components that are released in the catalog must have a `README.md` file at the root directory of the repository. -The `README.md` represents the documentation for the specific component, hence it's recommended -even when not releasing versions in the catalog. - ### The component version The version of the component can be (in order of highest priority first): 1. A commit SHA - For example: `gitlab.com/gitlab-org/dast@e3262fdd0914fa823210cdb79a8c421e2cef79d8` -1. A released tag - For example: `gitlab.com/gitlab-org/dast@1.0` -1. A special moving target version that points to the most recent released tag - For example: `gitlab.com/gitlab-org/dast@~latest` -1. An unreleased tag - For example: `gitlab.com/gitlab-org/dast@rc-1.0` +1. A tag - For example: `gitlab.com/gitlab-org/dast@1.0` +1. A special moving target version that points to the most recent released tag. The target project must be +explicitly marked as a [catalog resource](#catalog-resource) - For example: `gitlab.com/gitlab-org/dast@~latest` 1. A branch name - For example: `gitlab.com/gitlab-org/dast@master` If a tag and branch exist with the same name, the tag takes precedence over the branch. @@ -237,26 +238,31 @@ takes precedence over the tag. As we want to be able to reference any revisions (even those not released), a component must be defined in a Git repository. -NOTE: When referencing a component by local path (for example `./path/to/component`), its version is implicit and matches the commit SHA of the current pipeline context. -## Components project +## Components repository -A components project is a GitLab project/repository that exclusively hosts one or more pipeline components. +A components repository is a GitLab project/repository that exclusively hosts one or more pipeline components. -For components projects it's highly recommended to set an appropriate avatar and project description -to improve discoverability in the catalog. +A components repository can be a catalog resource. For a components repository it's highly recommended to set +an appropriate avatar and project description to improve discoverability in the catalog. -### Structure of a components project +Components repositories that are released in the catalog must have a `README.md` file at the root directory of the repository. +The `README.md` represents the documentation of the components repository, hence it's recommended +even when not listing the repository in the catalog. -A project can host one or more components depending on whether the author wants to define a single component -per project or include multiple cohesive components under the same project. +### Structure of a components repository -Let's imagine we are developing a component that runs RSpec tests for a Rails app. We create a component project +A components repository can host one or more components. The author can decide whether to define a single component +per repository or include multiple cohesive components in the same repository. + +A components repository is identified by the project full path. + +Let's imagine we are developing a component that runs RSpec tests for a Rails app. We create a project called `myorg/rails-rspec`. -The following directory structure would support 1 component per project: +The following directory structure would support 1 component per repository: ```plaintext . @@ -267,10 +273,10 @@ The following directory structure would support 1 component per project: The `.gitlab-ci.yml` is recommended for the project to ensure changes are verified accordingly. -The component is now identified by the path `gitlab.com/myorg/rails-rspec` and we expect a `template.yml` file -and `README.md` located in the root directory of the repository. +The component is now identified by the path `gitlab.com/myorg/rails-rspec` which also maps to the +project path. We expect a `template.yml` file and `README.md` to be located in the root directory of the repository. -The following directory structure would support multiple components per project: +The following directory structure would support multiple components per repository: ```plaintext . @@ -319,7 +325,6 @@ This limitation encourages cohesion at project level and keeps complexity low. If the component takes any input parameters they must be specified according to the following schema: ```yaml ---- spec: inputs: website: # by default all declared inputs are mandatory. @@ -346,13 +351,13 @@ When using the component we pass the input parameters as follows: ```yaml include: - component: gitlab.com/org/my-component@1.0 - with: + inputs: website: ${MY_WEBSITE} # variables expansion test_run: system environment: $[[ inputs.environment ]] # interpolation of upstream inputs ``` -Variables expansion must be supported for `with:` syntax as well as interpolation of +Variables expansion must be supported for `include:inputs` syntax as well as interpolation of possible [inputs provided upstream](#input-parameters-for-pipelines). Input parameters are validated as soon as possible: @@ -363,7 +368,6 @@ Input parameters are validated as soon as possible: 1. Interpolate input parameters inside the component's content. ```yaml ---- spec: inputs: environment: @@ -383,8 +387,8 @@ With `$[[ inputs.XXX ]]` inputs are interpolated immediately after parsing the c ### CI configuration interpolation perspectives and limitations -With `spec:` users will be able to define input arguments for CI configuration. -With `with:` keywords, they will pass these arguments to CI components. +With `spec:inputs` users will be able to define input arguments for CI configuration. +With `include:inputs`, they will pass these arguments to CI components. `inputs` in `$[[ inputs.something ]]` is going to be an initial "object" or "container" that we will provide, to allow users to access their arguments in @@ -427,32 +431,31 @@ enforce contracts. ### Input parameters for existing `include:` syntax Because we are adding input parameters to components used via `include:component` we have an opportunity to -extend it to other `include:` types support inputs via `with:` syntax: +extend it to other `include:` types support inputs through `inputs:` syntax: ```yaml include: - component: gitlab.com/org/my-component@1.0 - with: + inputs: foo: bar - local: path/to/file.yml - with: + inputs: foo: bar - project: org/another file: .gitlab-ci.yml - with: + inputs: foo: bar - remote: http://example.com/ci/config - with: + inputs: foo: bar - template: Auto-DevOps.gitlab-ci.yml - with: + inputs: foo: bar ``` Then the configuration being included must specify the inputs by defining a specification section in the YAML: ```yaml ---- spec: inputs: foo: @@ -460,15 +463,15 @@ spec: # rest of the configuration ``` -If a YAML includes content using `with:` but the including YAML doesn't define `inputs:` in the specifications, +If a YAML includes content using `include:inputs` but the including YAML doesn't define `spec:inputs` in the specifications, an error should be raised. -|`with:`| `inputs:` | result | -| --- | --- | --- | -| specified | | raise error | -| specified | specified | validate inputs | -| | specified | use defaults | -| | | legacy `include:` without input passing | +| `include:inputs` | `spec:inputs` | result | +|------------------|---------------|-----------------------------------------| +| specified | | raise error | +| specified | specified | validate inputs | +| | specified | use defaults | +| | | legacy `include:` without input passing | ### Input parameters for pipelines @@ -488,7 +491,7 @@ Today we have different use cases where using explicit input parameters would be deploy-app: trigger: project: org/deployer - with: + inputs: provider: aws deploy_environment: staging ``` @@ -496,7 +499,6 @@ deploy-app: To solve the problem of `Run Pipeline` UI form we could fully leverage the `inputs` specifications: ```yaml ---- spec: inputs: concurrency: @@ -516,35 +518,114 @@ spec: # rest of the pipeline config ``` -### Limits +## CI Catalog -Any MVC that exposes a feature should be added with limitations from the beginning. -It's safer to add new features with restrictions than trying to limit a feature after it's being used. -We can always soften the restrictions later depending on user demand. +The CI Catalog is an index of resources that users can leverage in CI/CD. It initially +contains a list of components repositories that users can discover and use in their pipelines. -Some limits we could consider adding: +In the future, the Catalog could contain also other types of resources (for example: +integrations, project templates, etc.). -- number of components that a single project can contain/export -- number of imports that a `.gitlab-ci.yml` file can use -- number of imports that a component can declare/use -- max level of nested imports -- max length of the exported component name +To list a components repository in the Catalog we need to mark the project as being a +catalog resource. We do that initially with an API endpoint, similar to changing a project setting. + +Once a project is marked as a "catalog resource" it can be displayed in the Catalog. + +We could create a database record when the API endpoint is used and remove the record when +the same is disabled/removed. + +## Catalog resource + +Upon publishing, a catalog resource should have at least following attributes: + +- `path`: to be uniquely identified. +- `name`: for components repository this could be the project name. +- `documentation`: we would use the `README.md` file which would be mandatory. +- `versions`: one or more releases of the resource. + +Other properties of a catalog resource: + +- `description`: for components repository this could be the project description. +- `avatar image`: we could use the project avatar. +- indicators of popularity (stars, forks). +- categorization: user should select a category and or define search tags + +As soon as a components repository is marked as being a "catalog resource" +we should be seeing the resource listed in the Catalog. + +Initially for the resource, the project may not have any released tags. +Users would be able to use the components repository by specifying a branch name or +commit SHA for the version. However, these types of version qualifiers should not +be listed in the catalog resource's page for various reasons: + +- The list of branches and tags can get very large. +- Branches and tags may not be meaningful for the end-user. +- Branches and tags don't communicate versioning thoroughly. -## Publishing components +## Releasing new resource versions to the Catalog -Users will be able to publish CI Components into a CI Catalog. This can happen -in a CI pipeline job, similarly to how software is being deployed following -Continuous Delivery principles. This will allow us to guardrail the quality of -components being deployed. To ensure that the CI Components meet quality -standards users will be able to test them before publishing new versions in the +The versions that should be displayed for the resource should be the project [releases](../../../user/project/releases/index.md). +Creating project releases is an official act of versioning a resource. + +A resource page would have: + +- The latest release in evidence (for example: the default version). +- The ability to inspect and use past releases of the resource. +- The documentation represented by the `README.md`. + +Users should be able to release new versions of the resource in a CI pipeline job, +similar to how software is being deployed following Continuous Delivery principles. + +To ensure that the components repository and the including components +meet quality standards, users can test them before releasing new versions in the CI Catalog. -Once a project containing components gets published we will index components' +Some examples of checks we can run during the release of a new resource version: + +- Ensure the project contains a `README.md` in the root directory. +- Ensure the project description exists. +- If an index of available components is present for a components repository, ensure each + component has valid YAML. + +Once a new release for the project gets created we index the resource's metadata. We want to initially index as much metadata as possible, to gain more flexibility in how we design CI Catalog's main page. We don't want to be -constrained by the lack of data available to properly visualize CI Components -in CI Catalog. In order to do that, we may need to find all components that are -being published, read their `spec` metadata and index what we find there. +constrained by the lack of data available to properly visualize resources in +the CI Catalog. To do that, we may need to find all resources that are +being released and index their data and metadata. +For example: index the content of `spec:` section for CI components. + +See an [example of development workflow](dev_workflow.md) for a components repository. + +## Note about future resource types + +In the future, to support multiple types of resources in the Catalog we could +require a file `catalog-resource.yml` to be defined in the root directory of the project: + +```yaml +name: DAST +description: Scan a web endpoint to find vulnerabilities +category: security +tags: [dynamic analysis, security scanner] +type: components_repository +``` + +This file could also be used for indexing metadata about the content of the resource. +For example, users could list the components in the repository and we can index +further data for search purpose: + +```yaml +name: DAST +description: Scan a web endpoint to find vulnerabilities +category: security +tags: [dynamic analysis, security scanner] +type: components_repository +metadata: + components: + - all-scans + - scan-x + - scan-y +``` ## Implementation guidelines @@ -585,6 +666,20 @@ being published, read their `spec` metadata and index what we find there. components from GitLab.com or from repository exports. - Iterate on feedback. +## Limits + +Any MVC that exposes a feature should be added with limitations from the beginning. +It's safer to add new features with restrictions than trying to limit a feature after it's being used. +We can always soften the restrictions later depending on user demand. + +Some limits we could consider adding: + +- number of components that a single project can contain/export +- number of imports that a `.gitlab-ci.yml` file can use +- number of imports that a component can declare/use +- max level of nested imports +- max length of the exported component name + ## Who Proposal: @@ -612,6 +707,6 @@ Domain experts: | Area | Who |------------------------------|------------------------| | Verify / Pipeline authoring | Avielle Wolfe | -| Verify / Pipeline authoring | Laura Montemayor-Rodriguez | +| Verify / Pipeline authoring | Laura Montemayor | diff --git a/doc/architecture/blueprints/ci_scale/index.md b/doc/architecture/blueprints/ci_scale/index.md index cf7065f7c07..3a6ed4ae9b1 100644 --- a/doc/architecture/blueprints/ci_scale/index.md +++ b/doc/architecture/blueprints/ci_scale/index.md @@ -7,12 +7,14 @@ approvers: [ "@cheryl.li", "@jreporter" ] owning-stage: "~devops::verify" --- + + # CI/CD Scaling ## Summary GitLab CI/CD is one of the most data and compute intensive components of GitLab. -Since its [initial release in November 2012](https://about.gitlab.com/blog/2012/11/13/continuous-integration-server-from-gitlab/), +Since its initial release in 2012, the CI/CD subsystem has evolved significantly. It was [integrated into GitLab in September 2015](https://about.gitlab.com/releases/2015/09/22/gitlab-8-0-released/) and has become [one of the most beloved CI/CD solutions](https://about.gitlab.com/blog/2017/09/27/gitlab-leader-continuous-integration-forrester-wave/). diff --git a/doc/architecture/blueprints/clickhouse_ingestion_pipeline/clickhouse_dbwriter.png b/doc/architecture/blueprints/clickhouse_ingestion_pipeline/clickhouse_dbwriter.png new file mode 100644 index 00000000000..fc65830d3ee Binary files /dev/null and b/doc/architecture/blueprints/clickhouse_ingestion_pipeline/clickhouse_dbwriter.png differ diff --git a/doc/architecture/blueprints/clickhouse_ingestion_pipeline/index.md b/doc/architecture/blueprints/clickhouse_ingestion_pipeline/index.md new file mode 100644 index 00000000000..94714e7b245 --- /dev/null +++ b/doc/architecture/blueprints/clickhouse_ingestion_pipeline/index.md @@ -0,0 +1,289 @@ +--- +status: proposed +creation-date: "2023-01-10" +authors: [ "@ankitbhatnagar", "@ahegyi", "@mikolaj_wawrzyniak" ] +coach: "@grzesiek" +approvers: [ "@nhxnguyen", "@stkerr" ] +owning-stage: "~workinggroup::clickhouse" +participating-stages: [ "~section::ops", "~section::dev" ] +--- + +# Scalable data ingestion abstraction for ClickHouse + +## Table of Contents + +- [Summary](#summary) + - [Why](#why) + - [How](#how) +- [Motivation](#motivation) +- [Case Studies](#case-studies) + - [Replicating existing data into ClickHouse](#1-replicating-existing-data-into-clickhouse) + - [Ingesting large volumes of data into ClickHouse](#2-ingesting-large-volumes-of-data-into-clickhouse) +- [Goals](#goals) +- [Non-goals](#non-goals) +- [General considerations](#general-considerations) +- [Challenges building this](#major-challenges-around-building-such-a-capability) +- [Proposed solution](#proposed-solution) +- [Design & Implementation](#design--implementation) +- [References](#references) + +## Summary + +Develop a scalable & reliable data ingestion abstraction to help efficiently ingest large volumes of data from high throughput systems into ClickHouse. + +### Why + +To enable any application at GitLab to write necessary data into ClickHouse regardless of the scale at which they generate data today, or in the future. Refer to [Motivation](#motivation) for why ClickHouse in the first place. + +### How + +By building a write abstraction (API/Library) that allows a user to write data into ClickHouse and has all necessary configurations, conventions and best-practices around instrumentation, service-discovery, etc, built into it out of the box. + +## Motivation + +ClickHouse is an online, analytical processing (OLAP) database that powers use-cases that require fetching real-time, aggregated data that does not mutate a lot. ClickHouse is highly performant and can scale to large volumes of data as compared to traditional transactional relational databases (OLTP) such as Postgres, MySQL. For further reading around ClickHouse's capabilities, see [[1]](https://about.gitlab.com/blog/2022/04/29/two-sizes-fit-most-postgresql-and-clickhouse/), [[2]](https://clickhouse.com/blog/migrating-data-between-clickhouse-postgres) and [[3]](https://posthog.com/blog/clickhouse-vs-postgres). + +At GitLab, [our current and future ClickHouse uses/capabilities](https://gitlab.com/groups/gitlab-com/-/epics/2075) reference & describe multiple use-cases that could be facilitated by using ClickHouse as a backing datastore. A majority of these talk about the following two major areas of concern: + +1. Being able to leverage [ClickHouse's OLAP capabilities](https://clickhouse.com/docs/en/faq/general/olap/) enabling underlying systems to perform an aggregated analysis of data, both over short and long periods of time. +1. The fact that executing these operations with our currently existing datasets primarily in Postgres, is starting to become challenging and non-performant. + +Looking forward, assuming a larger volume of data being produced by our application(s) and the rate at which it gets produced, the ability to ingest it into a *more* capable system, both effectively and efficiently helps us scale our applications and prepare for business growth. + +## Case studies + +From an initial assessment of all (reported) use-cases that intend to utilise ClickHouse, the following broad patterns of usage can be observed: + +1. Efficiently replicating existing data from other databases into ClickHouse, most prominently Postgres. +1. Directly ingesting large volumes of data into ClickHouse for asynchronous processing, data aggregation & analysis. + +The following section(s) explain details of each problem-domain: + +### 1. Replicating existing data into ClickHouse + +With due reference to our prior work around this, it has been established that logical replication from Postgres is too slow. Instead, we'll need to be able to emit data change events within database transactions which can then get processed asynchronously to write or update corresponding data in ClickHouse. + +The following case-studies describe how these groups intend to solve the underlying problem: + +- ~group::optimize has been working towards a scalable PostgreSQL data replication strategy which can be implemented on the application layer. + + - [Proposal: Scalable data sync/replication strategy](https://gitlab.com/gitlab-org/gitlab/-/issues/382172) talks about such a strategy and the additional challenges with using Sidekiq for queueing/batching needs. + + - It has been observed that pumping data from `PostgreSQL` into `ClickHouse` directly might not be the right way to approach the problem at hand. + + - In addition to the problems described above, another class of problems when replicating data across systems is also the handling of data backfill and/or data migrations that happen upstream. + +- [group::data](https://about.gitlab.com/handbook/business-technology/data-team/) has been working around syncing data from some of our Postgres databases into a Snowflake-based data warehouse. See this issue for optioned considered: [List down all possible options for postgres to snowflake pipeline](https://gitlab.com/gitlab-data/gitlab.com-saas-data-pipeline/-/issues/13) before designing the current system in place. + + - With the work done around our [Next Gen GitLab SaaS Data Pipeline](https://docs.google.com/presentation/d/1hVaCY42YhaO5UvgLzp3mbuMYJIFuTFYFJjdhixFTxPE/edit#slide=id.g143a48de8a3_0_0), the data team owns a "custom" pipeline that does incremental data extractions based on an `updated_at` timestamp column. This helps import a significant subset of operational database relations into Snowflake data-warehouse. + + - As the volume of data grows, we can foresee this (ETL) pipeline warranting more time and resources to execute resulting in delays across the time between data being produced and being available in Snowflake data-warehouse. + + - We might also see data inconsistency/incompleteness issues emanating from the current setup since row deletions are not transferred into Snowflake, inflating data volume and skewing analysis. Any information about multiple updates happening between import interval period are also lost. + + - Having a scalable ingestion pipeline that can help replicate data from our databases into an intermediate system and/or ClickHouse in near real-time would help improve the operational characteristics around this system. + +### 2. Ingesting large volumes of data into ClickHouse + +We need to be able to ingest large volumes of potentially unaggregated data into ClickHouse which may result into a large number of small writes as well. This can have an adverse effect on how ClickHouse processes and stores incoming data. To mitigate this problem, we need to queue & batch smaller writes into larger ones to keep the ingestion pipeline efficient at all times. + +The following case-studies describe how each group intends to solve the underlying problem: + +- ~group::observability explains their need of ingesting large amounts of data into ClickHouse, with the following two issues: + + - [Proposal: GitLab Observability Platform - Data Ingestion](https://gitlab.com/gitlab-org/opstrace/opstrace/-/issues/1878) talks about using an external events store, such as Kafka, to first ingest data as received from users, then writing it into ClickHouse in larger batches thereby eliminating the need to write a large number of small writes without hampering write performance from how ClickHouse `MergeTree` processes ingested data. + + - In addition, [ClickHouse: Investigate client-side buffering to batch writes into ClickHouse](https://gitlab.com/gitlab-org/opstrace/opstrace/-/issues/2044) talks about their experimentation with using application-local queueing/batching to work around the problems mentioned above. + +- ~"group::product intelligence" has been working on building our analytics offering and recently looking at building and/or improving parts of the system. + + - [Product Analytics Collector Component](https://gitlab.com/groups/gitlab-org/-/epics/9346) talks about replacing Jitsu with Snowplow for collecting and processing tracking events. For more details of the proposal, see [Jitsu replacement](https://gitlab.com/gitlab-org/analytics-section/product-intelligence/proposals/-/blob/62d332baf5701810d9e7a0b2c00df18431e82f22/doc/jitsu_replacement.md). + + - The initial design was prototyped with [Snowplow as Jitsu Replacement PoC](https://gitlab.com/gitlab-org/analytics-section/product-analytics/devkit/-/merge_requests/37). + + - From the design, it is easy to observe how large amounts of data will be ingested into ClickHouse and could potentially benefit from the use of a scalable ingestion pipeline. + +## Goals + +### Well-defined, established client abstractions + +We want to define and establish a fully-functional application-side abstraction that can help ingest data into ClickHouse without getting in the way of how an application itself is designed while keeping the underlying code backend-agnostic. The proposed abstraction should become the default choice for any applications, core or satellite, at GitLab. + +### Support for high throughput in volume of writes + +A solution here should enable an application to write any amount of inserts (order of upto 1000-5000 writes per second) to the underlying database efficiently while also allowing for growth as the application scales out. Considering how ClickHouse processes incoming writes, a proposed solution should be able to batch a number of very small writes into larger batches. + +### Reliable, consistent delivery of data + +A solution here should also ensure reliable & consistent delivery of ingested data into the underlying database minimising undue loss of data before being eventually persisted into ClickHouse. + +## Non-goals + +### Addressing data types, schemas or formats + +At this stage of this proposal, we're not optimizing for addressing which data types, schemas or formats we receive ingested data in. It should be delegated to the backend-specific implementations themselves and not handled within the write abstraction. + +### Addressing where our data sources exist today + +We're also not addressing any client-side specific details into the design at this point. The write abstraction should only remain a tool for the language in which it is written. As long as an application can use it to write data as any other third-party library, we should be good to build on top of it. + +## General Considerations + +Having addressed the details of the two aformentioned problem-domains, we can model a proposed solution with the following logical structure: + +- Ingestion + - APIs/SDKs + - HTTP2/gRPC Sidecar +- Transport & Routing + - Multi-destination +- Digestion/Compute + - Enrichment + - Processing + - Persisting + +## Major challenges around building such a capability + +### Self-managed environments + +The single, biggest challenge around introducing ClickHouse and related systems would be the ability to make it avaiable to our users running GitLab in self-managed environments. The intended goals of this proposal are intentionally kept within those constraints. It is also prudent to establish that what we're *proposing* here be applicable to applications consuming ClickHouse from inside self-managed environments. + +There are ongoing efforts to streamline distribution and deployment of ClickHouse instances for managed environment within the larger scope of [ClickHouse Usage at GitLab](../clickhouse_usage/index.md). A few other issues tackling parts of the aforementioned problem are: + +- [Research and understand component costs and maintenance requirements of running a ClickHouse instance with GitLab](https://gitlab.com/gitlab-com/www-gitlab-com/-/issues/14384) +- [ClickHouse maintenance and cost research](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/116669) + +### Wide variety of data sources, their structures & usage patterns + +The data that we intend to ingest into ClickHouse can come from a wide variety of data sources and be structured in different schemas or formats. With that considered, it's non-trivial effort to draft a solution that suffices all use-cases efficiently. + +Should we decide to build an intermediate ingestion system, any solution should help provide a source/schema/format-agnostic data transport layer with an established, matured client-abstraction to maximise the number of applications that can use it. + +### Building on top of our current database infrastructure + +Our current database infrastructure operates at a fairly large scale and adding more applications that continuously read/write against it adds to the pressure on the existing resources. It's important we move away any workloads and/or datasets that can be safely processed in a different context altogether. + +### Service Discovery + +We're still normalising the details around distribution and deployment of ClickHouse clusters and/or instances for our applications. Subject to how we end up doing it, for a client to be able to discover which ClickHouse cluster, shard or table would need to become a part any such solution. + +## Proposed Solution + +In light of the problems discussed earlier, it'd be in our better interests to allow the usage of an external, intermediate system subject to what one's needs might be especially around the volume & scale of data being writen from an application into ClickHouse. + +Therefore, we intend to develop an abstraction that can enable an application to store data into ClickHouse regardless of the scale that they (currently) operate at. It also: + +- Facilitates an application to switch from one *technology* to another should their performance and/or scale requirements change over time. +- Allows for backend-specific conventions, configurations & best practices such as instrumentation, service-discovery, etc. to be encoded in one place for all applications to leverage consistently. + +## Design & Implementation + +### Core assumptions + +- We're only going to focus on writing data into ClickHouse as mentioned in aforementioned non-goals. With details of how our data lands into ClickHouse, this document does not (intentionally) address where this data comes from. Some of those details are delegated to the applications generating this data i.e as long as they can consume this abstraction, they should be able to write data into ClickHouse. + +- We're going to delegate the choice of different storage backends to a following blueprint or epic since that's outside the scope of this design. With ClickHouse as the eventual destination for our data, this document only talks about writing data into it - either directly or indirectly via a queueing/batching system. + +### Architecture + +![Architecture](clickhouse_dbwriter.png) + +Having an abstraction around writing data help client-side instrumentation to stay backend-agnostic allowing them to switch code paths depending on where it runs. + +An example setup should look like: + +```ruby +Gitlab::Database::Writer.config do |config| + # + # when using sync mode, data gets written directly into ClickHouse, + # therefore, it's also assumed the backend here is ClickHouse + config.mode = :sync OR :async + config.backend = :clickhouse # optional + # OR + # + # when using async mode, data is written to an intermediate system + # first, then written into ClickHouse asynchronously + config.mode = :async + config.backend = :pubsub OR :kafka OR :otherbackend + # + # then backend-specific configurations hereafter + # + config.url = 'tcp://user:pwd@localhost:9000/database' + # e.g. a serializer helps define how data travels over the wire + config.json_serializer = ClickHouse::Serializer::JsonSerializer + # ... +end +# do application-specific processing +# eventually, write data using the object you just built +Gitlab::Database::Writer.write( + Gitlab::Database::Model::MyTable, + [{ id: 1, foo: 'bar' }], +) +``` + +We intend to keep `Gitlab::Database::Writer.backend` to be as close to the backend-specific client implementation as possible. Having a wrapper around a vanilla client helps us address peripheral concerns such as service-discovery for the backends while still allowing the user to leverage features of a given client. + +### Iterations + +Considering the large scope of this undertaking and the need for feedback around actual usage, we intend to build the proposed abstraction(s) across multiple iterations which can be described as follows: + +#### Iteration 1 - Develop write abstraction with sync mode enabled + +First, research and develop a simple write abstraction that our users can begin to use to write data into ClickHouse. This ensures our choice of the underlying client is well-researched and suffices to fulfill needs of as many reported use-cases as possible. Being able to see this running would help gather user-feedback and improve the write APIs/interfaces accordingly. + +Given this feedback and more development with how we aim to deploy ClickHouse across our environments, it'd then be prudent to build into this abstraction necessary conventions, best practices and abstract away details around connection-pooling, service-discovery, etc. + +#### Iteration 2 - Add support for schemas & data validation + +In the next iteration, we plan to add support for schema usage and validation. This helps keep model definitions sane and allows for validating data to be inserted. + +#### Iteration 3 - Add support for async mode, PoC with one backend + +With the above two iterations well-executed, we can start to scale up our write abstractions adding the support for writing data into intermediate data stores before writing it into ClickHouse asynchronously. We aim to prototype such an implementation with atleast one such backend. + +#### Further iterations + +With a backend-agnostic abstraction becoming the ingestion interface a client interacts with, there's various other use-cases that can be solved from within this abstraction. Some of them are: + +- Zero-configuration data ingestion from multiple sources +- Dynamically enriching data from multiple sources +- Offloading data to long-term retention data stores + +### Possible backend implementations + +- Applications writing directly to ClickHouse + - Application-local in-memory queueing/batching of data + - Application-local persistent queueing/batching of data +- Non-local queueing/batching of data before eventually writing into ClickHouse + - Managed cloud backends: + - [Google PubSub](https://cloud.google.com/pubsub) + - [AWS Kinesis](https://aws.amazon.com/kinesis/) + - Self-managed backends: + - [CHProxy](https://www.chproxy.org/) + - [Kafka](https://kafka.apache.org/) + - [RedPanda](https://redpanda.com/) + - [Vector](https://vector.dev/) + - [RabbitMQ](https://www.rabbitmq.com/) + +### Additional complexity when using a non-local backend + +- The need for running an additional process/sub-system that reads data from the concerned backend and writes it into ClickHouse efficiently and reliably. +- The additional hop across the backend also means that there might be potential delays in how soon this data lands into ClickHouse. + +Though the points above describe additional complexity for an application, they can be treated as valid trade-off(s) assuming their need for data ingestion at scale. + +### Comparing backends across multiple dimensions + +| Dimension | CHProxy | Redis | Google PubSub | Apache Kafka | +|---|---|---|---|---| +| Operations | Trivial | Trivial | Managed | Non-trivial, complex | +| Data Retention | Non-durable | Non-durable | Durable | Durable | +| Performance | Good | Good | High | High | +| Data Streaming | None | Minimal | Good | Best | +| Suitable for self-managed environments | Trivial | Trivial | - | Complex | + +## References + +- [ClickHouse use-cases within Manage](https://gitlab.com/groups/gitlab-org/-/epics/7964) +- [List down all possible options for postgres to snowflake pipeline](https://gitlab.com/gitlab-data/gitlab.com-saas-data-pipeline/-/issues/13) +- [Design Spike for Snowplow For Data Event capture](https://gitlab.com/gitlab-data/analytics/-/issues/12397) +- [Audit Events Performance Limits](https://gitlab.com/gitlab-org/gitlab/-/issues/375545) diff --git a/doc/architecture/blueprints/clickhouse_read_abstraction_layer/index.md b/doc/architecture/blueprints/clickhouse_read_abstraction_layer/index.md new file mode 100644 index 00000000000..8290641b7a4 --- /dev/null +++ b/doc/architecture/blueprints/clickhouse_read_abstraction_layer/index.md @@ -0,0 +1,318 @@ +--- +status: proposed +creation-date: "2023-02-23" +authors: [ "@mikolaj_wawrzyniak", "@jdrpereira", "@pskorupa" ] +coach: "@DylanGriffith" +approvers: [ "@nhxnguyen" ] +owning-stage: "~workinggroup::clickhouse" +participating-stages: [] +--- + +# Consider an abstraction layer to interact with ClickHouse or alternatives + +## Table of Contents + +- [Summary](#summary) +- [Motivation](#motivation) +- [Goals](#goals) +- [Non-goals](#non-goals) +- [Possible solutions](#possible-solutions) + - [Recommended approach](#recommended-approach) + - [Overview of open source tools](#overview-of-open-source-tools) +- [Open Questions](#open-questions) + +## Summary + +Provide a solution standardizing read access to ClickHouse or its alternatives for GitLab installations that will not opt-in to install ClickHouse. After analyzing different [open-source tools](#overview-of-open-source-tools) and weighing them against an option to [build a solution internally](#recommended-approach). The current recommended approach proposes to use dedicated database-level drivers to connect to each data source. Additionally, it proposes the usage of [repository pattern](https://martinfowler.com/eaaCatalog/repository.html) to confine optionally database availability complexity to a single application layer. + +## Motivation + +ClickHouse requires significant resources to be run, and smaller installations of GitLab might not get a return from investment with provided performance improvement. That creates a risk that ClickHouse might not be globally available for all installations and features might need to alternate between different data stores available. Out of all [present & future ClickHouse use cases](https://gitlab.com/groups/gitlab-com/-/epics/2075) that have been already proposed as part of the working group 7 out of 10 uses data stores different than ClickHouse. Considering that context it is important to +support those use cases in their effort to adopt ClickHouse by providing them with tools and guidelines that will standardize interactions with available data stores. + +The proposed solution can take different forms from stand-alone tooling +offering a unified interface for interactions with underlying data stores, to a set of libraries supporting each of the data stored individually backed by implementation guidelines that will describe rules and limitations placed around data stores interactions, and drawing borders of encapsulation. + +## Goals + +- Limit the impact of optionally available data stores on the overall GitLab application codebase to [single abstraction layer](../../../development/reusing_abstractions.md#abstractions) +- Support all data store specific features +- Support communication for satellite services of the main GitLab application + +## Non-goals + +- This proposal does not directly consider write communication with database, as this is a subject of [complementary effort](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/111148) +- This proposal does not directly consider schema changes and data migration challenges + +Despite above points being non goals, it is acknowledge that they might impose some alterations to final solution which is expressed at the end of this document in the [Open questions](#open-questions) section. + +## Possible Solutions + +High-level goals described in the previous paragraph can be achieved by both in-house-built solutions as well as by adopting open-source tools. +The following sections will take a closer look into both of those avenues + +### Recommended approach + +In the spirit of MVC and iteration, it is proposed to start with a solution that would rely on drivers that directly interact +with corresponding data stores, like ActiveRecord for Ruby. For this solution to be able to achieve goals set for +this exit criteria and help mitigate the issue listed in the _Motivation_ section of this document, such drivers need to be supported +by a set of development guidelines enforced with static code analysis. + +Such a solution was selected as preferred upon receiving feedback from different members of the working group concerned +about the risk of limitations that might be imposed by open-source tools, preventing groups from taking advantage of ClickHouse +features to their fullest. Members collaborating around working group criteria presented in this document, agree that +concerns around limitations could be mitigated by building a comprehensive set of prototypes, however time and effort +required to achieve that surpass the limits of this working group. It is also important to notice that ClickHouse adoption +is in an exploratory stage, and groups might not being even able to state what are their requirements just yet. + +#### Proposed drivers + +Following ClickHouse documentation there are the following drivers for Ruby and Go + +##### Ruby + +1. [ClickHouse Ruby driver](https://github.com/shlima/click_house) - Previously selected for use in GitLab as part of the Observability grup's research (see: [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/358158)) +1. [Clickhouse::Activerecord](https://github.com/PNixx/clickhouse-activerecord) + +##### Go + +1. [ClickHouse/clickhouse-go](https://github.com/ClickHouse/clickhouse-go) - Official SQL database client. +1. [uptrace/go-clickhouse](https://clickhouse.uptrace.dev/) - Alternative client. + +##### Proposed client architecture + +To keep the codebase well organized and limit coupling to any specific database engine it is important to encapsulate +interactions, including querying data to a single application layer, that would present its interface to layers above in +similar vain to [ActiveRecord interface propagation through abstraction layers](../../../development/reusing_abstractions.md) + +Keeping underlying database engines encapsulated makes the recommended solution a good two-way door decision that +keeps the opportunity to introduce other tools later on, while giving groups time to explore and understand their use cases. + +At the lowest abstraction layer, it can be expected that there will be a family of classes directly interacting with the ClickHouse driver, those classes +following MVC pattern implemented by Rails should be classified as _Models_. + +Models-level abstraction builds well into existing patterns and guidelines but unfortunately does not solve the challenge of the optional availability of the ClickHouse database engine for self-managed instances. It is required to design a dedicated entity that will house responsibility of selecting best database to serve business logic request. +From the already mentioned existing abstraction [guidelines](../../../development/reusing_abstractions.md) `Finders` seems to be the closest to the given requirements, due to the fact that `Finders` encapsulate database specific interaction behind their own public API, hiding database vendors detail from all layers above them. + +However, they are closely coupled to `ActiveRecord` ORM framework, and are bound by existing GitLab convention to return `ActiveRecord::Relation` objects, that might be used to compose even more complex queries. That coupling makes `Finders` unfit to deal with the optional availability of ClickHouse because returned data might come from two different databases, and might not be compatible with each other. + +With all that above in mind it might be worth considering adding a new entity into the codebase that would exist on a similar level of abstraction as `Finders` yet it would be required to return an `Array` of data objects instead. + +Required level of isolation can be achieved with usage of a [repository pattern](https://martinfowler.com/eaaCatalog/repository.html). The repository pattern is designed to separates business / domain logic from data access concerns, which is exactly what this proposal is looking for. +What is more the repository pattern does not limits operations performed on underlying databases allowing for full utilization of their features. + +To implement the repository pattern following things needs to be created: + +1. A **strategy** for each of supported databases, for example: `MyAwesomeFeature::Repository::Strategies::ClickHouseStrategy` and `MyAwesomeFeature::Repository::Strategies::PostgreSQLStrategy`. Strategies are responsible for implementing communication with underlying database ie: composing queries +1. A **repository** that is responsible for exposing high level interface to interact with database using one of available strategies selected with some predefined criteria ie: database availability. Strategies used by single repository must share the same public interface so they can be used interchangeable +1. A **Plain Old Ruby Object(PORO) Model** that represents data in business logic implemented by application layers using repository. It have to be database agnostic + +It is important to notice that the repository pattern based solution has already been implemented by Observability group (kudos to: @ahegyi, @splattael and @brodock). [`ErrorTracking::ErrorRepository`](https://gitlab.com/gitlab-org/gitlab/-/blob/1070c008b9e72626e25296480f82f2ee2b93f847/lib/gitlab/error_tracking/error_repository.rb) is being used to support migration of error tracking features from PostgreSQL to ClickHouse (integrated via API), and uses feature flag toggle as database selection criteria, that is great example of optional availability of database. + +`ErrorRepository` is using two strategies: + +1. [`OpenApiStrategy`](https://gitlab.com/gitlab-org/gitlab/-/blob/d0bdc8370ef17891fd718a4578e41fef97cf065d/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb) to interact with ClickHouse using API proxy entity +1. [`ActiveRecordStrategy`](https://gitlab.com/gitlab-org/gitlab/-/blob/d0bdc8370ef17891fd718a4578e41fef97cf065d/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb) to interact with PostgreSQL using `ActiveRecord` framework + +Each of those strategies return data back to abstraction layers above using following PORO Models: + +1. [`Gitlab::ErrorTracking::Error`](https://gitlab.com/gitlab-org/gitlab/-/blob/a8ea29d51ff23cd8f5b467de9063b64716c81879/lib/gitlab/error_tracking/error.rb) +1. [`Gitlab::ErrorTracking::DetailedError`](https://gitlab.com/gitlab-org/gitlab/-/blob/a8ea29d51ff23cd8f5b467de9063b64716c81879/lib/gitlab/error_tracking/detailed_error.rb) + +Additionally `ErrorRepository` is great example of remarkable flexibility offered by the repository pattern in terms of supported types of data stores, allowing to integrate solutions as different as a library and external service API under single unified interface. That example presents opportunity that the repository pattern in the future might be expanded beyond needs of ClickHouse and PostgreSQL when some use case would call for it. + +Following [merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85907/diffs) documents changes done by observability group in order to migrate from using current GitLab architecture based on ActiveRecord Models, Services and Finders to the repository pattern. + +##### Possible ways to enforce client architecture + +It is not enough to propose a client-side architecture for it to fully be established as common practice it needs +to be automatically enforced, reducing the risk of developers unconsciously going against it. There are multiple ways to +introduce automated verification of repository pattern implementation including: + +1. Utilize `ActiveRecord` query subscribers in a similar way to[Database::PreventCrossJoins](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/support/database/prevent_cross_joins.rb) in order to detect queries to ClickHouse executed outside of _Strategies_ +1. Expanding [`CodeReuse`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/rubocop/cop/code_reuse) rubocop rules to flag all usage of ClickHouse driver outside of _Strategies_ +1. Create rubocop rule that detects calls to utility method that checks the presence of ClickHouse instance (ie: `CurrentSettings.click_house_enabled?`) that are being made outside of _Repositories_ + +At this development stage, authors see all of the listed options as viable and promising, therefore a decision about which ones to use would be deferred to the moment when the first repository pattern implementation for ClickHouse will emerge. + +### Overview of open-source tools + +In this section authors provide an overview of existing 3rd party open-source solutions that were considered as alternative approaches to achieve stated goal, but was not selected as recommended approach. + +#### Evaluation criteria + +##### 1. License (MUST HAVE) + +1. Solutions must be open source under an [acceptable license](https://about.gitlab.com/handbook/engineering/open-source/#acceptable-licenses). + +##### 2. Support for different data stores (MUST HAVE) + +1. It focuses on the fact whether the proposed abstraction layer can support both ClickHouse and PostgreSQL (must have) +1. Additional consideration might be if more than the two must-have storages are supported +1. The solution must support the [minimum required versions](../../../install/requirements.md#postgresql-requirements) for PostgreSQL + +##### 3. Protocol compatibility + +Every abstraction layer comes at the cost of limited API compared to direct access to the tool. This exit criterion is trying to bring understanding to the degree of trade-off being made on limiting tools API for the sake of a common abstraction. + +1. List what read operations can be done via PostgreSQL and ClickHouse (`selects`, `joins`, `group by`, `order by`, `union` etc) +1. List what operations can be done with the proposed abstraction layer, how complicated it is to do such operations, and whether are there any performance concerns when compared to running operations natively +1. Does it still allow for direct access to a data source in case the required operation is not supported by the abstraction layer, eg: `ActiveRecord` allows for raw SQL strings to be run with `#execute` + +##### 4. Operational effort + +1. Deployment process: how complex is it? Is the proposed tool a library tool that is being added into the stack, or does it require additional services to be deployed independently along the GitLab system. What deployment types does the tool support (Kubernetes/VMs, SaaS/self-managed, supported OS, cloud providers). Does it support offline installation. +1. How many hardware resources does it need to operate +1. Does it require complex monitoring and operations to assure stable and performant services +1. Matured maintenance process and documentation around it: upgrades, backup and restore, scaling +1. High-availability support. Does the tool have documentation how to build HA cluster and perform failovers for self-managed? Does the tool support zero-downtime upgrade? +1. FIPS and FedRAMP compliance +1. Replication process and how the new tool would fit in GitLab Geo. + +##### 5. Developer experience + +1. Solutions must have well-structured, clear, and thoroughly documented APIs to ease adoption and reduce the learning curve. + +##### 6. Maturity (nice to have) + +1. How long does the solution exist? Is it used often? Does it have a stable community? If the license permits forking tool is also a considerable option + +##### 7. Tech fit + +1. Is the solution written in one of the programming languages we use at GitLab so that we can more easily contribute with bug fixes and new features? + +##### 8. Interoperability (Must have) + +1. Can the solution support both the main GitLab application written in Ruby on Rails also satellite services like container registry that might be written in Go + +#### Open - Source solutions + +##### 1. [Cube.dev](https://cube.dev/) + +**Evaluation** + +1. License + Apache 2.0 + MIT ✅ +1. Support for different data stores + Yes ✅ +1. Protocol compatibility + It uses OLAP theory concepts to aggregate data. This might be useful in some use cases like aggregating usage metrics, but not in others. It has APIs for both SQL queries and their own query format. +1. Operational effort + Separate service to be deployed using Docker or k8s. Uses Redis as a cache and data structure store. +1. Developer experience + Good [documentation](https://cube.dev/docs) +1. Maturity + Headless BI tools themselves are a fairly new idea, but Cube.js seems to be the leading open-source solution in this space. + The Analytics section uses it internally for our Product Analytics stack. +1. Tech fit + Uses REST and GraphQL APIs. It has its own query and data schema formats, but they are well-documented. Data definitions in either YAML or JavaScript. + +**Comment** + +The solution is already being used as a read interface for ClickHouse by ~"group::product analytics", +to gather first hand experience there was a conversation held with @mwoolf with key conclusions being: + +1. ClickHouse driver for cube.dev is community-sourced, and it does not have a maintainer as of now, which means there is no active development. It is a small and rather simple repository that should work at least until a new major version of ClickHouse will arrive with some breaking changes +1. Cube.dev is written in Type Script and JavaScript which are part of GitLab technical stack, and there are engineers here with expertise in them, however Cube.dev is expected to be mostly used by backend developers, which does not have that much experience in mentioned technologies +1. Abstraction layer for simple SQL works, based on JSON will build correct query depending on the backend +1. Data store-specific functions (like window funnel ClickHouse) are not being translated to other engines, which requires additional cube schemas to be built to represent the same data. +1. Performance so far was not an issue both on local dev and on AWS VPS millions of rows import load testing +1. It expose postgres SQL like interface for most engines, but not for ClickHouse unfortunately so for sake of working group use case JSON API might be more feasible +1. Cube.dev can automatically generate schemas on the fly, which can be used conditionally in the runtime handling optional components like ClickHouse + +There is also a [recording](https://youtu.be/iBPTCrvOBBs) of that conversation available. + +##### 2. [ClickHouse FDW](https://github.com/ildus/clickhouse_fdw) + +**Evaluation** + +A ClickHouse Foreign Data Wrapper for PostgreSQL. It allows ClickHouse tables to be queried as if they were stored in PostgreSQL. +Could be a viable option to easily introduce ClickHouse as a drop-in replacement when Postgres stops scaling. + +1. License + Apache 2.0 ✅ +1. Support for different data stores + Yes, by calling ClickHouse through a PostgreSQL instance. ✅ +1. Protocol compatibility + Supports SELECT, INSERT statements at a first glance. Not sure about joins. Allows for raw SQL by definition. +1. Operational effort + 1. A PostgreSQL extension. Requires some mapping between the two DBs. + 1. Might have adversary impact on PostgreSQL performance, when execution would wait for response from ClickHouse waisting CPU cycles on waiting + 1. Require exposing and managing connection between deployments of PostgreSQL and ClickHouse +1. Developer experience + TBD +1. Maturity + It's been around for a few years and is listed in ClickHouse docs, but doesn't seem to be widely used. +1. Tech fit + Raw SQL statements. + +**Comment** + +##### 3. [Clickhouse::Activerecord](https://github.com/PNixx/clickhouse-activerecord) + +**Evaluation** + +1. License + MIT License ✅ +1. Support for different data stores + Yes, in the sense that it provides a Clickhouse adapter for ActiveRecord in the application layer so that it can be used to query along PostgreSQL. ✅ +1. Protocol compatibility + Not sure about joins - no examples. +1. Operational effort + Ruby on Rails library tool - ORM interface in a form of an ActiveRecord adapter. +1. Developer experience + Easy to work with for developers familiar with Rails. +1. Maturity + Has been around for a few years, but repo activity is scarce (not a bad thing by itself, however). +1. Tech fit + Rails library, so yes. + +**Comment** + +##### 4. [Metriql](https://metriql.com/) + +**Evaluation** + +A headless BI solution using DBT to source data. Similar to Cube.dev in terms of defining metrics from data and transforming them with aggregations. +The authors explain the differences between Metriql and other BI tools like Cube.js in this FAQ entry. + +1. License + Apache 2.0 ✅ +1. Support for different data stores + Uses DBT to read from data sources, so CH and PostgreSQL are possible. +1. Protocol compatibility + It uses OLAP theory concepts to aggregate data. It does allow for impromptu SQL queries through a REST API. +1. Operational effort + It's a separate service to deploy and requires DBT. +1. Developer experience + I assume it requires DBT knowledge to set up and use. It has a fairly simple REST API documented here. +1. Maturity + First release May 2021, but repo activity is scarce (not a bad thing by itself). +1. Tech fit + Connects with BI tools through a REST API or JDBC Adapter. Allows querying using SQL or MQL (which is a SQL flavor/subset). + +**Comment** + +##### 5. Notable rejected 3rd party solutions + +ETL only solutions like Airflow and Meltano, as well as visualization tools like Tableau and Apache Superset, were excluded from the prospect list as they are usually clearly outside our criteria. + +**[pg2ch](https://github.com/mkabilov/pg2ch)** +PostgreSQL to ClickHouse mirroring using logical replication. +Repo archived; explicitly labeled not for production use. Logical replication might not be performant enough at our scale - we don't use it in our PostgreSQL DBs because of performance concerns. + +**Looker** +BI tooling. +Closed-source; proprietary. + +**[Hasura](https://github.com/hasura/graphql-engine)** +GraphQL interface for database sources. +No ClickHouse support yet. + +**[dbt Server](https://github.com/dbt-labs/dbt-server)** +HTTP API for dbt. MariaDB Business Source License (BSL) ❌ + +### Open questions + +1. This proposal main focus is read interface, however depending on outcome of [complementary effort](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/111148) that focus on write interface similar concerns around optional availability might be applicable to write interaction. In case if ingestion pipeline would not resolve optional availability challenges for write interface it might be considerable to include write interactions into repository pattern implementation proposed in this document. +1. Concerns around ClickHouse schema changes and data migrations is not covered by any existing working group criteria, even though solving this challenges as a whole is outside of the scope of this document it is prudent to raise awareness that some alterations to proposed repository pattern based implementation might be required in order to support schema changes. diff --git a/doc/architecture/blueprints/clickhouse_usage/index.md b/doc/architecture/blueprints/clickhouse_usage/index.md new file mode 100644 index 00000000000..8a5530313e5 --- /dev/null +++ b/doc/architecture/blueprints/clickhouse_usage/index.md @@ -0,0 +1,58 @@ +--- +status: proposed +creation-date: "2023-02-02" +authors: [ "@nhxnguyen" ] +coach: "@grzesiek" +approvers: [ "@dorrino", "@nhxnguyen" ] +owning-stage: "~devops::data_stores" +participating-stages: ["~section::ops", "~section::dev"] +--- + + + +# ClickHouse Usage at GitLab + +## Summary + +[ClickHouse](https://clickhouse.com/) is an open-source column-oriented database management system. It can efficiently filter, aggregate, and sum across large numbers of rows. In FY23, GitLab selected ClickHouse as its standard data store for features with big data and insert-heavy requirements such as Observability and Analytics. This blueprint is a product of the [ClickHouse working group](https://about.gitlab.com/company/team/structure/working-groups/clickhouse-datastore/). It serves as a high-level blueprint to ClickHouse adoption at GitLab and references other blueprints addressing specific ClickHouse-related technical challenges. + +## Motivation + +In FY23-Q2, the Monitor:Observability team developed and shipped a [ClickHouse data platform](https://gitlab.com/groups/gitlab-org/-/epics/7772) to store and query data for Error Tracking and other observability features. Other teams have also begun to incorporate ClickHouse into their current or planned architectures. Given the growing interest in ClickHouse across product development teams, it is important to have a cohesive strategy for developing features using ClickHouse. This will allow teams to more efficiently leverage ClickHouse and ensure that we can maintain and support this functionality effectively for SaaS and self-managed customers. + +### Goals + +As ClickHouse has already been selected for use at GitLab, our main goal now is to ensure successful adoption of ClickHouse across GitLab. It is helpful to break down this goal according to the different phases of the product development workflow. + +1. Plan: Make it easy for development teams to understand if ClickHouse is the right fit for their feature. +1. Develop and Test: Give teams the best practices and frameworks to develop ClickHouse-backed features. +1. Launch: Support ClickHouse-backed features for SaaS and self-managed. +1. Improve: Successfully scale our usage of ClickHouse. + +### Non-Goals + +## Proposals + +The following are links to proposals in the form of blueprints that address technical challenges to using ClickHouse across a wide variety of features. + +1. Scalable data ingestion pipeline. + - How do we ingest large volumes of data from GitLab into ClickHouse either directly or by replicating existing data? +1. Supporting ClickHouse for self-managed installations. + - For which use-cases and scales does it make sense to run ClickHouse for self-managed and what are the associated costs? + - How can we best support self-managed installation of ClickHouse for different types/sizes of environments? + - Consider using the [Opstrace ClickHouse operator](https://gitlab.com/gitlab-org/opstrace/opstrace/-/tree/main/clickhouse-operator) as the basis for a canonical distribution. + - Consider exposing Clickhouse backend as [GitLab Plus](https://gitlab.com/groups/gitlab-org/-/epics/308) to combine benefits of using self-managed instance and GitLab-managed database. + - Should we develop abstractions for querying and data ingestion to avoid requiring ClickHouse for small-scale installations? +1. Abstraction layer for features to leverage both ClickHouse or PostreSQL. + - What are the benefits and tradeoffs? For example, how would this impact our automated migration and query testing? +1. Security recommendations and secure defaults for ClickHouse usage. + +Note that we are still formulating proposals and will update the blueprint accordingly. + +## Best Practices + +Best practices and guidelines for developing performant and scalable features using ClickHouse are located in the [ClickHouse developer documentation](../../../development/database/clickhouse/index.md). + +## Cost and maintenance analysis + +ClickHouse components cost and maintenance analysis is located in the [ClickHouse Self-Managed component costs and maintenance requirements](self_managed_costs_and_requirements/index.md). diff --git a/doc/architecture/blueprints/clickhouse_usage/self_managed_costs_and_requirements/index.md b/doc/architecture/blueprints/clickhouse_usage/self_managed_costs_and_requirements/index.md new file mode 100644 index 00000000000..d8c9c0b25d5 --- /dev/null +++ b/doc/architecture/blueprints/clickhouse_usage/self_managed_costs_and_requirements/index.md @@ -0,0 +1,65 @@ +--- +status: proposed +creation-date: "2023-04-04" +authors: [ "@niskhakova", "@dmakovey" ] +coach: "@grzesiek" +approvers: [ "@dorrino", "@nhxnguyen" ] +owning-stage: "~workinggroup::clickhouse" +participating-stages: ["~section::enablement"] +--- + +# ClickHouse Self-Managed component costs and maintenance requirements + +## Summary + +[ClickHouse](https://clickhouse.com/) requires additional cost and maintenance for self-managed customers: + +- **Resource allocation cost**: ClickHouse requires a considerable amount of resources to run optimally. + - [Minimum cost estimation](#minimum-self-managed-component-costs) shows that setting up ClickHouse can be applicable only for very large Reference Architectures: 25k and up. +- **High availability**: ClickHouse SaaS supports HA. No documented HA configuration for self-managed at the moment. +- **Geo setups**: Sync and replication complexity for GitLab Geo setups. +- **Upgrades**: An additional database to maintain and upgrade along with existing Postgres database. This also includes compatibility issues of mapping GitLab version to ClickHouse version and keeping them up-to-date. +- **Backup and restore:** Self-managed customers need to have an engineer who is familiar with backup strategies and disaster recovery process in ClickHouse or switch to ClickHouse SaaS. +- **Monitoring**: ClickHouse can use Prometheus, additional component to monitor and troubleshoot. +- **Limitations**: Azure object storage is not supported. GitLab does not have the documentation or support expertise to assist customers with deployment and operation of self-managed ClickHouse. +- **ClickHouse SaaS**: Customers using a self-managed GitLab instance with regulatory or compliance requirements, or latency concerns likely cannot use ClickHouse SaaS. + +### Minimum self-managed component costs + +Based on [ClickHouse spec requirements](https://gitlab.com/gitlab-com/www-gitlab-com/-/issues/14384#note_1307456092) analysis +and collaborating with ClickHouse team, we identified the following minimal configurations for ClickHouse self-managed: + +1. ClickHouse High Availability (HA) + - ClickHouse - 2 machines with >=16-cores, >=64 GB RAM, SSD, 10 GB Internet. Each machine also runs Keeper. + - [Keeper](https://clickhouse.com/docs/en/guides/sre/keeper/clickhouse-keeper) - 1 machine with 2 CPU, 4 GB of RAM, SSD with high IOPS +1. ClickHouse non-HA + - ClickHouse - 1 machine with >=16-cores, >=64 GB RAM, SSD, 10 GB Internet. + +The following [cost table](https://gitlab.com/gitlab-com/www-gitlab-com/-/issues/14384#note_1324085466) was compiled using the machine CPU and memory requirements for ClickHouse, and comparing them to the +GitLab Reference Architecture sizes and [costs](../../../../administration/reference_architectures/index.md#cost-to-run) from the GCP calculator. + +| Reference Architecture | ClickHouse type | ClickHouse cost / (GitLab cost + ClickHouse cost) | +|-------------|-----------------|-----------------------------------| +| [1k - non HA](https://cloud.google.com/products/calculator#id=a6d6a94a-c7dc-4c22-85c4-7c5747f272ed) | [non-HA](https://cloud.google.com/products/calculator#id=9af5359e-b155-451c-b090-5f0879bb591e) | 78.01% | +| [2k - non HA](https://cloud.google.com/products/calculator#id=0d3aff1f-ea3d-43f9-aa59-df49d27c35ca) | [non-HA](https://cloud.google.com/products/calculator#id=9af5359e-b155-451c-b090-5f0879bb591e) | 44.50% | +| [3k - HA](https://cloud.google.com/products/calculator/#id=15fc2bd9-5b1c-479d-bc46-d5ce096b8107) | [HA](https://cloud.google.com/products/calculator#id=9909f5af-d41a-4da2-b8cc-a0347702a823) | 37.87% | +| [5k - HA](https://cloud.google.com/products/calculator/#id=9a798136-53f2-4c35-be43-8e1e975a6663) | [HA](https://cloud.google.com/products/calculator#id=9909f5af-d41a-4da2-b8cc-a0347702a823) | 30.92% | +| [10k - HA](https://cloud.google.com/products/calculator#id=cbe61840-31a1-487f-88fa-631251c2fde5) | [HA](https://cloud.google.com/products/calculator#id=9909f5af-d41a-4da2-b8cc-a0347702a823) | 20.47% | +| [25k - HA](https://cloud.google.com/products/calculator#id=b4b8b587-508a-4433-adc8-dc506bbe924f) | [HA](https://cloud.google.com/products/calculator#id=9909f5af-d41a-4da2-b8cc-a0347702a823) | 14.30% | +| [50k - HA](https://cloud.google.com/products/calculator/#id=48b4d817-d6cd-44b8-b069-0ba9a5d123ea) | [HA](https://cloud.google.com/products/calculator#id=9909f5af-d41a-4da2-b8cc-a0347702a823) | 8.16% | + +NOTE: +The ClickHouse Self-Managed component evaluation is the minimum estimation for the costs +with a simplified architecture. + +The following components increase the cost, and were not considered in the minimum calculation: + +- Disk size - depends on data size, hard to estimate. +- Disk types - ClickHouse recommends [fast SSDs](https://clickhouse.com/docs/ru/operations/tips#storage-subsystem). +- Network usage - ClickHouse recommends using [10 GB network, if possible](https://clickhouse.com/docs/en/operations/tips#network). +- For HA we sum minimum cost across all reference architectures from 3k to 50k users, but HA specs tend to increase with user count. + +### Resources + +- [Research and understand component costs and maintenance requirements of running a ClickHouse instance with GitLab](https://gitlab.com/gitlab-com/www-gitlab-com/-/issues/14384) +- [ClickHouse for Error Tracking on GitLab.com](https://gitlab.com/gitlab-com/gl-infra/readiness/-/blob/master/library/database/clickhouse/index.md) diff --git a/doc/architecture/blueprints/code_search_with_zoekt/index.md b/doc/architecture/blueprints/code_search_with_zoekt/index.md new file mode 100644 index 00000000000..db608b763b8 --- /dev/null +++ b/doc/architecture/blueprints/code_search_with_zoekt/index.md @@ -0,0 +1,307 @@ +--- +status: ongoing +creation-date: "2022-12-28" +authors: [ "@dgruzd", "@DylanGriffith" ] +coach: "@DylanGriffith" +approvers: [ "@joshlambert", "@changzhengliu" ] +owning-stage: "~devops::enablement" +participating-stages: [] +--- + + + +# Use Zoekt For code search + +## Summary + +We will be implementing an additional code search functionality in GitLab that +is backed by [Zoekt](https://github.com/sourcegraph/zoekt), an open source +search engine that is specifically designed for code search. Zoekt will be used as +an API by GitLab and remain an implementation detail while the user interface +in GitLab will not change much except for some new features made available by +Zoekt. + +This will be rolled out in phases to ensure that the system will actually meet +our scaling and cost expectations and will run alongside code search backed by +Elasticsearch until we can be sure it is a viable replacement. The first step +will be making it available for `gitlab-org` for internal and expanding +customer by customer based on customer interest. + +## Motivation + +GitLab code search functionality today is backed by Elasticsearch. +Elasticsearch has proven useful for other types of search (issues, merge +requests, comments and so-on) but is by design not a good choice for code +search where users expect matches to be precise (ie. no false positives) and +flexible (e.g. support +[substring matching](https://gitlab.com/gitlab-org/gitlab/-/issues/325234) +and +[regexes](https://gitlab.com/gitlab-org/gitlab/-/issues/4175)). We have +[investigated our options](https://gitlab.com/groups/gitlab-org/-/epics/7404) +and [Zoekt](https://github.com/sourcegraph/zoekt) is pretty much the only well +maintained open source technology that is suited to code search. Based on our +research we believe it will be better to adopt a well maintained open source +database than attempt to build our own. This is mostly due to the fact that our +research indicates that the fundamental architecture of Zoekt is what we would +implement again if we tried to implement something ourselves. + +Our +[early benchmarking](https://gitlab.com/gitlab-org/gitlab/-/issues/370832#note_1183611955) +suggests that Zoekt will be viable at our scale, but we feel strongly +that investing in building a beta integration with Zoekt and rolling it out +group by group on GitLab.com will provide better insights into scalability and +cost than more accurate benchmarking efforts. It will also be relatively low +risk as it will be rolled out internally first and later rolled out to +customers that wish to participate in the trial. + +### Goals + +The main goals of this integration will be to implement the following highly +requested improvements to code search: + +1. [Exact match (substring match) code searches in advanced search](https://gitlab.com/gitlab-org/gitlab/-/issues/325234) +1. [Support regular expressions with Advanced Global Search](https://gitlab.com/gitlab-org/gitlab/-/issues/4175) +1. [Support multiple line matches in the same file](https://gitlab.com/gitlab-org/gitlab/-/issues/668) + +The initial phases of the rollout will be designed to catch and resolve scaling +or infrastructure cost issues as early as possible so that we can pivot early +before investing too much in this technology if it is not suitable. + +### Non-Goals + +The following are not goals initially but could theoretically be built upon +this solution: + +1. Improving security scanning features by having access to quickly perform + regex scans across many repositories +1. Saving money on our search infrastructure - this may be possible with + further optimizations, but initial estimates suggest the cost is similar +1. AI/ML features of search used to predict what users might be interested in + finding +1. Code Intelligence and Navigation - likely code intelligence and navigation + features should be built on structured data rather than a trigram index but + regex based searches (using Zoekt) may be a suitable fallback for code which + does not have structured metadata enabled or dynamic languages where static + analysis is not very accurate. Zoekt in particular may not be well suited + initially, despite existing symbol extraction using ctags, because ctags + symbols may not contain enough data for accurate navigation and Zoekt + doesn't undersand dependencies which would be necessary for cross-project + navigation. + +## Proposal + +An +[initial implementation of a Zoekt integration](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/105049) +was created to demonstrate the feasibility of using Zoekt as a drop-in +replacement for Elasticsearch code searches. This blueprint will extend on all +the details needed to provide a minimum viable change as well steps needed to +scale this to a larger customer rollout on GitLab.com. + +## Design and implementation details + +### User Experience + +When a user performs an advanced search on a group or project that is part +of the Zoekt rollout we will present a toggle somewhere in the UI to change +to "precise search" (or some other UX TBD) which switches them from +Elasticsearch to Zoekt. Early user feedback will help us assess the best way +to present these choices to users and ultimately we will want to remove the +Elasticsearch option if we find Zoekt is a suitable long term option. + +### Indexing + +Similar to our Elasticsearch integration, GitLab will notify Zoekt every time +there are updates to a repository. Zoekt, unlike Elasticsearch, is designed to +clone and index Git repositories so we will simply notify Zoekt of the URL of +the repository that has changed and it will update its local copy of the Git +repo and then update its local index files. The Zoekt side of this logic will +be implemented in a new server-side indexing endpoint we add to Zoekt which is +currently in +[an open Pull request](https://github.com/sourcegraph/zoekt/pull/496). +While the details of +this pull request are still being debated, we may choose to deploy a fork with +the functionality we need, but our strongest intention is not to maintain a +fork of Zoekt and the maintainers have already expressed they are open to this +new functionality. + +The rails side of the integration will be a Sidekiq worker that is scheduled +every time there is an update to a repository and it will simply call this +`/index` endpoint in Zoekt. This will also need to generate a one-time token +that can allow Zoekt to clone a private repository. + +```mermaid +sequenceDiagram + participant user as User + participant gitlab_git as GitLab Git + participant gitlab_sidekiq as GitLab Sidekiq + participant zoekt as Zoekt + user->>gitlab_git: git push git@gitlab.com:gitlab-org/gitlab.git + gitlab_git->>gitlab_sidekiq: ZoektIndexerWorker.perform_async(278964) + gitlab_sidekiq->>zoekt: POST /index {"RepoUrl":"https://zoekt:SECRET_TOKEN@gitlab.com/gitlab-org/gitlab.git","RepoId":278964}' + zoekt->>gitlab_git: git clone https://zoekt:SECRET_TOKEN@gitlab.com/gitlab-org/gitlab.git +``` + +The Sidekiq worker can leverage de-duplication based on the `project_id`. + +Zoekt supports indexing multiple projects we'll likely need to, eventually, +allow a way for users to configure additional branches (beyond the default +branch) and this will need to be sent to Zoekt. We will need to decide if these +branch lists are sent every time we index the project or only when they change +configuration. + +There may be race conditions with multiple Zoekt processes indexing the same +repo at the same time. For this reason we should implement a locking mechanism +somewhere to ensure we are only indexing 1 project in 1 place at a time. We +could make use of the same Redis locking we use for indexing projects in +Elasticsearch. + +### Searching + +Searching will be implemented using the `/api/search` functionality in +Zoekt. There is also +[an open PR to fix this endpoint in Zoekt](https://github.com/sourcegraph/zoekt/pull/506), +and again we may consider working from a fork until this is fixed. GitLab will +prepend all searches with the appropriate filter for repositories based on the +user's search context (group or project) in the same way we do for +Elasticsearch. For Zoekt this will be implemented as a query string regex that +matches all the searched repositories. + +### Zoekt infrastructure + +Each Zoekt node will need to run a +[zoekt-dynamic-indexserver](https://github.com/sourcegraph/zoekt/pull/496) and +a +[zoekt-webserver](https://github.com/sourcegraph/zoekt/blob/main/cmd/zoekt-webserver/main.go). +These are both webservers with different responsibilities. Considering that the +Zoekt indexing process needs to keep a full clone of the bare repo +([unless we come up with a better option](https://gitlab.com/gitlab-org/gitlab/-/issues/384722)) +these bare repos will be stored on spinning disks to save space. These are only +used as an intermediate step to generate the actual `.zoekt` index files which +will be stored on an SSD for fast searches. These web servers need to run on +the same node as they access the same files. The `zoekt-dynamic-indexserver` is +responsible for writing the `.zoekt` index files. The `zoekt-webserver` is +responsible for responding to searches that it performs by reading these +`.zoekt` index files. + +### Rollout strategy + +Initially Zoekt code search will only be available to `gitlab-org`. After that +we'll start rolling it out to specific customers that have requested better +code search experience. As we learn about scaling and make improvements we will +gradually roll it out to all licensed groups on GitLab.com. We will use a +similar approach to Elasticsearch for keeping track of which groups are indexed +and which are not. This will be based on a new table `zoekt_indexed_namespaces` +with a `namespace_id` reference. We will only allow rolling out to top level +namespaces to simplify the logic of checking for all layers of group +inheritance. Once we've rolled out to all licensed groups we'll enable logic to +automatically enroll newly licensed groups. This table also may be a place to +store per-namespace sharding and replication data as described below. + +### Sharding and replication strategy + +Zoekt does not have any inbuilt sharding, and we expect that we'll need +multiple Zoekt servers to reach the scale to provide search functionality to +all of GitLab licensed customers. + +There are 2 clear ways to implement sharding: + +1. Build it on top of, or in front of Zoekt, as an independent component. Building + all the complexities of a distributed database into Zoekt is not likely to + be a good direction for the project so most likely this would be an + independent piece of infrastructure that proxied requests to the correct + shard. +1. Manage the shards inside GitLab. This would be an application layer in + GitLab which chooses the correct shard to send indexing and search requests + to. + +Likewise, there are a few ways to implement replication: + +1. Server-side where Zoekt replicas are aware of other Zoekt replicas and they + stream updates from some primary to remain in sync +1. Client-side replication where clients send indexing requests to all replicas + and search requests to any replica + +We plan to implement sharding inside GitLab application but replication may be +best served at the level of the filesystem of Zoekt servers rather than sending +duplicated updates from GitLab to all replicas. This could be some process on +Zoekt servers that monitors for changes to the `.zoekt` files in a specific +directory and syncs those updates to the replicas. This will need to be +slightly more sophisticated than `rsync` because the files are constantly +changing and files may be getting deleted while the sync is happening so we +would want to be syncing the updates in batches somehow without slowing down +indexing. + +Implementing sharding in GitLab simplifies the additional infrastructure +components that need to be deployed and allows more flexibility to control our +rollout to many customers alongside our rollout of multiple shards. + +Implementing syncing from primary -> replica on Zoekt nodes at the filesystem +level optimizes that overall resource usage. We only need to sync the index +files to replicas as the bare repo is just a cache. This saves on: + +1. Disk space on replicas +1. CPU usage on replicas as it does not need to rebuild the index +1. Load on Gitaly to clone the repos + +We plan to defer the implementation of these high availability aspects until +later, but a preliminary plan would be: + +1. GitLab is configured with a pool of Zoekt servers +1. GitLab assigns groups randomly a Zoekt primary server +1. There will also be Zoekt replica servers +1. Periodically Zoekt primary servers will sync their `.zoekt` index files to + their respective replicas +1. There will need to be some process by which to promote a replica to a + primary if the primary is having issues. We will be using Consul for + keeping track of which is the primary and which are the replicas. +1. When indexing a project GitLab will queue a Sidekiq job to update the index + on the primary +1. When searching we will randomly select one of the Zoekt primaries or replica + servers for the group being searched. We don't care which is "more up to + date" as code search will be "eventually consistent" and all reads may read + slightly out of date indexes. We will have a target of maximum latency of + index updates and may consider removing nodes from rotation if they are too + far out of date. +1. We will shard everything by top level group as this ensures group search can + always search a single Zoekt server. Aggregation may be possible for global + searches at some point in future if this turns out to be important. Smaller + self-managed instances may use a single Zoekt server allowing global + searches to work without any aggregation being implemented. Depending on our + largest group sizes and scaling limitations of a single node Zoekt server we + may consider implementing an approach where a group can be assigned multiple + shards. + +The downside of the chosen path will be added complexity of managing all these +Zoekt servers from GitLab when compared with a "proxy" layer outside of GitLab +that is managing all of these shards. We will consider this decision a work in +progress and reassess if it turns out to add too much complexity to GitLab. + +#### Sharding proposal using GitLab `::Zoekt::Shard` model + +This is already implemented as the `::Zoekt::IndexedNamespace` +implements a many-to-many relationship between namespaces and shards. + +#### Replication and service discovery using Consul + +If we plan to replicate at the Zoekt node level as described above we need to +change our data model to use a one-to-many relationship from `zoekt_shards -> +namespaces`. This means making the `namespace_id` column unique in +`zoekt_indexed_namespaces`. Then we need to implement a service discovery +approach where the `index_url` always points at a primary Zoekt node and the +`search_url` is a DNS record with N replicas and the primary. We then choose +randomly from `search_url` records when searching. + +### Iterations + +1. Make available for `gitlab-org` +1. Improve monitoring +1. Improve performance +1. Make available for select customers +1. Implement sharding +1. Implement replication +1. Make available to many more licensed groups +1. Implement automatic (re)balancing of shards +1. Estimate costs for rolling out to all licensed groups and decide if it's worth it or if we need to optimize further or adjust our plan +1. Rollout to all licensed groups +1. Improve performance +1. Assess costs and decide whether we should roll out to all free customers diff --git a/doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md b/doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md index 7fecbd1de71..5b82716cb21 100644 --- a/doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md +++ b/doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md @@ -8,6 +8,8 @@ owning-stage: "~devops::non_devops" participating-stages: [] --- + + # Composable GitLab Codebase NOTE: @@ -340,7 +342,7 @@ What was done? spec.add_dependency 'graphql-docs' spec.add_dependency 'grape' end - ``` + ``` 1. Move routes to the `engines/web_engine/config/routes.rb` file @@ -380,59 +382,59 @@ What was done? 1. Configure GitLab when to load the engine. - In GitLab `config/engines.rb`, we can configure when do we want to load our engines by relying on our `Gitlab::Runtime` + In GitLab `config/engines.rb`, we can configure when do we want to load our engines by relying on our `Gitlab::Runtime` - ```ruby - # config/engines.rb - # Load only in case we are running web_server or rails console - if Gitlab::Runtime.puma? || Gitlab::Runtime.console? - require 'web_engine' - end - ``` + ```ruby + # config/engines.rb + # Load only in case we are running web_server or rails console + if Gitlab::Runtime.puma? || Gitlab::Runtime.console? + require 'web_engine' + end + ``` 1. Configure Engine - Our Engine inherits from the `Rails::Engine` class. This way this gem notifies Rails that - there's an engine at the specified path so it will correctly mount the engine inside - the application, performing tasks such as adding the app directory of the engine to - the load path for models, mailers, controllers, and views. - A file at `lib/web_engine/engine.rb`, is identical in function to a standard Rails - application's `config/application.rb` file. This way engines can access a configuration - object which contains configuration shared by all railties and the application. - Additionally, each engine can access `autoload_paths`, `eager_load_paths`, and `autoload_once_paths` - settings which are scoped to that engine. - - ```ruby - module WebEngine - class Engine < ::Rails::Engine - config.eager_load_paths.push(*%W[#{config.root}/lib - #{config.root}/app/graphql/resolvers/concerns - #{config.root}/app/graphql/mutations/concerns - #{config.root}/app/graphql/types/concerns]) - - if Gitlab.ee? - ee_paths = config.eager_load_paths.each_with_object([]) do |path, memo| - ee_path = config.root - .join('ee', Pathname.new(path).relative_path_from(config.root)) - memo << ee_path.to_s - end - # Eager load should load CE first - config.eager_load_paths.push(*ee_paths) - end - end - end - ``` + Our Engine inherits from the `Rails::Engine` class. This way this gem notifies Rails that + there's an engine at the specified path so it will correctly mount the engine inside + the application, performing tasks such as adding the app directory of the engine to + the load path for models, mailers, controllers, and views. + A file at `lib/web_engine/engine.rb`, is identical in function to a standard Rails + application's `config/application.rb` file. This way engines can access a configuration + object which contains configuration shared by all railties and the application. + Additionally, each engine can access `autoload_paths`, `eager_load_paths`, and `autoload_once_paths` + settings which are scoped to that engine. + + ```ruby + module WebEngine + class Engine < ::Rails::Engine + config.eager_load_paths.push(*%W[#{config.root}/lib + #{config.root}/app/graphql/resolvers/concerns + #{config.root}/app/graphql/mutations/concerns + #{config.root}/app/graphql/types/concerns]) + + if Gitlab.ee? + ee_paths = config.eager_load_paths.each_with_object([]) do |path, memo| + ee_path = config.root + .join('ee', Pathname.new(path).relative_path_from(config.root)) + memo << ee_path.to_s + end + # Eager load should load CE first + config.eager_load_paths.push(*ee_paths) + end + end + end + ``` 1. Testing - We adapted CI to test `engines/web_engine/` as a self-sufficient component of stack. + We adapted CI to test `engines/web_engine/` as a self-sufficient component of stack. - - We moved `spec` as-is files to the `engines/web_engine/spec` folder - - We moved `ee/spec` as-is files to the `engines/web_engine/ee/spec` folder - - We control specs from main application using environment variable `TEST_WEB_ENGINE` - - We added new CI job that will run `engines/web_engine/spec` tests separately using `TEST_WEB_ENGINE` environment variable. - - We added new CI job that will run `engines/web_engine/ee/spec` tests separately using `TEST_WEB_ENGINE` environment variable. - - We are running all white box frontend tests with `TEST_WEB_ENGINE=true` + - We moved `spec` as-is files to the `engines/web_engine/spec` folder + - We moved `ee/spec` as-is files to the `engines/web_engine/ee/spec` folder + - We control specs from main application using environment variable `TEST_WEB_ENGINE` + - We added new CI job that will run `engines/web_engine/spec` tests separately using `TEST_WEB_ENGINE` environment variable. + - We added new CI job that will run `engines/web_engine/ee/spec` tests separately using `TEST_WEB_ENGINE` environment variable. + - We are running all white box frontend tests with `TEST_WEB_ENGINE=true` #### Results diff --git a/doc/architecture/blueprints/consolidating_groups_and_projects/index.md b/doc/architecture/blueprints/consolidating_groups_and_projects/index.md index 97853075607..f5bd53627cb 100644 --- a/doc/architecture/blueprints/consolidating_groups_and_projects/index.md +++ b/doc/architecture/blueprints/consolidating_groups_and_projects/index.md @@ -4,16 +4,18 @@ creation-date: "2021-02-07" authors: [ "@alexpooley", "@ifarkas" ] coach: "@grzesiek" approvers: [ "@m_gill", "@mushakov" ] -owning-stage: "~devops::plan" +author-stage: "~devops::plan" +owning-stage: "~devops::data_stores" participating-stages: [] --- + + # Consolidating Groups and Projects -There are numerous features that exist exclusively within groups or -projects. The boundary between group and project features used to be clear. -However, there is growing demand to have group features within projects, and -project features within groups. For example, having issues in groups, and epics +Numerous features exist exclusively within groups or projects. The boundary between group and project features used to be clear. +However, there is growing demand to have group features in projects, and +project features in groups. For example, having issues in groups, and epics in projects. The [Simplify Groups & Projects Working Group](https://about.gitlab.com/company/team/structure/working-groups/simplify-groups-and-projects/) @@ -31,12 +33,12 @@ no established process in place. This results in the reimplementation of the same feature. Those implementations diverge from each other over time as they all live on their own. A few more problems with this approach: -- Features are coupled to their container. In practice it is not straight +- Features are coupled to their container. In practice, it is not straight forward to decouple a feature from its container. The degree of coupling varies across features. - Naive duplication of features will result in a more complex and fragile codebase. - Generalizing solutions across groups and projects may degrade system performance. -- The range of features span across many teams, and these changes will need to +- The range of features spans across many teams, and these changes will need to manage development interference. - The group/project hierarchy creates a natural feature hierarchy. When features exist across containers the feature hierarchy becomes ambiguous. @@ -48,33 +50,33 @@ remains consistent. ### Performance -Resources can only be queried in elaborate / complicated ways. This caused +Resources can only be queried in elaborate/complicated ways. This caused performance issues with authorization, epics, and many other places. As an example, to query the projects a user has access to, the following sources need to be considered: -- personal projects -- direct group membership -- direct project membership -- inherited group membership -- inherited project membership -- group sharing -- inherited membership via group sharing -- project sharing +- Personal projects +- Direct group membership +- Direct project membership +- Inherited group membership +- Inherited project membership +- Group sharing +- Inherited membership via group sharing +- Project sharing -Group / project membership, group / project sharing are also examples of +Group/project membership, group/project sharing are also examples of duplicated features. ## Goals -For now this blueprint strictly relates to the engineering challenges. +For now, this blueprint strictly relates to the engineering challenges. - Consolidate the group and project container architecture. - Develop a set of solutions to decouple features from their container. - Decouple engineering changes from product changes. - Develop a strategy to make architectural changes without adversely affecting other teams. -- Provide a solution for requests asking for features availability of other levels. +- Provide a solution for requests asking for features to be made available at other levels. ## Proposal @@ -102,9 +104,9 @@ New features should be implemented on `Namespace`. Similarly, when a feature need to be reimplemented on a different level, moving it to `Namespace` essentially makes it available on all levels: -- personal namespaces -- groups -- projects +- Personal namespaces +- Groups +- Projects Various traversal queries are already available on `Namespaces` to query the group hierarchy. `Projects` represent the leaf nodes in the hierarchy, but with @@ -113,14 +115,14 @@ retrieve projects as well. This also enables further simplification of some of our core features: -- routes should be generated based on the `Namespace` hierarchy, instead of - mixing project with the group hierarchy. -- there is no need to differentiate between `GroupMembers` and `ProjectMembers`. +- Routes should be generated based on the `Namespace` hierarchy, instead of + mixing the project with the group hierarchy. +- There is no need to differentiate between `GroupMembers` and `ProjectMembers`. All `Members` should be related to a `Namespace`. This can lead to simplified querying, and potentially deduplicating policies. -As more and more features will be migrated to `Namespace`, the role of `Project` -model will diminish over time to essentially a container around repository +As more and more features will be migrated to `Namespace`, the role of the `Project` +model will diminish over time to essentially a container around the repository related functionality. ## Iterations @@ -129,9 +131,103 @@ The work required to establish `Namespace` as a container for our features is tracked under [Consolidate Groups and Projects](https://gitlab.com/groups/gitlab-org/-/epics/6473) epic. +### Phase 1 (complete) + +- [Phase 1 epic](https://gitlab.com/groups/gitlab-org/-/epics/6697). +- **Goals**: + 1. Ensure every project receives a corresponding record in the `namespaces` + table with `type='Project'`. + 1. For user namespaces, the type changes from `NULL` to `User`. + +We should make sure that projects, and the project namespace, are equivalent: + +- **Create project:** Use Rails callbacks to ensure a new project namespace is + created for each project. Project namespace records should contain `created_at` and + `updated_at` attributes equal to the project's `created_at`/`updated_at` attributes. +- **Update project:** Use the `after_save` callback in Rails to ensure some + attributes are kept in sync between project and project namespaces. + Read [`project#after_save`](https://gitlab.com/gitlab-org/gitlab/blob/6d26634e864d7b748dda0e283eb2477362263bc3/app/models/project.rb#L101-L101) + for more information. +- **Delete project:** Use FKs cascade delete or Rails callbacks to ensure when a `Project` + or its `ProjectNamespace` is removed, its corresponding `ProjectNamespace` or `Project` + is also removed. +- **Transfer project to a different group:** Make sure that when a project is transferred, + its corresponding project namespace is transferred to the same group. +- **Transfer group:** Make sure when transferring a group that all of its sub-projects, + either direct or through descendant groups, have their corresponding project + namespaces transferred correctly as well. +- **Export or import project** + - **Export project** continues to export only the project, and not its project namespace, + in this phase. The project namespace does not contain any specific information + to export at this point. Eventually, we want the project namespace to be exported as well. + - **Import project** creates a new project, so the project namespace is created through + Rails `after_save` callback on the project model. +- **Export or import group:** When importing or exporting a `Group`, projects are not + included in the operation. If that feature is changed to include `Project` when its group is + imported or exported, the logic must include their corresponding project namespaces + in the import or export. + +After ensuring these points, run a database migration to create a `ProjectNamespace` +record for every `Project`. Project namespace records created during the migration +should have `created_at` and `updated_at` attributes set to the migration runtime. +The project namespaces' `created_at` and `updated_at` attributes would not match +their corresponding project's `created_at` and `updated_at` attributes. We want +the different dates to help audit any of the created project namespaces, in case we need it. +After this work completes, we must migrate data as described in +[Backfill `ProjectNamespace` for every Project](https://gitlab.com/gitlab-org/gitlab/-/issues/337100). + +### Phase 2 (complete) + +- [Phase 2 epic](https://gitlab.com/groups/gitlab-org/-/epics/6768). +- **Goal**: Link `ProjectNamespace` to other entities on the database level. + +In this phase: + +- Communicate the changes company-wide at the engineering level. We want to make + engineers aware of the upcoming changes, even though teams are not expected to + collaborate actively until phase 3. +- Raise awareness to avoid regressions and conflicting or duplicate work that + can be dealt with before phase 3. + +### Phase 3 (ongoing) + +- [Phase 3 epic](https://gitlab.com/groups/gitlab-org/-/epics/6585). + +In this phase we are migrating basic, high-priority project functionality from `Project` to `ProjectNamespace`, or directly to `Namespace`. Problems to solve as part of this phase: + +- [Unify members/members actions](https://gitlab.com/groups/gitlab-org/-/epics/8010) - on UI and API level. +- Starring: Right now only projects can be starred. We want to bring this to the group level. +- Common actions: Destroying, transferring, restoring. This can be unified on the controller level and then propagated lower. +- Archiving currently only works on the project level. This can be brought to the group level, similar to the mechanism for “pending deletion”. +- Avatar's serving and actions. + +### Phase 4 + +- [Phase 4 epic](https://gitlab.com/groups/gitlab-org/-/epics/8687) + +In this phase we are migrating additional functionality from `Project` to `ProjectNamespace`/`Namespace`: + +- Replace usages of `Project` with `ProjectNamespace` in the code. +- API changes to expose namespaces and namespace features. + - Investigate if we extend API for `groups` or we introduce a `namespaces` endpoint and slowly deprecate `groups` and `projects` endpoints. +- Break down each feature that needs to be migrated from `Project` to `ProjectNamespace` or `Namespace`. + - Investigate if we can move a feature from `Project -> Namespace` directly vs `Project -> ProjectNamespace -> Namespace`. This can be decided on a feature by feature case. +- [Migrate Project#namespace to reference ProjectNamespace](https://gitlab.com/groups/gitlab-org/-/epics/6581). +- [Routes consolidation between Project & ProjectNamespace](https://gitlab.com/gitlab-org/gitlab/-/issues/337103). +- [Policies consolidation](https://gitlab.com/groups/gitlab-org/-/epics/6689). + +### Phase 5 + +- [Phase 5 epic](https://gitlab.com/groups/gitlab-org/-/epics/6944) + +We should strive to do the code clean up as we move through the phases. However, not everything can be cleaned up while something is still being developed. For example, dropping database columns can be done as the last task when we are sure everything is working. This phase will focus on: + +- Code cleanup +- Database cleanup + ## Migrating features to Namespaces -The initial iteration will provide a framework to house features under `Namespaces`. Stage groups will eventually need to migrate their own features and functionality over to `Namespaces`. This may impact these features in unexpected ways. Therefore, to minimize UX debt and maintain product consistency, stage groups will have to consider a number of factors when migrating their features over to `Namespaces`: +The initial iteration will provide a framework to house features under `Namespaces`. Stage groups will eventually need to migrate their own features and functionality over to `Namespaces`. This may impact these features in unexpected ways. Therefore, to minimize UX debt and maintain product consistency, stage groups will have to consider several factors when migrating their features over to `Namespaces`: 1. **Conceptual model**: What are the current and future state conceptual models of these features ([see object modeling for designers](https://hpadkisson.medium.com/object-modeling-for-designers-an-introduction-7871bdcf8baf))? These should be documented in Pajamas (example: [merge requests](https://design.gitlab.com/objects/merge-request/)). 1. **Merge conflicts**: What inconsistencies are there across project, group, and administrator levels? How might these be addressed? For an example of how we rationalized this for labels, please see [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/338820). @@ -147,4 +243,5 @@ The initial iteration will provide a framework to house features under `Namespac ## Related topics -- [Workspace developer documentation](../../../development/workspace/index.md) +- [Organization developer documentation](../../../development/organization/index.md) +- [Organization user documentation](../../../user/organization/index.md) diff --git a/doc/architecture/blueprints/container_registry_metadata_database/index.md b/doc/architecture/blueprints/container_registry_metadata_database/index.md index f3bcf1e4e59..b77aaf598e6 100644 --- a/doc/architecture/blueprints/container_registry_metadata_database/index.md +++ b/doc/architecture/blueprints/container_registry_metadata_database/index.md @@ -8,6 +8,8 @@ owning-stage: "~devops::package" participating-stages: [] --- + + # Container Registry Metadata Database ## Usage of the GitLab Container Registry diff --git a/doc/architecture/blueprints/database/scalability/patterns/index.md b/doc/architecture/blueprints/database/scalability/patterns/index.md index ec00d757377..d28734ce511 100644 --- a/doc/architecture/blueprints/database/scalability/patterns/index.md +++ b/doc/architecture/blueprints/database/scalability/patterns/index.md @@ -2,7 +2,6 @@ stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false description: 'Learn how to scale the database through the use of best-of-class database scalability patterns' --- diff --git a/doc/architecture/blueprints/database/scalability/patterns/read_mostly.md b/doc/architecture/blueprints/database/scalability/patterns/read_mostly.md index ec236c9bfe3..3a3fd2f33c2 100644 --- a/doc/architecture/blueprints/database/scalability/patterns/read_mostly.md +++ b/doc/architecture/blueprints/database/scalability/patterns/read_mostly.md @@ -2,7 +2,6 @@ stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false description: 'Learn how to scale operating on read-mostly data at scale' --- diff --git a/doc/architecture/blueprints/database/scalability/patterns/time_decay.md b/doc/architecture/blueprints/database/scalability/patterns/time_decay.md index 2b36a43a6db..24fc3f45717 100644 --- a/doc/architecture/blueprints/database/scalability/patterns/time_decay.md +++ b/doc/architecture/blueprints/database/scalability/patterns/time_decay.md @@ -2,7 +2,6 @@ stage: Data Stores group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false description: 'Learn how to operate on large time-decay data' --- diff --git a/doc/architecture/blueprints/database_scaling/size-limits.md b/doc/architecture/blueprints/database_scaling/size-limits.md index e530bd6eff0..b6b9cda8827 100644 --- a/doc/architecture/blueprints/database_scaling/size-limits.md +++ b/doc/architecture/blueprints/database_scaling/size-limits.md @@ -1,7 +1,6 @@ --- stage: Data Stores group: Database -comments: false description: 'Database Scalability / Limit table sizes' --- diff --git a/doc/architecture/blueprints/database_testing/index.md b/doc/architecture/blueprints/database_testing/index.md index fe6dcf1723d..79560dd3959 100644 --- a/doc/architecture/blueprints/database_testing/index.md +++ b/doc/architecture/blueprints/database_testing/index.md @@ -1,5 +1,5 @@ --- -status: accepted +status: implemented creation-date: "2021-02-08" authors: [ "@abrandl" ] coach: "@glopezfernandez" @@ -8,8 +8,15 @@ owning-stage: "~devops::data_stores" participating-stages: [] --- + + # Database Testing +**Notice:** This blueprint has been partially implemented. We still plan to +iterate on the tooling. The content below is a historical version of the +blueprint, written prior to incorporating database testing into our development +workflow. + We have identified [common themes of reverted migrations](https://gitlab.com/gitlab-org/gitlab/-/issues/233391) and discovered failed migrations breaking in both production and staging even when successfully tested in a developer environment. We have also experienced production incidents even with successful testing in staging. These failures are quite expensive: they can have a significant effect on availability, block deployments, and generate incident escalations. These escalations must be triaged and either reverted or fixed forward. Often, this can take place without the original author's involvement due to time zones and/or the criticality of the escalation. With our increased deployment speeds and stricter uptime requirements, the need for improving database testing is critical, particularly earlier in the development process (shift left). From a developer's perspective, it is hard, if not unfeasible, to validate a migration on a large enough dataset before it goes into production. @@ -86,13 +93,13 @@ The short-term focus is on testing regular migrations (typically schema changes) In order to secure this process and meet compliance goals, the runner environment is treated as a *production* environment and similarly locked down, monitored and audited. Only Database Maintainers have access to the CI pipeline and its job output. Everyone else can only see the results and statistics posted back on the merge request. -We implement a secured CI pipeline on that adds the execution steps outlined above. The goal is to secure this pipeline to solve the following problem: +We implement a secured CI pipeline on [Internal GitLab for Operations](https://ops.gitlab.net/users/sign_in) that adds the execution steps outlined above. The goal is to secure this pipeline to solve the following problem: Make sure we strongly protect production data, even though we allow everyone (GitLab team/developers) to execute arbitrary code on the thin-clone which contains production data. This is in principle achieved by locking down the GitLab Runner instance executing the code and its containers on a network level, such that no data can escape over the network. We make sure no communication can happen to the outside world from within the container executing the GitLab Rails code (and its database migrations). -Furthermore, we limit the ability to view the results of the jobs (including the output printed from code) to Maintainer and Owner level on the pipeline and provide only a high level summary back to the original MR. If there are issues or errors in one of the jobs run, the database Maintainer assigned to review the MR can check the original job for more details. +Furthermore, we limit the ability to view the results of the jobs (including the output printed from code) to Maintainer and Owner level on the [Internal GitLab for Operations](https://ops.gitlab.net/users/sign_in) pipeline and provide only a high level summary back to the original MR. If there are issues or errors in one of the jobs run, the database Maintainer assigned to review the MR can check the original job for more details. With this step implemented, we already have the ability to execute database migrations on the thin-cloned GitLab.com database automatically from GitLab CI and provide feedback back to the merge request and the developer. The content of that feedback is expected to evolve over time and we can continuously add to this. diff --git a/doc/architecture/blueprints/gitlab_agent_deployments/index.md b/doc/architecture/blueprints/gitlab_agent_deployments/index.md index 96e361d7531..d8d26389d7d 100644 --- a/doc/architecture/blueprints/gitlab_agent_deployments/index.md +++ b/doc/architecture/blueprints/gitlab_agent_deployments/index.md @@ -8,6 +8,8 @@ owning-stage: "~devops::release" participating-stages: [Configure, Release] --- + + # View and manage resources deployed by GitLab Agent For Kuberenetes ## Summary @@ -374,6 +376,8 @@ Here is an example of GraphQL query: lastDeployment(status: SUCCESS) { agent { id + name + project kubernetesNamespace } } diff --git a/doc/architecture/blueprints/gitlab_ci_events/index.md b/doc/architecture/blueprints/gitlab_ci_events/index.md new file mode 100644 index 00000000000..7ce8fea9410 --- /dev/null +++ b/doc/architecture/blueprints/gitlab_ci_events/index.md @@ -0,0 +1,63 @@ +--- +status: proposed +creation-date: "2023-03-15" +authors: [ "@furkanayhan" ] +coach: "@grzesiek" +approvers: [ "@jreporter", "@cheryl.li" ] +owning-stage: "~devops::verify" +participating-stages: [ "~devops::package", "~devops::deploy" ] +--- + +# GitLab CI Events + +## Summary + +In order to unlock innovation and build more value, GitLab is expected to be +the center of automation related to DevSecOps processes. We want to transform +GitLab into a programming environment, that will make it possible for engineers +to model various workflows on top of CI/CD pipelines. Today, users must create +custom automation around webhooks or scheduled pipelines to build required +workflows. + +In order to make this automation easier for our users, we want to build a +powerful CI/CD eventing system, that will make it possible to run pipelines +whenever something happens inside or outside of GitLab. + +A typical use-case is to run a CI/CD job whenever someone creates an issue, +posts a comment, changes a merge request status from "draft" to "ready for +review" or adds a new member to a group. + +To build that new technology, we should: + +1. Emit many hierarchical events from within GitLab in a more advanced way than we do it today. +1. Make it affordable to run this automation, that will react to GitLab events, at scale. +1. Provide a set of conventions and libraries to make writing the automation easier. + +## Goals + +While ["GitLab Events Platform"](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/113700) +aims to build new abstractions around emitting events in GitLab, "GitLab CI +Events" blueprint is about making it possible to: + +1. Define a way in which users will configure when an event emitted will result in a CI pipeline being run. +1. Describe technology required to match subscriptions with events at GitLab.com scale and beyond. +1. Describe technology we could use to reduce the cost of running automation jobs significantly. + +## Proposals + +For now, we have technical 4 proposals; + +1. [Proposal 1: Using the `.gitlab-ci.yml` file](proposal-1-using-the-gitlab-ci-file.md) + Based on; + - [GitLab CI Workflows PoC](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91244) + - [PoC NPM CI events](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/111693) +1. [Proposal 2: Using the `rules` keyword](proposal-2-using-the-rules-keyword.md) + Highly inefficient way. +1. [Proposal 3: Using the `.gitlab/ci/events` folder](proposal-3-using-the-gitlab-ci-events-folder.md) + Involves file reading for every event. +1. [Proposal 4: Creating events via CI files](proposal-4-creating-events-via-ci-files.md) + Combination of some proposals. + +Each of them has its pros and cons. There could be many more proposals and we +would like to discuss them all. We can combine the best part of those proposals +and create a new one. diff --git a/doc/architecture/blueprints/gitlab_ci_events/proposal-1-using-the-gitlab-ci-file.md b/doc/architecture/blueprints/gitlab_ci_events/proposal-1-using-the-gitlab-ci-file.md new file mode 100644 index 00000000000..7dfc3873ada --- /dev/null +++ b/doc/architecture/blueprints/gitlab_ci_events/proposal-1-using-the-gitlab-ci-file.md @@ -0,0 +1,60 @@ +--- +owning-stage: "~devops::verify" +description: 'GitLab CI Events Proposal 1: Using the .gitlab-ci.yml file' +--- + +# GitLab CI Events Proposal 1: Using the `.gitlab-ci.yml` file + +Currently, we have two proof-of-concept (POC) implementations: + +- [GitLab CI Workflows PoC](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91244) +- [PoC NPM CI events](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/111693) + +They both have similar ideas; + +1. Find a new CI Config syntax to define the pipeline events. + + Example 1: + + ```yaml + workflow: + events: + - events/package/published + + # or + + workflow: + on: + - events/package/published + ``` + + Example 2: + + ```yaml + spec: + on: + - events/package/published + - events/package/removed + # on: + # package: [published, removed] + --- + do_something: + script: echo "Hello World" + ``` + +1. Upsert an event to the database when creating a pipeline. +1. Create [EventStore subscriptions](../../../development/event_store.md) to handle the events. + +## Problems & Questions + +1. The CI config of a project can be anything; + - `.gitlab-ci.yml` by default + - another file in the project + - another file in another project + - completely a remote/external file + + How do we handle these cases? +1. Since we have these problems above, should we keep the events in its own file? (`.gitlab-ci-events.yml`) +1. Do we only accept the changes in the main branch? +1. We try to create event subscriptions every time a pipeline is created. +1. Can we move the existing workflows into the new CI events, for example, `merge_request_event`? diff --git a/doc/architecture/blueprints/gitlab_ci_events/proposal-2-using-the-rules-keyword.md b/doc/architecture/blueprints/gitlab_ci_events/proposal-2-using-the-rules-keyword.md new file mode 100644 index 00000000000..6f69a0f11f0 --- /dev/null +++ b/doc/architecture/blueprints/gitlab_ci_events/proposal-2-using-the-rules-keyword.md @@ -0,0 +1,38 @@ +--- +owning-stage: "~devops::verify" +description: 'GitLab CI Events Proposal 2: Using the rules keyword' +--- + +# GitLab CI Events Proposal 2: Using the `rules` keyword + +Can we do it with our current [`rules`](../../../ci/yaml/index.md#rules) system? + +```yaml +workflow: + rules: + - events: ["package/*"] + +test_package_published: + script: echo testing published package + rules: + - events: ["package/published"] + +test_package_removed: + script: echo testing removed package + rules: + - events: ["package/removed"] +``` + +1. We don't upsert anything to the database. +1. We'll have a single worker which subcribes to events +like `store.subscribe ::Ci::CreatePipelineFromEventWorker, to: ::Issues::CreatedEvent`. +1. The worker just runs `Ci::CreatePipelineService` with the correct parameters, the rest +will be handled by the `rules` system. Of course, we'll need modifications to the `rules` system to support `events`. + +## Problems & Questions + +1. For every defined event run, we need to enqueue a new `Ci::CreatePipelineFromEventWorker` job. +1. The worker will need to run `Ci::CreatePipelineService` for every event run. +This may be costly because we go through every cycle of `Ci::CreatePipelineService`. +1. This would be highly inefficient. +1. Can we move the existing workflows into the new CI events, for example, `merge_request_event`? diff --git a/doc/architecture/blueprints/gitlab_ci_events/proposal-3-using-the-gitlab-ci-events-folder.md b/doc/architecture/blueprints/gitlab_ci_events/proposal-3-using-the-gitlab-ci-events-folder.md new file mode 100644 index 00000000000..ad76b7f8dd4 --- /dev/null +++ b/doc/architecture/blueprints/gitlab_ci_events/proposal-3-using-the-gitlab-ci-events-folder.md @@ -0,0 +1,64 @@ +--- +owning-stage: "~devops::verify" +description: 'GitLab CI Events Proposal 3: Using the .gitlab/ci/events folder' +--- + +# GitLab CI Events Proposal 3: Using the `.gitlab/ci/events` folder + +We can also approach this problem by creating separate files for events. + +Let's say we'll have the `.gitlab/ci/events` folder (or `.gitlab/workflows/ci`). + +We can define events in the following format: + +```yaml +# .gitlab/ci/events/package-published.yml + +spec: + events: + - name: package/published + +--- + +include: + - local: .gitlab-ci.yml + with: + event: $[[ gitlab.event.name ]] +``` + +And in the `.gitlab-ci.yml` file, we can use the input; + +```yaml +# .gitlab-ci.yml + +spec: + inputs: + event: + default: push + +--- + +job1: + script: echo "Hello World" + +job2: + script: echo "Hello World" + +job-for-package-published: + script: echo "Hello World" + rules: + - if: $[[ inputs.event ]] == "package/published" +``` + +When an event happens; + +1. We'll enqueue a new job for the event. +1. The job will search for the event file in the `.gitlab/ci/events` folder. +1. The job will run `Ci::CreatePipelineService` for the event file. + +## Problems & Questions + +1. For every defined event run, we need to enqueue a new job. +1. Every event-job will need to search for files. +1. This would be only for the project-scope events. +1. This can be inefficient because of searching for files for the project for every event. diff --git a/doc/architecture/blueprints/gitlab_ci_events/proposal-4-creating-events-via-ci-files.md b/doc/architecture/blueprints/gitlab_ci_events/proposal-4-creating-events-via-ci-files.md new file mode 100644 index 00000000000..5f10ba1fbb2 --- /dev/null +++ b/doc/architecture/blueprints/gitlab_ci_events/proposal-4-creating-events-via-ci-files.md @@ -0,0 +1,73 @@ +--- +owning-stage: "~devops::verify" +description: 'GitLab CI Events Proposal 4: Creating events via CI files' +--- + +# GitLab CI Events Proposal 4: Creating events via CI files + +Each project can have its own event configuration file. Let's call it `.gitlab-ci-event.yml` for now. +In this file, we can define events in the following format: + +```yaml +events: + - package/published + - issue/created +``` + +When this file is changed in the project repository, it is parsed and the events are created, updated, or deleted. +This is highly similar to [Proposal 1](proposal-1-using-the-gitlab-ci-file.md) except that we don't need to +track pipeline creations every time. + +1. Upsert events to the database when `.gitlab-ci-event.yml` is updated. +1. Create [EventStore subscriptions](../../../development/event_store.md) to handle the events. + +## Filtering jobs + +We can filter jobs by using the `rules` keyword. For example: + +```yaml +test_package_published: + script: echo testing published package + rules: + - events: ["package/published"] + +test_package_removed: + script: echo testing removed package + rules: + - events: ["package/removed"] +``` + +Otherwise, we can make it work either a CI variable; + +```yaml +test_package_published: + script: echo testing published package + rules: + - if: $CI_EVENT == "package/published" + +test_package_removed: + script: echo testing removed package + rules: + - if: $CI_EVENT == "package/removed" +``` + +or an input like in the [Proposal 3](proposal-3-using-the-gitlab-ci-events-folder.md); + +```yaml +spec: + inputs: + event: + default: push + +--- + +test_package_published: + script: echo testing published package + rules: + - if: $[[ inputs.event ]] == "package/published" + +test_package_removed: + script: echo testing removed package + rules: + - if: $[[ inputs.event ]] == "package/removed" +``` diff --git a/doc/architecture/blueprints/gitlab_ml_experiments/index.md b/doc/architecture/blueprints/gitlab_ml_experiments/index.md new file mode 100644 index 00000000000..90adfc41257 --- /dev/null +++ b/doc/architecture/blueprints/gitlab_ml_experiments/index.md @@ -0,0 +1,170 @@ +--- +status: proposed +creation-date: "2023-04-13" +authors: [ "@andrewn" ] +coach: "@grzesiek" +--- + +# GitLab Service-Integration: AI and Beyond + +This document is an abbreviated proposal for Service-Integration to allow teams within GitLab to rapidly build new application features that leverage AI, ML, and data technologies. + +## Executive Summary + +This document proposes a service-integration approach to setting up infrastructure to allow teams within GitLab to build new application features that leverage AI, ML, and data technologies at a rapid pace. The scope of the document is limited specifically to internally hosted features, not third-party APIs. The current application architecture runs most GitLab application features in Ruby. However, many ML/AI experiments require different resources and tools, implemented in different languages, with huge libraries that do not always play nicely together, and have different hardware requirements. Adding all these features to the existing infrastructure will increase the size of the GitLab application container rapidly, resulting in slower startup times, increased number of dependencies, security risks, negatively impacting development velocity, and increasing complexity due to different hardware requirements. As an alternative, the proposal suggests adding services to avoid overloading GitLabs main workloads. These services will run independently with isolated resources and dependencies. By adding services, GitLab can maintain the availability and security of GitLab.com, and enable engineers to rapidly iterate on new ML/AI experiments. + +## Scope + +The infrastructure, platform, and other changes related to ML/AI experiments is broad. This blueprint is limited specifically to the following scope: + +1. Production workloads, running (directly or indirectly) as a result of requests into the GitLab application (`gitlab.com`), or an associated subdomains (for example, `codesuggestions.gitlab.com`). +1. Excludes requests from the GitLab application, made to third-party APIs outside of our infrastructure. From an Infrastructure point-of-view, external AI/ML API requests are no different from other API (non ML/AI) requests and generally follow the existing guidelines that are in place for calling external APIs. +1. Excludes training and tuning workloads not _directly_ connected to our production workloads. Training and tuning workloads are distinct from production workloads and will be covered by their own blueprint(s). + +## Running Production ML/AI experiment workloads + +### Why Not Simply Continue To Use The Existing Application Architecture? + +Let's start with some background on how the application is deployed: + +1. Most GitLab application features are implemented in Ruby and run in one of two types of Ruby deployments: broadly Rails and Sidekiq (although we do partition this traffic further for different workloads). +1. These Ruby workloads have two main container images `gitlab-webservice-ee` and `gitlab-sidekiq-ee`. All the code, libraries, binaries, and other resources that we use to support the main Ruby part of the codebase are embedded within these images. +1. There are thousands of pods running these containers in production for GitLab.com at any moment in time. They are started up and shut down at a high rate throughout the day as traffic demands on the site fluctuate. +1. For _most_ new features developed, any new supporting resources need to be added to either one, or both of these containers. + +![current containers](https://docs.google.com/drawings/d/e/2PACX-1vQh9ToJDy6ceKVMZxSJK5kjBjgKUKdnHcigqTz-Jte1G65aV9js5XZhCC-VYNtkJ_gnoNfob4z-DCui/pub?w=692&h=286)\ +[source](https://docs.google.com/drawings/d/1RiTUnsDSkTGaMqK_RfUlCd_rQ6CgSInhfQJNewIKf1M/edit) + +Many of the initial discussions focus on adding supporting resources to these existing containers ([example](https://gitlab.com/gitlab-org/gitlab/-/issues/403630#note_1345192671)). Choosing this approach would have many downsides, in terms of both the velocity at which new features can be iterated on, and in terms of the availability of GitLab.com. + +Many of the AI experiments that GitLab is considering integrating into the application are substantially different from other libraries and tools that have been integrated in the past. + +1. ML toolkits are **implemented in a plethora of languages**, each requiring separate runtimes. Python, C, C++ are the most common, but there is a long tail of languages used. +1. There are a very large number of tools that we're looking to integrate with and **no single tool will support all the features that are being investigated**. Tensorflow, PyTorch, Keras, Scikit-learn, Alpaca are just a few examples. +1. **These libraries are huge**. Tensorflow's container image with GPU support is 3GB, PyTorch is 5GB, Keras is 300MB. Prophet is ~250MB. +1. Many of these **libraries do not play nicely together**: they may have dependencies that are not compatible, or require different versions of Python, or GPU driver versions. + +It's likely that in the next few months, GitLab will experiment with many different features, using many different libraries. + +Trying to deploy all of these features into the existing infrastructure would have many downsides: + +1. **The size of the GitLab application container would expand very rapidly** as each new experiment introduces a new set of supporting libraries, each library is as big, or bigger, than the existing GitLab application within the container. +1. **Startup times for new workloads would increase**, potentially impacting the availability of GitLab.com during high-traffic periods. +1. The number of dependencies within the container would increase rapidly, putting pressure on the engineering teams to **keep ahead of exploits and vulnerabilities**. +1. **The security attack surface within the container would be greatly increased** with each new dependency. These containers include secrets which, if leaked via an exploit would need costly application-wide secret rotation to be done. +1. **Development velocity will be negatively impacted** as engineers work to avoid dependency conflicts between libraries. +1. Additionally there may be **extra complexity due to different hardware requirements** for different libraries with appropriate drivers etc for GPUs, TPUs, CUDA versions, etc. +1. Our Kubernetes workloads have been tuned for the existing multithreaded Ruby request (Rails) and message (Sidekiq) processes. Adding extremely resource-intensive applications into these workloads would affect unrelated requests, **starving requests of CPU and memory and requiring complex tuning to ensure fairness**. Failure to do this would impact our availability of GitLab.com. + +![fat containers](https://docs.google.com/drawings/d/e/2PACX-1vSW0Pm_7yZV-0JNmgfOHhQlvh6XsJYtrrzkPPhURf5sCbsQDKc0I0kCIbfios3ifD5tmcNvuchXSVUB/pub?w=686&h=364) +\ +[source](https://docs.google.com/drawings/d/1aYffBzzea5QuZ-mTMteowefbV7VmsOuq2v4BqbPd6KE/edit) + +### Proposal: Avoid Overfilling GitLabs Application Containers with Service-Integration + +GitLab.com migrated to Kubernetes several years back, but for numerous good reasons, the application architecture deployed for GitLab.com remains fairly simple. + +Instead of embedding these applications directly into the Rails and/or Sidekiq containers, we run them as small, independent Kubernetes deployments, isolated from the main workload. + +![use services instead of fat containers](https://docs.google.com/drawings/d/e/2PACX-1vSRrPo0TNtXG8Yqj37TO2PaND9PojGZzNRs2rcTA37-vBZm5WZlfxLDCKVJD1vYHTbGy1KY1rDYHwlg/pub?w=1008&h=564)\ +[source](https://docs.google.com/drawings/d/1ZPprcSYH5Oqp8T46I0p1Hhr-GD55iREDvFWcpQq9dTQ/edit) + +The service-integration approach has already been used for the [Suggested Reviewers feature](https://gitlab.com/gitlab-com/gl-infra/readiness/-/merge_requests/114) that has been deployed to GitLab.com. + +This approach would have many advantages: + +1. **Componentization and Replaceability**: some of these AI feature experiments will likely be short-lived. Being able to shut them down (possibly quickly, in an emergency, such as a security breach) is important. If they are terminated, they are less likely to leave technical debt behind in our main application workloads. +1. **Security Isolation**: experimental services can run with access to a minimal set of secrets, or possibly none. Ideally, the services would be stateless, with data being passed in, processed, and returned to the caller without access to PostgreSQL or other data sources. In the event of a remote code exploit or other security breach, the attacker would have limited access to sensitive data. + 1. In lieu of direct access to the main or CI Postgres clusters, services would be provided with access to the internal GitLab API through a predefined internal URL. The platform should provide instrumentation and monitoring on this address. + 1. In future iterations, but out of scope for the initial delivery, the platform could facilitate automatic authentication against the internal API, for example by managing and injecting short-lived API tokens into internal API calls, or OIDC etc. +1. **Resource Isolation**: resource-intensive workloads would be isolated to individual containers. OOM failures would not impact requests outside of the experiment. CPU saturation would not slow down unrelated requests. +1. **Dependency Isolation**: different AI libraries will have conflicting dependencies. This will not be an issue if they're run as separate services in Kubernetes. +1. **Container Size**: the size of the main application containers is not drastically increased, placing a burden on the application. +1. **Distribution Team Bottleneck**: The Distribution team avoids becoming a bottleneck as demands for many different libraries to be included in the main application containers increase. +1. **Stronger Ownership of Workloads**: teams can better understand how their workloads are running as they run in isolation. + +However, there are several outstanding questions: + +1. **Availability Requirements**: would experimental services have the same availability requirements (and alerting requirements) as the main application? +1. **Oncall**: would teams be responsible for handling pager alerts for their services? +1. **Support for non-SAAS GitLab instances**: initially all experiments would target GitLab.com, but eventually we may need to consider how to support other instances. + 1. There are three possible modes for services: + 1. `M1`: GitLab.com only: only GitLab.com supports the service. + 1. `M2`: SAAS-hosted for use with self-managed instance and instance-hosted: a singular SAAS-hosted service supports self-managed instances and GitLab.com. This is similar to the [GitLab Plus proposal](https://gitlab.com/groups/gitlab-org/-/epics/308). + 1. `M3`: Instance-hosted: each instance has a copy of the service. GitLab.com has a copy for GitLab.com. Self-managed instances host their copy of the service. This is similar to the container registry or Gitaly today. + 1. Initially, most experiments will probably be option 1 but may be promoted to 2 or 3 as they mature. +1. **Promotion Process**: ML/AI experimental features will need to be promoted to non-experimental status as they mature. A process for this will need to be established. + +#### Proposed Guidelines for Building ML/AI Services + +1. Avoid adding any large ML/AI libraries needed to support experimentation to the main application. +1. Create an platform to support individual ML/AI experiments. +1. Encourage supporting services to be stateless (excluding deployed models and other resources generated during ML training). +1. ML/AI experiment support services must not access main application datastores, including but not limited to main PostgreSQL, CI PostgreSQL, and main application Redis instances. +1. In the main application, client code for services should reside behind a feature-flag toggle, for fine-grained control of the feature. + +#### Technical Details + +Some points, in greater detail: + +##### Traffic Access + +1. Ideally these services should not be exposed externally to Internet traffic: only internally to our existing Rails and Sidekiq workloads should be routed. + 1. For services intended to run at `M2`: "SAAS-hosted for use with self-managed instance and instance-hosted", we would expect to migrate the service to a public endpoint once sufficient security review has been performed. + +##### Platform Requirements + +In order to quickly deploy and manage experiments, an minimally viable platform will need to be provided to stage-group teams. The technical implementation details of this platform are out of scope for this blueprint and will require their own blueprint (to follow). + +However, Service-Integration will establish certain necessary and optional requirements that the platform will need to satisfy. + +###### Ease of Use, Ownership Requirements + +1. `R100`: Required: the platform should be easy to use: imagine Heroku with [GitLab Production Readiness-approved](https://about.gitlab.com/handbook/engineering/infrastructure/production/readiness/) defaults. +1. `R110`: Required: with the exception of an Infrastructure-led onboarding process, services are owned, deployed and managed by stage-group teams. In other words,services follow a "You Build It, You Run It" model of ownership. +1. `R120`: Required: programming-language agnostic: no requirements for services. Services should be packaged as container images. +1. `R130`: Recommended: Each service should be evaluated against the GitLab.com [Service Maturity Model](https://about.gitlab.com/handbook/engineering/infrastructure/service-maturity-model/). +1. `R140`: Recommended: services using the platform have expedited production-readiness processes. + 1. Production-readiness requirements graded by service maturity: low-traffic, low-maturity experimental services will have lower requirement thresholds than more mature services. + 1. By default, the platform should provide services with defaults that would pass production-readiness review for the lowest service maturity-level. + 1. At introduction, lowest maturity services can be deployed without production readiness, provided the meet certain automatically validated requirements. This removes Infrastructure gate-keeping from being a blocker to experimental service delivery. + +###### Observability Requirements + +1. `R200`: Required: the platform must provide SLIs for services out-of-the-box. + 1. While it is recommended that services expose internal metrics, it is not mandatory. The platform will provide monitoring from the load-balancer. This is to speed up deployment by removing barriers to experimentation. + 1. For services that provide internal metrics scrape endpoints, the platform must be configurable to collect these. + 1. The platform must provide generic load-balancer level SLIs for all services. Service owners must be able to select from constructing SLIs from internal application metrics, the platform-provided external SLIs, or a combination of both. +1. `R210`: Required: Observability dashboards, rules, alerts (with per-term routing) must be generated from a manifest. +1. `R220`:Required: standardized logging infrastructure. + 1. Mandate that all logging emitted from services must be Structured JSON. Text logs are permitted but not recommended. + 1. See [Common Service Libraries](#common-service-libraries) for more details of building common SDKs for observability. + +###### Deployment Requirements + +1. `R300`: Required: No secrets stored in CI/CD. + 1. Authentication with Cloud Provider Resources should be exclusively via OIDC, managed as part of the platform. + 1. Secrets should be stored in the Infrastructure-provided Hashicorp Vault for the environment and passed to applications through files or environment variables. + 1. Generation and management of service account tokens should be done declaratively, without manual interaction. +1. `R310`: Required: multiple environment should be supported, eg Staging and Production. +1. `R320`: Required: the platform should be cost-effective. Kubernetes clusters should support multiple services and teams. +1. `R330`: Recommended: gradual rollouts, rollbacks, blue-green deployments. +1. `R340`: Required: services should be isolated from one another. +1. `R350`: Recommended: services should have the ability to specify node characteristic requirements (eg, GPU). +1. `R360`: Required: Developers should not need knowledge of Helm, Kubernetes, Prometheus in order to deploy. All required values are configured and validated in project-hosted manifest before generating Kubernetes manifests, Prometheus rules, etc. +1. `R370`: Initially services should be synchronous only - using REST or GRPC requests. + 1. This does not however preclude long-running HTTP(s) requests, for example long-polling or Websocket requests. +1. `R390`: Each service hosted in its own GitLab repository with deployment manifest stored in the repository. + 1. Continuous deployments that are initiated from the CI pipeline of the corresponding GitLab repository. + +##### Security Requirements + +1. `R400`: stateful services deployed on the platform that utilize their own stateful storage (for example, custom deployed Postgres instance), must not store application security tokens, cloud-provider service keys or other long-lived security tokens in their stateful stores. +1. `R410`: long-lived shared secrets are discouraged, and should be referenced in the service manifest as such, to allow for accounting and monitoring. +1. `R420`: services using long-lived shared secrets should ensure that secret rotation can take place without downtime. + 1. During a rotation, old and new generations of secrets should pass authentication, allowing gradual roll-out of new secrets. + +##### Common Service Libraries + +1. `R500`: Experimental services would be strongly encouraged to adopt and use [LabKit](https://gitlab.com/gitlab-org/labkit) (for Go services), or [LabKit-Ruby](https://gitlab.com/gitlab-org/ruby/gems/labkit-ruby) for observability, context, correlation, FIPs verification, etc. + 1. At present, there is no LabKit-Python library, but some experiments will run in Python, so building a library to providing observability, context, correlation services in Python will be required. diff --git a/doc/architecture/blueprints/gitlab_observability_backend/metrics/index.md b/doc/architecture/blueprints/gitlab_observability_backend/metrics/index.md index c5bd2440b0c..3edb01d9140 100644 --- a/doc/architecture/blueprints/gitlab_observability_backend/metrics/index.md +++ b/doc/architecture/blueprints/gitlab_observability_backend/metrics/index.md @@ -8,6 +8,8 @@ owning-stage: "~monitor::observability" participating-stages: [] --- + + # GitLab Observability Backend - Metrics ## Summary @@ -93,9 +95,9 @@ Additionally, since we intend to ingest data via Prometheus `remote_write` API, We also need to make sure to avoid writing a lot of small writes into Clickhouse, therefore it’d be prudent to batch data before writing it into Clickhouse. -We must also make sure ingestion remains decoupled with `Storage` so as to reduce undue dependence on a given storage implementation. While we do intend to use Clickhouse as our backing storage for any foreseeable future, this ensures we do not tie ourselves in into Clickhouse too much should future business requirements warrant the usage of a different backend/technology. A good way to implement this in Golang would be our implementations adhering to a standard interface, the following for example: +We must also make sure ingestion remains decoupled with `Storage` so as to reduce undue dependence on a given storage implementation. While we do intend to use Clickhouse as our backing storage for any foreseeable future, this ensures we do not tie ourselves in into Clickhouse too much should future business requirements warrant the usage of a different backend/technology. A good way to implement this in Go would be our implementations adhering to a standard interface, the following for example: -```golang +```go type Storage interface { Read( ctx context.Context, diff --git a/doc/architecture/blueprints/graphql_api/index.md b/doc/architecture/blueprints/graphql_api/index.md index 95ff834cd27..2c277049434 100644 --- a/doc/architecture/blueprints/graphql_api/index.md +++ b/doc/architecture/blueprints/graphql_api/index.md @@ -8,6 +8,8 @@ owning-stage: "~devops::manage" participating-stages: [] --- + + # GraphQL API [GraphQL](https://graphql.org/) is a data query and manipulation language for diff --git a/doc/architecture/blueprints/object_pools/index.md b/doc/architecture/blueprints/object_pools/index.md new file mode 100644 index 00000000000..3f3a0341e4a --- /dev/null +++ b/doc/architecture/blueprints/object_pools/index.md @@ -0,0 +1,495 @@ +--- +status: proposed +creation-date: "2023-03-30" +authors: [ "@pks-gitlab" ] +coach: [ ] +approvers: [ ] +owning-stage: "~devops::systems" +participating-stages: [ "~devops::create" ] +--- + +# Iterate on the design of object pools + +## Summary + +Forking repositories is at the heart of many modern workflows for projects +hosted in GitLab. As most of the objects between a fork and its upstream project +will typically be the same, this opens up potential for optimizations: + +- Creating forks can theoretically be lightning fast if we reuse much of the + parts of the upstream repository. + +- We can save on storage space by deduplicating objects which are shared. + +This architecture is currently implemented with object pools which hold objects +of the primary repository. But the design of object pools has organically grown +and is nowadays showing its limits. + +This blueprint explores how we can iterate on the design of object pools to fix +long standing issues with it. Furthermore, the intent is to arrive at a design +that lets us iterate more readily on the exact implementation details of object +pools. + +## Motivation + +The current design of object pools is showing problems with scalability in +various different ways. For a large part the problems come from the fact that +object pools have organically grown and that we learned as we went by. + +It is proving hard to fix the overall design of object pools because there is no +clear ownership. While Gitaly provides the low-level building blocks to make +them work, it does not have enough control over them to be able to iterate on +their implementation details. + +There are thus two major goals: taking ownership of object pools so that it +becomes easier to iterate on the design, and fixing scalability issues once we +can iterate. + +### Lifecycle ownership + +While Gitaly provides the interfaces to manage object pools, the actual life +cycle of them is controlled by the client. A typical lifecycle of an object pool +looks as following: + +1. An object pool is created via `CreateObjectPool()`. The caller provides the + path where the object pool shall be created as well as the origin repository + from which the repository shall be created. + +1. The origin repository needs to be linked to the object pool explicitly by + calling `LinkRepositoryToObjectPool()`. + +1. The object pool needs to be regularly updated via `FetchIntoObjectPool()` + that fetches all changes from the primary pool member into the object pool. + +1. To create forks, the client needs to call `CreateFork()` followed by + `LinkRepositoryToObjectPool()`. + +1. Repositories of forks are unlinked by calling `DisconnectGitAlternates()`. + This will reduplicate objects. + +1. The object pool is deleted via `DeleteObjectPool()`. + +This lifecycle is complex and leaks a lot of implementation details to the +caller. This was originally done in part to give the Rails side control and +management over Git object visibility. GitLab project visibility rules are +complex and not a Gitaly concern. By exposing these details Rails can control +when pool membership links are created and broken. It is not clear at the +current point in time how the complete system works and its limits are not +explicitly documented. + +In addition to the complexity of the lifecycle we also have multiple sources of +truth for pool membership. Gitaly never tracks the set of members of a pool +repository but can only tell for a specific repository that it is part of said +pool. Consequently, Rails is forced to maintain this information in a database, +but it is hard to maintain that information without becoming stale. + +### Repository maintenance + +Related to the lifecycle ownership issues is the issue of repository +maintenance. As mentioned, keeping an object pool up to date requires regular +calls to `FetchIntoObjectPool()`. This is leaking implementation details to the +client, but was done to give the client control over syncing the primary +repository with its object pool. With this control, private repositories can be +prevented from syncing and consquently leaking objects to other repositories in +the fork network. + +We have had good success with moving repository maintenance into Gitaly so that +clients do not need to know about on-disk details. Ideally, we would do the same +for repositories that are the primary member of an object pool: if we optimize +its on-disk state, we will also automatically update the object pool. + +There are two issues that keep us from doing so: + +- Gitaly does not know about the relationship between an object pool and its + members. + +- Updating object pools is expensive. + +By making Gitaly the single source of truth for object pool memberships we would +be in a position to fix both issues. + +### Fast forking + +In the current implementation, Rails first invokes `CreateFork()` which results +in a complete `git-clone(1)` being performed to generate the fork repository. +This is followed by `LinkRepositoryToObjectPool()` to link the fork with the +object pool. It is not until housekeeping is performed on the fork repository +that objects are deduplicated. This is not only leaking implementation details +to clients, but it also keeps us from reaping the full potential benefit of +object pools. + +In particular, creating forks is a lot slower than it could be since a clone is +always performed before linking. If the steps of creating the fork and linking +the fork to the pool repository were unified, the initial clone could be +avoided. + +### Clustered object pools + +Gitaly Cluster and object pools development overlapped. Consequently they are +known to not work well together. Praefect does neither ensure that repositories +with object pools have their object pools present on all nodes, nor does it +ensure that object pools are in a known state. If at all, object pools only work +by chance. + +The current state has led to cases where object pools were missing or had +different contents per node. This can result in inconsistently observed state in +object pool members and writes that depend on the object pool's contents to +fail. + +One way object pools might be handled for clustered Gitaly could be to have the +pool repositories duplicated on nodes that contain repositories dependent on +them. This would allow members of a fork network to exist of different nodes. To +make this work, repository replciation would have to be aware of object pools +and know when it needs to duplicate them onto a particular node. + +## Requirements + +There are a set of requirements and invariants that must be given for any +particular solution. + +### Private upstream repositories should not leak objects to forks + +When a project has a visibility setting that is not public, the objects in the +repository should not be fetched into an object pool. An object pool should only +ever contain objects from the upstream repository that were at one point public. +This prevents private upstream repositories from having objects leaked to forks +through a shared object pool. + +### Forks cannot sneak objects into upstream projects + +It should not be possible to make objects uploaded in a fork repository +accessible in the upstream repository via a shared object pool. Otherwise +potentially unauthorized users would be able to "sneak in" objects into +repositories by simply forking them. + +Despite leading to confusion, this could also serve as a mechanism to corrupt +upstream repositories by introducing objects that are known to be broken. + +### Object pool lifetime exceeds upstream repository lifetime + +If the upstream repository gets deleted, its object pool should remain in place +to provide continued deduplication of shared objects between the other +repositories in the fork network. Thus it can be said that the lifetime of the +object pool is longer than the lifetime of the upstream repository. An object +pool should only be deleted if there are no longer any repositories referencing +it. + +### Object lifetime + +By deduplicating objects in a fork network, repositories become dependent on the +object pool. Missing objects in the pooled repository could lead to corruption +of repositories in the fork network. Therefore, objects in the pooled repository +must continue to exist as long as there are repositories referencing them. + +Without a mechanism to accurately determine if a pooled object is referenenced +by one of more repositories, all objects in the pooled repository must remain. +Only when there are no repositories referencing the object pool can the pooled +repository, and therfore all its objects, be removed. + +### Object sharing + +An object that is deduplicated will become accessible from all forks of a +particular repository, even if it has never been reachable in any of the forks. +The consequence is that any write to an object pool immediately influences all +of its members. + +We need to be mindful of this property when repositories connected to an object +pool are replicated. As the user-observable state should be the same on all +replicas, we need to ensure that both the repository and its object pool are +consistent across the different nodes. + +## Proposal + +In the current design, management of object pools mostly happens on the client +side as they need to manage their complete lifecyclethem. This requires Rails to +store the object pool relationships in the Rails database, perform fine-grained +management of every single step of an object pool's life, and perform periodic +Sidekiq jobs to enforce state by calling idempotent Gitaly RPCs. This design +significantly increases complexity of an already-complex mechanism. + +Instead of handling the full lifecycle of object pools on the client-side, this +document proposes to instead encapsulate the object pool lifecycle management +inside of Gitaly. Instead of performing low-level actions to maintain object +pools, clients would only need to tell Gitaly about updated relationships +between a repository and its object pool. + +This brings us multiple advantages: + +- The inherent complexity of the lifecycle management is encapsulated in a + single place, namely Gitaly. + +- Gitaly is in a better position to iterate on the low-level technical design of + object pools in case we find a better solution compared to "alternates" in the + future. + +- We can ensure better interplay between Gitaly Cluster, object pools and + repository housekeeping. + +- Gitaly becomes the single source of truth for object pool relationships and + can thus start to manage it better. + +Overall, the goal is to raise the abstraction level so that clients need to +worry less about the technical details while Gitaly is in a better position to +iterate on them. + +### Move lifecycle management of pools into Gitaly + +The lifecycle management of object pools is leaking too many details to the +client, and by doing so makes parts things both hard to understand and +inefficient. + +The current solution relies on a set of fine-grained RPCs that manage the +relationship between repositories and their object pools. Instead, we are aiming +for a simplified approach that only exposes the high-level concept of forks to +the client. This will happen in the form of three RPCs: + +- `ForkRepository()` will create a fork of a given repository. If the upstream + repository does not yet have an object pool, Gitaly will create it. It will + then create the new repository and automatically link it to the object pool. + The upstream repository will be recorded as primary member of the object pool, + the fork will be recorded as a secondary member of the object pool. + +- `UnforkRepository()` will remove a repository from the object pool it is + connected to. This will stop deduplication of objects. For the primary object + pool member this also means that Gitaly will stop pulling new objects into the + object pool. + +- `GetObjectPool()` returns the object pool for a given repository. The pool + description will contain information about the pool's primary object pool + member as well as all secondary object pool members. + +Furthermore, the following changes will be implemented: + +- `RemoveRepository()` will remove the repository from its object pool. If it + was the last object pool member, the pool will be removed. + +- `OptimizeRepository()`, when executed on the primary object pool member, will + also update and optimize the object pool. + +- `ReplicateRepository()` needs to be aware of object pools and replicate them + correctly. Repositories shall be linked to and unlink from object pools as + required. While this is a step towards fixing the Praefect world, which may + seem redundant given that we plan to deprecate Praefect anyway, this RPC call + is also used for other use cases like repository rebalancing. + +With these changes, Gitaly will have much tighter control over the lifecycle of +object pools. Furthermore, as it starts to track the membership of repositories +in object pools it can become the single source of truth for fork networks. + +### Fix inefficient maintenance of object pools + +In order to update object pools, Gitaly performs a fetch of new objects from the +primary object pool member into the object pool. This fetch is inefficient as it +needs to needlessly negotiate objects that are new in the primary object pool +member. But given that objects are deduplicated already in the primary object +pool member it means that it should only have objects in its object database +that do not yet exist in the object pool. Consequently, we should be able to +skip the negotiation completely and instead link all objects into the object +pool that exist in the source repository. + +In the current design, these objects are kept alive by creating references to +the just-fetched objects. If the fetch deleted references or force-updated any +references, then it may happen that previously-referenced objects become +unreferenced. Gitaly thus creates keep-around references so that they cannot +ever be deleted. Furthermore, those references are required in order to properly +replicate object pools as the replication is reference-based. + +These two things can be solved in different ways: + +- We can set the `preciousObjects` repository extension. This will instruct all + versions of Git which understand this extension to never delete any objects + even if `git-prune(1)` or similar commands were executed. Versions of Git that + do not understand this extension would refuse to work in this repository. + +- Instead of replicating object pools via `git-fetch(1)`, we can instead + replicate them by sending over all objects part of the object database. + +Taken together this means that we can stop writing references in object pools +altogether. This leads to efficient updates of object pools by simply linking +all new objects into place, and it fixes issues we have seen with unbounded +growth of references in object pools. + +## Design and implementation details + + + +## Problems with the design + +As mentioned before, object pools are not a perfect solution. This section goes +over the most important issues. + +### Complexity of lifecycle management + +Even though the lifecycle of object pools becomes easier to handle once it is +fully owned by Gitaly, it is still complex and needs to be considered in many +ways. Handling object pools in combination with their repositories is not an +atomic operation as any action by necessity spans over at least two different +resources. + +### Performance issues + +As object pools deduplicate objects, the end result is that object pool members +never have the full closure of objects in a single packfile. This is not +typically an issue for the primary object pool member, which by definition +cannot diverge from the object pool's contents. But secondary object pool +members can and often will diverge from the original contents of the upstream +repository. + +This leads to two different sets of reachable objects in secondary object pool +members. Unfortunately, due to limitations in Git itself, this precludes the use +of a subset of optimizations: + +- Packfiles cannot be reused as efficiently when serving fetches to serve + already-deltified objects. This requires Git to recompute deltas on the fly + for object pool members which have diverged from object pools. + +- Packfile bitmaps can only exist in object pools as it is not possible nor + easily feasible for these bitmaps to cover multiple object databases. This + requires Git to traverse larger parts of the object graph for many operations + and especially when serving fetches. + +### Dependent writes across repositories + +The design of object pools introduces significant complexity into the Raft world +where we use a write-ahead log for all changes to repositories. In the ideal +case, a Raft-based design would only need to care about the write-ahead log of a +single repository when considering requests. But with object pools, we are +forced to consider both reads and writes for a pooled repository to be dependent +on all writes in its object pool having been applied. + +## Alternative Solutions + +The proposed solution is not obviously the best choice as it has issues both +with complexity (management of the lifecycle) and performance (inefficiently +served fetches for pool members). + +This section explores alternatives to object pools and why they have not been +chosen as the new target architecture. + +### Stop using object pools altogether + +An obvious way to avoid all of the complexity is to stop using object pools +altogether. While it is charming from an engineering point of view as we can +significantly simplify the architecture, it is not a viable approach from the +product perspective as it would mean that we cannot support efficient forking +workflows. + +### Primary repository as object pool + +Instead of creating an explicit object pool repository, we could just use the +upstream repository as an alternate object database of all forks. This avoids a +lot of complexity around managing the lifetime of the object pool, at least +superficially. Furthermore, it circumvents the issue of how to update object +pools as it will always match the contents of the upstream repository. + +It has a number of downsides though: + +- Normal repositories can now have different states, where some of the + repositories are allowed to prune objects and others aren't. This introduces a + source of uncertainty and makes it easy to accidentally delete objects in a + normal repository and thus corrupt its forks. + +- When upstream repositories go private we must stop updating objects which are + supposed to be deduplicated across members of the fork network. This means + that we would ultimately still be forced to create object pools once this + happens in order to freeze the set of deduplicated objects at the point in + time where the repository goes private. + +- Deleting repositories becomes more complex as we need to take into account + whether a repository is linked to by forks. + +### Reference namespaces + +With `gitnamespaces(7)`, Git provides a mechanism to partition references into +different sets of namespaces. This allows us to serve all forks from a single +repository that contains all objects. + +One neat property is that we have the global view of objects referenced by all +forks together in a single object database. We can thus easily perform shared +housekeeping across all forks at once, including deletion of objects that are +not used by any of the forks anymore. Regarding objects, this is likely to be +the most efficient solution we could potentially aim for. + +There are again some downsides though: + +- Calculating usage quotas must by necessity use actual reachability of objects + into account, which is expensive to compute. This is not a showstopper, but + something to keep in mind. + +- One stated requirement is that it must not be possible to make objects + reachable in other repositories from forks. This property could theoretically + be enforced by only allowing access to reachable objects. That way an object + can only be accessed through virtual repository if the object is reachable from + its references. Reachability checks are too compute heavy for this to be practical. + +- Even though references are partitioned, large fork networks would still easily + end up with multiple millions of references. It is unclear what the impact on + performance would be. + +- The blast radius for any repository-level attacks significantly increases as + you would not only impact your own repository, but also all forks. + +- Custom hooks would have to be isolated for each of the virtual repositories. + Since the execution of Git hooks is controled it should be possible to handle + this for each of the namespaces. + +### Filesystem-based deduplication + +The idea of deduplicating objects on the filesystem level was floating around at +several points in time. While it would be nice if we could shift the burden of +this to another component, it is likely not easy to implement due to the nature +of how Git works. + +The most important contributing factor to repository sizes are Git objects. +While it would be possible to store the objects in their loose representation +and thus deduplicate on that level, this is infeasible: + +- Git would not be able to deltify objects, which is an extremely important + mechanism to reduce on-disk size. It is unlikely that the size reduction + caused by deduplication would outweigh the size reduction gained from the + deltification mechanism. + +- Loose objects are significantly less efficient when accessing the repository. + +- Serving fetches requires us to send a packfile to the client. Usually, Git is + able to reuse large parts of already-existing packfiles, which significantly + reduces the computational overhead. + +Deduplicating on the loose-object level is thus infeasible. + +The other unit that one could try to deduplicate is packfiles. But packfiles are +not deterministically generated by Git and will furthermore be different once +repositories start to diverge from each other. So packfiles are not a natural +fit for filesystem-level deduplication either. + +An alternative could be to use hard links of packfiles across repositories. This +would cause us to duplicate storage space whenever any repository decides to +perform a repack of objects and would thus be unpredictable and hard to manage. + +### Custom object backend + +In theory, it would be possible to implement a custom object backend that allows +us to store objects in such a way that we can deduplicate them across forks. +There are several technical hurdles though that keep us from doing so without +significant upstream investments: + +- Git is not currently designed to have different backends for objects. Accesses + to files part of the object database are littered across the code base with no + abstraction level. This is in contrast to the reference database, which has at + least some level of abstraction. + +- Implementing a custom object backend would likely necessitate a fork of the + Git project. Even if we had the resources to do so, it would introduce a major + risk factor due to potential incompatibilities with upstream changes. It would + become impossible to use vanilla Git, which is often a requirement that exists + in the context of Linux distributions that package GitLab. + +Both the initial and the operational risk of ongoing maintenance are too high to +really justify this approach for now. We might revisit this approach in the +future. diff --git a/doc/architecture/blueprints/object_storage/index.md b/doc/architecture/blueprints/object_storage/index.md index 4a8eeaf86a9..3f649960554 100644 --- a/doc/architecture/blueprints/object_storage/index.md +++ b/doc/architecture/blueprints/object_storage/index.md @@ -8,6 +8,8 @@ owning-stage: "~devops::data_stores" participating-stages: [] --- + + # Object storage: `direct_upload` consolidation ## Abstract @@ -53,7 +55,7 @@ This has led to increased complexity across the board, from development [we no longer recommend](../../../administration/nfs.md) to our users and is no longer in use on GitLab.com. - Understanding all the moving parts and the flow is extremely - complicated: we have CarrierWave, Fog, Golang S3/Azure SDKs, all + complicated: we have CarrierWave, Fog, Go S3/Azure SDKs, all being used, and that complicates testing as well. - Fog and CarrierWave are not maintained to the level of the native SDKs (for example, AWS S3 SDK), so we have to maintain or monkey @@ -124,7 +126,7 @@ infrastructure. It also makes the initial installation more complex feature after feature. Implementing a direct upload by default, with a -[consolidated object storage configuration](../../../administration/object_storage.md#consolidated-object-storage-configuration) +[consolidated object storage configuration](../../../administration/object_storage.md#configure-a-single-storage-connection-for-all-object-types-consolidated-form) will reduce the number of merge requests needed to ship a new feature from four to only one. It will also remove the need for SRE intervention as the bucket will always be the same. diff --git a/doc/architecture/blueprints/organization/index.md b/doc/architecture/blueprints/organization/index.md new file mode 100644 index 00000000000..bd8d085413c --- /dev/null +++ b/doc/architecture/blueprints/organization/index.md @@ -0,0 +1,175 @@ +--- +status: ongoing +creation-date: "2023-04-05" +authors: [ "@lohrc" ] +coach: "@ayufan" +approvers: [ "@lohrc" ] +owning-stage: "~devops::data stores" +participating-stages: [] +--- + + + +# Organization + +This document is a work in progress and represents the current state of the Organization design. + +## Glossary + +- Organization: An Organization is the umbrella for one or multiple top-level groups. Organizations are isolated from each other by default meaning that cross-namespace features will only work for namespaces that exist in a single Organization. +- Top-level group: Top-level group is the name given to the topmost group of all other groups. Groups and projects are nested underneath the top-level group. +- Cell: A Cell is a set of infrastructure components that contains multiple Organizations. The infrastructure components provided in a Cell are shared among Organizations, but not shared with other Cells. This isolation of infrastructure components means that Cells are independent from each other. +- User: An Organization has many users. Joining an Organization makes someone a user of that Organization. +- Member: Adding a user to a group or project within an Organization makes them a member. Members are always users, but users are not necessarily members of a group or project within an Organization. For instance, a user could just have accepted the invitation to join an Organization, but not be a member of any group or project it contains. +- Non-user: A non-user of an Organization means a user is not part of that specific Organization. + +## Summary + +Organizations solve the following problems: + +1. Enables grouping of top-level groups. For example, the following top-level groups would belong to the Organization `GitLab`: + 1. `https://gitlab.com/gitlab-org/` + 1. `https://gitlab.com/gitlab-com/` +1. Allows different Organizations to be isolated. Top-level groups of the same Organization can interact with each other but not with groups in other Organizations, providing clear boundaries for an Organization, similar to a self-managed instance. Isolation should have a positive impact on performance and availability as things like user dashboards can be scoped to Organizations. +1. Allows integration with Cells. Isolating Organizations makes it possible to allocate and distribute them across different Cells. +1. Removes the need to define hierarchies. An Organization is a container that could be filled with whatever hierarchy/entity set makes sense (Organization, top-level groups, etc.) +1. Enables centralized control of user profiles. With an Organization-specific user profile, administrators can control the user's role in a company, enforce user emails, or show a graphical indicator that a user as part of the Organization. An example could be adding a "GitLab employee" stamp on comments. +1. Organizations bring an on-premise-like experience to SaaS (GitLab.com). The Organization admin will have access to instance-equivalent Admin Area settings with most of the configuration controlled on Organization level. + +## Motivation + +### Goals + +The Organization focuses on creating a better experience for Organizations to manage their GitLab experience. By introducing Organizations and [Cells](../cells/index.md) we can improve the reliability, performance and availability of our SaaS Platforms. + +- Wider audience: Many instance-level features are admin only. We do not want to lock out users of GitLab.com in that way. We want to make administrative capabilities that previously only existed for self-managed users available to our SaaS users as well. This also means we would give users of GitLab.com more independence from GitLab.com admins in the long run. Today, there are actions that self-managed admins can perform that GitLab.com users have to request from GitLab.com admins. +- Improved UX: Inconsistencies between the features available at the project and group levels create navigation and usability issues. Moreover, there isn't a dedicated place for Organization-level features. +- Aggregation: Data from all groups and projects in an Organization can be aggregated. +- An Organization includes settings, data, and features from all groups and projects under the same owner (including personal namespaces). +- Cascading behavior: Organization cascades behavior to all the projects and groups that are owned by the same Organization. It can be decided at the Organization level whether a setting can be overridden or not on the levels beneath. + +### Non-Goals + +Due to urgency of delivering Organizations as a prerequisite for Cells, it is currently not a goal to build Organization functionality on the namespace framework. + +## Proposal + +We create Organizations as a new lightweight entity, with just the features and workflows which it requires. We already have much of the functionality present in groups and projects, and groups themselves are essentially already the top-level entity. It is unlikely that we need to add significant features to Organizations outside of some key settings, as top-level groups can continue to serve this purpose at least on SaaS. + +```mermaid +graph TD + o[Organization] -. has many .- g + ns[Namespace] --> g[Group] + ns[Namespace] --> pns[ProjectNamespace] -. has one .- p[Project] + ns --> un[UserNamespace] + g -. has many .- p + un -. has many .- p + ns[Namespace] -. has many .- ns[Namespace] +``` + +Self-managed instances would set a default Organization. + +### Benefits + +- No changes to URL's for groups moving under an Organization, which makes moving around top-level groups very easy. +- Low risk rollout strategy, as there is no conversion process for existing top-level groups. +- Organization becomes the key for identifying what is part of an Organization, which is likely on its own table for performance and clarity. + +### Drawbacks + +- It is unclear right now how we would avoid continuing to spend effort to build instance (or not Organization) features, in particular much of the reporting. This is not an issue on SaaS as top-level groups already have this capability, however it is a challenge on self-managed. If we introduce a built-in Organization (or just none at all) for self-managed, it seems like we would need to continue to build instance/Organization level reporting features as we would not get that for free along with the work to add to groups. +- Billing may need to be moved from top-level groups to Organization level. + +## Design and Implementation Details + +### Organization MVC + +The Organization MVC will contain the following functionality: + +- Instance setting to allow the creation of multiple Organizations. This will be enabled by default on GitLab.com, and disabled for self-managed GitLab. +- Every instance will have a default organization. Initially, all users will be managed by this default Organization. +- Organization Owner. The creation of an Organization appoints that user as the Organization Owner. Once established, the Organization Owner can appoint other Organization Owners. +- Organization users. A user is managed by one Organization, but can be part of multiple Organizations. +- Setup settings. Containing the Organization name, ID, description, README, and avatar. Settings are editable by the Organization Owner. +- Setup flow. Users are able to build an Organization on top of an existing top-level group. New users are able to create an Organization from scratch and to start building top-level groups from there. +- Visibility. Options will be `public` and `private`. A nonuser of a specific Organization will not see private Organizations in the explore section. Visibility is editable by the Organization Owner. +- Organization settings page with the added ability to remove an Organization. Deletion of the default Organization is prevented. +- Groups. This includes the ability to create, edit, and delete groups, as well as a Groups overview that can be accessed by the Organization Owner. +- Projects. This includes the ability to create, edit, and delete projects, as well as a Projects overview that can be accessed by the Organization Owner. + +### Organization Access + +#### Organization Users + +Organization Users can get access to groups and projects as: + +- A group member: this grants access to the group and all its projects, regardless of their visibility. +- A project member: this grants access to the project, and limited access to parent groups, regardless of their visibility. +- A non-member: this grants access to public and internal groups and projects of that Organization. To access a private group or project in an Organization, a user must become a member. + +Organization Users can be managed by the Organization as: + +- Enterprise Users, managed by the Organization. This includes control over their user account and the ability to block the user. +- Non-Enterprise Users, managed by the User. Non-Enterprise Users can be removed from an Organization, but their user account remains in their control. + +Enterprise Users are only available to Organizations with a Premium or Ultimate subscription. Organizations on the free tier will only be able to host Non-Enterprise Users. + +#### Organization Non-Users + +Non-users are external to the Organization and can only access the public resources of an Organization, such as public projects. + +## Iteration Plan + +The following iteration plan outlines how we intend to arrive at the Organization MVC. We are following the guidelines for [Experiment, Beta, and Generally Available features](../../../policy/alpha-beta-support.md). + +### Iteration 1: Organization Prototype (FY24Q2) + +In iteration 1, we introduce the concept of an Organization as a way to group top-level groups together. Support for Organizations does not require any [Cells](../cells/index.md) work, but having them will make all subsequent iterations of Cells simpler. The goal of iteration 1 will be to generate a prototype that can be used by GitLab teams to test moving functionality to the Organization. It contains everything that is necessary to move an Organization to a Cell: + +- The Organization can be named, has an ID and an avatar. +- Only non-enterprise user can be part of an Organization. +- A user can be part of multiple Organizations. +- A single Organization Owner can be assigned. +- Groups can be created in an Organization. Groups are listed in the Groups overview. +- Projects can be created in a Group. Projects are listed in the Projects overview. + +### Iteration 2: Organization MVC Experiment (FY24Q3) + +In iteration 2, an Organization MVC Experiment will be released. We will test the functionality with a select set of customers and improve the MVC based on these learnings. Users will be able to build an Organization on top of their existing top-level group. + +- The Organization has a description and a README. + +### Iteration 3: Organization MVC Beta (FY24Q4) + +In iteration 3, the Organization MVC Beta will be released. + +- Multiple Organization Owners can be assigned. +- Enterprise users can be added to an Organization. + +### Iteration 4: Organization MVC GA (FY25Q1) + +### Post-MVC Iterations + +After the initial rollout of Organizations, the following functionality will be added to address customer needs relating to their implementation of GitLab: + +1. Internal visibility will be made available on Organizations that are part of GitLab.com. +1. Move billing from top-level group to Organization. +1. Audit events at the Organization level. +1. Set merge request approval rules at the Organization level and cascade to all groups and projects. +1. Security policies at the Organization level. +1. Vulnerability reports at the Organization level. +1. Cascading Organization setting to enforce security scans. +1. Scan result policies at the Organization level. +1. Compliance frameworks. + +## Alternative Solutions + +An alternative approach to building Organizations is to convert top-level groups into Organizations. The main advantage of this approach is that features could be built on top of the namespace framework and therewith leverage functionality that is already available at the group level. We would avoid building the same feature multiple times. However, Organizations have been identified as a critical driver of Cells. Due to the urgency of delivering Cells, we decided to opt for the quickest and most straightforward solution to deliver an Organization, which is the lightweight design described above. More details on comparing the two Organization proposals can be found [here](https://gitlab.com/gitlab-org/tenant-scale-group/group-tasks/-/issues/56). + +## Decision Log + +- 2023-05-10: [Billing is not part of the Organization MVC](https://gitlab.com/gitlab-org/gitlab/-/issues/406614#note_1384055365) + +## Links + +- [Organization epic](https://gitlab.com/groups/gitlab-org/-/epics/9265) diff --git a/doc/architecture/blueprints/pods/images/iteration0-organizations-introduction.png b/doc/architecture/blueprints/pods/images/iteration0-organizations-introduction.png deleted file mode 100644 index 5725b0fa71f..00000000000 Binary files a/doc/architecture/blueprints/pods/images/iteration0-organizations-introduction.png and /dev/null differ diff --git a/doc/architecture/blueprints/pods/images/pods-and-fulfillment.png b/doc/architecture/blueprints/pods/images/pods-and-fulfillment.png deleted file mode 100644 index fea32d1800e..00000000000 Binary files a/doc/architecture/blueprints/pods/images/pods-and-fulfillment.png and /dev/null differ diff --git a/doc/architecture/blueprints/pods/images/term-cluster.png b/doc/architecture/blueprints/pods/images/term-cluster.png deleted file mode 100644 index 87e4d631551..00000000000 Binary files a/doc/architecture/blueprints/pods/images/term-cluster.png and /dev/null differ diff --git a/doc/architecture/blueprints/pods/images/term-organization.png b/doc/architecture/blueprints/pods/images/term-organization.png deleted file mode 100644 index 4c82c62b8f4..00000000000 Binary files a/doc/architecture/blueprints/pods/images/term-organization.png and /dev/null differ diff --git a/doc/architecture/blueprints/pods/images/term-pod.png b/doc/architecture/blueprints/pods/images/term-pod.png deleted file mode 100644 index d8f79df2f29..00000000000 Binary files a/doc/architecture/blueprints/pods/images/term-pod.png and /dev/null differ diff --git a/doc/architecture/blueprints/pods/images/term-top-level-namespace.png b/doc/architecture/blueprints/pods/images/term-top-level-namespace.png deleted file mode 100644 index c1cd317d878..00000000000 Binary files a/doc/architecture/blueprints/pods/images/term-top-level-namespace.png and /dev/null differ diff --git a/doc/architecture/blueprints/pods/index.md b/doc/architecture/blueprints/pods/index.md index 077303be30f..5c15f880a54 100644 --- a/doc/architecture/blueprints/pods/index.md +++ b/doc/architecture/blueprints/pods/index.md @@ -1,356 +1,11 @@ --- -status: accepted -creation-date: "2022-09-07" -authors: [ "@ayufan", "@fzimmer", "@DylanGriffith" ] -coach: "@ayufan" -approvers: [ "@fzimmer" ] -owning-stage: "~devops::enablement" -participating-stages: [] +redirect_to: '../cells/index.md' +remove_date: '2023-06-13' --- -# Pods +This document was moved to [another location](../cells/index.md). -This document is a work-in-progress and represents a very early state of the Pods design. Significant aspects are not documented, though we expect to add them in the future. - -## Summary - -Pods is a new architecture for our Software as a Service platform that is horizontally-scalable, resilient, and provides a more consistent user experience. It may also provide additional features in the future, such as data residency control (regions) and federated features. - -## Terminology - -We use the following terms to describe components and properties of the Pods architecture. - -### Pod - -A Pod is a set of infrastructure components that contains multiple top-level namespaces that belong to different organizations. The components include both datastores (PostgreSQL, Redis etc.) and stateless services (web etc.). The infrastructure components provided within a Pod are shared among organizations and their top-level namespaces but not shared with other Pods. This isolation of infrastructure components means that Pods are independent from each other. - -![Term Pod](images/term-pod.png) - -#### Pod properties - -- Each pod is independent from the others -- Infrastructure components are shared by organizations and their top-level namespaces within a Pod -- More Pods can be provisioned to provide horizontal scalability -- A failing Pod does not lead to failure of other Pods -- Noisy neighbor effects are limited to within a Pod -- Pods are not visible to organizations; it is an implementation detail -- Pods may be located in different geographical regions (for example, EU, US, JP, UK) - -Discouraged synonyms: GitLab instance, cluster, shard - -### Cluster - -A cluster is a collection of Pods. - -![Term Cluster](images/term-cluster.png) - -#### Cluster properties - -- A cluster holds cluster-wide metadata, for example Users, Routes, Settings. - -Discouraged synonyms: whale - -### Organizations - -GitLab references [Organizations in the initial set up](../../../topics/set_up_organization.md) and users can add a (free text) organization to their profile. There is no Organization entity established in the GitLab codebase. - -As part of delivering Pods, we propose the introduction of an `organization` entity. Organizations would represent billable entities or customers. - -Organizations are a known concept, present for example in [AWS](https://docs.aws.amazon.com/whitepapers/latest/organizing-your-aws-environment/core-concepts.html) and [GCP](https://cloud.google.com/resource-manager/docs/cloud-platform-resource-hierarchy#organizations). - -Organizations work under the following assumptions: - -1. Users care about what happens within their organizations. -1. Features need to work within an organization. -1. Only few features need to work across organizations. -1. Users understand that the majority of pages they view are only scoped to a single organization at a time. -1. Organizations are located on a single pod. - -![Term Organization](images/term-organization.png) - -#### Organization properties - -- Top-level namespaces belong to organizations -- Users can be members of different organizations -- Organizations are isolated from each other by default meaning that cross-namespace features will only work for namespaces that exist within a single organization -- User namespaces must not belong to an organization - -Discouraged synonyms: Billable entities, customers - -### Top-Level namespace - -A top-level namespace is the logical object container in the code that represents all groups, subgroups and projects that belong to an organization. - -A top-level namespace is the root of nested collection namespaces and projects. The namespace and its related entities form a tree-like hierarchy: Namespaces are the nodes of the tree, projects are the leaves. - -Example: - -`https://gitlab.com/gitlab-org/gitlab/`: - -- `gitlab-org` is a `top-level namespace`; the root for all groups and projects of an organization -- `gitlab` is a `project`; a project of the organization. - -Top-level namespaces may [be replaced by workspaces](https://gitlab.com/gitlab-org/gitlab/-/issues/368237#high-level-goals). This proposal only uses the term top-level namespaces as the workspace definition is ongoing. - -Discouraged synonyms: Root-level namespace - -![Term Top-level Namespace](images/term-top-level-namespace.png) - -#### Top-level namespace properties - -- Top-level namespaces belonging to an organization are located on the same Pod -- Top-level namespaces can interact with other top-level namespaces that belong to the same organization - -### Users - -Users are available globally and not restricted to a single Pod. Users can be members of many different organizations with varying permissions. Inside organizations, users can create multiple top-level namespaces. User activity is not limited to a single organization but their contributions (for example TODOs) are only aggregated within an organization. This avoids the need for aggregating across pods. - -#### User properties - -- Users are shared globally across all Pods -- Users can create multiple top-level namespaces -- Users can be a member of multiple top-level namespaces -- Users can be a member of multiple organizations -- Users can administer organizations -- User activity is aggregated in an organization -- Every user has one personal namespace - -## Goals - -### Scalability - -The main goal of this new shared-infrastructure architecture is to provide additional scalability for our SaaS Platform. GitLab.com is largely monolithic and we have estimated (internal) that the current architecture has scalability limitations, even when database partitioning and decomposition are taken into account. - -Pods provide a horizontally scalable solution because additional Pods can be created based on demand. Pods can be provisioned and tuned as needed for optimal scalability. - -### Increased availability - -A major challenge for shared-infrastructure architectures is a lack of isolation between top-level namespaces. This can lead to noisy neighbor effects. A organization's behavior inside a top-level namespace can impact all other organizations. This is highly undesirable. Pods provide isolation at the pod level. A group of organizations is fully isolated from other organizations located on a different Pod. This minimizes noisy neighbor effects while still benefiting from the cost-efficiency of shared infrastructure. - -Additionally, Pods provide a way to implement disaster recovery capabilities. Entire Pods may be replicated to read-only standbys with automatic failover capabilities. - -### A consistent experience - -Organizations should have the same user experience on our SaaS platform as they do on a self-managed GitLab instance. - -### Regions - -GitLab.com is only hosted within the United States of America. Organizations located in other regions have voiced demand for local SaaS offerings. Pods provide a path towards [GitLab Regions](https://gitlab.com/groups/gitlab-org/-/epics/6037) because Pods may be deployed within different geographies. Depending on which of the organization's data is located outside a Pod, this may solve data residency and compliance problems. - -## Market segment - -Pods would provide a solution for organizations in the small to medium business (up to 100 users) and the mid-market segment (up to 2000 users). -(See [segmentation definitions](https://about.gitlab.com/handbook/sales/field-operations/gtm-resources/#segmentation).) -Larger organizations may benefit substantially from [GitLab Dedicated](../../../subscriptions/gitlab_dedicated/index.md). - -At this moment, GitLab.com has "social-network"-like capabilities that may not fit well into a more isolated organization model. Removing those features, however, possesses some challenges: - -1. How will existing `gitlab-org` contributors contribute to the namespace?? -1. How do we move existing top-level namespaces into the new model (effectively breaking their social features)? - -We should evaluate if the SMB and mid market segment is interested in these features, or if not having them is acceptable in most cases. - -### Self-managed - -For reasons of consistency, it is expected that self-managed instances will -adopt the pods architecture as well. To expand, self-managed instances can -continue with just a single Pod while supporting the option of adding additional -Pods. Organizations, and possible User decomposition will also be adopted for -self-managed instances. - -## High-level architecture problems to solve - -A number of technical issues need to be resolved to implement Pods (in no particular order). This section will be expanded. - -1. How are users of an organization routed to the correct Pod? -1. How do users authenticate? -1. How are Pods rebalanced? -1. How are Pods provisioned? -1. How can Pods implement disaster recovery capabilities? - -## Cross-section impact - -Pods is a fundamental architecture change that impacts other sections and stages. This section summarizes and links to other groups that may be impacted and highlights potential conflicts that need to be resolved. The Pods group is not responsible for achieving the goals of other groups but we want to ensure that dependencies are resolved. - -### Summary - -Based on discussions with other groups the net impact of introducing Pods and a new entity called organizations is mostly neutral. It may slow down development in some areas. We did not discover major blockers for other teams. - -1. We need to resolve naming conflicts (proposal is TBD) -1. Pods requires introducing Organizations. Organizations are a new entity **above** top-level groups. Because this is a new entity, it may impact the ability to consolidate settings for Group Workspace and influence their decision on [how to approach introducing a workspace](https://gitlab.com/gitlab-org/gitlab/-/issues/376285#approach-2-workspace-is-built-on-top-of-top-level-groups) -1. Organizations may make it slightly easier for Fulfillment to realize their billing plans. - -### Impact on Group Manage Workspace - -We synced with the Workspace PM and Designer ([recording](https://youtu.be/b5Opn9cFWFk)) and discussed the similarities and differences between the Pods and Workspace proposal ([presentation](https://docs.google.com/presentation/d/1FsUi22Up15b_tu6p2m-yLML3hCZ3rgrZrmzJAxUsNmU/edit?usp=sharing)). - -#### Goals of Group Manage Workspace - -As defined in the [workspace documentation](../../../user/workspace/index.md): - -1. Create an entity to manage everything you do as a GitLab administrator, including: - 1. Defining and applying settings to all of your groups, subgroups, and projects. - 1. Aggregating data from all your groups, subgroups, and projects. -1. Reach feature parity between SaaS and self-managed installations, with all Admin Area settings moving to groups (?). Hardware controls remain on the instance level. - -The [workspace roadmap outlines](https://gitlab.com/gitlab-org/gitlab/-/issues/368237#high-level-goals) the current goals in detail. - -#### Potential conflicts with Pods - -- Workspace and Organization are different terms for the same entity. Both define a new entity as the primary organizational object for groups and projects. This is mainly a semantic difference and **we need to decide on a name** following [user research to decide if workspace](https://gitlab.com/gitlab-org/ux-research/-/issues/2147). This is also driven by the fact that the Remote Development team is looking at better names and [are considering the term Workspace as well](https://gitlab.com/gitlab-com/Product/-/issues/4812). -- We will only introduce one entity -- Group workspace highlighted the need to further validate the key assumption that users only care about what happens within their organization. - -### Impact on Fulfillment - -We synced with Fulfillment ([recording](https://youtu.be/FkQF3uF7vTY)) to discuss how Pods would impact them. Fulfillment is supportive of an entity above top-level namespaces. Their perspective is outline in [!5639](https://gitlab.com/gitlab-org/customers-gitlab-com/-/merge_requests/5639/diffs). - -#### Goals of Fulfillment - -- Fulfillment has a longstanding plan to move billing from the top-level namespace to a level above. This would mean that a license applies for an organization and all its top-level namespaces. -- Fulfillment uses Zuora for billing and would like to have a 1-to-1 relationship between an organization and their Zuora entity called BillingAccount. They want to move away from tying a license to a single user. -- If a customer needs multiple organizations, the corresponding BillingAccounts can be rolled up into a consolidated billing account (similar to [AWS consolidated billing](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/consolidated-billing.html)) -- Ideally, a self-managed instance has a single Organization by default, which should be enough for most customers. -- Fulfillment prefers only one additional entity. - -A rough representation of this is: - -![Pods and Fulfillment](images/pods-and-fulfillment.png) - -#### Potential conflicts with Pods - -- There are no known conflicts between Fulfillment's plans and Pods - -## Iteration plan - -We can't ship the entire Pods architecture in one go - it is too large. Instead, we are adopting an iteration plan that provides value along the way. - -1. Introduce organizations -1. Migrate existing top-level namespaces to organizations -1. Create new organizations on `pod_0` -1. Migrate existing organizations from `pod_0` to `pod_n` -1. Add additional Pod capabilities (DR, Regions) - -### Iteration 0: Introduce organizations - -In the first iteration, we introduce the concept of an organization -as a way to group top-level namespaces together. Support for organizations **does not require any Pods work** but having them will make all subsequent iterations of Pods simpler. This is mainly because we can group top-level namespaces for a single organization onto a Pod. Within an organization all interactions work as normal but we eliminate any cross-organizational interactions except in well defined cases (e.g. forking). - -This means that we don't have a large number of cross-pod interactions. - -Introducing organizations allows GitLab to move towards a multi-tenant system that is similar to Discord's with a single user account but many different "servers" - our organizations - that allow users to switch context. This model harmonizes the UX across self-managed and our SaaS Platforms and is a good fit for Pods. - -Organizations solve the following problems: - -1. We can group top-level namespaces by organization. It is very similar to the initial concept of "instance groups". For example these two top-level namespaces would belong to the organization `GitLab`: - 1. `https://gitlab.com/gitlab-org/` - 1. `https://gitlab.com/gitlab-com/` -1. We can isolate organizations from each other. Top-level namespaces of the same organization can interact within organizations but are not allowed to interact with other namespaces in other organizations. This is useful for customers because it means an organization provides clear boundaries - similar to a self-managed instance. This means we don't have to aggregate user dashboards across everything and can locally scope them to organizations. -1. We don't need to define hierarchies inside an organization. It is a container that could be filled with whatever hierarchy / entity set makes sense (workspaces, top-level namespaces etc.) -1. Self-managed instances would set a default organization. -1. Organizations can control user-profiles in a central way. This could be achieved by having an organization specific user-profile. Such a profile makes it possible for the organization administrators to control the user role in a company, enforce user emails, or show a graphical indicator of a user being part of the organization. An example would be a "GitLab Employee stamp" on comments. - -![Move to Organizations](images/iteration0-organizations-introduction.png) - -#### Why would customers opt-in to Organizations? - -By introducing organizations and Pods we can improve the reliability, performance and availability of our SaaS Platforms. - -The first iteration of organizations would also have some benefits by providing more isolation. A simple example would be that `@` mentions could be scoped to an organization. - -Future iterations would create additional value but are beyond the scope of this blueprint. - -Organizations will likely be required in the future as well. - -#### Initial user experience - -1. We create a default `GitLab.com public` organization and assign all public top-level namespaces to it. This allows existing users to access all the data on GitLab.com, exactly as it does now. -1. Any user wanting to opt-in to the benefits of organizations will need to set a single default organization. Any attempts for these users to load a global page like `/dashboard` will end up redirecting to `/-/organizations//dashboard`. -1. New users that opted in to organizations will only ever see data that is related to a single organization. Upon login, data is shown for the default organization. It will be clear to the user how they can switch to a different organization. Users can still navigate to the `GitLab.com` organization but they won't see TODOs from their new organizations in any such views. Instead they'd need to navigate directly to `/organizations/my-company/-/dashboard`. - -### Migrating to Organizations - -Existing customers could also opt-in to migrate their existing top-level paid namespaces to become part of an organization. In most cases this will be a 1-to-1 mapping. But in some cases it may allow a customer to move multiple top-level namespaces into one organization (for example GitLab). - -Migrating to Organizations would be optional. We could even recruit a few beta testers early on to see if this works for them. GitLab itself could dogfood organizations and we'd surface a lot of issues restricting interactions with other namespaces. - -## Iteration 1 - Introduce Pod US 0 - -### GitLab.com as Pod US0 - -GitLab.com will be treated as the first pod `Pod US 0`. It will be unique and much larger compared to newly created pods. All existing top-level namespaces and organizations will remain on `Pod US 0` in the first iteration. - -### Users are globally available - -Users are globally available and the same for all pods. This means that user data needs to be handled separately, for example via decomposition, see [!95941](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/95941). - -### Pod groundwork - -In this iteration, we'll lay all the groundwork to support a second Pod for new organizations. This will be transparent to customers. - -## Iteration 2 - Introduce Pod US 1 - -### Add new organizations to Pod US 1 - -After we are ready to support a second Pod, newly created organizations are located by default on `Pod US 1`. The user experience for organizations is already well established. - -### Migrate existing organizations from Pod US 0 to Pod US 1 - -We know that we'll have to move organizations from `Pod US 0` to other pods to reduce its size and ultimately retire the existing GitLab.com architecture. - -By introducing organizations early, we should be able to draw strong "boundaries" across organizations and support migrating existing organizations to a new Pod. - -This is likely going to be GitLab itself - if we can dogfood this, we are likely going to be successful with other organizations as well. - -## Iteration 3 - Introduce Regions - -We can now leverage the Pods architecture to introduce Regions. - -## Iteration 4 - Introduce cross-organizational interactions as needed - -Based on user research, we may want to change certain features to work across organizations. Examples include: - -- Specific features allow for cross-organization interactions, for example forking, search. - -## Technical Proposals - -The Pods architecture do have long lasting implications to data processing, location, scalability and the GitLab architecture. -This section links all different technical proposals that are being evaluated. - -- [Stateless Router That Uses a Cache to Pick Pod and Is Redirected When Wrong Pod Is Reached](proposal-stateless-router-with-buffering-requests.md) - -- [Stateless Router That Uses a Cache to Pick Pod and pre-flight `/api/v4/pods/learn`](proposal-stateless-router-with-routes-learning.md) - -## Impacted features - -The Pods architecture will impact many features requiring some of them to be rewritten, or changed significantly. -This is the list of known affected features with the proposed solutions. - -- [Pods: Git Access](pods-feature-git-access.md) -- [Pods: Data Migration](pods-feature-data-migration.md) -- [Pods: Database Sequences](pods-feature-database-sequences.md) -- [Pods: GraphQL](pods-feature-graphql.md) -- [Pods: Organizations](pods-feature-organizations.md) -- [Pods: Router Endpoints Classification](pods-feature-router-endpoints-classification.md) -- [Pods: Schema changes (Postgres and Elasticsearch migrations)](pods-feature-schema-changes.md) -- [Pods: Backups](pods-feature-backups.md) -- [Pods: Global Search](pods-feature-global-search.md) -- [Pods: CI Runners](pods-feature-ci-runners.md) -- [Pods: Admin Area](pods-feature-admin-area.md) -- [Pods: Secrets](pods-feature-secrets.md) -- [Pods: Container Registry](pods-feature-container-registry.md) -- [Pods: Contributions: Forks](pods-feature-contributions-forks.md) -- [Pods: Personal Namespaces](pods-feature-personal-namespaces.md) -- [Pods: Dashboard: Projects, Todos, Issues, Merge Requests, Activity, ...](pods-feature-dashboard.md) -- [Pods: Snippets](pods-feature-snippets.md) -- [Pods: Uploads](pods-feature-uploads.md) -- [Pods: GitLab Pages](pods-feature-gitlab-pages.md) -- [Pods: Agent for Kubernetes](pods-feature-agent-for-kubernetes.md) - -## Links - -- [Internal Pods presentation](https://docs.google.com/presentation/d/1x1uIiN8FR9fhL7pzFh9juHOVcSxEY7d2_q4uiKKGD44/edit#slide=id.ge7acbdc97a_0_155) -- [Pods Epic](https://gitlab.com/groups/gitlab-org/-/epics/7582) -- [Database Group investigation](https://about.gitlab.com/handbook/engineering/development/enablement/data_stores/database/doc/root-namespace-sharding.html) -- [Shopify Pods architecture](https://shopify.engineering/a-pods-architecture-to-allow-shopify-to-scale) -- [Opstrace architecture](https://gitlab.com/gitlab-org/opstrace/opstrace/-/blob/main/docs/architecture/overview.md) + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-admin-area.md b/doc/architecture/blueprints/pods/pods-feature-admin-area.md index 7efaa383510..0f02a4a88ba 100644 --- a/doc/architecture/blueprints/pods/pods-feature-admin-area.md +++ b/doc/architecture/blueprints/pods/pods-feature-admin-area.md @@ -1,58 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: Admin Area' +redirect_to: '../cells/cells-feature-admin-area.md' +remove_date: '2023-06-13' --- -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. +This document was moved to [another location](../cells/cells-feature-admin-area.md). -# Pods: Admin Area - -In our Pods architecture proposal we plan to share all admin related tables in -GitLab. This allows simpler management of all Pods in one interface and reduces -the risk of settings diverging in different Pods. This introduces challenges -with admin pages that allow you to manage data that will be spread across all -Pods. - -## 1. Definition - -There are consequences for admin pages that contain data that spans "the whole -instance" as the Admin pages may be served by any Pod or possibly just 1 pod. -There are already many parts of the Admin interface that will have data that -spans many pods. For example lists of all Groups, Projects, Topics, Jobs, -Analytics, Applications and more. There are also administrative monitoring -capabilities in the Admin page that will span many pods such as the "Background -Jobs" and "Background Migrations" pages. - -## 2. Data flow - -## 3. Proposal - -We will need to decide how to handle these exceptions with a few possible -options: - -1. Move all these pages out into a dedicated per-pod Admin section. Probably - the URL will need to be routable to a single Pod like `/pods//admin`, - then we can display this data per Pod. These pages will be distinct from - other Admin pages which control settings that are shared across all Pods. We - will also need to consider how this impacts self-managed customers and - whether, or not, this should be visible for single-pod instances of GitLab. -1. Build some aggregation interfaces for this data so that it can be fetched - from all Pods and presented in a single UI. This may be beneficial to an - administrator that needs to see and filter all data at a glance, especially - when they don't know which Pod the data is on. The downside, however, is - that building this kind of aggregation is very tricky when all the Pods are - designed to be totally independent, and it does also enforce more strict - requirements on compatibility between Pods. - -## 4. Evaluation - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-agent-for-kubernetes.md b/doc/architecture/blueprints/pods/pods-feature-agent-for-kubernetes.md index f390c751b8b..f28cc447e0a 100644 --- a/doc/architecture/blueprints/pods/pods-feature-agent-for-kubernetes.md +++ b/doc/architecture/blueprints/pods/pods-feature-agent-for-kubernetes.md @@ -1,29 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: Agent for Kubernetes' +redirect_to: '../cells/cells-feature-agent-for-kubernetes.md' +remove_date: '2023-06-13' --- -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. +This document was moved to [another location](../cells/cells-feature-agent-for-kubernetes.md). -# Pods: Agent for Kubernetes - -> TL;DR - -## 1. Definition - -## 2. Data flow - -## 3. Proposal - -## 4. Evaluation - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-backups.md b/doc/architecture/blueprints/pods/pods-feature-backups.md index 5e4de42f473..db22317cf75 100644 --- a/doc/architecture/blueprints/pods/pods-feature-backups.md +++ b/doc/architecture/blueprints/pods/pods-feature-backups.md @@ -1,61 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: Backups' +redirect_to: '../cells/cells-feature-backups.md' +remove_date: '2023-06-13' --- -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. +This document was moved to [another location](../cells/cells-feature-backups.md). -# Pods: Backups - -Each pods will take its own backups, and consequently have its own isolated -backup / restore procedure. - -## 1. Definition - -GitLab Backup takes a backup of the PostgreSQL database used by the application, -and also Git repository data. - -## 2. Data flow - -Each pod has a number of application databases to back up (e.g. `main`, and `ci`). - -Additionally, there may be cluster-wide metadata tables (e.g. `users` table) -which is directly accesible via PostgreSQL. - -## 3. Proposal - -### 3.1. Cluster-wide metadata - -It is currently unknown how cluster-wide metadata tables will be accessible. We -may choose to have cluster-wide metadata tables backed up separately, or have -each pod back up its copy of cluster-wide metdata tables. - -### 3.2 Consistency - -#### 3.2.1 Take backups independently - -As each pod will communicate with each other via API, and there will be no joins -to the users table, it should be acceptable for each pod to take a backup -independently of each other. - -#### 3.2.2 Enforce snapshots - -We can require that each pod take a snapshot for the PostgreSQL databases at -around the same time to allow for a consistent-enough backup. - -## 4. Evaluation - -As the number of pods increases, it will likely not be feasible to take a -snapshot at the same time for all pods. Hence taking backups independently is -the better option. - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-ci-runners.md b/doc/architecture/blueprints/pods/pods-feature-ci-runners.md index b75515a916f..1985bb21884 100644 --- a/doc/architecture/blueprints/pods/pods-feature-ci-runners.md +++ b/doc/architecture/blueprints/pods/pods-feature-ci-runners.md @@ -1,169 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: CI Runners' +redirect_to: '../cells/cells-feature-ci-runners.md' +remove_date: '2023-06-13' --- -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. +This document was moved to [another location](../cells/cells-feature-ci-runners.md). -# Pods: CI Runners - -GitLab in order to execute CI jobs [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner/), -very often managed by customer in their infrastructure. - -All CI jobs created as part of CI pipeline are run in a context of project -it poses a challenge how to manage GitLab Runners. - -## 1. Definition - -There are 3 different types of runners: - -- instance-wide: runners that are registered globally with specific tags (selection criteria) -- group runners: runners that execute jobs from a given top-level group or subprojects of that group -- project runners: runners that execute jobs from projects or many projects: some runners might - have projects assigned from projects in different top-level groups. - -This alongside with existing data structure where `ci_runners` is a table describing -all types of runners poses a challenge how the `ci_runners` should be managed in a Pods environment. - -## 2. Data flow - -GitLab Runners use a set of globally scoped endpoints to: - -- registration of a new runner via registration token `https://gitlab.com/api/v4/runners` - ([subject for removal](../runner_tokens/index.md)) (`registration token`) -- requests jobs via an authenticated `https://gitlab.com/api/v4/jobs/request` endpoint (`runner token`) -- upload job status via `https://gitlab.com/api/v4/jobs/:job_id` (`build token`) -- upload trace via `https://gitlab.com/api/v4/jobs/:job_id/trace` (`build token`) -- download and upload artifacts via `https://gitlab.com/api/v4/jobs/:job_id/artifacts` (`build token`) - -Currently three types of authentication tokens are used: - -- runner registration token ([subject for removal](../runner_tokens/index.md)) -- runner token representing an registered runner in a system with specific configuration (`tags`, `locked`, etc.) -- build token representing an ephemeral token giving a limited access to updating a specific - job, uploading artifacts, downloading dependent artifacts, downloading and uploading - container registry images - -Each of those endpoints do receive an authentication token via header (`JOB-TOKEN` for `/trace`) -or body parameter (`token` all other endpoints). - -Since the CI pipeline would be created in a context of a specific Pod it would be required -that pick of a build would have to be processed by that particular Pod. This requires -that build picking depending on a solution would have to be either: - -- routed to correct Pod for a first time -- be made to be two phase: request build from global pool, claim build on a specific Pod using a Pod specific URL - -## 3. Proposal - -This section describes various proposals. Reader should consider that those -proposals do describe solutions for different problems. Many or some aspects -of those proposals might be the solution to the stated problem. - -### 3.1. Authentication tokens - -Even though the paths for CI Runners are not routable they can be made routable with -those two possible solutions: - -- The `https://gitlab.com/api/v4/jobs/request` uses a long polling mechanism with - a ticketing mechanism (based on `X-GitLab-Last-Update` header). Runner when first - starts sends a request to GitLab to which GitLab responds with either a build to pick - by runner. This value is completely controlled by GitLab. This allows GitLab - to use JWT or any other means to encode `pod` identifier that could be easily - decodable by Router. -- The majority of communication (in terms of volume) is using `build token` making it - the easiest target to change since GitLab is sole owner of the token that Runner later - uses for specific job. There were prior discussions about not storing `build token` - but rather using `JWT` token with defined scopes. Such token could encode the `pod` - to which router could easily route all requests. - -### 3.2. Request body - -- The most of used endpoints pass authentication token in request body. It might be desired - to use HTTP Headers as an easier way to access this information by Router without - a need to proxy requests. - -### 3.3. Instance-wide are Pod local - -We can pick a design where all runners are always registered and local to a given Pod: - -- Each Pod has it's own set of instance-wide runners that are updated at it's own pace -- The project runners can only be linked to projects from the same organization - creating strong isolation. -- In this model the `ci_runners` table is local to the Pod. -- In this model we would require the above endpoints to be scoped to a Pod in some way - or made routable. It might be via prefixing them, adding additional Pod parameter, - or providing much more robust way to decode runner token and match it to Pod. -- If routable token is used, we could move away from cryptographic random stored in - database to rather prefer to use JWT tokens that would encode -- The Admin Area showing registered Runners would have to be scoped to a Pod - -This model might be desired since it provides strong isolation guarantees. -This model does significantly increase maintenance overhead since each Pod is managed -separately. - -This model may require adjustments to runner tags feature so that projects have consistent runner experience across pods. - -### 3.4. Instance-wide are cluster-wide - -Contrary to proposal where all runners are Pod local, we can consider that runners -are global, or just instance-wide runners are global. - -However, this requires significant overhaul of system and to change the following aspects: - -- `ci_runners` table would likely have to be split decomposed into `ci_instance_runners`, ... -- all interfaces would have to be adopted to use correct table -- build queuing would have to be reworked to be two phase where each Pod would know of all pending - and running builds, but the actual claim of a build would happen against a Pod containing data -- likely `ci_pending_builds` and `ci_running_builds` would have to be made `cluster-wide` tables - increasing likelihood of creating hotspots in a system related to CI queueing - -This model makes it complex to implement from engineering side. Does make some data being shared -between Pods. Creates hotspots / scalability issues in a system (ex. during abuse) that -might impact experience of organizations on other Pods. - -### 3.5. GitLab CI Daemon - -Another potential solution to explore is to have a dedicated service responsible for builds queueing -owning it's database and working in a model of either sharded or podded service. There were prior -discussions about [CI/CD Daemon](https://gitlab.com/gitlab-org/gitlab/-/issues/19435). - -If the service would be sharded: - -- depending on a model if runners are cluster-wide or pod-local this service would have to fetch - data from all Pods -- if the sharded service would be used we could adapt a model of either sharing database containing - `ci_pending_builds/ci_running_builds` with the service -- if the sharded service would be used we could consider a push model where each Pod pushes to CI/CD Daemon - builds that should be picked by Runner -- the sharded service would be aware which Pod is responsible for processing the given build and could - route processing requests to designated Pod - -If the service would be podded: - -- all expectations of routable endpoints are still valid - -In general usage of CI Daemon does not help significantly with the stated problem. However, this offers -a few upsides related to more efficient processing and decoupling model: push model and it opens a way -to offer stateful communication with GitLab Runners (ex. gRPC or Websockets). - -## 4. Evaluation - -Considering all solutions it appears that solution giving the most promise is: - -- use "instance-wide are Pod local" -- refine endpoints to have routable identities (either via specific paths, or better tokens) - -Other potential upsides is to get rid of `ci_builds.token` and rather use a `JWT token` -that can much better and easier encode wider set of scopes allowed by CI runner. - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-container-registry.md b/doc/architecture/blueprints/pods/pods-feature-container-registry.md index d47913fbc2a..9d2bbb3febe 100644 --- a/doc/architecture/blueprints/pods/pods-feature-container-registry.md +++ b/doc/architecture/blueprints/pods/pods-feature-container-registry.md @@ -1,131 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: Container Registry' +redirect_to: '../cells/cells-feature-container-registry.md' +remove_date: '2023-06-13' --- -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. +This document was moved to [another location](../cells/cells-feature-container-registry.md). -# Pods: Container Registry - -GitLab Container Registry is a feature allowing to store Docker Container Images -in GitLab. You can read about GitLab integration [here](../../../user/packages/container_registry/index.md). - -## 1. Definition - -GitLab Container Registry is a complex service requiring usage of PostgreSQL, Redis -and Object Storage dependencies. Right now there's undergoing work to introduce -[Container Registry Metadata](../container_registry_metadata_database/index.md) -to optimize data storage and image retention policies of Container Registry. - -GitLab Container Registry is serving as a container for stored data, -but on it's own does not authenticate `docker login`. The `docker login` -is executed with user credentials (can be `personal access token`) -or CI build credentials (ephemeral `ci_builds.token`). - -Container Registry uses data deduplication. It means that the same blob -(image layer) that is shared between many projects is stored only once. -Each layer is hashed by `sha256`. - -The `docker login` does request JWT time-limited authentication token that -is signed by GitLab, but validated by Container Registry service. The JWT -token does store all authorized scopes (`container repository images`) -and operation types (`push` or `pull`). A single JWT authentication token -can be have many authorized scopes. This allows container registry and client -to mount existing blobs from another scopes. GitLab responds only with -authorized scopes. Then it is up to GitLab Container Registry to validate -if the given operation can be performed. - -The GitLab.com pages are always scoped to project. Each project can have many -container registry images attached. - -Currently in case of GitLab.com the actual registry service is served -via `https://registry.gitlab.com`. - -The main identifiable problems are: - -- the authentication request (`https://gitlab.com/jwt/auth`) that is processed by GitLab.com -- the `https://registry.gitlab.com` that is run by external service and uses it's own data store -- the data deduplication, the Pods architecture with registry run in a Pod would reduce - efficiency of data storage - -## 2. Data flow - -### 2.1. Authorization request that is send by `docker login` - -```shell -curl \ - --user "username:password" \ - "https://gitlab/jwt/auth?client_id=docker&offline_token=true&service=container_registry&scope=repository:gitlab-org/gitlab-build-images:push,pull" -``` - -Result is encoded and signed JWT token. Second base64 encoded string (split by `.`) contains JSON with authorized scopes. - -```json -{"auth_type":"none","access":[{"type":"repository","name":"gitlab-org/gitlab-build-images","actions":["pull"]}],"jti":"61ca2459-091c-4496-a3cf-01bac51d4dc8","aud":"container_registry","iss":"omnibus-gitlab-issuer","iat":1669309469,"nbf":166} -``` - -### 2.2. Docker client fetching tags - -```shell -curl \ - -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ - -H "Authorization: Bearer token" \ - https://registry.gitlab.com/v2/gitlab-org/gitlab-build-images/tags/list - -curl \ - -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ - -H "Authorization: Bearer token" \ - https://registry.gitlab.com/v2/gitlab-org/gitlab-build-images/manifests/danger-ruby-2.6.6 -``` - -### 2.3. Docker client fetching blobs and manifests - -```shell -curl \ - -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ - -H "Authorization: Bearer token" \ - https://registry.gitlab.com/v2/gitlab-org/gitlab-build-images/blobs/sha256:a3f2e1afa377d20897e08a85cae089393daa0ec019feab3851d592248674b416 -``` - -## 3. Proposal - -### 3.1. Shard Container Registry separately to Pods architecture - -Due to it's architecture it extensive architecture and in general highly scalable -horizontal architecture it should be evaluated if the GitLab Container Registry -should be run not in Pod, but in a Cluster and be scaled independently. - -This might be easier, but would definitely not offer the same amount of data isolation. - -### 3.2. Run Container Registry within a Pod - -It appears that except `/jwt/auth` which would likely have to be processed by Router -(to decode `scope`) the container registry could be run as a local service of a Pod. - -The actual data at least in case of GitLab.com is not forwarded via registry, -but rather served directly from Object Storage / CDN. - -Its design encodes container repository image in a URL that is easily routable. -It appears that we could re-use the same stateless Router service in front of Container Registry -to serve manifests and blobs redirect. - -The only downside is increased complexity of managing standalone registry for each Pod, -but this might be desired approach. - -## 4. Evaluation - -There do not seem any theoretical problems with running GitLab Container Registry in a Pod. -Service seems that can be easily made routable to work well. - -The practical complexities are around managing complex service from infrastructure side. - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-contributions-forks.md b/doc/architecture/blueprints/pods/pods-feature-contributions-forks.md index 566ae50ec49..38bdef35329 100644 --- a/doc/architecture/blueprints/pods/pods-feature-contributions-forks.md +++ b/doc/architecture/blueprints/pods/pods-feature-contributions-forks.md @@ -1,120 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: Contributions: Forks' +redirect_to: '../cells/cells-feature-contributions-forks.md' +remove_date: '2023-06-13' --- -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. +This document was moved to [another location](../cells/cells-feature-contributions-forks.md). -# Pods: Contributions: Forks - -[Forking workflow](../../../user/project/repository/forking_workflow.md) allows users -to copy existing project sources into their own namespace of choice (personal or group). - -## 1. Definition - -[Forking workflow](../../../user/project/repository/forking_workflow.md) is common workflow -with various usage patterns: - -- allows users to contribute back to upstream project -- persist repositories into their personal namespace -- copy to make changes and release as modified project - -Forks allow users not having write access to parent project to make changes. The forking workflow -is especially important for the Open Source community which is able to contribute back -to public projects. However, it is equally important in some companies which prefer the strong split -of responsibilites and tighter access control. The access to project is restricted -to designated list of developers. - -Forks enable: - -- tigther control of who can modify the upstream project -- split of the responsibilites: parent project might use CI configuration connecting to production systems -- run CI pipelines in context of fork in much more restrictive environment -- consider all forks to be unveted which reduces risks of leaking secrets, or any other information - tied with the project - -The forking model is problematic in Pods architecture for following reasons: - -- Forks are clones of existing repositories, forks could be created across different organizations, Pods and Gitaly shards. -- User can create merge request and contribute back to upstream project, this upstream project might in a different organization and Pod. -- The merge request CI pipeline is to executed in a context of source project, but presented in a context of target project. - -## 2. Data flow - -## 3. Proposals - -### 3.1. Intra-Cluster forks - -This proposal makes us to implement forks as a intra-ClusterPod forks where communication is done via API -between all trusted Pods of a cluster: - -- Forks when created, they are created always in context of user choice of group. -- Forks are isolated from Organization. -- Organization or group owner could disable forking across organizations or forking in general. -- When a Merge Request is created it is created in context of target project, referencing - external project on another Pod. -- To target project the merge reference is transfered that is used for presenting information - in context of target project. -- CI pipeline is fetched in context of source project as it-is today, the result is fetched into - Merge Request of target project. -- The Pod holding target project internally uses GraphQL to fetch status of source project - and include in context of the information for merge request. - -Upsides: - -- All existing forks continue to work as-is, as they are treated as intra-Cluster forks. - -Downsides: - -- The purpose of Organizations is to provide strong isolation between organizations - allowing to fork across does break security boundaries. -- However, this is no different to ability of users today to clone repository to local computer - and push it to any repository of choice. -- Access control of source project can be lower than those of target project. System today - requires that in order to contribute back the access level needs to be the same for fork and upstream. - -### 3.2. Forks are created in a personal namespace of the current organization - -Instead of creating projects across organizations, the forks are created in a user personal namespace -tied with the organization. Example: - -- Each user that is part of organization receives their personal namespace. For example for `GitLab Inc.` - it could be `gitlab.com/organization/gitlab-inc/@ayufan`. -- The user has to fork into it's own personal namespace of the organization. -- The user has that many personal namespaces as many organizations it belongs to. -- The personal namespace behaves similar to currently offered personal namespace. -- The user can manage and create projects within a personal namespace. -- The organization can prevent or disable usage of personal namespaces disallowing forks. -- All current forks are migrated into personal namespace of user in Organization. -- All forks are part of to the organization. -- The forks are not federated features. -- The personal namespace and forked project do not share configuration with parent project. - -### 3.3. Forks are created as internal projects under current project - -Instead of creating projects across organizations, the forks are attachments to existing projects. -Each user forking a project receives their unique project. Example: - -- For project: `gitlab.com/gitlab-org/gitlab`, forks would be created in `gitlab.com/gitlab-org/gitlab/@kamil-gitlab`. -- Forks are created in a context of current organization, they do not cross organization boundaries - and are managed by the organization. -- Tied to the user (or any other user-provided name of the fork). -- The forks are not federated features. - -Downsides: - -- Does not answer how to handle and migrate all exisiting forks. -- Might share current group / project settings - breaking some security boundaries. - -## 4. Evaluation - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-dashboard.md b/doc/architecture/blueprints/pods/pods-feature-dashboard.md index e63d912b4c9..1d92b891aff 100644 --- a/doc/architecture/blueprints/pods/pods-feature-dashboard.md +++ b/doc/architecture/blueprints/pods/pods-feature-dashboard.md @@ -1,29 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: Dashboard' +redirect_to: '../cells/cells-feature-dashboard.md' +remove_date: '2023-06-13' --- -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. +This document was moved to [another location](../cells/cells-feature-dashboard.md). -# Pods: Dashboard - -> TL;DR - -## 1. Definition - -## 2. Data flow - -## 3. Proposal - -## 4. Evaluation - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-data-migration.md b/doc/architecture/blueprints/pods/pods-feature-data-migration.md index fbe97316dcc..c06006a86dc 100644 --- a/doc/architecture/blueprints/pods/pods-feature-data-migration.md +++ b/doc/architecture/blueprints/pods/pods-feature-data-migration.md @@ -1,130 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: Data migration' +redirect_to: '../cells/cells-feature-data-migration.md' +remove_date: '2023-06-13' --- -DISCLAIMER: -This page may contain information related to upcoming products, features and -functionality. It is important to note that the information presented is for -informational purposes only, so please do not rely on the information for -purchasing or planning purposes. Just like with all projects, the items -mentioned on the page are subject to change or delay, and the development, -release, and timing of any products, features, or functionality remain at the -sole discretion of GitLab Inc. +This document was moved to [another location](../cells/cells-feature-data-migration.md). -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. - -# Pods: Data migration - -It is essential for Pods architecture to provide a way to migrate data out of big Pods -into smaller ones. This describes various approaches to provide this type of split. - -We also need to handle for cases where data is already violating the expected -isolation constraints of Pods (ie. references cannot span multiple -organizations). We know that existing features like linked issues allowed users -to link issues across any projects regardless of their hierarchy. There are many -similar features. All of this data will need to be migrated in some way before -it can be split across different pods. This may mean some data needs to be -deleted, or the feature changed and modelled slightly differently before we can -properly split or migrate the organizations between pods. - -Having schema deviations across different Pods, which is a necessary -consequence of different databases, will also impact our ability to migrate -data between pods. Different schemas impact our ability to reliably replicate -data across pods and especially impact our ability to validate that the data is -correctly replicated. It might force us to only be able to move data between -pods when the schemas are all in sync (slowing down deployments and the -rebalancing process) or possibly only migrate from newer to older schemas which -would be complex. - -## 1. Definition - -## 2. Data flow - -## 3. Proposal - -### 3.1. Split large Pods - -A single Pod can only be divided into many Pods. This is based on principle -that it is easier to create exact clone of an existing Pod in many replicas -out of which some will be made authoritative once migrated. Keeping those -replicas up-to date with Pod 0 is also much easier due to pre-existing -replication solutions that can replicate the whole systems: Geo, PostgreSQL -physical replication, etc. - -1. All data of an organization needs to not be divided across many Pods. -1. Split should be doable online. -1. New Pods cannot contain pre-existing data. -1. N Pods contain exact replica of Pod 0. -1. The data of Pod 0 is live replicated to as many Pods it needs to be split. -1. Once consensus is achieved between Pod 0 and N-Pods the organizations to be migrated away - are marked as read-only cluster-wide. -1. The `routes` is updated on for all organizations to be split to indicate an authoritative - Pod holding the most recent data, like `gitlab-org` on `pod-100`. -1. The data for `gitlab-org` on Pod 0, and on other non-authoritative N-Pods are dormant - and will be removed in the future. -1. All accesses to `gitlab-org` on a given Pod are validated about `pod_id` of `routes` - to ensure that given Pod is authoritative to handle the data. - -#### More challenges of this proposal - -1. There is no streaming replication capability for Elasticsearch, but you could - snapshot the whole Elasticsearch index and recreate, but this takes hours. - It could be handled by pausing Elasticsearch indexing on the initial pod during - the migration as indexing downtime is not a big issue, but this still needs - to be coordinated with the migration process -1. Syncing Redis, Gitaly, CI Postgres, Main Postgres, registry Postgres, other - new data stores snapshots in an online system would likely lead to gaps - without a long downtime. You need to choose a sync point and at the sync - point you need to stop writes to perform the migration. The more data stores - there are to migrate at the same time the longer the write downtime for the - failover. We would also need to find a reliable place in the application to - actually block updates to all these systems with a high degree of - confidence. In the past we've only been confident by shutting down all rails - services because any rails process could write directly to any of these at - any time due to async workloads or other surprising code paths. -1. How to efficiently delete all the orphaned data. Locating all `ci_builds` - associated with half the organizations would be very expensive if we have to - do joins. We haven't yet determined if we'd want to store an `organization_id` - column on every table, but this is the kind of thing it would be helpful for. - -### 3.2. Migrate organization from an existing Pod - -This is different to split, as we intend to perform logical and selective replication -of data belonging to a single organization. - -Today this type of selective replication is only implemented by Gitaly where we can migrate -Git repository from a single Gitaly node to another with minimal downtime. - -In this model we would require identifying all resources belonging to a given organization: -database rows, object storage files, Git repositories, etc. and selectively copy them over -to another (likely) existing Pod importing data into it. Ideally ensuring that we can -perform logical replication live of all changed data, but change similarly to split -which Pod is authoritative for this organization. - -1. It is hard to identify all resources belonging to organization. -1. It requires either downtime for organization or a robust system to identify - live changes made. -1. It likely will require a full database structure analysis (more robust than project import/export) - to perform selective PostgreSQL logical replication. - -#### More challenges of this proposal - -1. Logical replication is still not performant enough to keep up with our - scale. Even if we could use logical replication we still don't have an - efficient way to filter data related to a single organization without - joining all the way to the `organizations` table which will slow down - logical replication dramatically. - -## 4. Evaluation - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-database-sequences.md b/doc/architecture/blueprints/pods/pods-feature-database-sequences.md index 0a8bb4d250e..9c4d6c5e290 100644 --- a/doc/architecture/blueprints/pods/pods-feature-database-sequences.md +++ b/doc/architecture/blueprints/pods/pods-feature-database-sequences.md @@ -1,94 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: Database Sequences' +redirect_to: '../cells/cells-feature-database-sequences.md' +remove_date: '2023-06-13' --- -DISCLAIMER: -This page may contain information related to upcoming products, features and -functionality. It is important to note that the information presented is for -informational purposes only, so please do not rely on the information for -purchasing or planning purposes. Just like with all projects, the items -mentioned on the page are subject to change or delay, and the development, -release, and timing of any products, features, or functionality remain at the -sole discretion of GitLab Inc. +This document was moved to [another location](../cells/cells-feature-database-sequences.md). -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. - -# Pods: Database Sequences - -GitLab today ensures that every database row create has unique ID, allowing -to access Merge Request, CI Job or Project by a known global ID. - -Pods will use many distinct and not connected databases, each of them having -a separate IDs for most of entities. - -It might be desirable to retain globally unique IDs for all database rows -to allow migrating resources between Pods in the future. - -## 1. Definition - -## 2. Data flow - -## 3. Proposal - -This are some preliminary ideas how we can retain unique IDs across the system. - -### 3.1. UUID - -Instead of using incremental sequences use UUID (128 bit) that is stored in database. - -- This might break existing IDs and requires adding UUID column for all existing tables. -- This makes all indexes larger as it requires storing 128 bit instead of 32/64 bit in index. - -### 3.2. Use Pod index encoded in ID - -Since significant number of tables already use 64 bit ID numbers we could use MSB to encode -Pod ID effectively enabling - -- This might limit amount of Pods that can be enabled in system, as we might decide to only - allocate 1024 possible Pod numbers. -- This might make IDs to be migratable between Pods, since even if entity from Pod 1 is migrated to Pod 100 - this ID would still be unique. -- If resources are migrated the ID itself will not be enough to decode Pod number and we would need - lookup table. -- This requires updating all IDs to 32 bits. - -### 3.3. Allocate sequence ranges from central place - -Each Pod might receive its own range of the sequences as they are consumed from a centrally managed place. -Once Pod consumes all IDs assigned for a given table it would be replenished and a next range would be allocated. -Ranges would be tracked to provide a faster lookup table if a random access pattern is required. - -- This might make IDs to be migratable between Pods, since even if entity from Pod 1 is migrated to Pod 100 - this ID would still be unique. -- If resources are migrated the ID itself will not be enough to decode Pod number and we would need - much more robust lookup table as we could be breaking previously assigned sequence ranges. -- This does not require updating all IDs to 64 bits. -- This adds some performance penalty to all `INSERT` statements in Postgres or at least from Rails as we need to check for the sequence number and potentially wait for our range to be refreshed from the ID server -- The available range will need to be stored and incremented in a centralized place so that concurrent transactions cannot possibly get the same value. - -### 3.4. Define only some tables to require unique IDs - -Maybe this is acceptable only for some tables to have a globally unique IDs. It could be projects, groups -and other top-level entities. All other tables like `merge_requests` would only offer Pod-local ID, -but when referenced outside it would rather use IID (an ID that is monotonic in context of a given resource, like project). - -- This makes the ID 10000 for `merge_requests` be present on all Pods, which might be sometimes confusing - as for uniqueness of the resource. -- This might make random access by ID (if ever needed) be impossible without using composite key, like: `project_id+merge_request_id`. -- This would require us to implement a transformation/generation of new ID if we need to migrate records to another pod. This can lead to very difficult migration processes when these IDs are also used as foreign keys for other records being migrated. -- If IDs need to change when moving between pods this means that any links to records by ID would no longer work even if those links included the `project_id`. -- If we plan to allow these ids to not be unique and change the unique constraint to be based on a composite key then we'd need to update all foreign key references to be based on the composite key - -## 4. Evaluation - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-git-access.md b/doc/architecture/blueprints/pods/pods-feature-git-access.md index 9bda2d1de9c..1a0df0f9569 100644 --- a/doc/architecture/blueprints/pods/pods-feature-git-access.md +++ b/doc/architecture/blueprints/pods/pods-feature-git-access.md @@ -1,163 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: Git Access' +redirect_to: '../cells/cells-feature-git-access.md' +remove_date: '2023-06-13' --- -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. +This document was moved to [another location](../cells/cells-feature-git-access.md). -# Pods: Git Access - -This document describes impact of Pods architecture on all Git access (over HTTPS and SSH) -patterns providing explanation of how potentially those features should be changed -to work well with Pods. - -## 1. Definition - -Git access is done through out the application. It can be an operation performed by the system -(read Git repository) or by user (create a new file via Web IDE, `git clone` or `git push` via command line). - -The Pods architecture defines that all Git repositories will be local to the Pod, -so no repository could be shared with another Pod. - -The Pods architecture will require that any Git operation done can only be handled by a Pod holding -the data. It means that any operation either via Web interface, API, or GraphQL needs to be routed -to the correct Pod. It means that any `git clone` or `git push` operation can only be performed -in a context of a Pod. - -## 2. Data flow - -The are various operations performed today by the GitLab on a Git repository. This describes -the data flow how they behave today to better represent the impact. - -It appears that Git access does require changes only to a few endpoints that are scoped to project. -There appear to be different types of repositories: - -- Project: assigned to Group -- Wiki: additional repository assigned to Project -- Design: similar to Wiki, additional repository assigned to Project -- Snippet: creates a virtual project to hold repository, likely tied to the User - -### 2.1. Git clone over HTTPS - -Execution of: `git clone` over HTTPS - -```mermaid -sequenceDiagram - User ->> Workhorse: GET /gitlab-org/gitlab.git/info/refs?service=git-upload-pack - Workhorse ->> Rails: GET /gitlab-org/gitlab.git/info/refs?service=git-upload-pack - Rails ->> Workhorse: 200 OK - Workhorse ->> Gitaly: RPC InfoRefsUploadPack - Gitaly ->> User: Response - User ->> Workhorse: POST /gitlab-org/gitlab.git/git-upload-pack - Workhorse ->> Gitaly: RPC PostUploadPackWithSidechannel - Gitaly ->> User: Response -``` - -### 2.2. Git clone over SSH - -Execution of: `git clone` over SSH - -```mermaid -sequenceDiagram - User ->> Git SSHD: ssh git@gitlab.com - Git SSHD ->> Rails: GET /api/v4/internal/authorized_keys - Rails ->> Git SSHD: 200 OK (list of accepted SSH keys) - Git SSHD ->> User: Accept SSH - User ->> Git SSHD: git clone over SSH - Git SSHD ->> Rails: POST /api/v4/internal/allowed?project=/gitlab-org/gitlab.git&service=git-upload-pack - Rails ->> Git SSHD: 200 OK - Git SSHD ->> Gitaly: RPC SSHUploadPackWithSidechannel - Gitaly ->> User: Response -``` - -### 2.3. Git push over HTTPS - -Execution of: `git push` over HTTPS - -```mermaid -sequenceDiagram - User ->> Workhorse: GET /gitlab-org/gitlab.git/info/refs?service=git-receive-pack - Workhorse ->> Rails: GET /gitlab-org/gitlab.git/info/refs?service=git-receive-pack - Rails ->> Workhorse: 200 OK - Workhorse ->> Gitaly: RPC PostReceivePack - Gitaly ->> Rails: POST /api/v4/internal/allowed?gl_repository=project-111&service=git-receive-pack - Gitaly ->> Rails: POST /api/v4/internal/pre_receive?gl_repository=project-111 - Gitaly ->> Rails: POST /api/v4/internal/post_receive?gl_repository=project-111 - Gitaly ->> User: Response -``` - -### 2.4. Git push over SSHD - -Execution of: `git clone` over SSH - -```mermaid -sequenceDiagram - User ->> Git SSHD: ssh git@gitlab.com - Git SSHD ->> Rails: GET /api/v4/internal/authorized_keys - Rails ->> Git SSHD: 200 OK (list of accepted SSH keys) - Git SSHD ->> User: Accept SSH - User ->> Git SSHD: git clone over SSH - Git SSHD ->> Rails: POST /api/v4/internal/allowed?project=/gitlab-org/gitlab.git&service=git-receive-pack - Rails ->> Git SSHD: 200 OK - Git SSHD ->> Gitaly: RPC ReceivePack - Gitaly ->> Rails: POST /api/v4/internal/allowed?gl_repository=project-111 - Gitaly ->> Rails: POST /api/v4/internal/pre_receive?gl_repository=project-111 - Gitaly ->> Rails: POST /api/v4/internal/post_receive?gl_repository=project-111 - Gitaly ->> User: Response -``` - -### 2.5. Create commit via Web - -Execution of `Add CHANGELOG` to repository: - -```mermaid -sequenceDiagram - Web ->> Puma: POST /gitlab-org/gitlab/-/create/main - Puma ->> Gitaly: RPC TreeEntry - Gitaly ->> Rails: POST /api/v4/internal/allowed?gl_repository=project-111 - Gitaly ->> Rails: POST /api/v4/internal/pre_receive?gl_repository=project-111 - Gitaly ->> Rails: POST /api/v4/internal/post_receive?gl_repository=project-111 - Gitaly ->> Puma: Response - Puma ->> Web: See CHANGELOG -``` - -## 3. Proposal - -The Pods stateless router proposal requires that any ambiguous path (that is not routable) -will be made to be routable. It means that at least the following paths will have to be updated -do introduce a routable entity (project, group, or organization). - -Change: - -- `/api/v4/internal/allowed` => `/api/v4/internal/projects//allowed` -- `/api/v4/internal/pre_receive` => `/api/v4/internal/projects//pre_receive` -- `/api/v4/internal/post_receive` => `/api/v4/internal/projects//post_receive` -- `/api/v4/internal/lfs_authenticate` => `/api/v4/internal/projects//lfs_authenticate` - -Where: - -- `gl_repository` can be `project-1111` (`Gitlab::GlRepository`) -- `gl_repository` in some cases might be a full path to repository as executed by GitLab Shell (`/gitlab-org/gitlab.git`) - -## 4. Evaluation - -Supporting Git repositories if a Pod can access only its own repositories does not appear to be complex. - -The one major complication is supporting snippets, but this likely falls in the same category as for the approach -to support user's personal namespaces. - -## 4.1. Pros - -1. The API used for supporting HTTPS/SSH and Hooks are well defined and can easily be made routable. - -## 4.2. Cons - -1. The sharing of repositories objects is limited to the given Pod and Gitaly node. -1. The across-Pods forks are likely impossible to be supported (discover: how this work today across different Gitaly node). + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-gitlab-pages.md b/doc/architecture/blueprints/pods/pods-feature-gitlab-pages.md index 932f996d8ba..4c7f162434e 100644 --- a/doc/architecture/blueprints/pods/pods-feature-gitlab-pages.md +++ b/doc/architecture/blueprints/pods/pods-feature-gitlab-pages.md @@ -1,29 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: GitLab Pages' +redirect_to: '../cells/cells-feature-gitlab-pages.md' +remove_date: '2023-06-13' --- -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. +This document was moved to [another location](../cells/cells-feature-gitlab-pages.md). -# Pods: GitLab Pages - -> TL;DR - -## 1. Definition - -## 2. Data flow - -## 3. Proposal - -## 4. Evaluation - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-global-search.md b/doc/architecture/blueprints/pods/pods-feature-global-search.md index 5ea863ac646..035e95219e4 100644 --- a/doc/architecture/blueprints/pods/pods-feature-global-search.md +++ b/doc/architecture/blueprints/pods/pods-feature-global-search.md @@ -1,47 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: Global search' +redirect_to: '../cells/cells-feature-global-search.md' +remove_date: '2023-06-13' --- -DISCLAIMER: -This page may contain information related to upcoming products, features and -functionality. It is important to note that the information presented is for -informational purposes only, so please do not rely on the information for -purchasing or planning purposes. Just like with all projects, the items -mentioned on the page are subject to change or delay, and the development, -release, and timing of any products, features, or functionality remain at the -sole discretion of GitLab Inc. +This document was moved to [another location](../cells/cells-feature-global-search.md). -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. - -# Pods: Global search - -When we introduce multiple Pods we intend to isolate all services related to -those Pods. This will include Elasticsearch which means our current global -search functionality will not work. It may be possible to implement aggregated -search across all pods, but it is unlikely to be performant to do fan-out -searches across all pods especially once you start to do pagination which -requires setting the correct offset and page number for each search. - -## 1. Definition - -## 2. Data flow - -## 3. Proposal - -Likely first versions of Pods will simply not support global searches and then -we may later consider if building global searches to support popular use cases -is worthwhile. - -## 4. Evaluation - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-graphql.md b/doc/architecture/blueprints/pods/pods-feature-graphql.md index 87c8391fbb3..f0f01a2b120 100644 --- a/doc/architecture/blueprints/pods/pods-feature-graphql.md +++ b/doc/architecture/blueprints/pods/pods-feature-graphql.md @@ -1,94 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: GraphQL' +redirect_to: '../cells/cells-feature-graphql.md' +remove_date: '2023-06-13' --- -DISCLAIMER: -This page may contain information related to upcoming products, features and -functionality. It is important to note that the information presented is for -informational purposes only, so please do not rely on the information for -purchasing or planning purposes. Just like with all projects, the items -mentioned on the page are subject to change or delay, and the development, -release, and timing of any products, features, or functionality remain at the -sole discretion of GitLab Inc. +This document was moved to [another location](../cells/cells-feature-graphql.md). -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. - -# Pods: GraphQL - -GitLab extensively uses GraphQL to perform efficient data query operations. -GraphQL due to it's nature is not directly routable. The way how GitLab uses -it calls the `/api/graphql` endpoint, and only query or mutation of body request -might define where the data can be accessed. - -## 1. Definition - -## 2. Data flow - -## 3. Proposal - -There are at least two main ways to implement GraphQL in Pods architecture. - -### 3.1. GraphQL routable by endpoint - -Change `/api/graphql` to `/api/organization//graphql`. - -- This breaks all existing usages of `/api/graphql` endpoint - since the API URI is changed. - -### 3.2. GraphQL routable by body - -As part of router parse GraphQL body to find a routable entity, like `project`. - -- This still makes the GraphQL query be executed only in context of a given Pod - and not allowing the data to be merged. - -```json -# Good example -{ - project(fullPath:"gitlab-org/gitlab") { - id - description - } -} - -# Bad example, since Merge Request is not routable -{ - mergeRequest(id: 1111) { - iid - description - } -} -``` - -### 3.3. Merging GraphQL Proxy - -Implement as part of router GraphQL Proxy which can parse body -and merge results from many Pods. - -- This might make pagination hard to achieve, or we might assume that - we execute many queries of which results are merged across all Pods. - -```json -{ - project(fullPath:"gitlab-org/gitlab"){ - id, description - } - group(fullPath:"gitlab-com") { - id, description - } -} -``` - -## 4. Evaluation - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-organizations.md b/doc/architecture/blueprints/pods/pods-feature-organizations.md index a0a87458767..f801f739374 100644 --- a/doc/architecture/blueprints/pods/pods-feature-organizations.md +++ b/doc/architecture/blueprints/pods/pods-feature-organizations.md @@ -1,58 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: Organizations' +redirect_to: '../cells/cells-feature-organizations.md' +remove_date: '2023-06-13' --- -DISCLAIMER: -This page may contain information related to upcoming products, features and -functionality. It is important to note that the information presented is for -informational purposes only, so please do not rely on the information for -purchasing or planning purposes. Just like with all projects, the items -mentioned on the page are subject to change or delay, and the development, -release, and timing of any products, features, or functionality remain at the -sole discretion of GitLab Inc. +This document was moved to [another location](../cells/cells-feature-organizations.md). -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. - -# Pods: Organizations - -One of the major designs of Pods architecture is strong isolation between Groups. -Organizations as described by this blueprint provides a way to have plausible UX -for joining together many Groups that are isolated from the rest of systems. - -## 1. Definition - -Pods do require that all groups and projects of a single organization can -only be stored on a single Pod since a Pod can only access data that it holds locally -and has very limited capabilities to read information from other Pods. - -Pods with Organizations do require strong isolation between organizations. - -It will have significant implications on various user-facing features, -like Todos, dropdowns allowing to select projects, references to other issues -or projects, or any other social functions present at GitLab. Today those functions -were able to reference anything in the whole system. With the introduction of -organizations such will be forbidden. - -This problem definition aims to answer effort and implications required to add -strong isolation between organizations to the system. Including features affected -and their data processing flow. The purpose is to ensure that our solution when -implemented consistently avoids data leakage between organizations residing on -a single Pod. - -## 2. Data flow - -## 3. Proposal - -## 4. Evaluation - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-personal-namespaces.md b/doc/architecture/blueprints/pods/pods-feature-personal-namespaces.md index f78044bb551..237eb5f9d64 100644 --- a/doc/architecture/blueprints/pods/pods-feature-personal-namespaces.md +++ b/doc/architecture/blueprints/pods/pods-feature-personal-namespaces.md @@ -1,29 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: Personal Namespaces' +redirect_to: '../cells/cells-feature-personal-namespaces.md' +remove_date: '2023-06-13' --- -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. +This document was moved to [another location](../cells/cells-feature-personal-namespaces.md). -# Pods: Personal Namespaces - -> TL;DR - -## 1. Definition - -## 2. Data flow - -## 3. Proposal - -## 4. Evaluation - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-router-endpoints-classification.md b/doc/architecture/blueprints/pods/pods-feature-router-endpoints-classification.md index bf0969fcb38..b9e85c29481 100644 --- a/doc/architecture/blueprints/pods/pods-feature-router-endpoints-classification.md +++ b/doc/architecture/blueprints/pods/pods-feature-router-endpoints-classification.md @@ -1,46 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: Router Endpoints Classification' +redirect_to: '../cells/cells-feature-router-endpoints-classification.md' +remove_date: '2023-06-13' --- -DISCLAIMER: -This page may contain information related to upcoming products, features and -functionality. It is important to note that the information presented is for -informational purposes only, so please do not rely on the information for -purchasing or planning purposes. Just like with all projects, the items -mentioned on the page are subject to change or delay, and the development, -release, and timing of any products, features, or functionality remain at the -sole discretion of GitLab Inc. +This document was moved to [another location](../cells/cells-feature-router-endpoints-classification.md). -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. - -# Pods: Router Endpoints Classification - -Classification of all endpoints is essential to properly route request -hitting load balancer of a GitLab installation to a Pod that can serve it. - -Each Pod should be able to decode each request and classify for which Pod -it belongs to. - -GitLab currently implements hundreds of endpoints. This document tries -to describe various techniques that can be implemented to allow the Rails -to provide this information efficiently. - -## 1. Definition - -## 2. Data flow - -## 3. Proposal - -## 4. Evaluation - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-schema-changes.md b/doc/architecture/blueprints/pods/pods-feature-schema-changes.md index ae7c288028d..a57f76ad9d4 100644 --- a/doc/architecture/blueprints/pods/pods-feature-schema-changes.md +++ b/doc/architecture/blueprints/pods/pods-feature-schema-changes.md @@ -1,55 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: Schema changes' +redirect_to: '../cells/cells-feature-schema-changes.md' +remove_date: '2023-06-13' --- -DISCLAIMER: -This page may contain information related to upcoming products, features and -functionality. It is important to note that the information presented is for -informational purposes only, so please do not rely on the information for -purchasing or planning purposes. Just like with all projects, the items -mentioned on the page are subject to change or delay, and the development, -release, and timing of any products, features, or functionality remain at the -sole discretion of GitLab Inc. +This document was moved to [another location](../cells/cells-feature-schema-changes.md). -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. - -# Pods: Schema changes - -When we introduce multiple Pods that own their own databases this will -complicate the process of making schema changes to Postgres and Elasticsearch. -Today we already need to be careful to make changes comply with our zero -downtime deployments. For example, -[when removing a column we need to make changes over 3 separate deployments](../../../development/database/avoiding_downtime_in_migrations.md#dropping-columns). -We have tooling like `post_migrate` that helps with these kinds of changes to -reduce the number of merge requests needed, but these will be complicated when -we are dealing with deploying multiple rails applications that will be at -different versions at any one time. This problem will be particularly tricky to -solve for shared databases like our plan to share the `users` related tables -among all Pods. - -A key benefit of Pods may be that it allows us to run different -customers on different versions of GitLab. We may choose to update our own pod -before all our customers giving us even more flexibility than our current -canary architecture. But doing this means that schema changes need to have even -more versions of backward compatibility support which could slow down -development as we need extra steps to make schema changes. - -## 1. Definition - -## 2. Data flow - -## 3. Proposal - -## 4. Evaluation - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-secrets.md b/doc/architecture/blueprints/pods/pods-feature-secrets.md index f18a41dc0fb..f33b98add21 100644 --- a/doc/architecture/blueprints/pods/pods-feature-secrets.md +++ b/doc/architecture/blueprints/pods/pods-feature-secrets.md @@ -1,48 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: Secrets' +redirect_to: '../cells/cells-feature-secrets.md' +remove_date: '2023-06-13' --- -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. +This document was moved to [another location](../cells/cells-feature-secrets.md). -# Pods: Secrets - -Where possible, each pod should have its own distinct set of secrets. -However, there will be some secrets that will be required to be the same for all -pods in the cluster - -## 1. Definition - -GitLab has a lot of -[secrets](https://docs.gitlab.com/charts/installation/secrets.html) that needs -to be configured. - -Some secrets are for inter-component communication, e.g. `GitLab Shell secret`, -and used only within a pod. - -Some secrets are used for features, e.g. `ci_jwt_signing_key`. - -## 2. Data flow - -## 3. Proposal - -1. Secrets used for features will need to be consistent across all pods, so that the UX is consistent. - 1. This is especially true for the `db_key_base` secret which is used for - encrypting data at rest in the database - so that projects that are - transferred to another pod will continue to work. We do not want to have - to re-encrypt such rows when we move projects/groups between pods. -1. Secrets which are used for intra-pod communication only should be uniquely generated - per-pod. - -## 4. Evaluation - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-snippets.md b/doc/architecture/blueprints/pods/pods-feature-snippets.md index 1bb866ca958..42d3c401dba 100644 --- a/doc/architecture/blueprints/pods/pods-feature-snippets.md +++ b/doc/architecture/blueprints/pods/pods-feature-snippets.md @@ -1,29 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: Snippets' +redirect_to: '../cells/cells-feature-snippets.md' +remove_date: '2023-06-13' --- -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. +This document was moved to [another location](../cells/cells-feature-snippets.md). -# Pods: Snippets - -> TL;DR - -## 1. Definition - -## 2. Data flow - -## 3. Proposal - -## 4. Evaluation - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-template.md b/doc/architecture/blueprints/pods/pods-feature-template.md index dfae21b5406..acc8e329725 100644 --- a/doc/architecture/blueprints/pods/pods-feature-template.md +++ b/doc/architecture/blueprints/pods/pods-feature-template.md @@ -1,29 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: Problem A' +redirect_to: '../cells/cells-feature-template.md' +remove_date: '2023-06-13' --- -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. +This document was moved to [another location](../cells/cells-feature-template.md). -# Pods: A - -> TL;DR - -## 1. Definition - -## 2. Data flow - -## 3. Proposal - -## 4. Evaluation - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/pods-feature-uploads.md b/doc/architecture/blueprints/pods/pods-feature-uploads.md index 634f3ef9560..1de4c138843 100644 --- a/doc/architecture/blueprints/pods/pods-feature-uploads.md +++ b/doc/architecture/blueprints/pods/pods-feature-uploads.md @@ -1,29 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods: Uploads' +redirect_to: '../cells/cells-feature-uploads.md' +remove_date: '2023-06-13' --- -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. +This document was moved to [another location](../cells/cells-feature-uploads.md). -# Pods: Uploads - -> TL;DR - -## 1. Definition - -## 2. Data flow - -## 3. Proposal - -## 4. Evaluation - -## 4.1. Pros - -## 4.2. Cons + + + + diff --git a/doc/architecture/blueprints/pods/proposal-stateless-router-with-buffering-requests.md b/doc/architecture/blueprints/pods/proposal-stateless-router-with-buffering-requests.md index adc523e90c2..4c135c5dbc3 100644 --- a/doc/architecture/blueprints/pods/proposal-stateless-router-with-buffering-requests.md +++ b/doc/architecture/blueprints/pods/proposal-stateless-router-with-buffering-requests.md @@ -1,648 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods Stateless Router Proposal' +redirect_to: '../cells/proposal-stateless-router-with-buffering-requests.md' +remove_date: '2023-06-13' --- -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. +This document was moved to [another location](../cells/proposal-stateless-router-with-buffering-requests.md). -# Proposal: Stateless Router - -We will decompose `gitlab_users`, `gitlab_routes` and `gitlab_admin` related -tables so that they can be shared between all pods and allow any pod to -authenticate a user and route requests to the correct pod. Pods may receive -requests for the resources they don't own, but they know how to redirect back -to the correct pod. - -The router is stateless and does not read from the `routes` database which -means that all interactions with the database still happen from the Rails -monolith. This architecture also supports regions by allowing for low traffic -databases to be replicated across regions. - -Users are not directly exposed to the concept of Pods but instead they see -different data dependent on their chosen "organization". -[Organizations](index.md#organizations) will be a new model introduced to enforce isolation in the -application and allow us to decide which request route to which pod, since an -organization can only be on a single pod. - -## Differences - -The main difference between this proposal and the one [with learning routes](proposal-stateless-router-with-routes-learning.md) -is that this proposal always sends requests to any of the Pods. If the requests cannot be processed, -the requests will be bounced back with relevant headers. This requires that request to be buffered. -It allows that request decoding can be either via URI or Body of request by Rails. -This means that each request might be sent more than once and be processed more than once as result. - -The [with learning routes proposal](proposal-stateless-router-with-routes-learning.md) requires that -routable information is always encoded in URI, and the router sends a pre-flight request. - -## Summary in diagrams - -This shows how a user request routes via DNS to the nearest router and the router chooses a pod to send the request to. - -```mermaid -graph TD; - user((User)); - dns[DNS]; - router_us(Router); - router_eu(Router); - pod_us0{Pod US0}; - pod_us1{Pod US1}; - pod_eu0{Pod EU0}; - pod_eu1{Pod EU1}; - user-->dns; - dns-->router_us; - dns-->router_eu; - subgraph Europe - router_eu-->pod_eu0; - router_eu-->pod_eu1; - end - subgraph United States - router_us-->pod_us0; - router_us-->pod_us1; - end -``` - -
More detail - -This shows that the router can actually send requests to any pod. The user will -get the closest router to them geographically. - -```mermaid -graph TD; - user((User)); - dns[DNS]; - router_us(Router); - router_eu(Router); - pod_us0{Pod US0}; - pod_us1{Pod US1}; - pod_eu0{Pod EU0}; - pod_eu1{Pod EU1}; - user-->dns; - dns-->router_us; - dns-->router_eu; - subgraph Europe - router_eu-->pod_eu0; - router_eu-->pod_eu1; - end - subgraph United States - router_us-->pod_us0; - router_us-->pod_us1; - end - router_eu-.->pod_us0; - router_eu-.->pod_us1; - router_us-.->pod_eu0; - router_us-.->pod_eu1; -``` - -
- -
Even more detail - -This shows the databases. `gitlab_users` and `gitlab_routes` exist only in the -US region but are replicated to other regions. Replication does not have an -arrow because it's too hard to read the diagram. - -```mermaid -graph TD; - user((User)); - dns[DNS]; - router_us(Router); - router_eu(Router); - pod_us0{Pod US0}; - pod_us1{Pod US1}; - pod_eu0{Pod EU0}; - pod_eu1{Pod EU1}; - db_gitlab_users[(gitlab_users Primary)]; - db_gitlab_routes[(gitlab_routes Primary)]; - db_gitlab_users_replica[(gitlab_users Replica)]; - db_gitlab_routes_replica[(gitlab_routes Replica)]; - db_pod_us0[(gitlab_main/gitlab_ci Pod US0)]; - db_pod_us1[(gitlab_main/gitlab_ci Pod US1)]; - db_pod_eu0[(gitlab_main/gitlab_ci Pod EU0)]; - db_pod_eu1[(gitlab_main/gitlab_ci Pod EU1)]; - user-->dns; - dns-->router_us; - dns-->router_eu; - subgraph Europe - router_eu-->pod_eu0; - router_eu-->pod_eu1; - pod_eu0-->db_pod_eu0; - pod_eu0-->db_gitlab_users_replica; - pod_eu0-->db_gitlab_routes_replica; - pod_eu1-->db_gitlab_users_replica; - pod_eu1-->db_gitlab_routes_replica; - pod_eu1-->db_pod_eu1; - end - subgraph United States - router_us-->pod_us0; - router_us-->pod_us1; - pod_us0-->db_pod_us0; - pod_us0-->db_gitlab_users; - pod_us0-->db_gitlab_routes; - pod_us1-->db_gitlab_users; - pod_us1-->db_gitlab_routes; - pod_us1-->db_pod_us1; - end - router_eu-.->pod_us0; - router_eu-.->pod_us1; - router_us-.->pod_eu0; - router_us-.->pod_eu1; -``` - -
- -## Summary of changes - -1. Tables related to User data (including profile settings, authentication credentials, personal access tokens) are decomposed into a `gitlab_users` schema -1. The `routes` table is decomposed into `gitlab_routes` schema -1. The `application_settings` (and probably a few other instance level tables) are decomposed into `gitlab_admin` schema -1. A new column `routes.pod_id` is added to `routes` table -1. A new Router service exists to choose which pod to route a request to. -1. A new concept will be introduced in GitLab called an organization and a user can select a "default organization" and this will be a user level setting. The default organization is used to redirect users away from ambiguous routes like `/dashboard` to organization scoped routes like `/organizations/my-organization/-/dashboard`. Legacy users will have a special default organization that allows them to keep using global resources on `Pod US0`. All existing namespaces will initially move to this public organization. -1. If a pod receives a request for a `routes.pod_id` that it does not own it returns a `302` with `X-Gitlab-Pod-Redirect` header so that the router can send the request to the correct pod. The correct pod can also set a header `X-Gitlab-Pod-Cache` which contains information about how this request should be cached to remember the pod. For example if the request was `/gitlab-org/gitlab` then the header would encode `/gitlab-org/* => Pod US0` (for example, any requests starting with `/gitlab-org/` can always be routed to `Pod US0` -1. When the pod does not know (from the cache) which pod to send a request to it just picks a random pod within it's region -1. Writes to `gitlab_users` and `gitlab_routes` are sent to a primary PostgreSQL server in our `US` region but reads can come from replicas in the same region. This will add latency for these writes but we expect they are infrequent relative to the rest of GitLab. - -## Detailed explanation of default organization in the first iteration - -All users will get a new column `users.default_organization` which they can -control in user settings. We will introduce a concept of the -`GitLab.com Public` organization. This will be set as the default organization for all existing -users. This organization will allow the user to see data from all namespaces in -`Pod US0` (for example, our original GitLab.com instance). This behavior can be invisible to -existing users such that they don't even get told when they are viewing a -global page like `/dashboard` that it's even scoped to an organization. - -Any new users with a default organization other than `GitLab.com Public` will have -a distinct user experience and will be fully aware that every page they load is -only ever scoped to a single organization. These users can never -load any global pages like `/dashboard` and will end up being redirected to -`/organizations//-/dashboard`. This may also be the case -for legacy APIs and such users may only ever be able to use APIs scoped to a -organization. - -## Detailed explanation of Admin Area settings - -We believe that maintaining and synchronizing Admin Area settings will be -frustrating and painful so to avoid this we will decompose and share all Admin Area -settings in the `gitlab_admin` schema. This should be safe (similar to other -shared schemas) because these receive very little write traffic. - -In cases where different pods need different settings (for example, the -Elasticsearch URL), we will either decide to use a templated -format in the relevant `application_settings` row which allows it to be dynamic -per pod. Alternatively if that proves difficult we'll introduce a new table -called `per_pod_application_settings` and this will have 1 row per pod to allow -setting different settings per pod. It will still be part of the `gitlab_admin` -schema and shared which will allow us to centrally manage it and simplify -keeping settings in sync for all pods. - -## Pros - -1. Router is stateless and can live in many regions. We use Anycast DNS to resolve to nearest region for the user. -1. Pods can receive requests for namespaces in the wrong pod and the user - still gets the right response as well as caching at the router that - ensures the next request is sent to the correct pod so the next request - will go to the correct pod -1. The majority of the code still lives in `gitlab` rails codebase. The Router doesn't actually need to understand how GitLab URLs are composed. -1. Since the responsibility to read and write `gitlab_users`, - `gitlab_routes` and `gitlab_admin` still lives in Rails it means minimal - changes will be needed to the Rails application compared to extracting - services that need to isolate the domain models and build new interfaces. -1. Compared to a separate routing service this allows the Rails application - to encode more complex rules around how to map URLs to the correct pod - and may work for some existing API endpoints. -1. All the new infrastructure (just a router) is optional and a single-pod - self-managed installation does not even need to run the Router and there are - no other new services. - -## Cons - -1. `gitlab_users`, `gitlab_routes` and `gitlab_admin` databases may need to be - replicated across regions and writes need to go across regions. We need to - do an analysis on write TPS for the relevant tables to determine if this is - feasible. -1. Sharing access to the database from many different Pods means that they are - all coupled at the Postgres schema level and this means changes to the - database schema need to be done carefully in sync with the deployment of all - Pods. This limits us to ensure that Pods are kept in closely similar - versions compared to an architecture with shared services that have an API - we control. -1. Although most data is stored in the right region there can be requests - proxied from another region which may be an issue for certain types - of compliance. -1. Data in `gitlab_users` and `gitlab_routes` databases must be replicated in - all regions which may be an issue for certain types of compliance. -1. The router cache may need to be very large if we get a wide variety of URLs - (for example, long tail). In such a case we may need to implement a 2nd level of - caching in user cookies so their frequently accessed pages always go to the - right pod the first time. -1. Having shared database access for `gitlab_users` and `gitlab_routes` - from multiple pods is an unusual architecture decision compared to - extracting services that are called from multiple pods. -1. It is very likely we won't be able to find cacheable elements of a - GraphQL URL and often existing GraphQL endpoints are heavily dependent on - ids that won't be in the `routes` table so pods won't necessarily know - what pod has the data. As such we'll probably have to update our GraphQL - calls to include an organization context in the path like - `/api/organizations//graphql`. -1. This architecture implies that implemented endpoints can only access data - that are readily accessible on a given Pod, but are unlikely - to aggregate information from many Pods. -1. All unknown routes are sent to the latest deployment which we assume to be `Pod US0`. - This is required as newly added endpoints will be only decodable by latest pod. - This Pod could later redirect to correct one that can serve the given request. - Since request processing might be heavy some Pods might receive significant amount - of traffic due to that. - -## Example database configuration - -Handling shared `gitlab_users`, `gitlab_routes` and `gitlab_admin` databases, while having dedicated `gitlab_main` and `gitlab_ci` databases should already be handled by the way we use `config/database.yml`. We should also, already be able to handle the dedicated EU replicas while having a single US primary for `gitlab_users` and `gitlab_routes`. Below is a snippet of part of the database configuration for the Pod architecture described above. - -
Pod US0 - -```yaml -# config/database.yml -production: - main: - host: postgres-main.pod-us0.primary.consul - load_balancing: - discovery: postgres-main.pod-us0.replicas.consul - ci: - host: postgres-ci.pod-us0.primary.consul - load_balancing: - discovery: postgres-ci.pod-us0.replicas.consul - users: - host: postgres-users-primary.consul - load_balancing: - discovery: postgres-users-replicas.us.consul - routes: - host: postgres-routes-primary.consul - load_balancing: - discovery: postgres-routes-replicas.us.consul - admin: - host: postgres-admin-primary.consul - load_balancing: - discovery: postgres-admin-replicas.us.consul -``` - -
- -
Pod EU0 - -```yaml -# config/database.yml -production: - main: - host: postgres-main.pod-eu0.primary.consul - load_balancing: - discovery: postgres-main.pod-eu0.replicas.consul - ci: - host: postgres-ci.pod-eu0.primary.consul - load_balancing: - discovery: postgres-ci.pod-eu0.replicas.consul - users: - host: postgres-users-primary.consul - load_balancing: - discovery: postgres-users-replicas.eu.consul - routes: - host: postgres-routes-primary.consul - load_balancing: - discovery: postgres-routes-replicas.eu.consul - admin: - host: postgres-admin-primary.consul - load_balancing: - discovery: postgres-admin-replicas.eu.consul -``` - -
- -## Request flows - -1. `gitlab-org` is a top level namespace and lives in `Pod US0` in the `GitLab.com Public` organization -1. `my-company` is a top level namespace and lives in `Pod EU0` in the `my-organization` organization - -### Experience for paying user that is part of `my-organization` - -Such a user will have a default organization set to `/my-organization` and will be -unable to load any global routes outside of this organization. They may load other -projects/namespaces but their MR/Todo/Issue counts at the top of the page will -not be correctly populated in the first iteration. The user will be aware of -this limitation. - -#### Navigates to `/my-company/my-project` while logged in - -1. User is in Europe so DNS resolves to the router in Europe -1. They request `/my-company/my-project` without the router cache, so the router chooses randomly `Pod EU1` -1. `Pod EU1` does not have `/my-company`, but it knows that it lives in `Pod EU0` so it redirects the router to `Pod EU0` -1. `Pod EU0` returns the correct response as well as setting the cache headers for the router `/my-company/* => Pod EU0` -1. The router now caches and remembers any request paths matching `/my-company/*` should go to `Pod EU0` - -```mermaid -sequenceDiagram - participant user as User - participant router_eu as Router EU - participant pod_eu0 as Pod EU0 - participant pod_eu1 as Pod EU1 - user->>router_eu: GET /my-company/my-project - router_eu->>pod_eu1: GET /my-company/my-project - pod_eu1->>router_eu: 302 /my-company/my-project X-Gitlab-Pod-Redirect={pod:Pod EU0} - router_eu->>pod_eu0: GET /my-company/my-project - pod_eu0->>user:

My Project... X-Gitlab-Pod-Cache={path_prefix:/my-company/} -``` - -#### Navigates to `/my-company/my-project` while not logged in - -1. User is in Europe so DNS resolves to the router in Europe -1. The router does not have `/my-company/*` cached yet so it chooses randomly `Pod EU1` -1. `Pod EU1` redirects them through a login flow -1. Still they request `/my-company/my-project` without the router cache, so the router chooses a random pod `Pod EU1` -1. `Pod EU1` does not have `/my-company`, but it knows that it lives in `Pod EU0` so it redirects the router to `Pod EU0` -1. `Pod EU0` returns the correct response as well as setting the cache headers for the router `/my-company/* => Pod EU0` -1. The router now caches and remembers any request paths matching `/my-company/*` should go to `Pod EU0` - -```mermaid -sequenceDiagram - participant user as User - participant router_eu as Router EU - participant pod_eu0 as Pod EU0 - participant pod_eu1 as Pod EU1 - user->>router_eu: GET /my-company/my-project - router_eu->>pod_eu1: GET /my-company/my-project - pod_eu1->>user: 302 /users/sign_in?redirect=/my-company/my-project - user->>router_eu: GET /users/sign_in?redirect=/my-company/my-project - router_eu->>pod_eu1: GET /users/sign_in?redirect=/my-company/my-project - pod_eu1->>user:

Sign in... - user->>router_eu: POST /users/sign_in?redirect=/my-company/my-project - router_eu->>pod_eu1: POST /users/sign_in?redirect=/my-company/my-project - pod_eu1->>user: 302 /my-company/my-project - user->>router_eu: GET /my-company/my-project - router_eu->>pod_eu1: GET /my-company/my-project - pod_eu1->>router_eu: 302 /my-company/my-project X-Gitlab-Pod-Redirect={pod:Pod EU0} - router_eu->>pod_eu0: GET /my-company/my-project - pod_eu0->>user:

My Project... X-Gitlab-Pod-Cache={path_prefix:/my-company/} -``` - -#### Navigates to `/my-company/my-other-project` after last step - -1. User is in Europe so DNS resolves to the router in Europe -1. The router cache now has `/my-company/* => Pod EU0`, so the router chooses `Pod EU0` -1. `Pod EU0` returns the correct response as well as the cache header again - -```mermaid -sequenceDiagram - participant user as User - participant router_eu as Router EU - participant pod_eu0 as Pod EU0 - participant pod_eu1 as Pod EU1 - user->>router_eu: GET /my-company/my-project - router_eu->>pod_eu0: GET /my-company/my-project - pod_eu0->>user:

My Project... X-Gitlab-Pod-Cache={path_prefix:/my-company/} -``` - -#### Navigates to `/gitlab-org/gitlab` after last step - -1. User is in Europe so DNS resolves to the router in Europe -1. The router has no cached value for this URL so randomly chooses `Pod EU0` -1. `Pod EU0` redirects the router to `Pod US0` -1. `Pod US0` returns the correct response as well as the cache header again - -```mermaid -sequenceDiagram - participant user as User - participant router_eu as Router EU - participant pod_eu0 as Pod EU0 - participant pod_us0 as Pod US0 - user->>router_eu: GET /gitlab-org/gitlab - router_eu->>pod_eu0: GET /gitlab-org/gitlab - pod_eu0->>router_eu: 302 /gitlab-org/gitlab X-Gitlab-Pod-Redirect={pod:Pod US0} - router_eu->>pod_us0: GET /gitlab-org/gitlab - pod_us0->>user:

GitLab.org... X-Gitlab-Pod-Cache={path_prefix:/gitlab-org/} -``` - -In this case the user is not on their "default organization" so their TODO -counter will not include their normal todos. We may choose to highlight this in -the UI somewhere. A future iteration may be able to fetch that for them from -their default organization. - -#### Navigates to `/` - -1. User is in Europe so DNS resolves to the router in Europe -1. Router does not have a cache for `/` route (specifically rails never tells it to cache this route) -1. The Router choose `Pod EU0` randomly -1. The Rails application knows the users default organization is `/my-organization`, so - it redirects the user to `/organizations/my-organization/-/dashboard` -1. The Router has a cached value for `/organizations/my-organization/*` so it then sends the - request to `POD EU0` -1. `Pod EU0` serves up a new page `/organizations/my-organization/-/dashboard` which is the same - dashboard view we have today but scoped to an organization clearly in the UI -1. The user is (optionally) presented with a message saying that data on this page is only - from their default organization and that they can change their default - organization if it's not right. - -```mermaid -sequenceDiagram - participant user as User - participant router_eu as Router EU - participant pod_eu0 as Pod EU0 - user->>router_eu: GET / - router_eu->>pod_eu0: GET / - pod_eu0->>user: 302 /organizations/my-organization/-/dashboard - user->>router: GET /organizations/my-organization/-/dashboard - router->>pod_eu0: GET /organizations/my-organization/-/dashboard - pod_eu0->>user:

My Company Dashboard... X-Gitlab-Pod-Cache={path_prefix:/organizations/my-organization/} -``` - -#### Navigates to `/dashboard` - -As above, they will end up on `/organizations/my-organization/-/dashboard` as -the rails application will already redirect `/` to the dashboard page. - -### Navigates to `/not-my-company/not-my-project` while logged in (but they don't have access since this project/group is private) - -1. User is in Europe so DNS resolves to the router in Europe -1. The router knows that `/not-my-company` lives in `Pod US1` so sends the request to this -1. The user does not have access so `Pod US1` returns 404 - -```mermaid -sequenceDiagram - participant user as User - participant router_eu as Router EU - participant pod_us1 as Pod US1 - user->>router_eu: GET /not-my-company/not-my-project - router_eu->>pod_us1: GET /not-my-company/not-my-project - pod_us1->>user: 404 -``` - -#### Creates a new top level namespace - -The user will be asked which organization they want the namespace to belong to. -If they select `my-organization` then it will end up on the same pod as all -other namespaces in `my-organization`. If they select nothing we default to -`GitLab.com Public` and it is clear to the user that this is isolated from -their existing organization such that they won't be able to see data from both -on a single page. - -### Experience for GitLab team member that is part of `/gitlab-org` - -Such a user is considered a legacy user and has their default organization set to -`GitLab.com Public`. This is a "meta" organization that does not really exist but -the Rails application knows to interpret this organization to mean that they are -allowed to use legacy global functionality like `/dashboard` to see data across -namespaces located on `Pod US0`. The rails backend also knows that the default pod to render any ambiguous -routes like `/dashboard` is `Pod US0`. Lastly the user will be allowed to -navigate to organizations on another pod like `/my-organization` but when they do the -user will see a message indicating that some data may be missing (for example, the -MRs/Issues/Todos) counts. - -#### Navigates to `/gitlab-org/gitlab` while not logged in - -1. User is in the US so DNS resolves to the US router -1. The router knows that `/gitlab-org` lives in `Pod US0` so sends the request - to this pod -1. `Pod US0` serves up the response - -```mermaid -sequenceDiagram - participant user as User - participant router_us as Router US - participant pod_us0 as Pod US0 - user->>router_us: GET /gitlab-org/gitlab - router_us->>pod_us0: GET /gitlab-org/gitlab - pod_us0->>user:

GitLab.org... X-Gitlab-Pod-Cache={path_prefix:/gitlab-org/} -``` - -#### Navigates to `/` - -1. User is in US so DNS resolves to the router in US -1. Router does not have a cache for `/` route (specifically rails never tells it to cache this route) -1. The Router chooses `Pod US1` randomly -1. The Rails application knows the users default organization is `GitLab.com Public`, so - it redirects the user to `/dashboards` (only legacy users can see - `/dashboard` global view) -1. Router does not have a cache for `/dashboard` route (specifically rails never tells it to cache this route) -1. The Router chooses `Pod US1` randomly -1. The Rails application knows the users default organization is `GitLab.com Public`, so - it allows the user to load `/dashboards` (only legacy users can see - `/dashboard` global view) and redirects to router the legacy pod which is `Pod US0` -1. `Pod US0` serves up the global view dashboard page `/dashboard` which is the same - dashboard view we have today - -```mermaid -sequenceDiagram - participant user as User - participant router_us as Router US - participant pod_us0 as Pod US0 - participant pod_us1 as Pod US1 - user->>router_us: GET / - router_us->>pod_us1: GET / - pod_us1->>user: 302 /dashboard - user->>router_us: GET /dashboard - router_us->>pod_us1: GET /dashboard - pod_us1->>router_us: 302 /dashboard X-Gitlab-Pod-Redirect={pod:Pod US0} - router_us->>pod_us0: GET /dashboard - pod_us0->>user:

Dashboard... -``` - -#### Navigates to `/my-company/my-other-project` while logged in (but they don't have access since this project is private) - -They get a 404. - -### Experience for non-authenticated users - -Flow is similar to authenticated users except global routes like `/dashboard` will -redirect to the login page as there is no default organization to choose from. - -### A new customers signs up - -They will be asked if they are already part of an organization or if they'd -like to create one. If they choose neither they end up no the default -`GitLab.com Public` organization. - -### An organization is moved from 1 pod to another - -TODO - -### GraphQL/API requests which don't include the namespace in the URL - -TODO - -### The autocomplete suggestion functionality in the search bar which remembers recent issues/MRs - -TODO - -### Global search - -TODO - -## Administrator - -### Loads `/admin` page - -1. Router picks a random pod `Pod US0` -1. Pod US0 redirects user to `/admin/pods/podus0` -1. Pod US0 renders an Admin Area page and also returns a cache header to cache `/admin/podss/podus0/* => Pod US0`. The Admin Area page contains a dropdown list showing other pods they could select and it changes the query parameter. - -Admin Area settings in Postgres are all shared across all pods to avoid -divergence but we still make it clear in the URL and UI which pod is serving -the Admin Area page as there is dynamic data being generated from these pages and -the operator may want to view a specific pod. - -## More Technical Problems To Solve - -### Replicating User Sessions Between All Pods - -Today user sessions live in Redis but each pod will have their own Redis instance. We already use a dedicated Redis instance for sessions so we could consider sharing this with all pods like we do with `gitlab_users` PostgreSQL database. But an important consideration will be latency as we would still want to mostly fetch sessions from the same region. - -An alternative might be that user sessions get moved to a JWT payload that encodes all the session data but this has downsides. For example, it is difficult to expire a user session, when their password changes or for other reasons, if the session lives in a JWT controlled by the user. - -### How do we migrate between Pods - -Migrating data between pods will need to factor all data stores: - -1. PostgreSQL -1. Redis Shared State -1. Gitaly -1. Elasticsearch - -### Is it still possible to leak the existence of private groups via a timing attack? - -If you have router in EU, and you know that EU router by default redirects -to EU located Pods, you know their latency (lets assume 10 ms). Now, if your -request is bounced back and redirected to US which has different latency -(lets assume that roundtrip will be around 60 ms) you can deduce that 404 was -returned by US Pod and know that your 404 is in fact 403. - -We may defer this until we actually implement a pod in a different region. Such timing attacks are already theoretically possible with the way we do permission checks today but the timing difference is probably too small to be able to detect. - -One technique to mitigate this risk might be to have the router add a random -delay to any request that returns 404 from a pod. - -## Should runners be shared across all pods? - -We have 2 options and we should decide which is easier: - -1. Decompose runner registration and queuing tables and share them across all - pods. This may have implications for scalability, and we'd need to consider - if this would include group/project runners as this may have scalability - concerns as these are high traffic tables that would need to be shared. -1. Runners are registered per-pod and, we probably have a separate fleet of - runners for every pod or just register the same runners to many pods which - may have implications for queueing - -## How do we guarantee unique ids across all pods for things that cannot conflict? - -This project assumes at least namespaces and projects have unique ids across -all pods as many requests need to be routed based on their ID. Since those -tables are across different databases then guaranteeing a unique ID will -require a new solution. There are likely other tables where unique IDs are -necessary and depending on how we resolve routing for GraphQL and other APIs -and other design goals it may be determined that we want the primary key to be -unique for all tables. + + + + diff --git a/doc/architecture/blueprints/pods/proposal-stateless-router-with-routes-learning.md b/doc/architecture/blueprints/pods/proposal-stateless-router-with-routes-learning.md index 1156e65f6aa..093d5d7acc6 100644 --- a/doc/architecture/blueprints/pods/proposal-stateless-router-with-routes-learning.md +++ b/doc/architecture/blueprints/pods/proposal-stateless-router-with-routes-learning.md @@ -1,672 +1,11 @@ --- -stage: enablement -group: pods -comments: false -description: 'Pods Stateless Router Proposal' +redirect_to: '../cells/proposal-stateless-router-with-routes-learning.md' +remove_date: '2023-06-13' --- -This document is a work-in-progress and represents a very early state of the -Pods design. Significant aspects are not documented, though we expect to add -them in the future. This is one possible architecture for Pods, and we intend to -contrast this with alternatives before deciding which approach to implement. -This documentation will be kept even if we decide not to implement this so that -we can document the reasons for not choosing this approach. +This document was moved to [another location](../cells/proposal-stateless-router-with-routes-learning.md). -# Proposal: Stateless Router - -We will decompose `gitlab_users`, `gitlab_routes` and `gitlab_admin` related -tables so that they can be shared between all pods and allow any pod to -authenticate a user and route requests to the correct pod. Pods may receive -requests for the resources they don't own, but they know how to redirect back -to the correct pod. - -The router is stateless and does not read from the `routes` database which -means that all interactions with the database still happen from the Rails -monolith. This architecture also supports regions by allowing for low traffic -databases to be replicated across regions. - -Users are not directly exposed to the concept of Pods but instead they see -different data dependent on their chosen "organization". -[Organizations](index.md#organizations) will be a new model introduced to enforce isolation in the -application and allow us to decide which request route to which pod, since an -organization can only be on a single pod. - -## Differences - -The main difference between this proposal and one [with buffering requests](proposal-stateless-router-with-buffering-requests.md) -is that this proposal uses a pre-flight API request (`/api/v4/pods/learn`) to redirect the request body to the correct Pod. -This means that each request is sent exactly once to be processed, but the URI is used to decode which Pod it should be directed. - -## Summary in diagrams - -This shows how a user request routes via DNS to the nearest router and the router chooses a pod to send the request to. - -```mermaid -graph TD; - user((User)); - dns[DNS]; - router_us(Router); - router_eu(Router); - pod_us0{Pod US0}; - pod_us1{Pod US1}; - pod_eu0{Pod EU0}; - pod_eu1{Pod EU1}; - user-->dns; - dns-->router_us; - dns-->router_eu; - subgraph Europe - router_eu-->pod_eu0; - router_eu-->pod_eu1; - end - subgraph United States - router_us-->pod_us0; - router_us-->pod_us1; - end -``` - -### More detail - -This shows that the router can actually send requests to any pod. The user will -get the closest router to them geographically. - -```mermaid -graph TD; - user((User)); - dns[DNS]; - router_us(Router); - router_eu(Router); - pod_us0{Pod US0}; - pod_us1{Pod US1}; - pod_eu0{Pod EU0}; - pod_eu1{Pod EU1}; - user-->dns; - dns-->router_us; - dns-->router_eu; - subgraph Europe - router_eu-->pod_eu0; - router_eu-->pod_eu1; - end - subgraph United States - router_us-->pod_us0; - router_us-->pod_us1; - end - router_eu-.->pod_us0; - router_eu-.->pod_us1; - router_us-.->pod_eu0; - router_us-.->pod_eu1; -``` - -### Even more detail - -This shows the databases. `gitlab_users` and `gitlab_routes` exist only in the -US region but are replicated to other regions. Replication does not have an -arrow because it's too hard to read the diagram. - -```mermaid -graph TD; - user((User)); - dns[DNS]; - router_us(Router); - router_eu(Router); - pod_us0{Pod US0}; - pod_us1{Pod US1}; - pod_eu0{Pod EU0}; - pod_eu1{Pod EU1}; - db_gitlab_users[(gitlab_users Primary)]; - db_gitlab_routes[(gitlab_routes Primary)]; - db_gitlab_users_replica[(gitlab_users Replica)]; - db_gitlab_routes_replica[(gitlab_routes Replica)]; - db_pod_us0[(gitlab_main/gitlab_ci Pod US0)]; - db_pod_us1[(gitlab_main/gitlab_ci Pod US1)]; - db_pod_eu0[(gitlab_main/gitlab_ci Pod EU0)]; - db_pod_eu1[(gitlab_main/gitlab_ci Pod EU1)]; - user-->dns; - dns-->router_us; - dns-->router_eu; - subgraph Europe - router_eu-->pod_eu0; - router_eu-->pod_eu1; - pod_eu0-->db_pod_eu0; - pod_eu0-->db_gitlab_users_replica; - pod_eu0-->db_gitlab_routes_replica; - pod_eu1-->db_gitlab_users_replica; - pod_eu1-->db_gitlab_routes_replica; - pod_eu1-->db_pod_eu1; - end - subgraph United States - router_us-->pod_us0; - router_us-->pod_us1; - pod_us0-->db_pod_us0; - pod_us0-->db_gitlab_users; - pod_us0-->db_gitlab_routes; - pod_us1-->db_gitlab_users; - pod_us1-->db_gitlab_routes; - pod_us1-->db_pod_us1; - end - router_eu-.->pod_us0; - router_eu-.->pod_us1; - router_us-.->pod_eu0; - router_us-.->pod_eu1; -``` - -## Summary of changes - -1. Tables related to User data (including profile settings, authentication credentials, personal access tokens) are decomposed into a `gitlab_users` schema -1. The `routes` table is decomposed into `gitlab_routes` schema -1. The `application_settings` (and probably a few other instance level tables) are decomposed into `gitlab_admin` schema -1. A new column `routes.pod_id` is added to `routes` table -1. A new Router service exists to choose which pod to route a request to. -1. If a router receives a new request it will send `/api/v4/pods/learn?method=GET&path_info=/group-org/project` to learn which Pod can process it -1. A new concept will be introduced in GitLab called an organization -1. We require all existing endpoints to be routable by URI, or be fixed to a specific Pod for processing. This requires changing ambiguous endpoints like `/dashboard` to be scoped like `/organizations/my-organization/-/dashboard` -1. Endpoints like `/admin` would be routed always to the specific Pod, like `pod_0` -1. Each Pod can respond to `/api/v4/pods/learn` and classify each endpoint -1. Writes to `gitlab_users` and `gitlab_routes` are sent to a primary PostgreSQL server in our `US` region but reads can come from replicas in the same region. This will add latency for these writes but we expect they are infrequent relative to the rest of GitLab. - -## Pre-flight request learning - -While processing a request the URI will be decoded and a pre-flight request -will be sent for each non-cached endpoint. - -When asking for the endpoint GitLab Rails will return information about -the routable path. GitLab Rails will decode `path_info` and match it to -an existing endpoint and find a routable entity (like project). The router will -treat this as short-lived cache information. - -1. Prefix match: `/api/v4/pods/learn?method=GET&path_info=/gitlab-org/gitlab-test/-/issues` - - ```json - { - "path": "/gitlab-org/gitlab-test", - "pod": "pod_0", - "source": "routable" - } - ``` - -1. Some endpoints might require an exact match: `/api/v4/pods/learn?method=GET&path_info=/-/profile` - - ```json - { - "path": "/-/profile", - "pod": "pod_0", - "source": "fixed", - "exact": true - } - ``` - -## Detailed explanation of default organization in the first iteration - -All users will get a new column `users.default_organization` which they can -control in user settings. We will introduce a concept of the -`GitLab.com Public` organization. This will be set as the default organization for all existing -users. This organization will allow the user to see data from all namespaces in -`Pod US0` (ie. our original GitLab.com instance). This behavior can be invisible to -existing users such that they don't even get told when they are viewing a -global page like `/dashboard` that it's even scoped to an organization. - -Any new users with a default organization other than `GitLab.com Public` will have -a distinct user experience and will be fully aware that every page they load is -only ever scoped to a single organization. These users can never -load any global pages like `/dashboard` and will end up being redirected to -`/organizations//-/dashboard`. This may also be the case -for legacy APIs and such users may only ever be able to use APIs scoped to a -organization. - -## Detailed explanation of Admin Area settings - -We believe that maintaining and synchronizing Admin Area settings will be -frustrating and painful so to avoid this we will decompose and share all Admin Area -settings in the `gitlab_admin` schema. This should be safe (similar to other -shared schemas) because these receive very little write traffic. - -In cases where different pods need different settings (eg. the -Elasticsearch URL), we will either decide to use a templated -format in the relevant `application_settings` row which allows it to be dynamic -per pod. Alternatively if that proves difficult we'll introduce a new table -called `per_pod_application_settings` and this will have 1 row per pod to allow -setting different settings per pod. It will still be part of the `gitlab_admin` -schema and shared which will allow us to centrally manage it and simplify -keeping settings in sync for all pods. - -## Pros - -1. Router is stateless and can live in many regions. We use Anycast DNS to resolve to nearest region for the user. -1. Pods can receive requests for namespaces in the wrong pod and the user - still gets the right response as well as caching at the router that - ensures the next request is sent to the correct pod so the next request - will go to the correct pod -1. The majority of the code still lives in `gitlab` rails codebase. The Router doesn't actually need to understand how GitLab URLs are composed. -1. Since the responsibility to read and write `gitlab_users`, - `gitlab_routes` and `gitlab_admin` still lives in Rails it means minimal - changes will be needed to the Rails application compared to extracting - services that need to isolate the domain models and build new interfaces. -1. Compared to a separate routing service this allows the Rails application - to encode more complex rules around how to map URLs to the correct pod - and may work for some existing API endpoints. -1. All the new infrastructure (just a router) is optional and a single-pod - self-managed installation does not even need to run the Router and there are - no other new services. - -## Cons - -1. `gitlab_users`, `gitlab_routes` and `gitlab_admin` databases may need to be - replicated across regions and writes need to go across regions. We need to - do an analysis on write TPS for the relevant tables to determine if this is - feasible. -1. Sharing access to the database from many different Pods means that they are - all coupled at the Postgres schema level and this means changes to the - database schema need to be done carefully in sync with the deployment of all - Pods. This limits us to ensure that Pods are kept in closely similar - versions compared to an architecture with shared services that have an API - we control. -1. Although most data is stored in the right region there can be requests - proxied from another region which may be an issue for certain types - of compliance. -1. Data in `gitlab_users` and `gitlab_routes` databases must be replicated in - all regions which may be an issue for certain types of compliance. -1. The router cache may need to be very large if we get a wide variety of URLs - (ie. long tail). In such a case we may need to implement a 2nd level of - caching in user cookies so their frequently accessed pages always go to the - right pod the first time. -1. Having shared database access for `gitlab_users` and `gitlab_routes` - from multiple pods is an unusual architecture decision compared to - extracting services that are called from multiple pods. -1. It is very likely we won't be able to find cacheable elements of a - GraphQL URL and often existing GraphQL endpoints are heavily dependent on - ids that won't be in the `routes` table so pods won't necessarily know - what pod has the data. As such we'll probably have to update our GraphQL - calls to include an organization context in the path like - `/api/organizations//graphql`. -1. This architecture implies that implemented endpoints can only access data - that are readily accessible on a given Pod, but are unlikely - to aggregate information from many Pods. -1. All unknown routes are sent to the latest deployment which we assume to be `Pod US0`. - This is required as newly added endpoints will be only decodable by latest pod. - Likely this is not a problem for the `/pods/learn` is it is lightweight - to process and this should not cause a performance impact. - -## Example database configuration - -Handling shared `gitlab_users`, `gitlab_routes` and `gitlab_admin` databases, while having dedicated `gitlab_main` and `gitlab_ci` databases should already be handled by the way we use `config/database.yml`. We should also, already be able to handle the dedicated EU replicas while having a single US primary for `gitlab_users` and `gitlab_routes`. Below is a snippet of part of the database configuration for the Pod architecture described above. - -**Pod US0**: - -```yaml -# config/database.yml -production: - main: - host: postgres-main.pod-us0.primary.consul - load_balancing: - discovery: postgres-main.pod-us0.replicas.consul - ci: - host: postgres-ci.pod-us0.primary.consul - load_balancing: - discovery: postgres-ci.pod-us0.replicas.consul - users: - host: postgres-users-primary.consul - load_balancing: - discovery: postgres-users-replicas.us.consul - routes: - host: postgres-routes-primary.consul - load_balancing: - discovery: postgres-routes-replicas.us.consul - admin: - host: postgres-admin-primary.consul - load_balancing: - discovery: postgres-admin-replicas.us.consul -``` - -**Pod EU0**: - -```yaml -# config/database.yml -production: - main: - host: postgres-main.pod-eu0.primary.consul - load_balancing: - discovery: postgres-main.pod-eu0.replicas.consul - ci: - host: postgres-ci.pod-eu0.primary.consul - load_balancing: - discovery: postgres-ci.pod-eu0.replicas.consul - users: - host: postgres-users-primary.consul - load_balancing: - discovery: postgres-users-replicas.eu.consul - routes: - host: postgres-routes-primary.consul - load_balancing: - discovery: postgres-routes-replicas.eu.consul - admin: - host: postgres-admin-primary.consul - load_balancing: - discovery: postgres-admin-replicas.eu.consul -``` - -## Request flows - -1. `gitlab-org` is a top level namespace and lives in `Pod US0` in the `GitLab.com Public` organization -1. `my-company` is a top level namespace and lives in `Pod EU0` in the `my-organization` organization - -### Experience for paying user that is part of `my-organization` - -Such a user will have a default organization set to `/my-organization` and will be -unable to load any global routes outside of this organization. They may load other -projects/namespaces but their MR/Todo/Issue counts at the top of the page will -not be correctly populated in the first iteration. The user will be aware of -this limitation. - -#### Navigates to `/my-company/my-project` while logged in - -1. User is in Europe so DNS resolves to the router in Europe -1. They request `/my-company/my-project` without the router cache, so the router chooses randomly `Pod EU1` -1. The `/pods/learn` is sent to `Pod EU1`, which responds that resource lives on `Pod EU0` -1. `Pod EU0` returns the correct response -1. The router now caches and remembers any request paths matching `/my-company/*` should go to `Pod EU0` - -```mermaid -sequenceDiagram - participant user as User - participant router_eu as Router EU - participant pod_eu0 as Pod EU0 - participant pod_eu1 as Pod EU1 - user->>router_eu: GET /my-company/my-project - router_eu->>pod_eu1: /api/v4/pods/learn?method=GET&path_info=/my-company/my-project - pod_eu1->>router_eu: {path: "/my-company", pod: "pod_eu0", source: "routable"} - router_eu->>pod_eu0: GET /my-company/my-project - pod_eu0->>user:

My Project... -``` - -#### Navigates to `/my-company/my-project` while not logged in - -1. User is in Europe so DNS resolves to the router in Europe -1. The router does not have `/my-company/*` cached yet so it chooses randomly `Pod EU1` -1. The `/pods/learn` is sent to `Pod EU1`, which responds that resource lives on `Pod EU0` -1. `Pod EU0` redirects them through a login flow -1. User requests `/users/sign_in`, uses random Pod to run `/pods/learn` -1. The `Pod EU1` responds with `pod_0` as a fixed route -1. User after login requests `/my-company/my-project` which is cached and stored in `Pod EU0` -1. `Pod EU0` returns the correct response - -```mermaid -sequenceDiagram - participant user as User - participant router_eu as Router EU - participant pod_eu0 as Pod EU0 - participant pod_eu1 as Pod EU1 - user->>router_eu: GET /my-company/my-project - router_eu->>pod_eu1: /api/v4/pods/learn?method=GET&path_info=/my-company/my-project - pod_eu1->>router_eu: {path: "/my-company", pod: "pod_eu0", source: "routable"} - router_eu->>pod_eu0: GET /my-company/my-project - pod_eu0->>user: 302 /users/sign_in?redirect=/my-company/my-project - user->>router_eu: GET /users/sign_in?redirect=/my-company/my-project - router_eu->>pod_eu1: /api/v4/pods/learn?method=GET&path_info=/users/sign_in - pod_eu1->>router_eu: {path: "/users", pod: "pod_eu0", source: "fixed"} - router_eu->>pod_eu0: GET /users/sign_in?redirect=/my-company/my-project - pod_eu0-->>user:

Sign in... - user->>router_eu: POST /users/sign_in?redirect=/my-company/my-project - router_eu->>pod_eu0: POST /users/sign_in?redirect=/my-company/my-project - pod_eu0->>user: 302 /my-company/my-project - user->>router_eu: GET /my-company/my-project - router_eu->>pod_eu0: GET /my-company/my-project - router_eu->>pod_eu0: GET /my-company/my-project - pod_eu0->>user:

My Project... -``` - -#### Navigates to `/my-company/my-other-project` after last step - -1. User is in Europe so DNS resolves to the router in Europe -1. The router cache now has `/my-company/* => Pod EU0`, so the router chooses `Pod EU0` -1. `Pod EU0` returns the correct response as well as the cache header again - -```mermaid -sequenceDiagram - participant user as User - participant router_eu as Router EU - participant pod_eu0 as Pod EU0 - participant pod_eu1 as Pod EU1 - user->>router_eu: GET /my-company/my-project - router_eu->>pod_eu0: GET /my-company/my-project - pod_eu0->>user:

My Project... -``` - -#### Navigates to `/gitlab-org/gitlab` after last step - -1. User is in Europe so DNS resolves to the router in Europe -1. The router has no cached value for this URL so randomly chooses `Pod EU0` -1. `Pod EU0` redirects the router to `Pod US0` -1. `Pod US0` returns the correct response as well as the cache header again - -```mermaid -sequenceDiagram - participant user as User - participant router_eu as Router EU - participant pod_eu0 as Pod EU0 - participant pod_us0 as Pod US0 - user->>router_eu: GET /gitlab-org/gitlab - router_eu->>pod_eu0: /api/v4/pods/learn?method=GET&path_info=/gitlab-org/gitlab - pod_eu0->>router_eu: {path: "/gitlab-org", pod: "pod_us0", source: "routable"} - router_eu->>pod_us0: GET /gitlab-org/gitlab - pod_us0->>user:

GitLab.org... -``` - -In this case the user is not on their "default organization" so their TODO -counter will not include their normal todos. We may choose to highlight this in -the UI somewhere. A future iteration may be able to fetch that for them from -their default organization. - -#### Navigates to `/` - -1. User is in Europe so DNS resolves to the router in Europe -1. Router does not have a cache for `/` route (specifically rails never tells it to cache this route) -1. The Router choose `Pod EU0` randomly -1. The Rails application knows the users default organization is `/my-organization`, so - it redirects the user to `/organizations/my-organization/-/dashboard` -1. The Router has a cached value for `/organizations/my-organization/*` so it then sends the - request to `POD EU0` -1. `Pod EU0` serves up a new page `/organizations/my-organization/-/dashboard` which is the same - dashboard view we have today but scoped to an organization clearly in the UI -1. The user is (optionally) presented with a message saying that data on this page is only - from their default organization and that they can change their default - organization if it's not right. - -```mermaid -sequenceDiagram - participant user as User - participant router_eu as Router EU - participant pod_eu0 as Pod EU0 - user->>router_eu: GET / - router_eu->>pod_eu0: GET / - pod_eu0->>user: 302 /organizations/my-organization/-/dashboard - user->>router: GET /organizations/my-organization/-/dashboard - router->>pod_eu0: GET /organizations/my-organization/-/dashboard - pod_eu0->>user:

My Company Dashboard... X-Gitlab-Pod-Cache={path_prefix:/organizations/my-organization/} -``` - -#### Navigates to `/dashboard` - -As above, they will end up on `/organizations/my-organization/-/dashboard` as -the rails application will already redirect `/` to the dashboard page. - -### Navigates to `/not-my-company/not-my-project` while logged in (but they don't have access since this project/group is private) - -1. User is in Europe so DNS resolves to the router in Europe -1. The router knows that `/not-my-company` lives in `Pod US1` so sends the request to this -1. The user does not have access so `Pod US1` returns 404 - -```mermaid -sequenceDiagram - participant user as User - participant router_eu as Router EU - participant pod_us1 as Pod US1 - user->>router_eu: GET /not-my-company/not-my-project - router_eu->>pod_us1: GET /not-my-company/not-my-project - pod_us1->>user: 404 -``` - -#### Creates a new top level namespace - -The user will be asked which organization they want the namespace to belong to. -If they select `my-organization` then it will end up on the same pod as all -other namespaces in `my-organization`. If they select nothing we default to -`GitLab.com Public` and it is clear to the user that this is isolated from -their existing organization such that they won't be able to see data from both -on a single page. - -### Experience for GitLab team member that is part of `/gitlab-org` - -Such a user is considered a legacy user and has their default organization set to -`GitLab.com Public`. This is a "meta" organization that does not really exist but -the Rails application knows to interpret this organization to mean that they are -allowed to use legacy global functionality like `/dashboard` to see data across -namespaces located on `Pod US0`. The rails backend also knows that the default pod to render any ambiguous -routes like `/dashboard` is `Pod US0`. Lastly the user will be allowed to -navigate to organizations on another pod like `/my-organization` but when they do the -user will see a message indicating that some data may be missing (eg. the -MRs/Issues/Todos) counts. - -#### Navigates to `/gitlab-org/gitlab` while not logged in - -1. User is in the US so DNS resolves to the US router -1. The router knows that `/gitlab-org` lives in `Pod US0` so sends the request - to this pod -1. `Pod US0` serves up the response - -```mermaid -sequenceDiagram - participant user as User - participant router_us as Router US - participant pod_us0 as Pod US0 - user->>router_us: GET /gitlab-org/gitlab - router_us->>pod_us0: GET /gitlab-org/gitlab - pod_us0->>user:

GitLab.org... -``` - -#### Navigates to `/` - -1. User is in US so DNS resolves to the router in US -1. Router does not have a cache for `/` route (specifically rails never tells it to cache this route) -1. The Router chooses `Pod US1` randomly -1. The Rails application knows the users default organization is `GitLab.com Public`, so - it redirects the user to `/dashboards` (only legacy users can see - `/dashboard` global view) -1. Router does not have a cache for `/dashboard` route (specifically rails never tells it to cache this route) -1. The Router chooses `Pod US1` randomly -1. The Rails application knows the users default organization is `GitLab.com Public`, so - it allows the user to load `/dashboards` (only legacy users can see - `/dashboard` global view) and redirects to router the legacy pod which is `Pod US0` -1. `Pod US0` serves up the global view dashboard page `/dashboard` which is the same - dashboard view we have today - -```mermaid -sequenceDiagram - participant user as User - participant router_us as Router US - participant pod_us0 as Pod US0 - participant pod_us1 as Pod US1 - user->>router_us: GET / - router_us->>pod_us1: GET / - pod_us1->>user: 302 /dashboard - user->>router_us: GET /dashboard - router_us->>pod_us1: /api/v4/pods/learn?method=GET&path_info=/dashboard - pod_us1->>router_us: {path: "/dashboard", pod: "pod_us0", source: "routable"} - router_us->>pod_us0: GET /dashboard - pod_us0->>user:

Dashboard... -``` - -#### Navigates to `/my-company/my-other-project` while logged in (but they don't have access since this project is private) - -They get a 404. - -### Experience for non-authenticated users - -Flow is similar to logged in users except global routes like `/dashboard` will -redirect to the login page as there is no default organization to choose from. - -### A new customers signs up - -They will be asked if they are already part of an organization or if they'd -like to create one. If they choose neither they end up no the default -`GitLab.com Public` organization. - -### An organization is moved from 1 pod to another - -TODO - -### GraphQL/API requests which don't include the namespace in the URL - -TODO - -### The autocomplete suggestion functionality in the search bar which remembers recent issues/MRs - -TODO - -### Global search - -TODO - -## Administrator - -### Loads `/admin` page - -1. The `/admin` is locked to `Pod US0` -1. Some endpoints of `/admin`, like Projects in Admin are scoped to a Pod - and users needs to choose the correct one in a dropdown, which results in endpoint - like `/admin/pods/pod_0/projects`. - -Admin Area settings in Postgres are all shared across all pods to avoid -divergence but we still make it clear in the URL and UI which pod is serving -the Admin Area page as there is dynamic data being generated from these pages and -the operator may want to view a specific pod. - -## More Technical Problems To Solve - -### Replicating User Sessions Between All Pods - -Today user sessions live in Redis but each pod will have their own Redis instance. We already use a dedicated Redis instance for sessions so we could consider sharing this with all pods like we do with `gitlab_users` PostgreSQL database. But an important consideration will be latency as we would still want to mostly fetch sessions from the same region. - -An alternative might be that user sessions get moved to a JWT payload that encodes all the session data but this has downsides. For example, it is difficult to expire a user session, when their password changes or for other reasons, if the session lives in a JWT controlled by the user. - -### How do we migrate between Pods - -Migrating data between pods will need to factor all data stores: - -1. PostgreSQL -1. Redis Shared State -1. Gitaly -1. Elasticsearch - -### Is it still possible to leak the existence of private groups via a timing attack? - -If you have router in EU, and you know that EU router by default redirects -to EU located Pods, you know their latency (lets assume 10 ms). Now, if your -request is bounced back and redirected to US which has different latency -(lets assume that roundtrip will be around 60 ms) you can deduce that 404 was -returned by US Pod and know that your 404 is in fact 403. - -We may defer this until we actually implement a pod in a different region. Such timing attacks are already theoretically possible with the way we do permission checks today but the timing difference is probably too small to be able to detect. - -One technique to mitigate this risk might be to have the router add a random -delay to any request that returns 404 from a pod. - -## Should runners be shared across all pods? - -We have 2 options and we should decide which is easier: - -1. Decompose runner registration and queuing tables and share them across all - pods. This may have implications for scalability, and we'd need to consider - if this would include group/project runners as this may have scalability - concerns as these are high traffic tables that would need to be shared. -1. Runners are registered per-pod and, we probably have a separate fleet of - runners for every pod or just register the same runners to many pods which - may have implications for queueing - -## How do we guarantee unique ids across all pods for things that cannot conflict? - -This project assumes at least namespaces and projects have unique ids across -all pods as many requests need to be routed based on their ID. Since those -tables are across different databases then guaranteeing a unique ID will -require a new solution. There are likely other tables where unique IDs are -necessary and depending on how we resolve routing for GraphQL and other APIs -and other design goals it may be determined that we want the primary key to be -unique for all tables. + + + + diff --git a/doc/architecture/blueprints/rate_limiting/index.md b/doc/architecture/blueprints/rate_limiting/index.md index b466a54e922..17808267032 100644 --- a/doc/architecture/blueprints/rate_limiting/index.md +++ b/doc/architecture/blueprints/rate_limiting/index.md @@ -8,6 +8,8 @@ owning-stage: "~devops::enablement" participating-stages: [] --- + + # Next Rate Limiting Architecture ## Summary @@ -375,7 +377,7 @@ hierarchy. Choosing a proper solution will require a thoughtful research. - Implementing a separate Go library which uses the same backend (for example, Redis) for rate limiting. 1. **SDK for Satellite Services (Owning Team)** - - Build Golang SDK. + - Build Go SDK. - Create examples showcasing usage of the new rate limits SDK. 1. **Team fan out for Satellite Services (Stage Groups)** diff --git a/doc/architecture/blueprints/remote_development/img/remote_dev_15_7.png b/doc/architecture/blueprints/remote_development/img/remote_dev_15_7.png index d0849ded94f..f36bfa24998 100644 Binary files a/doc/architecture/blueprints/remote_development/img/remote_dev_15_7.png and b/doc/architecture/blueprints/remote_development/img/remote_dev_15_7.png differ diff --git a/doc/architecture/blueprints/remote_development/index.md b/doc/architecture/blueprints/remote_development/index.md index 162ae04f6b6..e2647551a95 100644 --- a/doc/architecture/blueprints/remote_development/index.md +++ b/doc/architecture/blueprints/remote_development/index.md @@ -8,6 +8,8 @@ owning-stage: "~devops::create" participating-stages: [] --- + + # Remote Development ## Summary @@ -40,6 +42,64 @@ As a [new Software Developer to a team such as Sasha](https://about.gitlab.com/h ![User Flow](img/remote_dev_15_7.png) +## Architecture + +```plantuml +@startuml +node "Kubernetes" { + [Ingress Controller] --> [GitLab Workspaces Proxy] : Decrypt Traffic + + note right of "Ingress Controller" + Customers can choose + an ingress controller + of their choice + end note + + note top of "GitLab Workspaces Proxy" + Authenticate and + authorize user traffic + end note + + [GitLab Workspaces Proxy] ..> [Workspace n] : Forward traffic\nfor workspace n + [GitLab Workspaces Proxy] ..> [Workspace 2] : Forward traffic\nfor workspace 2 + [GitLab Workspaces Proxy] --> [Workspace 1] : Forward traffic\nfor workspace 1 + + [Agentk] .up.> [Workspace n] : Applies kubernetes resources\nfor workspace n + [Agentk] .up.> [Workspace 2] : Applies kubernetes resources\nfor workspace 2 + [Agentk] .up.> [Workspace 1] : Applies kubernetes resources\nfor workspace 1 + + [Agentk] --> [Kubernetes API Server] : Interact and get/apply\nKubernetes resources +} + +node "GitLab" { + [Nginx] --> [GitLab Rails] : Forward + [GitLab Rails] --> [Postgres] : Access database + [GitLab Rails] --> [Gitaly] : Fetch files + [KAS] -up-> [GitLab Rails] : Proxy +} + +[Agentk] -up-> [KAS] : Initiate reconciliation loop +"Load Balancer IP" --> [Ingress Controller] +[Browser] --> [Nginx] : Browse GitLab +[Browser] -right-> "Domain IP" : Browse workspace URL +"Domain IP" .right.> "Load Balancer IP" +[GitLab Workspaces Proxy] ..> [GitLab Rails] : Authenticate and authorize\nthe user accessing the workspace. + +note top of "Domain IP" + For local development, workspace URL + is [workspace-name].workspaces.localdev.me + which resolves to localhost (127.0.0.1) +end note + +note top of "Load Balancer IP" + For local development, + it includes all local loopback interfaces + e.g. 127.0.0.1, 172.16.123.1, 192.168.0.1, etc. +end note + +@enduml +``` + ## Terminology We use the following terms to describe components and properties of the Remote Development architecture. @@ -68,17 +128,6 @@ Container/VM-based developer machines providing all the tools and dependencies n - A workspace should be a combination of resources that support cloud-based development environment. - Workspaces are constrained by the amount of resources provided to them. -### Legacy Web IDE - -The current production [Web IDE](../../../user/project/web_ide/index.md). - -#### Legacy Web IDE properties - -An advanced editor with commit staging that currently supports: - -- [Live Preview](../../../user/project/web_ide/index.md#live-preview-removed) -- [Interactive Web Terminals](../../../user/project/web_ide/index.md#interactive-web-terminals-for-the-web-ide) - ### Web IDE VS Code for web - replacement of our current legacy Web IDE. @@ -137,145 +186,6 @@ As a zero-install development environment that runs in your browser, Remote Deve GitLab.com is only hosted within the United States of America. Organizations located in other regions have voiced demand for local SaaS offerings. BYO infrastructure helps work in conjunction with [GitLab Regions](https://gitlab.com/groups/gitlab-org/-/epics/6037) because a user's workspace may be deployed within different geographies. The ability to deploy workspaces to different geographies might also help to solve data residency and compliance problems. -## High-level architecture problems to solve - -A number of technical issues need to be resolved to implement a stable Remote Development offering. This section will be expanded. - -- Who is our main persona for BYO infrastructure? -- How do users authenticate? -- How do we support more than one IDE? -- How are workspaces provisioned? -- How can workspaces implement disaster recovery capabilities? -- If we cannot use SSH, what are the viable alternatives for establishing a secure WebSocket connection? -- Are we running into any limitations in functionality with the Web IDE by not having it running in the container itself? For example, are we going to get code completion, linting, and language server type features to work with our approach? -- How will our environments be provisioned, managed, created, destroyed, etc.? -- To what extent do we need to provide the user with a UI to interact with the provisioned environments? -- How will the files inside the workspace get live updated based on changes in the Web IDE? Are we going to use a [CRDT](https://en.wikipedia.org/wiki/Conflict-free_replicated_data_type)-like setup to patch files in a container? Are we going to generate a diff and send it though a WebSocket connection? - -## Iteration plan - -We can't ship the entire Remote Development architecture in one go - it is too large. Instead, we are adopting an iteration plan that provides value along the way. - -- Use GitLab Agent for Kubernetes Remote Development Module. -- Integrate Remote Development with the UI and Web IDE. -- Improve security and usability. - -### High-level approach - -The nuts and bolts are being worked out at [Remote Development GA4K Architecture](https://gitlab.com/gitlab-org/remote-development/gitlab-remote-development-docs/-/blob/main/doc/architecture.md) to keep a SSoT. Once we have hammered out the details, we'll replace this section with the diagram in the above repository. - -### Iteration 0: [GitLab Agent for Kubernetes Remote Development Module (plumbing)](https://gitlab.com/groups/gitlab-org/-/epics/9138) - -#### Goals - -- Use the [GitLab Agent](../../../user/clusters/agent/index.md) integration. -- Create a workspace in a Kubernetes cluster based on a `devfile` in a public repository. -- Install the IDE and dependencies as defined. -- Report the status of the environment (via the terminal or through an endpoint). -- Connect to an IDE in the workspace. - -#### Requirements - -- Remote environment running on a Kubernetes cluster based on a `devfile` in a repo. - -These are **not** part of Iteration 0: - -- Authentication/authorization with GitLab and a user. -- Integration of Remote Development with the GitLab UI and Web IDE. -- Using GA4K instead of an Ingress controller. - -#### Assumptions - -- We will use [`devworkspace-operator` v0.17.0 (latest version)](https://github.com/devfile/devworkspace-operator/releases/tag/v0.17.0). A prerequisite is [`cert-manager`](https://github.com/devfile/devworkspace-operator#with-yaml-resources). -- We have an Ingress controller ([Ingress-NGINX](https://github.com/kubernetes/ingress-nginx)), which is accessible over the network. -- The initial server is stubbed. - -#### Success criteria - -- Using GA4K to communicate with the Kubernetes API from the `remote_dev` agent module. -- All calls to the Kubernetes API are done through GA4K. -- A workspace in a Kubernetes cluster created using DevWorkspace Operator. - -### Iteration 1: [Rails endpoints, authentication, and authorization](https://gitlab.com/groups/gitlab-org/-/epics/9323) - -#### Goals - -- Add endpoints in Rails to accept work from a user. -- Poll Rails for work from KAS. -- Add authentication and authorization to the workspaces created in the Kubernetes cluster. -- Extend the GA4K `remote_dev` agent module to accept more types of work (get details of a workspace, list workspaces for a user, etc). -- Build an editor injector for the GitLab fork of VS Code. - -#### Requirements - -- [GitLab Agent for Kubernetes Remote Development Module (plumbing)](https://gitlab.com/groups/gitlab-org/-/epics/9138) is complete. - -These are **not** part of Iteration 1: - -- Integration of Remote Development with the GitLab UI and Web IDE. -- Using GA4K instead of an Ingress controller. - -#### Assumptions - -- TBA - -#### Success criteria - -- Poll Rails for work from KAS. -- Rails endpoints to create/delete/get/list workspaces. -- All requests are correctly authenticated and authorized except where the user has requested the traffic to be public (for example, opening a server while developing and making it public). -- A user can create a workspace, start a server on that workspace, and have that traffic become private/internal/public. -- We are using the GitLab fork of VS Code as an editor. - -### Iteration 2: [Integrate Remote Development with the UI and Web IDE](https://gitlab.com/groups/gitlab-org/-/epics/9169) - -#### Goals - -- Allow users full control of their workspaces via the GitLab UI. - -#### Requirements - -- [GitLab Agent for Kubernetes Remote Development Module](https://gitlab.com/groups/gitlab-org/-/epics/9138). - -These are **not** part of Iteration 2: - -- Usability improvements -- Security improvements - -#### Success criteria - -- Be able to list/create/delete/stop/start/restart workspaces from the UI. -- Be able to create workspaces for the user in the Web IDE. -- Allow the Web IDE terminal to connect to different containers in the workspace. -- Configure DevWorkspace Operator for user-expected configuration (30-minute workspace timeout, a separate persistent volume for each workspace that is deleted when the workspace is deleted, etc.). - -### Iteration 3: [Improve security and usability](https://gitlab.com/groups/gitlab-org/-/epics/9170) - -#### Goals - -- Improve security and usability of our Remote Development solution. - -#### Requirements - -- [Integrate Remote Development with the UI and Web IDE](https://gitlab.com/groups/gitlab-org/-/epics/9169) is complete. - -#### Assumptions - -- We are allowing for internal feedback and closed/early customer feedback that can be iterated on. -- We have explored or are exploring the feasibility of using GA4K with Ingresses in [Solving Ingress problems for Remote Development](https://gitlab.com/gitlab-org/gitlab/-/issues/378998). -- We have explored or are exploring Kata containers for providing root access to workspace users in [Investigate Kata Containers / Firecracker / gVisor](https://gitlab.com/gitlab-org/gitlab/-/issues/367043). -- We have explored or are exploring how Ingress/Egress requests cannot be misused from [resources within or outside the cluster](https://gitlab.com/gitlab-org/remote-development/gitlab-remote-development-docs/-/blob/main/doc/securing-the-workspace.md) (security hardening). - -#### Success criteria - -Add options to: - -- Create different classes of workspaces (1gb-2cpu, 4gb-8cpu, etc.). -- Vertically scale up workspace resources. -- Inject secrets from a GitLab user/group/repository. -- Configure timeouts of workspaces at multiple levels. -- Allow users to expose endpoints in their workspace (for example, not allow anyone in the organization to expose any endpoint publicly). - ## Market analysis We have conducted a market analysis to understand the broader market and what others can offer us by way of open-source libraries, integrations, or partnership opportunities. We have broken down the effort into a set of issues where we investigate each potential competitor/pathway/partnership as a spike. @@ -283,13 +193,15 @@ We have conducted a market analysis to understand the broader market and what ot - [Market analysis](https://gitlab.com/groups/gitlab-org/-/epics/8131) - [YouTube results](https://www.youtube.com/playlist?list=PL05JrBw4t0KrRQhnSYRNh1s1mEUypx67-) -### Next Steps +### Implementation -While our spike proved fruitful, we have paused this investigation until we reach our goals in [Viable Maturity](https://gitlab.com/groups/gitlab-org/-/epics/9190). +- [Viable Maturity Epic](https://gitlab.com/groups/gitlab-org/-/epics/9190) to track progress. +- [Documentation](https://gitlab.com/gitlab-org/remote-development/gitlab-remote-development-docs) +explaining the architecture and implementation details. -## Che versus a custom-built solution +## Che vs. DevWorkspace Operatoor vs. Custom-Built Solution -After an investigation into using [Che](https://gitlab.com/gitlab-org/gitlab/-/issues/366052) as our backend to accelerate Remote Development, we ultimately opted to [write our own custom-built solution](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/97449#note_1131215629). +After an investigation into using [Che](https://gitlab.com/gitlab-org/gitlab/-/issues/366052) as our backend to accelerate Remote Development, we ultimately opted to [write our own custom-built solution using DevWorkspace Operator](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/97449#note_1131215629). Some advantages of us opting to write our own custom-built solution are: @@ -297,6 +209,10 @@ Some advantages of us opting to write our own custom-built solution are: - It is easier to add support for other configurations apart from `devfile` in the future if the need arises. - We have the ability to choose which tech stack to use (for example, instead of using Traefik which is used in Che, explore NGINX itself or use GitLab Agent for Kubernetes). +After writing our own custom-built solution using DevWorkspace Operator, +we decided to [remove the dependency on DevWorkspace Operator](https://gitlab.com/groups/gitlab-org/-/epics/9895) +and thus the transitive dependency of Cert Manager. + ## Links - [Remote Development presentation](https://docs.google.com/presentation/d/1XHH_ZilZPufQoWVWViv3evipI-BnAvRQrdvzlhBuumw/edit#slide=id.g131f2bb72e4_0_8) @@ -304,7 +220,7 @@ Some advantages of us opting to write our own custom-built solution are: - [Minimal Maturity epic](https://gitlab.com/groups/gitlab-org/-/epics/9189) - [Viable Maturity epic](https://gitlab.com/groups/gitlab-org/-/epics/9190) - [Complete Maturity epic](https://gitlab.com/groups/gitlab-org/-/epics/9191) -- [Bi-weekly sync](https://docs.google.com/document/d/1hWVvksIc7VzZjG-0iSlzBnLpyr-OjwBVCYMxsBB3h_E/edit#) +- [Remote Development sync](https://docs.google.com/document/d/1hWVvksIc7VzZjG-0iSlzBnLpyr-OjwBVCYMxsBB3h_E/edit#) - [Market analysis and architecture](https://gitlab.com/groups/gitlab-org/-/epics/8131) - [GA4K Architecture](https://gitlab.com/gitlab-org/remote-development/gitlab-remote-development-docs/-/blob/main/doc/architecture.md) - [BYO infrastructure](https://gitlab.com/groups/gitlab-org/-/epics/8290) diff --git a/doc/architecture/blueprints/runner_scaling/index.md b/doc/architecture/blueprints/runner_scaling/index.md index 53401d80e34..de1203843aa 100644 --- a/doc/architecture/blueprints/runner_scaling/index.md +++ b/doc/architecture/blueprints/runner_scaling/index.md @@ -8,6 +8,8 @@ owning-stage: "~devops::verify" participating-stages: [] --- + + # Next Runner Auto-scaling Architecture ## Summary @@ -214,12 +216,12 @@ sequence diagram. ![GitLab Runner Autoscaling Overview](gitlab-autoscaling-overview.png) -On the diagrams above we see that currently a GitLab Runner Manager runs on a +On the diagrams above we see that currently a runner manager runs on a machine that has access to a cloud provider's API. It is using Docker Machine to provision new Virtual Machines with Docker Engine installed and it configures the Docker daemon there to allow external authenticated requests. It stores credentials to such ephemeral Docker environments on disk. Once a -machine has been provisioned and made available for GitLab Runner Manager to +machine has been provisioned and made available for the runner manager to run builds, it is using one of the existing executors to run a user-provided script. In auto-scaling, this is typically done using the Docker executor. @@ -281,7 +283,7 @@ coupled with the VM lifecycle and job routing logic. Creating idle capacity happens as a side-effect of calling [`Acquire`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/executors/docker/machine/provider.go#L449) on the `machineProvider` while binding a job to a VM. There is also no current abstraction for in-VM job execution. VM-specific -commands are generated by the Runner Manager using the [`GenerateShellScript`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/common/build.go#L336) +commands are generated by the runner manager using the [`GenerateShellScript`](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/common/build.go#L336) function and [injected](https://gitlab.com/gitlab-org/gitlab-runner/-/blob/267f40d871cd260dd063f7fbd36a921fedc62241/common/build.go#L373) into the VM as the manager drives the job execution stages. diff --git a/doc/architecture/blueprints/runner_tokens/index.md b/doc/architecture/blueprints/runner_tokens/index.md index 69a10674d7d..0d3cc9c3e17 100644 --- a/doc/architecture/blueprints/runner_tokens/index.md +++ b/doc/architecture/blueprints/runner_tokens/index.md @@ -8,6 +8,8 @@ owning-stage: "~devops::verify" participating-stages: [] --- + + # Next GitLab Runner Token Architecture ## Summary @@ -183,14 +185,17 @@ CREATE TABLE ci_runners ( ) ``` -The `ci_builds_metadata` table shall reference `ci_runner_machines`. +A new `p_ci_runner_machine_builds` table joins the `ci_runner_machines` and `ci_builds` tables, to avoid +adding more pressure to those tables. We might consider a more efficient way to store `contacted_at` than updating the existing record. ```sql -CREATE TABLE ci_builds_metadata ( - ... +CREATE TABLE p_ci_runner_machine_builds ( + partition_id bigint DEFAULT 100 NOT NULL, + build_id bigint NOT NULL, runner_machine_id bigint NOT NULL -); +) +PARTITION BY LIST (partition_id); CREATE TABLE ci_runner_machines ( id bigint NOT NULL, @@ -370,44 +375,55 @@ scope. | GitLab Rails app | `%15.8` | Create database migration to add `config` column to `ci_runner_machines` table. | | GitLab Runner | `%15.9` | Start sending `system_id` value in `POST /jobs/request` request and other follow-up requests that require identifying the unique system. | | GitLab Rails app | `%15.9` | Create service similar to `StaleGroupRunnersPruneCronWorker` service to clean up `ci_runner_machines` records instead of `ci_runners` records.
Existing service continues to exist but focuses only on legacy runners. | -| GitLab Rails app | `%15.9` | [Feature flag] Rollout of `create_runner_machine`. | +| GitLab Rails app | `%15.9` | Implement the `create_runner_machine` [feature flag](../../../administration/feature_flags.md). | | GitLab Rails app | `%15.9` | Create `ci_runner_machines` record in `POST /runners/verify` request if the runner token is prefixed with `glrt-`. | | GitLab Rails app | `%15.9` | Use runner token + `system_id` JSON parameters in `POST /jobs/request` request in the [heartbeat request](https://gitlab.com/gitlab-org/gitlab/blob/c73c96a8ffd515295842d72a3635a8ae873d688c/lib/api/ci/helpers/runner.rb#L14-20) to update the `ci_runner_machines` cache/table. | -| GitLab Rails app | `%15.9` | [Feature flag] Enable runner creation workflow (`create_runner_workflow`). | +| GitLab Rails app | `%15.9` | Implement the `create_runner_workflow_for_admin` [feature flag](../../../administration/feature_flags.md). | | GitLab Rails app | `%15.9` | Implement `create_{instance|group|project}_runner` permissions. | | GitLab Rails app | `%15.9` | Rename `ci_runner_machines.machine_xid` column to `system_xid` to be consistent with `system_id` passed in APIs. | -| GitLab Rails app | `%15.10` | Drop `ci_runner_machines.machine_xid` column. | -| GitLab Rails app | `%15.11` | Remove the ignore rule for `ci_runner_machines.machine_xid` column. | +| GitLab Rails app | `%15.10` | Remove the ignore rule for `ci_runner_machines.machine_xid` column. | +| GitLab Rails app | `%15.10` | Replace `ci_builds_metadata.runner_machine_id` with a new join table. | +| GitLab Rails app | `%15.11` | Drop `ci_builds_metadata.runner_machine_id` column. | +| GitLab Rails app | `%16.0` | Remove the ignore rule for `ci_builds_metadata.runner_machine_id` column. | ### Stage 4 - Create runners from the UI | Component | Milestone | Changes | |------------------|----------:|---------| -| GitLab Rails app | `%15.9` | Implement new GraphQL user-authenticated API to create a new runner. | | GitLab Rails app | `%15.9` | [Add prefix to newly generated runner authentication tokens](https://gitlab.com/gitlab-org/gitlab/-/issues/383198). | +| GitLab Rails app | `%15.9` | Add new runner field for with token that is used in registration | +| GitLab Rails app | `%15.9` | Implement new GraphQL user-authenticated API to create a new runner. | | GitLab Rails app | `%15.10` | Return token and runner ID information from `/runners/verify` REST endpoint. | | GitLab Runner | `%15.10` | [Modify register command to allow new flow with glrt- prefixed authentication tokens](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/29613). | -| GitLab Rails app | `%15.10` | Implement UI to create new runner. | -| GitLab Rails app | `%15.10` | GraphQL changes to `CiRunner` type. | -| GitLab Rails app | `%15.10` | UI changes to runner details view (listing of platform, architecture, IP address, etc.) (?) | +| GitLab Runner | `%15.10` | Make the `gitlab-runner register` command happen in a single operation. | +| GitLab Rails app | `%15.10` | Define feature flag and policies for "New Runner creation workflow" for groups and projects. | +| GitLab Rails app | `%15.10` | Only update runner `contacted_at` and `status` when polled for jobs. | +| GitLab Rails app | `%15.10` | Add GraphQL type to represent runner machines under `CiRunner`. | +| GitLab Rails app | `%15.11` | Implement UI to create new instance runner. | +| GitLab Rails app | `%15.11` | Update service and mutation to accept groups and projects. | +| GitLab Rails app | `%15.11` | Implement UI to create new group/project runners. | +| GitLab Rails app | `%15.11` | Add runner_machine field to CiJob GraphQL type. | +| GitLab Rails app | `%15.11` | UI changes to runner details view (listing of platform, architecture, IP address, etc.) (?) | | GitLab Rails app | `%15.11` | Adapt `POST /api/v4/runners` REST endpoint to accept a request from an authorized user with a scope instead of a registration token. | +| GitLab Runner | `%15.11` | Handle `glrt-` runner tokens in `unregister` command. | +| GitLab Runner | `%15.11` | Runner asks for registration token when a `glrt-` runner token is passed in `--token`. | +| GitLab Rails app | `%15.11` | Move from 'runner machine' terminology to 'runner manager'. | ### Stage 5 - Optional disabling of registration token | Component | Milestone | Changes | |------------------|----------:|---------| -| GitLab Rails app | `%15.11` | Adapt `register_{group|project}_runner` permissions to take [application setting](https://gitlab.com/gitlab-org/gitlab/-/issues/386712) in consideration. | -| GitLab Rails app | `%15.11` | Add UI to allow disabling use of registration tokens at project or group level. | -| GitLab Rails app | `%15.11` | Introduce `:enforce_create_runner_workflow` feature flag (disabled by default) to control whether use of registration tokens is allowed. | -| GitLab Rails app | `%15.11` | Make [`POST /api/v4/runners` endpoint](../../../api/runners.md#register-a-new-runner) permanently return `HTTP 410 Gone` if either `allow_runner_registration_token` setting or `:enforce_create_runner_workflow` feature flag disables registration tokens.
A future v5 version of the API should return `HTTP 404 Not Found`. | -| GitLab Rails app | `%15.11` | Start refusing job requests that don't include a unique ID, if either `allow_runner_registration_token` setting or `:enforce_create_runner_workflow` feature flag disables registration tokens. | -| GitLab Rails app | `%15.11` | Hide legacy UI showing registration with a registration token, if `:enforce_create_runner_workflow` feature flag disables registration tokens. | +| GitLab Rails app | `%16.0` | Adapt `register_{group|project}_runner` permissions to take [application setting](https://gitlab.com/gitlab-org/gitlab/-/issues/386712) in consideration. | +| GitLab Rails app | | Add UI to allow disabling use of registration tokens at project or group level. | +| GitLab Rails app | | Introduce `:enforce_create_runner_workflow` feature flag (disabled by default) to control whether use of registration tokens is allowed. | +| GitLab Rails app | | Make [`POST /api/v4/runners` endpoint](../../../api/runners.md#register-a-new-runner) permanently return `HTTP 410 Gone` if either `allow_runner_registration_token` setting or `:enforce_create_runner_workflow` feature flag disables registration tokens.
A future v5 version of the API should return `HTTP 404 Not Found`. | +| GitLab Rails app | | Start refusing job requests that don't include a unique ID, if either `allow_runner_registration_token` setting or `:enforce_create_runner_workflow` feature flag disables registration tokens. | +| GitLab Rails app | | Hide legacy UI showing registration with a registration token, if `:enforce_create_runner_workflow` feature flag disables registration tokens. | ### Stage 6 - Enforcement | Component | Milestone | Changes | |------------------|----------:|---------| -| GitLab Runner | `%16.0` | Do not allow runner to start if `.runner_system_id` file cannot be written. | | GitLab Rails app | `%16.6` | Enable `:enforce_create_runner_workflow` feature flag by default. | | GitLab Rails app | `%16.6` | Start reject job requests that don't include `system_id` value. | @@ -495,7 +511,7 @@ gitlab-runner register --executor "shell" \ --url "https://gitlab.com/" \ --non-interactive \ - --registration-token="grlt-2CR8_eVxiioB1QmzPZwa" + --registration-token="glrt-2CR8_eVxiioB1QmzPZwa" ``` ### How does this change impact auto-scaling scenarios? diff --git a/doc/architecture/blueprints/search/code_search_with_zoekt.md b/doc/architecture/blueprints/search/code_search_with_zoekt.md deleted file mode 100644 index d0d347f1ff4..00000000000 --- a/doc/architecture/blueprints/search/code_search_with_zoekt.md +++ /dev/null @@ -1,305 +0,0 @@ ---- -status: ongoing -creation-date: "2022-12-28" -authors: [ "@dgruzd", "@DylanGriffith" ] -coach: "@DylanGriffith" -approvers: [ "@joshlambert", "@changzhengliu" ] -owning-stage: "~devops::enablement" -participating-stages: [] ---- - -# Use Zoekt For code search - -## Summary - -We will be implementing an additional code search functionality in GitLab that -is backed by [Zoekt](https://github.com/sourcegraph/zoekt), an open source -search engine that is specifically designed for code search. Zoekt will be used as -an API by GitLab and remain an implementation detail while the user interface -in GitLab will not change much except for some new features made available by -Zoekt. - -This will be rolled out in phases to ensure that the system will actually meet -our scaling and cost expectations and will run alongside code search backed by -Elasticsearch until we can be sure it is a viable replacement. The first step -will be making it available for `gitlab-org` for internal and expanding -customer by customer based on customer interest. - -## Motivation - -GitLab code search functionality today is backed by Elasticsearch. -Elasticsearch has proven useful for other types of search (issues, merge -requests, comments and so-on) but is by design not a good choice for code -search where users expect matches to be precise (ie. no false positives) and -flexible (e.g. support -[substring matching](https://gitlab.com/gitlab-org/gitlab/-/issues/325234) -and -[regexes](https://gitlab.com/gitlab-org/gitlab/-/issues/4175)). We have -[investigated our options](https://gitlab.com/groups/gitlab-org/-/epics/7404) -and [Zoekt](https://github.com/sourcegraph/zoekt) is pretty much the only well -maintained open source technology that is suited to code search. Based on our -research we believe it will be better to adopt a well maintained open source -database than attempt to build our own. This is mostly due to the fact that our -research indicates that the fundamental architecture of Zoekt is what we would -implement again if we tried to implement something ourselves. - -Our -[early benchmarking](https://gitlab.com/gitlab-org/gitlab/-/issues/370832#note_1183611955) -suggests that Zoekt will be viable at our scale, but we feel strongly -that investing in building a beta integration with Zoekt and rolling it out -group by group on GitLab.com will provide better insights into scalability and -cost than more accurate benchmarking efforts. It will also be relatively low -risk as it will be rolled out internally first and later rolled out to -customers that wish to participate in the trial. - -### Goals - -The main goals of this integration will be to implement the following highly -requested improvements to code search: - -1. [Exact match (substring match) code searches in Advanced Search](https://gitlab.com/gitlab-org/gitlab/-/issues/325234) -1. [Support regular expressions with Advanced Global Search](https://gitlab.com/gitlab-org/gitlab/-/issues/4175) -1. [Support multiple line matches in the same file](https://gitlab.com/gitlab-org/gitlab/-/issues/668) - -The initial phases of the rollout will be designed to catch and resolve scaling -or infrastructure cost issues as early as possible so that we can pivot early -before investing too much in this technology if it is not suitable. - -### Non-Goals - -The following are not goals initially but could theoretically be built upon -this solution: - -1. Improving security scanning features by having access to quickly perform - regex scans across many repositories -1. Saving money on our search infrastructure - this may be possible with - further optimizations, but initial estimates suggest the cost is similar -1. AI/ML features of search used to predict what users might be interested in - finding -1. Code Intelligence and Navigation - likely code intelligence and navigation - features should be built on structured data rather than a trigram index but - regex based searches (using Zoekt) may be a suitable fallback for code which - does not have structured metadata enabled or dynamic languages where static - analysis is not very accurate. Zoekt in particular may not be well suited - initially, despite existing symbol extraction using ctags, because ctags - symbols may not contain enough data for accurate navigation and Zoekt - doesn't undersand dependencies which would be necessary for cross-project - navigation. - -## Proposal - -An -[initial implementation of a Zoekt integration](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/105049) -was created to demonstrate the feasibility of using Zoekt as a drop-in -replacement for Elasticsearch code searches. This blueprint will extend on all -the details needed to provide a minimum viable change as well steps needed to -scale this to a larger customer rollout on GitLab.com. - -## Design and implementation details - -### User Experience - -When a user performs an advanced search on a group or project that is part -of the Zoekt rollout we will present a toggle somewhere in the UI to change -to "precise search" (or some other UX TBD) which switches them from -Elasticsearch to Zoekt. Early user feedback will help us assess the best way -to present these choices to users and ultimately we will want to remove the -Elasticsearch option if we find Zoekt is a suitable long term option. - -### Indexing - -Similar to our Elasticsearch integration, GitLab will notify Zoekt every time -there are updates to a repository. Zoekt, unlike Elasticsearch, is designed to -clone and index Git repositories so we will simply notify Zoekt of the URL of -the repository that has changed and it will update its local copy of the Git -repo and then update its local index files. The Zoekt side of this logic will -be implemented in a new server-side indexing endpoint we add to Zoekt which is -currently in -[an open Pull request](https://github.com/sourcegraph/zoekt/pull/496). -While the details of -this pull request are still being debated, we may choose to deploy a fork with -the functionality we need, but our strongest intention is not to maintain a -fork of Zoekt and the maintainers have already expressed they are open to this -new functionality. - -The rails side of the integration will be a Sidekiq worker that is scheduled -every time there is an update to a repository and it will simply call this -`/index` endpoint in Zoekt. This will also need to generate a one-time token -that can allow Zoekt to clone a private repository. - -```mermaid -sequenceDiagram - participant user as User - participant gitlab_git as GitLab Git - participant gitlab_sidekiq as GitLab Sidekiq - participant zoekt as Zoekt - user->>gitlab_git: git push git@gitlab.com:gitlab-org/gitlab.git - gitlab_git->>gitlab_sidekiq: ZoektIndexerWorker.perform_async(278964) - gitlab_sidekiq->>zoekt: POST /index {"RepoUrl":"https://zoekt:SECRET_TOKEN@gitlab.com/gitlab-org/gitlab.git","RepoId":278964}' - zoekt->>gitlab_git: git clone https://zoekt:SECRET_TOKEN@gitlab.com/gitlab-org/gitlab.git -``` - -The Sidekiq worker can leverage de-duplication based on the `project_id`. - -Zoekt supports indexing multiple projects we'll likely need to, eventually, -allow a way for users to configure additional branches (beyond the default -branch) and this will need to be sent to Zoekt. We will need to decide if these -branch lists are sent every time we index the project or only when they change -configuration. - -There may be race conditions with multiple Zoekt processes indexing the same -repo at the same time. For this reason we should implement a locking mechanism -somewhere to ensure we are only indexing 1 project in 1 place at a time. We -could make use of the same Redis locking we use for indexing projects in -Elasticsearch. - -### Searching - -Searching will be implemented using the `/api/search` functionality in -Zoekt. There is also -[an open PR to fix this endpoint in Zoekt](https://github.com/sourcegraph/zoekt/pull/506), -and again we may consider working from a fork until this is fixed. GitLab will -prepend all searches with the appropriate filter for repositories based on the -user's search context (group or project) in the same way we do for -Elasticsearch. For Zoekt this will be implemented as a query string regex that -matches all the searched repositories. - -### Zoekt infrastructure - -Each Zoekt node will need to run a -[zoekt-dynamic-indexserver](https://github.com/sourcegraph/zoekt/pull/496) and -a -[zoekt-webserver](https://github.com/sourcegraph/zoekt/blob/main/cmd/zoekt-webserver/main.go). -These are both webservers with different responsibilities. Considering that the -Zoekt indexing process needs to keep a full clone of the bare repo -([unless we come up with a better option](https://gitlab.com/gitlab-org/gitlab/-/issues/384722)) -these bare repos will be stored on spinning disks to save space. These are only -used as an intermediate step to generate the actual `.zoekt` index files which -will be stored on an SSD for fast searches. These web servers need to run on -the same node as they access the same files. The `zoekt-dynamic-indexserver` is -responsible for writing the `.zoekt` index files. The `zoekt-webserver` is -responsible for responding to searches that it performs by reading these -`.zoekt` index files. - -### Rollout strategy - -Initially Zoekt code search will only be available to `gitlab-org`. After that -we'll start rolling it out to specific customers that have requested better -code search experience. As we learn about scaling and make improvements we will -gradually roll it out to all licensed groups on GitLab.com. We will use a -similar approach to Elasticsearch for keeping track of which groups are indexed -and which are not. This will be based on a new table `zoekt_indexed_namespaces` -with a `namespace_id` reference. We will only allow rolling out to top level -namespaces to simplify the logic of checking for all layers of group -inheritance. Once we've rolled out to all licensed groups we'll enable logic to -automatically enroll newly licensed groups. This table also may be a place to -store per-namespace sharding and replication data as described below. - -### Sharding and replication strategy - -Zoekt does not have any inbuilt sharding, and we expect that we'll need -multiple Zoekt servers to reach the scale to provide search functionality to -all of GitLab licensed customers. - -There are 2 clear ways to implement sharding: - -1. Build it on top of, or in front of Zoekt, as an independent component. Building - all the complexities of a distributed database into Zoekt is not likely to - be a good direction for the project so most likely this would be an - independent piece of infrastructure that proxied requests to the correct - shard. -1. Manage the shards inside GitLab. This would be an application layer in - GitLab which chooses the correct shard to send indexing and search requests - to. - -Likewise, there are a few ways to implement replication: - -1. Server-side where Zoekt replicas are aware of other Zoekt replicas and they - stream updates from some primary to remain in sync -1. Client-side replication where clients send indexing requests to all replicas - and search requests to any replica - -We plan to implement sharding inside GitLab application but replication may be -best served at the level of the filesystem of Zoekt servers rather than sending -duplicated updates from GitLab to all replicas. This could be some process on -Zoekt servers that monitors for changes to the `.zoekt` files in a specific -directory and syncs those updates to the replicas. This will need to be -slightly more sophisticated than `rsync` because the files are constantly -changing and files may be getting deleted while the sync is happening so we -would want to be syncing the updates in batches somehow without slowing down -indexing. - -Implementing sharding in GitLab simplifies the additional infrastructure -components that need to be deployed and allows more flexibility to control our -rollout to many customers alongside our rollout of multiple shards. - -Implementing syncing from primary -> replica on Zoekt nodes at the filesystem -level optimizes that overall resource usage. We only need to sync the index -files to replicas as the bare repo is just a cache. This saves on: - -1. Disk space on replicas -1. CPU usage on replicas as it does not need to rebuild the index -1. Load on Gitaly to clone the repos - -We plan to defer the implementation of these high availability aspects until -later, but a preliminary plan would be: - -1. GitLab is configured with a pool of Zoekt servers -1. GitLab assigns groups randomly a Zoekt primary server -1. There will also be Zoekt replica servers -1. Periodically Zoekt primary servers will sync their `.zoekt` index files to - their respective replicas -1. There will need to be some process by which to promote a replica to a - primary if the primary is having issues. We will be using Consul for - keeping track of which is the primary and which are the replicas. -1. When indexing a project GitLab will queue a Sidekiq job to update the index - on the primary -1. When searching we will randomly select one of the Zoekt primaries or replica - servers for the group being searched. We don't care which is "more up to - date" as code search will be "eventually consistent" and all reads may read - slightly out of date indexes. We will have a target of maximum latency of - index updates and may consider removing nodes from rotation if they are too - far out of date. -1. We will shard everything by top level group as this ensures group search can - always search a single Zoekt server. Aggregation may be possible for global - searches at some point in future if this turns out to be important. Smaller - self-managed instances may use a single Zoekt server allowing global - searches to work without any aggregation being implemented. Depending on our - largest group sizes and scaling limitations of a single node Zoekt server we - may consider implementing an approach where a group can be assigned multiple - shards. - -The downside of the chosen path will be added complexity of managing all these -Zoekt servers from GitLab when compared with a "proxy" layer outside of GitLab -that is managing all of these shards. We will consider this decision a work in -progress and reassess if it turns out to add too much complexity to GitLab. - -#### Sharding proposal using GitLab `::Zoekt::Shard` model - -This is already implemented as the `::Zoekt::IndexedNamespace` -implements a many-to-many relationship between namespaces and shards. - -#### Replication and service discovery using Consul - -If we plan to replicate at the Zoekt node level as described above we need to -change our data model to use a one-to-many relationship from `zoekt_shards -> -namespaces`. This means making the `namespace_id` column unique in -`zoekt_indexed_namespaces`. Then we need to implement a service discovery -approach where the `index_url` always points at a primary Zoekt node and the -`search_url` is a DNS record with N replicas and the primary. We then choose -randomly from `search_url` records when searching. - -### Iterations - -1. Make available for `gitlab-org` -1. Improve monitoring -1. Improve performance -1. Make available for select customers -1. Implement sharding -1. Implement replication -1. Make available to many more licensed groups -1. Implement automatic (re)balancing of shards -1. Estimate costs for rolling out to all licensed groups and decide if it's worth it or if we need to optimize further or adjust our plan -1. Rollout to all licensed groups -1. Improve performance -1. Assess costs and decide whether we should roll out to all free customers diff --git a/doc/architecture/blueprints/secret_detection/index.md b/doc/architecture/blueprints/secret_detection/index.md index 26551367a7c..fc97ca71d7f 100644 --- a/doc/architecture/blueprints/secret_detection/index.md +++ b/doc/architecture/blueprints/secret_detection/index.md @@ -1,5 +1,5 @@ --- -status: proposed +status: ongoing creation-date: "2022-11-25" authors: [ "@theoretick" ] coach: "@DylanGriffith" @@ -8,6 +8,8 @@ owning-stage: "~devops::secure" participating-stages: [] --- + + # Secret Detection as a platform-wide experience ## Summary @@ -24,7 +26,7 @@ job logs, and project management features such as issues, epics, and MRs. ### Goals -- Support asynchronous secret detection for: +- Support asynchronous secret detection for the following scan targets: - push events - issuable creation - issuable updates @@ -47,6 +49,24 @@ Scanned object types beyond the scope of this MVC include: - Snippets - Wikis +#### Management UI + +Development of an independent interface for managing secrets is out of scope +for this blueprint. Any detections will be managed using the existing +Vulnerability Management UI. + +Management of detected secrets will remain distinct from the +[Secret Management feature capability](../../../ci/secrets/index.md) as +"detected" secrets are categorically distinct from actively "managed" secrets. +When a detected secret is identified, it has already been compromised due to +their presence in the target object (that is a repository). Alternatively, managed +secrets should be stored with stricter standards for secure storage, including +encryption and masking when visible (such as job logs or in the UI). + +As a long-term priority we should consider unifying the management of the two +secret types however that work is out of scope for the current blueprints goals, +which remain focused on active detection. + ## Proposal To achieve scalable secret detection for a variety of domain objects a dedicated @@ -67,6 +87,7 @@ as self-managed instances. - Secure authentication to GitLab.com infrastructure - Performance of scanning against large blobs - Performance of scanning against volume of domain objects (such as push frequency) +- Queueing of scan requests ## Design and implementation details @@ -74,13 +95,13 @@ The critical paths as outlined under [goals above](#goals) cover two major objec types: Git blobs (corresponding to push events) and arbitrary text blobs. The detection flow for push events relies on subscribing to the PostReceive hook -and enqueueing Sidekiq requests to the `SecretScanningService`. The `SecretScanningService` +to enqueue Sidekiq requests to the `SecretScanningService`. The `SecretScanningService` service fetches enqueued refs, queries Gitaly for the ref blob contents, scans the commit contents, and notifies the Rails application when a secret is detected. See [Push event detection flow](#push-event-detection-flow) for sequence. The detection flow for arbitrary text blobs, such as issue comments, relies on -subscribing to `Notes::PostProcessService` (or equivalent service) and enqueueing +subscribing to `Notes::PostProcessService` (or equivalent service) to enqueue Sidekiq requests to the `SecretScanningService` to process the text blob by object type and primary key of domain object. The `SecretScanningService` service fetches the relevant text blob, scans the contents, and notifies the Rails application when a secret @@ -92,7 +113,7 @@ around scanning during streaming and the added complexity in buffering lookbacks for arbitrary trace chunks. In any case of detection, the Rails application manually creates a vulnerability -using the `Vulnerabilities::ManuallyCreateService` to surface the finding within the +using the `Vulnerabilities::ManuallyCreateService` to surface the finding in the existing Vulnerability Management UI. See [technical discovery](https://gitlab.com/gitlab-org/gitlab/-/issues/376716) @@ -115,7 +136,7 @@ Token types to identify in order of importance: ### Detection engine Our current secret detection offering utilizes [Gitleaks](https://github.com/zricethezav/gitleaks/) -for all secret scanning within pipeline contexts. By using its `--no-git` configuration +for all secret scanning in pipeline contexts. By using its `--no-git` configuration we can scan arbitrary text blobs outside of a repository context and continue to utilize it for non-pipeline scanning. @@ -123,6 +144,23 @@ Given our existing familiarity with the tool and its extensibility, it should remain our engine of choice. Changes to the detection engine are out of scope unless benchmarking unveils performance concerns. +Notable alternatives include high-performance regex engines such as [hyperscan](https://github.com/intel/hyperscan) or it's portable fork [vectorscan](https://github.com/VectorCamp/vectorscan). + +### High-level architecture + +The implementation of the secret scanning service is highly dependent on the outcomes of our benchmarking +and capacity planning against both GitLab.com and our +[Reference Architectures](../../../administration/reference_architectures/index.md). +As the scanning capability must be an on-by-default component of both our SaaS and self-managed +instances [the PoC](#iterations), the deployment characteristics must be considered to determine whether +this is a standalone component or executed as a subprocess of the existing Sidekiq worker fleet +(similar to the implementation of our Elasticsearch indexing service). + +Similarly, the scan target volume will require a robust and scalable enqueueing system to limit resource consumption. + +See [this thread](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/105142#note_1194863310) +for past discussion around scaling approaches. + ### Push event detection flow ```mermaid @@ -151,17 +189,20 @@ sequenceDiagram ## Iterations -1. Requirements definition for detection coverage and actions -1. PoC of secret scanning service - 1. gRPC commit retrieval from Gitaly - 1. blob scanning - 1. benchmarking of issuables, comments, job logs and blobs to gain confidence that the total costs will be viable -1. Implementation of secret scanning service MVC (targeting individual commits) -1. Security and readiness review -1. Deployment and monitoring -1. Implementation of secret scanning service MVC (targeting arbitrary text blobs) -1. Deployment and monitoring -1. High priority domain object rollout (priority `TBD`) - 1. Issuable comments - 1. Issuable bodies - 1. Job logs +- ✓ Define [requirements for detection coverage and actions](https://gitlab.com/gitlab-org/gitlab/-/issues/376716) +- ✓ Implement [Clientside detection of GitLab tokens in comments/issues](https://gitlab.com/gitlab-org/gitlab/-/issues/368434) +- PoC of secret scanning service + - Benchmarking of issuables, comments, job logs and blobs to gain confidence that the total costs will be viable + - Capacity planning for addition of service component to Reference Architectures headroom + - Service capabilities + - gRPC commit retrieval from Gitaly + - blob scanning +- Implementation of secret scanning service MVC (targeting individual commits) +- Security and readiness review +- Deployment and monitoring +- Implementation of secret scanning service MVC (targeting arbitrary text blobs) +- Deployment and monitoring +- High priority domain object rollout (priority `TBD`) + - Issuable comments + - Issuable bodies + - Job logs diff --git a/doc/architecture/blueprints/work_items/index.md b/doc/architecture/blueprints/work_items/index.md index 2c854ecea59..f067d9fab52 100644 --- a/doc/architecture/blueprints/work_items/index.md +++ b/doc/architecture/blueprints/work_items/index.md @@ -8,6 +8,8 @@ owning-stage: "~devops::plan" participating-stages: [] --- + + # Work Items This document is a work-in-progress. Some aspects are not documented, though we expect to add them in the future. diff --git a/doc/architecture/index.md b/doc/architecture/index.md index 2467ba33fae..9fccd1c21d1 100644 --- a/doc/architecture/index.md +++ b/doc/architecture/index.md @@ -1,6 +1,5 @@ --- feedback: false -comments: false toc: false --- diff --git a/doc/ci/caching/index.md b/doc/ci/caching/index.md index 1aa579b842a..e9d3dae3837 100644 --- a/doc/ci/caching/index.md +++ b/doc/ci/caching/index.md @@ -38,7 +38,7 @@ can't link to files outside it. - Subsequent jobs in later stages of the same pipeline can use artifacts. - Different projects cannot share artifacts. - Artifacts expire after 30 days by default. You can define a custom [expiration time](../yaml/index.md#artifactsexpire_in). -- The latest artifacts do not expire if [keep latest artifacts](../pipelines/job_artifacts.md#keep-artifacts-from-most-recent-successful-jobs) is enabled. +- The latest artifacts do not expire if [keep latest artifacts](../jobs/job_artifacts.md#keep-artifacts-from-most-recent-successful-jobs) is enabled. - Use [dependencies](../yaml/index.md#dependencies) to control which jobs fetch the artifacts. ## Good caching practices @@ -65,7 +65,7 @@ For runners to work with caches efficiently, you must do one of the following: ## Use multiple caches > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32814) in GitLab 13.10. -> - [Feature Flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/321877), in GitLab 13.12. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/321877), in GitLab 13.12. You can have a maximum of four caches: @@ -91,10 +91,37 @@ test-job: ``` If multiple caches are combined with a fallback cache key, -the fallback cache is fetched every time a cache is not found. +the global fallback cache is fetched every time a cache is not found. ## Use a fallback cache key +### Per-cache fallback keys + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/110467) in GitLab 16.0 + +Each cache entry supports up-to 5 fallback keys: + +```yaml +test-job: + stage: build + cache: + - key: cache-$CI_COMMIT_REF_SLUG + fallback_keys: + - cache-$CI_DEFAULT_BRANCH + - cache-default + paths: + - vendor/ruby + script: + - bundle config set --local path 'vendor/ruby' + - bundle install + - yarn install --cache-folder .yarn-cache + - echo Run tests... +``` + +Fallback keys follows the same processing logic as `cache:key`, meaning that the fullname may include a `-$index` (based on cache clearance) and `-protected`/`-non_protected` (if cache separation enabled on protected branches) suffixes. + +### Global fallback key + > [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/1534) in GitLab Runner 13.4. You can use the `$CI_COMMIT_REF_SLUG` [predefined variable](../variables/predefined_variables.md) @@ -120,12 +147,20 @@ job1: - binaries/ ``` +The order of caches extraction is: + +1. Retrieval attempt for `cache:key` +1. Retrieval attemps for each entry in order in `fallback_keys` +1. Retrieval attempt for the global fallback key in `CACHE_FALLBACK_KEY` + +The cache extraction process stops after the first successful cache is retrieved. + ## Disable cache for specific jobs If you define the cache globally, each job uses the same definition. You can override this behavior for each job. -To disable it completely for a job, use an empty hash: +To disable it completely for a job, use an empty list: ```yaml job: @@ -139,13 +174,14 @@ You can override cache settings without overwriting the global cache by using `policy` for one job: ```yaml -cache: &global_cache - key: $CI_COMMIT_REF_SLUG - paths: - - node_modules/ - - public/ - - vendor/ - policy: pull-push +default: + cache: &global_cache + key: $CI_COMMIT_REF_SLUG + paths: + - node_modules/ + - public/ + - vendor/ + policy: pull-push job: cache: @@ -330,8 +366,8 @@ before_script: test: script: - python setup.py test - - pip install flake8 - - flake8 . + - pip install ruff + - ruff --format=gitlab . ``` ### Cache Ruby dependencies @@ -570,7 +606,7 @@ You can clear the cache in the GitLab UI: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **CI/CD > Pipelines**. -1. In the upper right, select **Clear runner caches**. +1. In the upper-right corner, select **Clear runner caches**. On the next commit, your CI/CD jobs use a new cache. @@ -589,6 +625,7 @@ If you have a cache mismatch, follow these steps to troubleshoot. | You use runners in autoscale mode without a distributed cache enabled. | Configure the autoscale runner to use a distributed cache. | | The machine the runner is installed on is low on disk space or, if you've set up distributed cache, the S3 bucket where the cache is stored doesn't have enough space. | Make sure you clear some space to allow new caches to be stored. There's no automatic way to do this. | | You use the same `key` for jobs where they cache different paths. | Use different cache keys so that the cache archive is stored to a different location and doesn't overwrite wrong caches. | +| You have not enabled the [distributed runner caching on your runners](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching). | Set `Shared = false` and re-provision your runners. | #### Cache mismatch example 1 @@ -621,10 +658,10 @@ job B: ``` 1. `job A` runs. -1. `public/` is cached as cache.zip. +1. `public/` is cached as `cache.zip`. 1. `job B` runs. 1. The previous cache, if any, is unzipped. -1. `vendor/` is cached as cache.zip and overwrites the previous one. +1. `vendor/` is cached as `cache.zip` and overwrites the previous one. 1. The next time `job A` runs it uses the cache of `job B` which is different and thus isn't effective. diff --git a/doc/ci/chatops/img/gitlab-chatops-icon-small.png b/doc/ci/chatops/img/gitlab-chatops-icon-small.png deleted file mode 100644 index 71cc5dba5cf..00000000000 Binary files a/doc/ci/chatops/img/gitlab-chatops-icon-small.png and /dev/null differ diff --git a/doc/ci/chatops/index.md b/doc/ci/chatops/index.md index f0efb5fc884..6dff87ed1c5 100644 --- a/doc/ci/chatops/index.md +++ b/doc/ci/chatops/index.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Manage +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: index, concepts, howto --- @@ -9,62 +9,70 @@ type: index, concepts, howto > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/4466) in GitLab Ultimate 10.6. > - [Moved](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/24780) to GitLab Free in 11.9. -> - `CHAT_USER_ID` was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/341798) in GitLab 14.4. +> - `CHAT_USER_ID` [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/341798) in GitLab 14.4. -GitLab ChatOps provides a method to interact with CI/CD jobs through chat services -like Slack. Many organizations' discussion, collaboration, and troubleshooting takes -place in chat services. Having a method to run CI/CD jobs with output -posted back to the channel can significantly augment your team's workflow. +Use GitLab ChatOps to interact with CI/CD jobs through chat services +like Slack. -## How GitLab ChatOps works +Many organizations use chat services to collaborate, troubleshoot, and plan work. With ChatOps, +you can discuss work with your team, run CI/CD jobs, and view job output, all from the same +application. -GitLab ChatOps is built upon [GitLab CI/CD](../index.md) and -[Slack Slash Commands](../../user/project/integrations/slack_slash_commands.md). -ChatOps provides a `run` action for [slash commands](../../integration/slash_commands.md) -with the following arguments: +## ChatOps workflow and CI/CD configuration -- A `` to execute. -- The ``. +ChatOps looks for the specified job in the +[`.gitlab-ci.yml`](../yaml/index.md) on the project's default +branch. If the job is found, ChatOps creates a pipeline that contains +only the specified job. If you set `when: manual`, ChatOps creates the +pipeline, but the job doesn't start automatically. + +A job run with ChatOps has the same functionality as a job run from +GitLab. The job can use existing [CI/CD variables](../variables/index.md#predefined-cicd-variables) like +`GITLAB_USER_ID` to perform additional rights validation, but these +variables can be [overridden](../variables/index.md#cicd-variable-precedence). + +You should set [`rules`](../yaml/index.md#rules) so the job does not +run as part of the standard CI/CD pipeline. ChatOps passes the following [CI/CD variables](../variables/index.md#predefined-cicd-variables) to the job: -- `CHAT_INPUT` contains any additional arguments. -- `CHAT_CHANNEL` is set to the name of channel the action was triggered in. -- `CHAT_USER_ID` is set to the chat service's user ID of the user who triggered the slash command. +- `CHAT_INPUT` - The arguments passed to `/project-name run`. +- `CHAT_CHANNEL` - The name of the chat channel the job is run from. +- `CHAT_USER_ID` - The chat service ID of the user who runs the job. -When executed, ChatOps looks up the specified job name and attempts to match it -to a corresponding job in [`.gitlab-ci.yml`](../yaml/index.md). If a matching job -is found on the default branch, a pipeline containing only that job is scheduled. After the -job completes: +When the job runs: -- If the job completes in *less than 30 minutes*, the ChatOps sends the job's output to Slack. -- If the job completes in *more than 30 minutes*, the job must use the +- If the job completes in less than 30 minutes, ChatOps sends the job output to the chat channel. +- If the job completes in more than 30 minutes, you must use a method like the [Slack API](https://api.slack.com/) to send data to the channel. -To use the `run` command, you must have at least the -Developer role. -If a job shouldn't be able to be triggered from chat, you can set the job to `except: [chat]`. +## Run a CI/CD job + +You can run a CI/CD job from chat with the `/project-name run` +[slash command](../../integration/slash_commands.md). + +Prerequisites: + +- You must have at least the Developer role for the project. + +To run a CI/CD job: + +- In the chat client, enter `/project-name run `. -## Best practices for ChatOps CI jobs +ChatOps schedules a pipeline that contains only the specified job. -Since ChatOps is built upon GitLab CI/CD, the job has all the same features and -functions available. Consider these best practices when creating ChatOps jobs: +### Exclude a job from ChatOps -- GitLab strongly recommends you set [`rules`](../yaml/index.md#rules) so the job does not run as part - of the standard CI pipeline. -- If the job is set to `when: manual`, ChatOps creates the pipeline, but the job waits to be started. -- ChatOps provides limited support for access control. If the user triggering the - slash command has at least the Developer role - in the project, the job runs. The job itself can use existing - [CI/CD variables](../variables/index.md#predefined-cicd-variables) like - `GITLAB_USER_ID` to perform additional rights validation, but - these variables can be [overridden](../variables/index.md#cicd-variable-precedence). +To prevent a job from being run from chat: -### Controlling the ChatOps reply +- In `.gitlab-ci.yml`, set the job to `except: [chat]`. -The output for jobs with a single command is sent to the channel as a reply. For -example, the chat reply of the following job is `Hello World` in the channel: +## Customize the ChatOps reply + +ChatOps sends the output for a job with a single command to the +channel as a reply. For example, when the following job runs, +the chat reply is `Hello world`: ```yaml stages: @@ -78,13 +86,12 @@ hello-world: - echo "Hello World" ``` -Jobs that contain multiple commands (or `before_script`) return additional -content in the chat reply. In these cases, both the commands and their output are -included, with the commands wrapped in ANSI color codes. +If the job contains multiple commands, or if `before_script` is set, ChatOps sends the commands +and their output to the channel. The commands are wrapped in ANSI color codes. -To selectively reply with the output of one command, its output must be bounded by -the `chat_reply` section. For example, the following job lists the files in the -current directory: +To selectively reply with the output of one command, place the output +in a `chat_reply` section. For example, the following job lists the +files in the current directory: ```yaml stages: @@ -99,28 +106,8 @@ ls: - echo -e "section_start:$( date +%s ):chat_reply\r\033[0K\n$( ls -la )\nsection_end:$( date +%s ):chat_reply\r\033[0K" ``` -## GitLab ChatOps examples - -The GitLab.com team created a repository of [common ChatOps scripts](https://gitlab.com/gitlab-com/chatops) -they use to interact with our Production instance of GitLab. Administrators of -other GitLab instances may find them useful. They can serve as inspiration for ChatOps -scripts you can write to interact with your own applications. - -## GitLab ChatOps icon - -The [official GitLab ChatOps icon](img/gitlab-chatops-icon.png) is available for download. -You can find and download the official GitLab ChatOps icon here. - -![GitLab ChatOps bot icon](img/gitlab-chatops-icon-small.png) - - +- [The official GitLab ChatOps icon](img/gitlab-chatops-icon.png) +- [A repository of common ChatOps scripts](https://gitlab.com/gitlab-com/chatops) + that GitLab uses to interact with GitLab.com diff --git a/doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md b/doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md index 920e7cca2cb..9ea5f76733a 100644 --- a/doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md +++ b/doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/ci/cloud_deployment/heroku.md b/doc/ci/cloud_deployment/heroku.md index 541ba89dca8..5cb6682debd 100644 --- a/doc/ci/cloud_deployment/heroku.md +++ b/doc/ci/cloud_deployment/heroku.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/ci/cloud_deployment/index.md b/doc/ci/cloud_deployment/index.md index a8826a6fdb5..ce03e9e3916 100644 --- a/doc/ci/cloud_deployment/index.md +++ b/doc/ci/cloud_deployment/index.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: howto --- diff --git a/doc/ci/cloud_services/aws/index.md b/doc/ci/cloud_services/aws/index.md index 484a159cd2b..b1148d3a258 100644 --- a/doc/ci/cloud_services/aws/index.md +++ b/doc/ci/cloud_services/aws/index.md @@ -1,11 +1,15 @@ --- stage: Verify -group: Pipeline Authoring +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Configure OpenID Connect in AWS to retrieve temporary credentials **(FREE)** +WARNING: +`CI_JOB_JWT_V2` was [deprecated in GitLab 15.9](../../../update/deprecations.md#old-versions-of-json-web-tokens-are-deprecated) +and is scheduled to be removed in GitLab 16.5. Use [ID tokens](../../yaml/index.md#id_tokens) instead. + In this tutorial, we'll show you how to use a GitLab CI/CD job with a JSON web token (JWT) to retrieve temporary credentials from AWS without needing to store secrets. To do this, you must configure OpenID Connect (OIDC) for ID federation between GitLab and AWS. For background and requirements for integrating GitLab using OIDC, see [Connect to cloud services](../index.md). @@ -62,21 +66,24 @@ After you configure the OIDC and role, the GitLab CI/CD job can retrieve a tempo ```yaml assume role: + id_tokens: + GITLAB_OIDC_TOKEN: + aud: https://gitlab.example.com script: - > export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" $(aws sts assume-role-with-web-identity --role-arn ${ROLE_ARN} --role-session-name "GitLabRunner-${CI_PROJECT_ID}-${CI_PIPELINE_ID}" - --web-identity-token $CI_JOB_JWT_V2 + --web-identity-token ${GITLAB_OIDC_TOKEN} --duration-seconds 3600 --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' --output text)) - aws sts get-caller-identity ``` -- `CI_JOB_JWT_V2`: Predefined variable. - `ROLE_ARN`: The role ARN defined in this [step](#configure-a-role-and-trust). +- `GITLAB_OIDC_TOKEN`: An OIDC [ID token](../../yaml/index.md#id_tokens). ## Working example diff --git a/doc/ci/cloud_services/azure/index.md b/doc/ci/cloud_services/azure/index.md index 321f9849632..912e6597985 100644 --- a/doc/ci/cloud_services/azure/index.md +++ b/doc/ci/cloud_services/azure/index.md @@ -1,11 +1,15 @@ --- stage: Verify -group: Pipeline Authoring +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Configure OpenID Connect in Azure to retrieve temporary credentials **(FREE)** +WARNING: +`CI_JOB_JWT_V2` was [deprecated in GitLab 15.9](../../../update/deprecations.md#old-versions-of-json-web-tokens-are-deprecated) +and is scheduled to be removed in GitLab 16.5. Use [ID tokens](../../yaml/index.md#id_tokens) instead. + This tutorial demonstrates how to use a JSON web token (JWT) in a GitLab CI/CD job to retrieve temporary credentials from Azure without needing to store secrets. diff --git a/doc/ci/cloud_services/google_cloud/index.md b/doc/ci/cloud_services/google_cloud/index.md index 516a2d05cd1..5ed22883518 100644 --- a/doc/ci/cloud_services/google_cloud/index.md +++ b/doc/ci/cloud_services/google_cloud/index.md @@ -1,13 +1,14 @@ --- stage: Verify -group: Pipeline Authoring +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Configure OpenID Connect with GCP Workload Identity Federation **(FREE)** WARNING: -The `CI_JOB_JWT_V2` variable is under development [(alpha)](../../../policy/alpha-beta-support.md#alpha-features) and is not yet suitable for production use. +`CI_JOB_JWT_V2` was [deprecated in GitLab 15.9](../../../update/deprecations.md#old-versions-of-json-web-tokens-are-deprecated) +and is scheduled to be removed in GitLab 16.5. Use [ID tokens](../../yaml/index.md#id_tokens) instead. This tutorial demonstrates authenticating to Google Cloud from a GitLab CI/CD job using a JSON Web Token (JWT) token and Workload Identity Federation. This configuration @@ -30,7 +31,7 @@ To complete this tutorial: ## Create the Google Cloud Workload Identity Pool -[Create a new Google Cloud Workload Identity Pool](https://cloud.google.com/iam/docs/workload-identity-federation-with-other-clouds#oidc) with the following options: +[Create a new Google Cloud Workload Identity Pool](https://cloud.google.com/iam/docs/workload-identity-federation-with-other-clouds#create_the_workload_identity_pool_and_provider) with the following options: - **Name**: Human-friendly name for the Workload Identity Pool, such as `GitLab`. - **Pool ID**: Unique ID in the Google Cloud project for the Workload Identity Pool, @@ -80,7 +81,7 @@ However, you have no permissions on Google Cloud (_authorization_). To grant your GitLab CI/CD job permissions on Google Cloud, you must: -1. [Create a Google Cloud Service Account](https://www.google.com/search?q=google+cloud+create+service+account). +1. [Create a Google Cloud Service Account](https://cloud.google.com/iam/docs/service-accounts-create). You can use whatever name and ID you prefer. 1. [Grant IAM permissions](https://cloud.google.com/iam/docs/granting-changing-revoking-access) to your service account on Google Cloud resources. These permissions vary significantly based on diff --git a/doc/ci/cloud_services/index.md b/doc/ci/cloud_services/index.md index 9304e3562d9..54cadc9e1b6 100644 --- a/doc/ci/cloud_services/index.md +++ b/doc/ci/cloud_services/index.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Authoring +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -8,7 +8,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - `CI_JOB_JWT` variable for reading secrets from Vault [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207125) in GitLab 12.10. > - `CI_JOB_JWT_V2` variable to support additional OIDC providers [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/346737) in GitLab 14.7. -> - [ID tokens](../yaml/index.md) to support any OIDC provider, including HashiCorp Vault, [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356986) in GitLab 15.7. +> - [ID tokens](../yaml/index.md#id_tokens) to support any OIDC provider, including HashiCorp Vault, [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356986) in GitLab 15.7. + +WARNING: +`CI_JOB_JWT` and `CI_JOB_JWT_V2` were [deprecated in GitLab 15.9](../../update/deprecations.md#old-versions-of-json-web-tokens-are-deprecated) +and are scheduled to be removed in GitLab 16.5. Use [ID tokens](../yaml/index.md#id_tokens) instead. GitLab CI/CD supports [OpenID Connect (OIDC)](https://openid.net/connect/faq/) to give your build and deployment jobs access to cloud credentials and services. @@ -19,20 +23,18 @@ in the CI/CD job allowing you to follow a scalable and least-privilege security In GitLab 15.6 and earlier, you must use `CI_JOB_JWT_V2` instead of an ID token, but it is not customizable. In GitLab 14.6 an earlier you must use the `CI_JOB_JWT`, which has limited support. -## Requirements +## Prerequisites - Account on GitLab. - Access to a cloud provider that supports OIDC to configure authorization and create roles. -ID tokens and `CI_JOB_JWT_V2` support cloud providers with OIDC, including: +ID tokens support cloud providers with OIDC, including: - AWS - Azure - GCP - HashiCorp Vault -The `CI_JOB_JWT` only supports the [HashiCorp Vault integration](../examples/authenticating-with-hashicorp-vault/index.md). - NOTE: Configuring OIDC enables JWT token access to the target environments for all pipelines. When you configure OIDC for a pipeline, you should complete a software supply chain security @@ -50,61 +52,7 @@ as a starting point, and for more information about supply chain attacks, see ## How it works -Each job can be configured with ID tokens, which are provided as a CI/CD variable. These JWTs can be used to authenticate with the OIDC-supported cloud provider such as AWS, Azure, GCP, or Vault. - -The following fields are included in the JWT: - -| Field | When | Description | -| ----------------------- | ------ | ----------- | -| `aud` | Always | Specified in the [ID tokens](../yaml/index.md#id_tokens) configuration | -| `jti` | Always | Unique identifier for this token | -| `iss` | Always | Issuer, the domain of your GitLab instance | -| `iat` | Always | Issued at | -| `nbf` | Always | Not valid before | -| `exp` | Always | Expires at | -| `sub` | Always |`project_path:{group}/{project}:ref_type:{type}:ref:{branch_name}` | -| `namespace_id` | Always | Use this to scope to group or user level namespace by ID | -| `namespace_path` | Always | Use this to scope to group or user level namespace by path | -| `project_id` | Always | Use this to scope to project by ID | -| `project_path` | Always | Use this to scope to project by path | -| `user_id` | Always | ID of the user executing the job | -| `user_login` | Always | Username of the user executing the job | -| `user_email` | Always | Email of the user executing the job | -| `pipeline_id` | Always | ID of this pipeline | -| `pipeline_source` | Always | [Pipeline source](../jobs/job_control.md#common-if-clauses-for-rules) | -| `job_id` | Always | ID of this job | -| `ref` | Always | Git ref for this job | -| `ref_type` | Always | Git ref type, either `branch` or `tag` | -| `ref_protected` | Always | `true` if this Git ref is protected, `false` otherwise | -| `environment` | Job is creating a deployment | Environment this job deploys to ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/294440) in GitLab 13.9) | -| `environment_protected` | Job is creating a deployment |`true` if deployed environment is protected, `false` otherwise ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/294440) in GitLab 13.9) | - -```json -{ - "jti": "c82eeb0c-5c6f-4a33-abf5-4c474b92b558", - "iss": "https://gitlab.example.com", - "aud": "https://vault.example.com", - "iat": 1585710286, - "nbf": 1585798372, - "exp": 1585713886, - "sub": "project_path:mygroup/myproject:ref_type:branch:ref:main", - "namespace_id": "1", - "namespace_path": "mygroup", - "project_id": "22", - "project_path": "mygroup/myproject", - "user_id": "42", - "user_login": "myuser", - "user_email": "myuser@example.com", - "pipeline_id": "1212", - "pipeline_source": "web", - "job_id": "1212", - "ref": "auto-deploy-2020-04-01", - "ref_type": "branch", - "ref_protected": "true", - "environment": "production", - "environment_protected": "true" -} -``` +Each job can be configured with ID tokens, which are provided as a CI/CD variable containing the [token payload](../secrets/id_token_authentication.md#token-payload). These JWTs can be used to authenticate with the OIDC-supported cloud provider such as AWS, Azure, GCP, or Vault. ### Authorization workflow @@ -115,7 +63,7 @@ sequenceDiagram Note right of Cloud: Create role with conditionals Note left of GitLab: CI/CD job with ID token GitLab->>+Cloud: Call cloud API with ID token - Note right of Cloud: Decode & verify JWT with public key (https://gitlab/-/jwks) + Note right of Cloud: Decode & verify JWT with public key (https://gitlab.com/oauth/discovery/keys) Note right of Cloud: Validate audience defined in OIDC Note right of Cloud: Validate conditional (sub, aud) role Note right of Cloud: Generate credential or fetch secret diff --git a/doc/ci/components/index.md b/doc/ci/components/index.md new file mode 100644 index 00000000000..95a513220a2 --- /dev/null +++ b/doc/ci/components/index.md @@ -0,0 +1,201 @@ +--- +stage: Verify +group: Pipeline Authoring +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +type: reference +--- + +# CI/CD Components (Experimental) + +> Introduced in GitLab 16.0 as an [experimental feature](../../policy/alpha-beta-support.md). + +FLAG: +On self-managed GitLab, by default this feature is not available. +To make it available, ask an administrator to enable the feature flag named `ci_namespace_catalog_experimental`. +On GitLab.com, this feature is not available. This feature is not ready for production use. + +This feature is an experimental feature and [an epic exists](https://gitlab.com/groups/gitlab-org/-/epics/9897) +to track future work. Tell us about your use case by leaving comments in the epic. + +## Components Repository + +A components repository is a GitLab project with a repository that hosts one or more pipeline components. A pipeline component is a reusable single pipeline configuration unit. You can use them to compose an entire pipeline configuration or a small part of a larger pipeline. It can optionally take [input parameters](../yaml/includes.md#define-input-parameters-with-specinputs). + +### Create a components repository + +To create a components repository, you must: + +1. [Create a new project](../../user/project/index.md#create-a-blank-project) with a `README.md` file. + +1. Create a `template.yml` file inside the project's root directory that contains the configuration you want to provide as a component. For example: + + ```yaml + spec: + inputs: + stage: + default: test + --- + component-job: + script: echo job 1 + stage: $[[ inputs.stage ]] + ``` + +### Directory structure + +A components repository can host one or more components. + +Components repositories must follow a mandatory file structure, containing: + +- `template.yml`: The component configuration, one file per component. If there is + only one component, this file can be in the root of the project. If there are multiple + components, each file must be in a separate subdirectory. +- `README.md`: A documentation file explaining the details of the all the components in the repository. + +For example, if the project is on GitLab.com, named `my-component`, and in a personal +namespace named `my-username`: + +- Containing a single component and a simple pipeline to test the component, then + the file structure might be: + + ```plaintext + ├── template.yml + ├── README.md + └── .gitlab-ci.yml + ``` + + This component is referenced with the path `gitlab.com/my-username/my-component@`. + +- Containing one default component and multiple sub-components, then the file structure + might be: + + ```plaintext + ├── template.yml + ├── README.md + ├── .gitlab-ci.yml + ├── unit/ + │ └── template.yml + └── integration/ + └── template.yml + ``` + + These components are identified by these paths: + + - `gitlab.com/my-username/my-component` + - `gitlab.com/my-username/my-component/unit` + - `gitlab.com/my-username/my-component/integration` + +It is possible to have a components repository with no default component, by having +no `template.yml` in the root directory. + +**Additional notes:** + +Nesting of components is not possible. For example: + +```plaintext +├── unit/ +│ └── template.yml +│ └── another_folder/ +│ └── nested_template.yml +``` + +### Test a component + +Testing components as part of the development workflow to ensure that quality maintains high standards is strongly recommended. + +Testing changes in a CI/CD pipeline can be done, like any other project, by creating a `.gitlab-ci.yml` in the root directory. + +For example: + +```yaml +include: + # include the component located in the current project from the current SHA + - component: gitlab.com/$CI_PROJECT_PATH@$CI_COMMIT_SHA + inputs: + stage: build + +stages: [build, test, release] + +# Expect `component-job` is added. +# This is an example of testing that the included component works as expected. +# You can leverage GitLab API endpoints or 3rd party tools to inspect data generated by the component. +ensure-job-added: + stage: test + image: badouralix/curl-jq + script: + - | + route="https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/pipelines/$CI_PIPELINE_ID/jobs" + count=`curl --silent --header "PRIVATE-TOKEN: $API_TOKEN" $route | jq 'map(select(.name | contains("component-job"))) | length'` + if [ "$count" != "1" ]; then + exit 1 + fi + +# If we are tagging a release with a specific convention ("v" + number) and all +# previous checks succeeded, we proceed with creating a release automatically. +create-release: + stage: release + image: registry.gitlab.com/gitlab-org/release-cli:latest + rules: + - if: $CI_COMMIT_TAG =~ /^v\d+/ + script: echo "Creating release $CI_COMMIT_TAG" + release: + tag_name: $CI_COMMIT_TAG + description: "Release $CI_COMMIT_TAG of components repository $CI_PROJECT_PATH" +``` + +After committing and pushing changes, the pipeline tests the component then releases it if the test passes. + +### Release a component + +Component repositories are released using the [`release`](../yaml/index.md#release) keyword within a CI pipeline. + +Like in the [example above](#test-a-component), after all tests pass in a pipeline running for a tag ref, we can release a new version of the components repository. + +All released versions of the components repository are displayed in the Components Catalog page for the given resource, providing users with information about official releases. + +### Use a component in a CI/CD configuration + +A pipeline component is identified by a unique address in the form `/@` +containing: + +- **A fully qualified domain name (FQDN)**: The FQDN must match the GitLab host. +- **A specific version**: The version of the component can be (in order of highest priority first): + - A commit SHA, for example `gitlab.com/gitlab-org/dast@e3262fdd0914fa823210cdb79a8c421e2cef79d8`. + - A tag. for example: `gitlab.com/gitlab-org/dast@1.0`. + - `~latest`, which is a special version that always points to the most recent released tag, + for example `gitlab.com/gitlab-org/dast@~latest`. + - A branch name, for example `gitlab.com/gitlab-org/dast@main`. +- **A component path**: Contains the project's full path and the directory where the component YAML file `template.yml` is located. + +For example, for a component repository located at `gitlab-org/dast` on `gitlab.com`: + +- The path `gitlab.com/gitlab-org/dast` tries to load the `template.yml` from the root directory. +- The path `gitalb.com/gitlab-org/dast/api-scan` tries to load the `template.yml` from the `/api-scan` directory. + +**Additional notes:** + +- If a tag and branch exist with the same name, the tag takes precedence over the branch. +- If a tag is named the same as a commit SHA that exists, like `e3262fdd0914fa823210cdb79a8c421e2cef79d8`, + the commit SHA takes precedence over the tag. + +## CI/CD Catalog + +The CI/CD Catalog is a list of [components repositories](#components-repository), +each containing resources that you can add to your CI/CD pipelines. + +### Mark the project as a catalog resource + +After components are added to a components repository, they can immediately be [used](#use-a-component-in-a-cicd-configuration) to build pipelines in other projects. + +However, this repository is not discoverable. You must mark this project as a catalog resource to allow it to be visible in the CI Catalog +so other users can discover it. + +To mark a project as a catalog resource, run the following [graphQL](../../api/graphql/index.md) +mutation: + +```graphql +mutation { + catalogResourcesCreate(input: { projectPath: "path-to-project"}) { + errors + } +} +``` diff --git a/doc/ci/directed_acyclic_graph/index.md b/doc/ci/directed_acyclic_graph/index.md index 7c741ef3842..f0384fb29c7 100644 --- a/doc/ci/directed_acyclic_graph/index.md +++ b/doc/ci/directed_acyclic_graph/index.md @@ -81,7 +81,7 @@ are certain use cases that you may need to work around. For more information, ch ## Needs Visualization -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/215517) in GitLab 13.1 as a [Beta feature](../../policy/alpha-beta-support.md#beta-features). +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/215517) in GitLab 13.1 as a [Beta feature](../../policy/alpha-beta-support.md#beta). > - It became a [standard feature](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38517) in 13.3. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/52208) in GitLab 13.9. diff --git a/doc/ci/docker/authenticate_registry.md b/doc/ci/docker/authenticate_registry.md new file mode 100644 index 00000000000..224d0cdf7aa --- /dev/null +++ b/doc/ci/docker/authenticate_registry.md @@ -0,0 +1,144 @@ +--- +stage: Verify +group: Pipeline Execution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +type: concepts, howto +--- + +# Authenticate with registry in Docker-in-Docker + +When you use Docker-in-Docker, the +[standard authentication methods](using_docker_images.md#access-an-image-from-a-private-container-registry) +do not work, because a fresh Docker daemon is started with the service. + +## Option 1: Run `docker login` + +In [`before_script`](../yaml/index.md#before_script), run `docker +login`: + +```yaml +image: docker:20.10.16 + +variables: + DOCKER_TLS_CERTDIR: "/certs" + +services: + - docker:20.10.16-dind + +build: + stage: build + before_script: + - echo "$DOCKER_REGISTRY_PASS" | docker login $DOCKER_REGISTRY --username $DOCKER_REGISTRY_USER --password-stdin + script: + - docker build -t my-docker-image . + - docker run my-docker-image /script/to/run/tests +``` + +To sign in to Docker Hub, leave `$DOCKER_REGISTRY` +empty or remove it. + +## Option 2: Mount `~/.docker/config.json` on each job + +If you are an administrator for GitLab Runner, you can mount a file +with the authentication configuration to `~/.docker/config.json`. +Then every job that the runner picks up is already authenticated. If you +are using the official `docker:20.10.16` image, the home directory is +under `/root`. + +If you mount the configuration file, any `docker` command +that modifies the `~/.docker/config.json` fails. For example, `docker login` +fails, because the file is mounted as read-only. Do not change it from +read-only, because this causes problems. + +Here is an example of `/opt/.docker/config.json` that follows the +[`DOCKER_AUTH_CONFIG`](using_docker_images.md#determine-your-docker_auth_config-data) +documentation: + +```json +{ + "auths": { + "https://index.docker.io/v1/": { + "auth": "bXlfdXNlcm5hbWU6bXlfcGFzc3dvcmQ=" + } + } +} +``` + +### Docker + +Update the +[volume mounts](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#volumes-in-the-runnersdocker-section) +to include the file. + +```toml +[[runners]] + ... + executor = "docker" + [runners.docker] + ... + privileged = true + volumes = ["/opt/.docker/config.json:/root/.docker/config.json:ro"] +``` + +### Kubernetes + +Create a [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) with the content +of this file. You can do this with a command like: + +```shell +kubectl create configmap docker-client-config --namespace gitlab-runner --from-file /opt/.docker/config.json +``` + +Update the [volume mounts](https://docs.gitlab.com/runner/executors/kubernetes.html#using-volumes) +to include the file. + +```toml +[[runners]] + ... + executor = "kubernetes" + [runners.kubernetes] + image = "alpine:3.12" + privileged = true + [[runners.kubernetes.volumes.config_map]] + name = "docker-client-config" + mount_path = "/root/.docker/config.json" + # If you are running GitLab Runner 13.5 + # or lower you can remove this + sub_path = "config.json" +``` + +## Option 3: Use `DOCKER_AUTH_CONFIG` + +If you already have +[`DOCKER_AUTH_CONFIG`](using_docker_images.md#determine-your-docker_auth_config-data) +defined, you can use the variable and save it in +`~/.docker/config.json`. + +You can define this authentication in several ways: + +- In [`pre_build_script`](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section) + in the runner configuration file. +- In [`before_script`](../yaml/index.md#before_script). +- In [`script`](../yaml/index.md#script). + +The following example shows [`before_script`](../yaml/index.md#before_script). +The same commands apply for any solution you implement. + +```yaml +image: docker:20.10.16 + +variables: + DOCKER_TLS_CERTDIR: "/certs" + +services: + - docker:20.10.16-dind + +build: + stage: build + before_script: + - mkdir -p $HOME/.docker + - echo $DOCKER_AUTH_CONFIG > $HOME/.docker/config.json + script: + - docker build -t my-docker-image . + - docker run my-docker-image /script/to/run/tests +``` diff --git a/doc/ci/docker/docker_layer_caching.md b/doc/ci/docker/docker_layer_caching.md new file mode 100644 index 00000000000..145ead9a212 --- /dev/null +++ b/doc/ci/docker/docker_layer_caching.md @@ -0,0 +1,61 @@ +--- +stage: Verify +group: Pipeline Execution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +type: concepts, howto +--- + +# Make Docker-in-Docker builds faster with Docker layer caching + +When using Docker-in-Docker, Docker downloads all layers of your image every +time you create a build. Recent versions of Docker (Docker 1.13 and later) can +use a pre-existing image as a cache during the `docker build` step. This significantly +accelerates the build process. + +## How Docker caching works + +When running `docker build`, each command in `Dockerfile` creates a layer. +These layers are retained as a cache and can be reused if there have been no changes. Change in one layer causes the recreation of all subsequent layers. + +To specify a tagged image to be used as a cache source for the `docker build` +command, use the `--cache-from` argument. Multiple images can be specified +as a cache source by using multiple `--cache-from` arguments. Any image that's used +with the `--cache-from` argument must be pulled +(using `docker pull`) before it can be used as a cache source. + +## Docker caching example + +This example `.gitlab-ci.yml` file shows how to use Docker caching: + +```yaml +image: docker:20.10.16 + +services: + - docker:20.10.16-dind + +variables: + # Use TLS https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled + DOCKER_HOST: tcp://docker:2376 + DOCKER_TLS_CERTDIR: "/certs" + +before_script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + +build: + stage: build + script: + - docker pull $CI_REGISTRY_IMAGE:latest || true + - docker build --cache-from $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA --tag $CI_REGISTRY_IMAGE:latest . + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + - docker push $CI_REGISTRY_IMAGE:latest +``` + +In the `script` section for the `build` stage: + +1. The first command tries to pull the image from the registry so that it can be + used as a cache for the `docker build` command. +1. The second command builds a Docker image by using the pulled image as a + cache (see the `--cache-from $CI_REGISTRY_IMAGE:latest` argument) if + available, and tags it. +1. The last two commands push the tagged Docker images to the container registry + so that they can also be used as cache for subsequent builds. diff --git a/doc/ci/docker/index.md b/doc/ci/docker/index.md index e976700c160..162d67286ad 100644 --- a/doc/ci/docker/index.md +++ b/doc/ci/docker/index.md @@ -2,7 +2,6 @@ stage: Verify group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false type: index --- diff --git a/doc/ci/docker/using_docker_build.md b/doc/ci/docker/using_docker_build.md index a8c192dc944..fe57b451146 100644 --- a/doc/ci/docker/using_docker_build.md +++ b/doc/ci/docker/using_docker_build.md @@ -9,12 +9,12 @@ type: concepts, howto You can use GitLab CI/CD with Docker to create Docker images. For example, you can create a Docker image of your application, -test it, and publish it to a container registry. +test it, and push it to a container registry. To run Docker commands in your CI/CD jobs, you must configure -GitLab Runner to support `docker` commands. +GitLab Runner to support `docker` commands. This method requires `privileged` mode. -If you want to build Docker images without enabling privileged mode on the runner, +If you want to build Docker images without enabling `privileged` mode on the runner, you can use a [Docker alternative](#docker-alternatives). ## Enable Docker commands in your CI/CD jobs @@ -23,7 +23,7 @@ To enable Docker commands for your CI/CD jobs, you can use: - [The shell executor](#use-the-shell-executor) - [Docker-in-Docker](#use-docker-in-docker) -- [Docker socket binding](#use-docker-socket-binding) +- [Docker socket binding](#use-the-docker-executor-with-docker-socket-binding) ### Use the shell executor @@ -58,7 +58,7 @@ the Docker commands, but needs permission to do so. sudo -u gitlab-runner -H docker info ``` -1. In GitLab, to verify that everything works, add `docker info` to `.gitlab-ci.yml`: +1. In GitLab, add `docker info` to `.gitlab-ci.yml` to verify that Docker is working: ```yaml before_script: @@ -70,10 +70,10 @@ the Docker commands, but needs permission to do so. - docker run my-docker-image /script/to/run/tests ``` -You can now use `docker` commands (and install `docker-compose` if needed). +You can now use `docker` commands (and install Docker Compose if needed). -When you add `gitlab-runner` to the `docker` group, you are effectively granting `gitlab-runner` full root permissions. -For more information, see the [security of the `docker` group](https://blog.zopyx.com/on-docker-security-docker-group-considered-harmful/). +When you add `gitlab-runner` to the `docker` group, you effectively grant `gitlab-runner` full root permissions. +For more information, see [security of the `docker` group](https://blog.zopyx.com/on-docker-security-docker-group-considered-harmful/). ### Use Docker-in-Docker @@ -83,15 +83,15 @@ For more information, see the [security of the `docker` group](https://blog.zopy - The executor uses a [container image of Docker](https://hub.docker.com/_/docker/), provided by Docker, to run your CI/CD jobs. -The Docker image has all of the `docker` tools installed and can run +The Docker image includes all of the `docker` tools and can run the job script in context of the image in privileged mode. -We recommend you use Docker-in-Docker with TLS enabled, +You should use Docker-in-Docker with TLS enabled, which is supported by [GitLab.com shared runners](../runners/index.md). -You should always specify a specific version of the image, like `docker:20.10.16`. +You should always pin a specific version of the image, like `docker:20.10.16`. If you use a tag like `docker:stable`, you have no control over which version is used. -Unpredictable behavior can result, especially when new versions are released. +This can cause incompatibility problems when new versions are released. #### Use the Docker executor with Docker-in-Docker @@ -101,14 +101,11 @@ You can use the Docker executor to run jobs in a Docker container. > Introduced in GitLab Runner 11.11. -The Docker daemon supports connections over TLS. In Docker 19.03.12 and later, -TLS is the default. +The Docker daemon supports connections over TLS. TLS is the default in Docker 19.03.12 and later. WARNING: -This task enables `--docker-privileged`. When you do this, you are effectively disabling all of -the security mechanisms of containers and exposing your host to privilege -escalation. Doing this can lead to container breakout. For more information, -see the official Docker documentation about +This task enables `--docker-privileged`, which effectively disables the container's security mechanisms and exposes your host to privilege +escalation. This action can cause container breakout. For more information, see [runtime privilege and Linux capabilities](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities). To use Docker-in-Docker with TLS enabled: @@ -134,10 +131,9 @@ To use Docker-in-Docker with TLS enabled: you must always use `privileged = true` in your Docker containers. - This command mounts `/certs/client` for the service and build container, which is needed for the Docker client to use the - certificates in that directory. For more information on how - Docker with TLS works, see . + certificates in that directory. For more information, see [the Docker image documentation](https://hub.docker.com/_/docker/). - The previous command creates a `config.toml` entry similar to this: + The previous command creates a `config.toml` entry similar to the following example: ```toml [[runners]] @@ -155,7 +151,7 @@ To use Docker-in-Docker with TLS enabled: [runners.cache.gcs] ``` -1. You can now use `docker` in the job script. Note the inclusion of the +1. You can now use `docker` in the job script. You should include the `docker:20.10.16-dind` service: ```yaml @@ -193,7 +189,7 @@ To use Docker-in-Docker with TLS enabled: ##### Docker-in-Docker with TLS disabled in the Docker executor -Sometimes you might have legitimate reasons to disable TLS. +Sometimes there are legitimate reasons to disable TLS. For example, you have no control over the GitLab Runner configuration that you are using. @@ -215,7 +211,7 @@ Assuming that the runner's `config.toml` is similar to: [runners.cache.gcs] ``` -You can now use `docker` in the job script. Note the inclusion of the +You can now use `docker` in the job script. You should include the `docker:20.10.16-dind` service: ```yaml @@ -254,7 +250,7 @@ build: #### Use the Kubernetes executor with Docker-in-Docker -You can use the Kubernetes executor to run jobs in a Docker container. +You can use the [Kubernetes executor](https://docs.gitlab.com/runner/executors/kubernetes.html) to run jobs in a Docker container. ##### Docker-in-Docker with TLS enabled in Kubernetes @@ -280,7 +276,7 @@ To use Docker-in-Docker with TLS enabled in Kubernetes: medium = "Memory" ``` -1. You can now use `docker` in the job script. Note the inclusion of the +1. You can now use `docker` in the job script. You should include the `docker:20.10.16-dind` service: ```yaml @@ -324,25 +320,23 @@ To use Docker-in-Docker with TLS enabled in Kubernetes: - docker run my-docker-image /script/to/run/tests ``` -#### Limitations of Docker-in-Docker +#### Known issues with Docker-in-Docker -Docker-in-Docker is the recommended configuration, but is -not without its own challenges: +Docker-in-Docker is the recommended configuration, but you should be aware of the following issues: - **The `docker-compose` command**: This command is not available in this configuration by default. - To use `docker-compose` in your job scripts, follow the `docker-compose` + To use `docker-compose` in your job scripts, follow the Docker Compose [installation instructions](https://docs.docker.com/compose/install/). -- **Cache**: Each job runs in a new environment. Concurrent jobs work fine, - because every build gets its own instance of Docker engine and they don't conflict with each other. - However, jobs can be slower because there's no caching of layers. +- **Cache**: Each job runs in a new environment. Because every build gets its own instance of the Docker engine, concurrent jobs do not cause conflicts. + However, jobs can be slower because there's no caching of layers. See [Docker layer caching](#make-docker-in-docker-builds-faster-with-docker-layer-caching). - **Storage drivers**: By default, earlier versions of Docker use the `vfs` storage driver, which copies the file system for each job. Docker 17.09 and later use `--storage-driver overlay2`, which is the recommended storage driver. See [Using the OverlayFS driver](#use-the-overlayfs-driver) for details. -- **Root file system**: Because the `docker:20.10.16-dind` container and the runner container don't share their +- **Root file system**: Because the `docker:20.10.16-dind` container and the runner container do not share their root file system, you can use the job's working directory as a mount point for child containers. For example, if you have files you want to share with a - child container, you might create a subdirectory under `/builds/$CI_PROJECT_PATH` - and use it as your mount point. For a more detailed explanation, view + child container, you could create a subdirectory under `/builds/$CI_PROJECT_PATH` + and use it as your mount point. For a more detailed explanation, see [issue #41227](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41227). ```yaml @@ -353,7 +347,7 @@ not without its own challenges: - docker run -v "$MOUNT_POINT:/mnt" my-docker-image ``` -### Use Docker socket binding +### Use the Docker executor with Docker socket binding To use Docker commands in your CI/CD jobs, you can bind-mount `/var/run/docker.sock` into the container. Docker is then available in the context of the image. @@ -362,16 +356,14 @@ NOTE: If you bind the Docker socket and you are [using GitLab Runner 11.11 or later](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/1261), you can no longer use `docker:20.10.16-dind` as a service. Volume bindings -are done to the services as well, making these incompatible. - -#### Use the Docker executor with Docker socket binding +also affect services, making them incompatible. To make Docker available in the context of the image, you need to mount `/var/run/docker.sock` into the launched containers. To do this with the Docker -executor, you need to add `"/var/run/docker.sock:/var/run/docker.sock"` to the +executor, add `"/var/run/docker.sock:/var/run/docker.sock"` to the [Volumes in the `[runners.docker]` section](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#volumes-in-the-runnersdocker-section). -Your configuration should look something like this: +Your configuration should look similar to this example: ```toml [[runners]] @@ -388,7 +380,7 @@ Your configuration should look something like this: Insecure = false ``` -You can also do this while registering your runner by providing the following options: +To mount `/var/run/docker.sock` while registering your runner, include the following options: ```shell sudo gitlab-runner register -n \ @@ -400,14 +392,14 @@ sudo gitlab-runner register -n \ --docker-volumes /var/run/docker.sock:/var/run/docker.sock ``` -##### Enable registry mirror for `docker:dind` service +#### Enable registry mirror for `docker:dind` service -When the Docker daemon starts inside of the service container, it uses -the default configuration. You may want to configure a +When the Docker daemon starts inside the service container, it uses +the default configuration. You might want to configure a [registry mirror](https://docs.docker.com/registry/recipes/mirror/) for -performance improvements and to ensure you don't reach Docker Hub rate limits. +performance improvements and to ensure you do not exceed Docker Hub rate limits. -###### The service in the `.gitlab-ci.yml` file +##### The service in the `.gitlab-ci.yml` file You can append extra CLI flags to the `dind` service to set the registry mirror: @@ -418,7 +410,7 @@ services: command: ["--registry-mirror", "https://registry-mirror.example.com"] # Specify the registry mirror to use ``` -###### The service in the GitLab Runner configuration file +##### The service in the GitLab Runner configuration file > [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27173) in GitLab Runner 13.6. @@ -455,7 +447,7 @@ Kubernetes: command = ["--registry-mirror", "https://registry-mirror.example.com"] ``` -###### The Docker executor in the GitLab Runner configuration file +##### The Docker executor in the GitLab Runner configuration file If you are a GitLab Runner administrator, you can use the mirror for every `dind` service. Update the @@ -474,9 +466,9 @@ content: ``` Update the `config.toml` file to mount the file to -`/etc/docker/daemon.json`. This would mount the file for **every** -container that is created by GitLab Runner. The configuration is -picked up by the `dind` service. +`/etc/docker/daemon.json`. This mounts the file for **every** +container created by GitLab Runner. The configuration is +detected by the `dind` service. ```toml [[runners]] @@ -488,7 +480,7 @@ picked up by the `dind` service. volumes = ["/opt/docker/daemon.json:/etc/docker/daemon.json:ro"] ``` -###### The Kubernetes executor in the GitLab Runner configuration file +##### The Kubernetes executor in the GitLab Runner configuration file > [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3223) in GitLab Runner 13.6. @@ -516,13 +508,12 @@ kubectl create configmap docker-daemon --namespace gitlab-runner --from-file /tm ``` NOTE: -Make sure to use the namespace that the Kubernetes executor for GitLab Runner uses -to create job pods in. +You must use the namespace that the Kubernetes executor for GitLab Runner uses to create job pods. After the ConfigMap is created, you can update the `config.toml` file to mount the file to `/etc/docker/daemon.json`. This update -mounts the file for **every** container that is created by GitLab Runner. -The configuration is picked up by the `dind` service. +mounts the file for **every** container created by GitLab Runner. +The `dind` service detects this configuration. ```toml [[runners]] @@ -537,22 +528,21 @@ The configuration is picked up by the `dind` service. sub_path = "daemon.json" ``` -##### Limitations of Docker socket binding +#### Known issues with Docker socket binding When you use Docker socket binding, you avoid running Docker in privileged mode. However, the implications of this method are: -- By sharing the Docker daemon, you are effectively disabling all - the security mechanisms of containers and exposing your host to privilege - escalation, which can lead to container breakout. For example, if a project - ran `docker rm -f $(docker ps -a -q)` it would remove the GitLab Runner +- By sharing the Docker daemon, you effectively disable all + the container's security mechanisms and expose your host to privilege + escalation. This can cause container breakout. For example, if a project + ran `docker rm -f $(docker ps -a -q)`, it would remove the GitLab Runner containers. -- Concurrent jobs may not work; if your tests - create containers with specific names, they may conflict with each other. -- Any containers spawned by Docker commands are siblings of the runner rather - than children of the runner. This may have complications and limitations that - are unsuitable for your workflow. -- Sharing files and directories from the source repository into containers may not +- Concurrent jobs might not work. If your tests + create containers with specific names, they might conflict with each other. +- Any containers created by Docker commands are siblings of the runner, rather + than children of the runner. This might cause complications for your workflow. +- Sharing files and directories from the source repository into containers might not work as expected. Volume mounting is done in the context of the host machine, not the build container. For example: @@ -560,8 +550,8 @@ the implications of this method are: docker run --rm -t -i -v $(pwd)/src:/home/app/src test-image:latest run_app_tests ``` -You don't need to include the `docker:20.10.16-dind` service, like you do when -you're using the Docker-in-Docker executor: +You do not need to include the `docker:20.10.16-dind` service, like you do when +you use the Docker-in-Docker executor: ```yaml image: docker:20.10.16 @@ -578,225 +568,37 @@ build: ## Authenticate with registry in Docker-in-Docker -When you use Docker-in-Docker, the -[standard authentication methods](using_docker_images.md#access-an-image-from-a-private-container-registry) -don't work because a fresh Docker daemon is started with the service. - -### Option 1: Run `docker login` - -In [`before_script`](../yaml/index.md#before_script), run `docker -login`: - -```yaml -image: docker:20.10.16 - -variables: - DOCKER_TLS_CERTDIR: "/certs" - -services: - - docker:20.10.16-dind - -build: - stage: build - before_script: - - echo "$DOCKER_REGISTRY_PASS" | docker login $DOCKER_REGISTRY --username $DOCKER_REGISTRY_USER --password-stdin - script: - - docker build -t my-docker-image . - - docker run my-docker-image /script/to/run/tests -``` - -To sign in to Docker Hub, leave `$DOCKER_REGISTRY` -empty or remove it. - -### Option 2: Mount `~/.docker/config.json` on each job - -If you are an administrator for GitLab Runner, you can mount a file -with the authentication configuration to `~/.docker/config.json`. -Then every job that the runner picks up is authenticated already. If you -are using the official `docker:20.10.16` image, the home directory is -under `/root`. - -If you mount the configuration file, any `docker` command -that modifies the `~/.docker/config.json` fails. For example, `docker login` -fails, because the file is mounted as read-only. Do not change it from -read-only, because problems occur. - -Here is an example of `/opt/.docker/config.json` that follows the -[`DOCKER_AUTH_CONFIG`](using_docker_images.md#determine-your-docker_auth_config-data) -documentation: - -```json -{ - "auths": { - "https://index.docker.io/v1/": { - "auth": "bXlfdXNlcm5hbWU6bXlfcGFzc3dvcmQ=" - } - } -} -``` - -#### Docker - -Update the -[volume mounts](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#volumes-in-the-runnersdocker-section) -to include the file. - -```toml -[[runners]] - ... - executor = "docker" - [runners.docker] - ... - privileged = true - volumes = ["/opt/.docker/config.json:/root/.docker/config.json:ro"] -``` - -#### Kubernetes - -Create a [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) with the content -of this file. You can do this with a command like: - -```shell -kubectl create configmap docker-client-config --namespace gitlab-runner --from-file /opt/.docker/config.json -``` - -Update the [volume mounts](https://docs.gitlab.com/runner/executors/kubernetes.html#using-volumes) -to include the file. - -```toml -[[runners]] - ... - executor = "kubernetes" - [runners.kubernetes] - image = "alpine:3.12" - privileged = true - [[runners.kubernetes.volumes.config_map]] - name = "docker-client-config" - mount_path = "/root/.docker/config.json" - # If you are running GitLab Runner 13.5 - # or lower you can remove this - sub_path = "config.json" -``` - -### Option 3: Use `DOCKER_AUTH_CONFIG` - -If you already have -[`DOCKER_AUTH_CONFIG`](using_docker_images.md#determine-your-docker_auth_config-data) -defined, you can use the variable and save it in -`~/.docker/config.json`. - -There are multiple ways to define this authentication: - -- In [`pre_build_script`](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section) - in the runner configuration file. -- In [`before_script`](../yaml/index.md#before_script). -- In [`script`](../yaml/index.md#script). - -The following example shows [`before_script`](../yaml/index.md#before_script). -The same commands apply for any solution you implement. - -```yaml -image: docker:20.10.16 - -variables: - DOCKER_TLS_CERTDIR: "/certs" - -services: - - docker:20.10.16-dind - -build: - stage: build - before_script: - - mkdir -p $HOME/.docker - - echo $DOCKER_AUTH_CONFIG > $HOME/.docker/config.json - script: - - docker build -t my-docker-image . - - docker run my-docker-image /script/to/run/tests -``` +When you use Docker-in-Docker, the [standard authentication methods](using_docker_images.md#access-an-image-from-a-private-container-registry) do not work, because a fresh Docker daemon is started with the service. You should [authenticate with registry](authenticate_registry.md). ## Make Docker-in-Docker builds faster with Docker layer caching -When using Docker-in-Docker, Docker downloads all layers of your image every -time you create a build. Recent versions of Docker (Docker 1.13 and later) can -use a pre-existing image as a cache during the `docker build` step. This considerably -speeds up the build process. - -### How Docker caching works - -When running `docker build`, each command in `Dockerfile` results in a layer. -These layers are kept around as a cache and can be reused if there haven't been -any changes. Change in one layer causes all subsequent layers to be recreated. - -You can specify a tagged image to be used as a cache source for the `docker build` -command by using the `--cache-from` argument. Multiple images can be specified -as a cache source by using multiple `--cache-from` arguments. Any image that's used -with the `--cache-from` argument must first be pulled -(using `docker pull`) before it can be used as a cache source. - -### Docker caching example - -Here's a `.gitlab-ci.yml` file that shows how to use Docker caching: - -```yaml -image: docker:20.10.16 - -services: - - docker:20.10.16-dind - -variables: - # Use TLS https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled - DOCKER_HOST: tcp://docker:2376 - DOCKER_TLS_CERTDIR: "/certs" - -before_script: - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - -build: - stage: build - script: - - docker pull $CI_REGISTRY_IMAGE:latest || true - - docker build --cache-from $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA --tag $CI_REGISTRY_IMAGE:latest . - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - - docker push $CI_REGISTRY_IMAGE:latest -``` - -In the `script` section for the `build` stage: - -1. The first command tries to pull the image from the registry so that it can be - used as a cache for the `docker build` command. -1. The second command builds a Docker image by using the pulled image as a - cache (see the `--cache-from $CI_REGISTRY_IMAGE:latest` argument) if - available, and tags it. -1. The last two commands push the tagged Docker images to the container registry - so that they may also be used as cache for subsequent builds. +When using Docker-in-Docker, Docker downloads all layers of your image every time you create a build. You can [make your builds faster with Docker layer caching](docker_layer_caching.md). ## Use the OverlayFS driver NOTE: The shared runners on GitLab.com use the `overlay2` driver by default. -By default, when using `docker:dind`, Docker uses the `vfs` storage driver which -copies the file system on every run. This is a disk-intensive operation -which can be avoided if a different driver is used, for example `overlay2`. +By default, when using `docker:dind`, Docker uses the `vfs` storage driver, which +copies the file system on every run. You can avoid this disk-intensive operation by using a different driver, for example `overlay2`. ### Requirements -1. Make sure a recent kernel is used, preferably `>= 4.2`. +1. Ensure a recent kernel is used, preferably `>= 4.2`. 1. Check whether the `overlay` module is loaded: ```shell sudo lsmod | grep overlay ``` - If you see no result, then it isn't loaded. To load it use: + If you see no result, then the module is not loaded. To load the module, use: ```shell sudo modprobe overlay ``` - If everything went fine, you need to make sure module is loaded on reboot. - On Ubuntu systems, this is done by editing `/etc/modules`. Just add the - following line into it: + If the module loaded, you must make sure the module loads on reboot. + On Ubuntu systems, do this by adding the following line to `/etc/modules`: ```plaintext overlay @@ -823,7 +625,7 @@ environment variable in the environment = ["DOCKER_DRIVER=overlay2"] ``` -If you're running multiple runners, you have to modify all configuration files. +If you're running multiple runners, you must modify all configuration files. Read more about the [runner configuration](https://docs.gitlab.com/runner/configuration/) and [using the OverlayFS storage driver](https://docs.docker.com/storage/storagedriver/overlayfs-driver/). @@ -833,8 +635,8 @@ and [using the OverlayFS storage driver](https://docs.docker.com/storage/storage To build Docker images without enabling privileged mode on the runner, you can use one of these alternatives: -- [`kaniko`](using_kaniko.md). -- [`buildah`](https://github.com/containers/buildah). +- [`kaniko`](using_kaniko.md) +- [`buildah`](https://github.com/containers/buildah) For example, with `buildah`: @@ -868,7 +670,7 @@ build: ## Use the GitLab Container Registry -After you've built a Docker image, you can push it up to the built-in +After you've built a Docker image, you can push it to the [GitLab Container Registry](../../user/packages/container_registry/build_and_push_images.md#use-gitlab-cicd). ## Troubleshooting @@ -879,18 +681,18 @@ This is a common error when you are using [Docker-in-Docker](#use-docker-in-docker) v19.03 or later. -This issue occurs because Docker starts on TLS automatically. +This error occurs because Docker starts on TLS automatically. -- If this is your first time setting it up, read +- If this is your first time setting it up, see [use the Docker executor with the Docker image](#use-docker-in-docker). -- If you are upgrading from v18.09 or earlier, read our +- If you are upgrading from v18.09 or earlier, see the [upgrade guide](https://about.gitlab.com/blog/2019/07/31/docker-in-docker-with-docker-19-dot-03/). -This error can also occur with the [Kubernetes executor](https://docs.gitlab.com/runner/executors/kubernetes.html#using-dockerdind) when attempts are made to access the Docker-in-Docker service before it has had time to fully start up. For a more detailed explanation, see [this issue](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27215). +This error can also occur with the [Kubernetes executor](https://docs.gitlab.com/runner/executors/kubernetes.html#using-dockerdind) when attempts are made to access the Docker-in-Docker service before it has fully started up. For a more detailed explanation, see [issue 27215](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27215). ### Docker `no such host` error -You may get an error that says +You might get an error that says `docker: error during connect: Post https://docker:2376/v1.40/containers/create: dial tcp: lookup docker on x.x.x.x:53: no such host`. This issue can occur when the service's image name @@ -914,3 +716,10 @@ services: - name: registry.hub.docker.com/library/docker:20.10.16-dind alias: docker ``` + +### Error response from daemon: Get "https://registry-1.docker.io/v2/": unauthorized: incorrect username or password + +This error appears when you use the deprecated variable, `CI_BUILD_TOKEN`. To prevent users from receiving this error, you should: + +- Use [CI_JOB_TOKEN](../jobs/ci_job_token.md) instead. +- Change from `gitlab-ci-token/CI_BUILD_TOKEN` to `$CI_REGISTRY_USER/$CI_REGISTRY_PASSWORD`. diff --git a/doc/ci/docker/using_kaniko.md b/doc/ci/docker/using_kaniko.md index b19e65ee396..32f95052980 100644 --- a/doc/ci/docker/using_kaniko.md +++ b/doc/ci/docker/using_kaniko.md @@ -20,7 +20,7 @@ method: to function, which is a significant security concern. - Docker-in-Docker generally incurs a performance penalty and can be quite slow. -## Requirements +## Prerequisites To use kaniko with GitLab, [a runner](https://docs.gitlab.com/runner/) with one of the following executors is required: @@ -113,7 +113,7 @@ build: You can build [multi-arch images](https://www.docker.com/blog/multi-arch-build-and-images-the-simple-way/) inside a container by using [`manifest-tool`](https://github.com/estesp/manifest-tool). -For a detailed guide on how to build a multi-arch image, read [Building a multi-arch container image in unprivileged containers](https://ingenuity.siemens.com/2022/07/building-a-multi-arch-container-image-in-unprivileged-containers/). +For a detailed guide on how to build a multi-arch image, read [Building a multi-arch container image in unprivileged containers](https://blog.siemens.com/2022/07/building-a-multi-arch-container-image-in-unprivileged-containers/). ## Using a registry with a custom certificate diff --git a/doc/ci/environments/deployment_approvals.md b/doc/ci/environments/deployment_approvals.md index 089ecefb373..96011d5ddff 100644 --- a/doc/ci/environments/deployment_approvals.md +++ b/doc/ci/environments/deployment_approvals.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments description: Require approvals prior to deploying to a Protected Environment --- @@ -17,7 +17,7 @@ When a protected environment requires one or more approvals, all deployments to NOTE: See the [epic](https://gitlab.com/groups/gitlab-org/-/epics/6832) for planned features. -## Requirements +## Prerequisites - Basic knowledge of [GitLab Environments and Deployments](index.md). - Basic knowledge of [Protected Environments](protected_environments.md). @@ -62,9 +62,11 @@ co-exist and multiple approval rules takes the precedence over the unified appro #### Unified approval setting -There are two ways to configure approvals for a protected environment: +> - UI configuration [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/378447) in GitLab +> 15.11. + +To configure approvals for a protected environment: -- Using the [UI](protected_environments.md#protecting-environments), set the **Required approvals** field to 1 or more. - Using the [REST API](../../api/protected_environments.md#protect-a-single-environment), set the `required_approval_count` field to 1 or more. @@ -87,14 +89,18 @@ Maintainer role. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345678) in GitLab 14.10 with a flag named `deployment_approval_rules`. Disabled by default. > - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/345678) in GitLab 15.0. [Feature flag `deployment_approval_rules`](https://gitlab.com/gitlab-org/gitlab/-/issues/345678) removed. +> - UI configuration [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/378445) in GitLab 15.11. - Using the [REST API](../../api/group_protected_environments.md#protect-a-single-environment). - `deploy_access_levels` represents which entity can execute the deployment job. - `approval_rules` represents which entity can approve the deployment job. +- Using the [UI](protected_environments.md#protecting-environments). + - **Allowed to deploy** sets which entities can execute the deployment job. + - **Approvers** sets which entities can approve the deployment job. After this is configured, all jobs deploying to this environment automatically go into a blocked state and wait for approvals before running. Ensure that the number of required approvals is less than the number of users allowed to deploy. -Example: +A configuration that uses the REST API might look like: ```shell curl --header 'Content-Type: application/json' --request POST \ diff --git a/doc/ci/environments/deployment_safety.md b/doc/ci/environments/deployment_safety.md index cf82238564e..8be46da3fa8 100644 --- a/doc/ci/environments/deployment_safety.md +++ b/doc/ci/environments/deployment_safety.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -126,6 +126,10 @@ vacation period when most employees are out, you can set up a [Deploy Freeze](.. During a deploy freeze period, no deployment can be executed. This is helpful to ensure that deployments do not happen unexpectedly. +The next configured deploy freeze is displayed at the top of the +[environment deployments list](index.md#view-environments-and-deployments) +page. + ## Protect production secrets Production secrets are needed to deploy successfully. For example, when deploying to the cloud, diff --git a/doc/ci/environments/environments_dashboard.md b/doc/ci/environments/environments_dashboard.md index 479b783202d..fd2c85819fe 100644 --- a/doc/ci/environments/environments_dashboard.md +++ b/doc/ci/environments/environments_dashboard.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: reference --- diff --git a/doc/ci/environments/external_deployment_tools.md b/doc/ci/environments/external_deployment_tools.md index ff3172f0e02..37c4cf05f13 100644 --- a/doc/ci/environments/external_deployment_tools.md +++ b/doc/ci/environments/external_deployment_tools.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: reference --- @@ -14,7 +14,7 @@ GitLab can receive deployment events from these external tools and allows you to For example, the following features are available by setting up tracking: - [See when an merge request has been deployed, and to which environment](../../user/project/merge_requests/widgets.md#post-merge-pipeline-status). -- [Filter merge requests by environment or deployment date](../../user/project/merge_requests/index.md#filter-merge-requests-by-environment-or-deployment-date). +- [Filter merge requests by environment or deployment date](../../user/project/merge_requests/index.md#by-environment-or-deployment-date). - [DevOps Research and Assessment (DORA) metrics](../../user/analytics/dora_metrics.md). - [View environments and deployments](index.md#view-environments-and-deployments). - [Track newly included merge requests per deployment](index.md#track-newly-included-merge-requests-per-deployment). @@ -42,45 +42,45 @@ Here is an example setup that creates a `success` deployment record in GitLab wh 1. Create a new webhook. You can save the following manifest file and apply it by `kubectl apply -n argocd -f `: - ```yaml - apiVersion: v1 - kind: ConfigMap - metadata: - name: argocd-notifications-cm - data: - trigger.on-deployed: | - - description: Application is synced and healthy. Triggered once per commit. - oncePer: app.status.sync.revision - send: - - gitlab-deployment-status - when: app.status.operationState.phase in ['Succeeded'] and app.status.health.status == 'Healthy' - template.gitlab-deployment-status: | - webhook: - gitlab: - method: POST - path: /projects//deployments - body: | - { - "status": "success", - "environment": "production", - "sha": "{{.app.status.operationState.operation.sync.revision}}", - "ref": "main", - "tag": "false" - } - service.webhook.gitlab: | - url: https://gitlab.com/api/v4 - headers: - - name: PRIVATE-TOKEN - value: - - name: Content-type - value: application/json - ``` + ```yaml + apiVersion: v1 + kind: ConfigMap + metadata: + name: argocd-notifications-cm + data: + trigger.on-deployed: | + - description: Application is synced and healthy. Triggered once per commit. + oncePer: app.status.sync.revision + send: + - gitlab-deployment-status + when: app.status.operationState.phase in ['Succeeded'] and app.status.health.status == 'Healthy' + template.gitlab-deployment-status: | + webhook: + gitlab: + method: POST + path: /projects//deployments + body: | + { + "status": "success", + "environment": "production", + "sha": "{{.app.status.operationState.operation.sync.revision}}", + "ref": "main", + "tag": "false" + } + service.webhook.gitlab: | + url: https://gitlab.com/api/v4 + headers: + - name: PRIVATE-TOKEN + value: + - name: Content-type + value: application/json + ``` 1. Create a new subscription in your application: - ```shell - kubectl patch app -n argocd -p '{"metadata": {"annotations": {"notifications.argoproj.io/subscribe.on-deployed.gitlab":""}}}' --type merge - ``` + ```shell + kubectl patch app -n argocd -p '{"metadata": {"annotations": {"notifications.argoproj.io/subscribe.on-deployed.gitlab":""}}}' --type merge + ``` NOTE: If a deployment wasn't created as expected, you can troubleshoot with [`argocd-notifications` tool](https://argocd-notifications.readthedocs.io/en/stable/troubleshooting/). diff --git a/doc/ci/environments/img/environment_auto_stop_v13_10.png b/doc/ci/environments/img/environment_auto_stop_v13_10.png deleted file mode 100644 index 50f268da27f..00000000000 Binary files a/doc/ci/environments/img/environment_auto_stop_v13_10.png and /dev/null differ diff --git a/doc/ci/environments/img/environments_project_home.png b/doc/ci/environments/img/environments_project_home.png index 36b33e260f0..10d9de4f415 100644 Binary files a/doc/ci/environments/img/environments_project_home.png and b/doc/ci/environments/img/environments_project_home.png differ diff --git a/doc/ci/environments/incremental_rollouts.md b/doc/ci/environments/incremental_rollouts.md index 10cda68c4b5..c032ae3be60 100644 --- a/doc/ci/environments/incremental_rollouts.md +++ b/doc/ci/environments/incremental_rollouts.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: concepts, howto --- diff --git a/doc/ci/environments/index.md b/doc/ci/environments/index.md index 60450692794..f4d155369e9 100644 --- a/doc/ci/environments/index.md +++ b/doc/ci/environments/index.md @@ -1,9 +1,8 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: reference -disqus_identifier: 'https://docs.gitlab.com/ee/ci/environments.html' --- # Environments and deployments **(FREE)** @@ -159,6 +158,20 @@ deploy_review_app: - main ``` +### Rename an environment + +> - Renaming an environment by using the UI was [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68550) in GitLab 14.3. +> - Renaming an environment by using the API was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/338897) in GitLab 15.9. +> - Renaming an environment with the API [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/338897) in GitLab 16.0. + +You cannot rename an environment. + +To achieve the same result as renaming an environment: + +1. [Stop the existing environment](#stop-an-environment-by-using-the-ui). +1. [Delete the existing environment](#delete-an-environment). +1. [Create a new environment](#create-a-static-environment) with the desired name. + ## Deployment tier of environments > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/300741) in GitLab 13.10. @@ -207,7 +220,7 @@ The `when: manual` action: You can find the play button in the pipelines, environments, deployments, and jobs views. -## Configure Kubernetes deployments (DEPRECATED) +## Configure Kubernetes deployments (deprecated) > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27630) in GitLab 12.6. > - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. @@ -440,28 +453,31 @@ For example: With GitLab [Route Maps](../review_apps/index.md#route-maps), you can go directly from source files to public pages in the environment set for Review Apps. -### Stop an environment - -When you stop an environment: - -- On the **Environments** page, it moves from the list of **Available** environments - to the list of **Stopped** environments. -- An [`on_stop` action](../yaml/index.md#environmenton_stop), if defined, is executed. +### Stopping an environment -There are multiple ways to clean up [dynamic environments](#create-a-dynamic-environment): +Stopping an environment means its deployments are not accessible on the target server. You must stop +an environment before it can be deleted. -- If you use [merge request pipelines](../pipelines/merge_request_pipelines.md), GitLab stops an environment [when a merge request is merged or closed](#stop-an-environment-when-a-merge-request-is-merged-or-closed). -- If you do _NOT_ use [merge request pipelines](../pipelines/merge_request_pipelines.md), GitLab stops an environment [when the associated feature branch is deleted](#stop-an-environment-when-a-branch-is-deleted). -- If you set [an expiry period to an environment](../yaml/index.md#environmentauto_stop_in), GitLab stops an environment [when it's expired](#stop-an-environment-after-a-certain-time-period). - -To stop stale environments, you can [use the API](../../api/environments.md#stop-stale-environments). +If the environment has an [`on_stop` action](../yaml/index.md#environmenton_stop) defined, it's +executed to stop the environment. #### Stop an environment when a branch is deleted You can configure environments to stop when a branch is deleted. -The following example shows a `deploy_review` job that calls a `stop_review` job -to clean up and stop the environment. +In the following example, a `deploy_review` job calls a `stop_review` job to clean up and stop the +environment. + +- Both jobs must have the same [`rules`](../yaml/index.md#rules) + or [`only/except`](../yaml/index.md#only--except) configuration. Otherwise, + the `stop_review` job might not be included in all pipelines that include the + `deploy_review` job, and you cannot trigger `action: stop` to stop the environment automatically. +- The job with [`action: stop` might not run](#the-job-with-action-stop-doesnt-run) + if it's in a later stage than the job that started the environment. +- If you can't use [merge request pipelines](../pipelines/merge_request_pipelines.md), + set the [`GIT_STRATEGY`](../runners/configure_runners.md#git-strategy) to `none` in the + `stop_review` job. Then the [runner](https://docs.gitlab.com/runner/) doesn't + try to check out the code after the branch is deleted. ```yaml deploy_review: @@ -483,30 +499,13 @@ stop_review: when: manual ``` -Both jobs must have the same [`rules`](../yaml/index.md#rules) -or [`only/except`](../yaml/index.md#only--except) configuration. Otherwise, -the `stop_review` job might not be included in all pipelines that include the -`deploy_review` job, and you cannot trigger `action: stop` to stop the environment automatically. - -The job with [`action: stop` might not run](#the-job-with-action-stop-doesnt-run) -if it's in a later stage than the job that started the environment. - -If you can't use [merge request pipelines](../pipelines/merge_request_pipelines.md), -set the [`GIT_STRATEGY`](../runners/configure_runners.md#git-strategy) to `none` in the -`stop_review` job. Then the [runner](https://docs.gitlab.com/runner/) doesn't -try to check out the code after the branch is deleted. - -Read more in the [`.gitlab-ci.yml` reference](../yaml/index.md#environmenton_stop). - #### Stop an environment when a merge request is merged or closed -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/60885) in GitLab 11.10. - -You can configure environments to stop when a merge request is merged or closed. -This stop trigger is automatically enabled when you use [merge request pipelines](../pipelines/merge_request_pipelines.md). +When you use the [merge request pipelines](../pipelines/merge_request_pipelines.md) configuration, +the `stop` trigger is automatically enabled. -The following example shows a `deploy_review` job that calls a `stop_review` job -to clean up and stop the environment. +In the following example, the `deploy_review` job calls a `stop_review` job to clean up and stop +the environment. ```yaml deploy_review: @@ -531,38 +530,35 @@ stop_review: when: manual ``` -#### Stop an environment when another job is finished +#### Run a pipeline job when environment is stopped -You can set an environment to stop when another job is finished. +You can specify a job to run when an environment is stopped. -In your `.gitlab-ci.yml` file, specify in the [`on_stop`](../yaml/index.md#environmenton_stop) -keyword the name of the job that stops the environment. - -The following example shows a `review_app` job that calls a `stop_review_app` job after the first -job is finished. The `stop_review_app` is triggered based on what is defined under `when`. In this -case, it is set to `manual`, so it needs a -[manual action](../jobs/job_control.md#create-a-job-that-must-be-run-manually) -from the GitLab UI to run. - -Both jobs must have the same rules or only/except configuration. -In this example, if the configuration is not identical: +Prerequisites: -- The `stop_review_app` job might not be included in all pipelines that include the `review_app` job. -- It is not possible to trigger the `action: stop` to stop the environment automatically. +- Both jobs must have the same rules or only/except configuration. +- The `stop_review_app` job **must** have the following keywords defined: + - `when`, defined at either: + - [The job level](../yaml/index.md#when). + - [In a rules clause](../yaml/index.md#rules). If you use `rules` and `when: manual`, you should + also set [`allow_failure: true`](../yaml/index.md#allow_failure) so the pipeline can complete + even if the job doesn't run. + - `environment:name` + - `environment:action` -Also in the example, `GIT_STRATEGY` is set to `none`. If the -`stop_review_app` job is [automatically triggered](../environments/index.md#stop-an-environment), -the runner doesn't try to check out the code after the branch is deleted. +In your `.gitlab-ci.yml` file, specify in the [`on_stop`](../yaml/index.md#environmenton_stop) +keyword the name of the job that stops the environment. -The `stop_review_app` job **must** have the following keywords defined: +In the following example: -- `when`, defined at either: - - [The job level](../yaml/index.md#when). - - [In a rules clause](../yaml/index.md#rules). If you use `rules` and `when: manual`, you should - also set [`allow_failure: true`](../yaml/index.md#allow_failure) so the pipeline can complete - even if the job doesn't run. -- `environment:name` -- `environment:action` +- A `review_app` job calls a `stop_review_app` job after the first job is finished. +- The `stop_review_app` is triggered based on what is defined under `when`. In this + case, it is set to `manual`, so it needs a + [manual action](../jobs/job_control.md#create-a-job-that-must-be-run-manually) + from the GitLab UI to run. +- The `GIT_STRATEGY` is set to `none`. If the `stop_review_app` job is + [automatically triggered](../environments/index.md#stopping-an-environment), + the runner doesn't try to check out the code after the branch is deleted. ```yaml review_app: @@ -586,21 +582,23 @@ stop_review_app: #### Stop an environment after a certain time period -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/20956) in GitLab 12.8. +You can set an environment to stop automatically after a certain time period. -You can set environments to stop automatically after a certain time period. +NOTE: +Due to resource limitations, a background worker for stopping environments runs only once every +hour. This means that environments may not be stopped after the exact time period specified, but are +instead stopped when the background worker detects expired environments. In your `.gitlab-ci.yml` file, specify the [`environment:auto_stop_in`](../yaml/index.md#environmentauto_stop_in) -keyword. You can specify a human-friendly date as the value, such as `1 hour and 30 minutes` or `1 day`. +keyword. Specify the time period in natural language, such as `1 hour and 30 minutes` or `1 day`. After the time period passes, GitLab automatically triggers a job to stop the environment. -Due to resource limitations, a background worker for stopping environments only runs once -every hour. This means that environments aren't stopped at the exact timestamp specified, but are -instead stopped when the hourly cron worker detects expired environments. +In the following example: -In the following example, each merge request creates a Review App environment. -Each push triggers the `review_app` job and an environment named `review/your-branch-name` -is created or updated. The environment runs until `stop_review_app` is executed: +- Each commit on a merge request triggers a `review_app` job that deploys the latest change to the + environment and resets its expiry period. +- If the environment is inactive for more than a week, GitLab automatically triggers the + `stop_review_app` job to stop the environment. ```yaml review_app: @@ -622,13 +620,33 @@ stop_review_app: when: manual ``` -As long as the merge request is active and keeps getting new commits, -the Review App doesn't stop. Developers don't need to worry about -re-initiating Review App. +##### View an environment's scheduled stop date and time + +When a environment has been [scheduled to stop after a specified time period](#stop-an-environment-after-a-certain-time-period), +you can view its expiration date and time. + +To view an environment's expiration date and time: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Deployments > Environments**. +1. Select the name of the environment. + +The expiration date and time is displayed in the upper-left corner, next to the environment's name. + +##### Override a environment's scheduled stop date and time -Because `stop_review_app` is set to `auto_stop_in: 1 week`, -if a merge request is inactive for more than a week, -GitLab automatically triggers the `stop_review_app` job to stop the environment. +When a environment has been [scheduled to stop after a specified time period](#stop-an-environment-after-a-certain-time-period), +you can override its expiration. + +To override an environment's expiration: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Deployments > Environments**. +1. Select the deployment name. +1. in the upper-right corner, select the thumbtack (**{thumbtack}**). + +The `auto_stop_in` setting is overridden and the environment remains active until it's stopped +manually. #### Stop an environment without running the `on_stop` action @@ -640,7 +658,7 @@ To stop an environment without running the defined `on_stop` action, execute the [Stop an environment API](../../api/environments.md#stop-an-environment) with the parameter `force=true`. -#### Stop an environment through the UI +#### Stop an environment by using the UI NOTE: To trigger an `on_stop` action and manually stop an environment from the @@ -659,15 +677,20 @@ To stop an environment in the GitLab UI: > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22456) in GitLab 14.10 [with a flag](../../administration/feature_flags.md) named `environment_multiple_stop_actions`. Disabled by default. > - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/358911) in GitLab 15.0. [Feature flag `environment_multiple_stop_actions`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86685) removed. -This feature is useful when you need to perform multiple **parallel** stop actions on an environment. +To configure multiple **parallel** stop actions on an environment, specify the +[`on_stop`](../yaml/index.md#environmenton_stop) keyword across multiple +[deployment jobs](../jobs/index.md#deployment-jobs) for the same `environment`, as defined in the +`.gitlab-ci.yml` file. + +When an environment is stopped, the matching `on_stop` actions from only successful deployment jobs are run in parallel, in no particular order. -To configure multiple stop actions on an environment, specify the [`on_stop`](../yaml/index.md#environmenton_stop) -keyword across multiple [deployment jobs](../jobs/index.md#deployment-jobs) for the same `environment`, as defined in the `.gitlab-ci.yml` file. +In the following example, for the `test` environment there are two deployment jobs: -When an environment is stopped, the matching `on_stop` actions from *successful deployment jobs* alone are run in parallel in no particular order. +- `deploy-to-cloud-a` +- `deploy-to-cloud-b` -In the following example, for the `test` environment there are two deployment jobs `deploy-to-cloud-a` -and `deploy-to-cloud-b`. +When the environment is stopped, the system runs `on_stop` actions `teardown-cloud-a` and +`teardown-cloud-b` in parallel. ```yaml deploy-to-cloud-a: @@ -697,32 +720,6 @@ teardown-cloud-b: when: manual ``` -When the environment is stopped, the system runs `on_stop` actions -`teardown-cloud-a` and `teardown-cloud-b` in parallel. - -#### View a deployment's scheduled stop time - -You can view a deployment's expiration date in the GitLab UI. - -1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Deployments > Environments**. -1. Select the name of the deployment. - -In the upper left, next to the environment name, the expiration date is displayed. - -#### Override a deployment's scheduled stop time - -You can manually override a deployment's expiration date. - -1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Deployments > Environments**. -1. Select the deployment name. -1. in the upper right, select the thumbtack (**{thumbtack}**). - -![Environment auto stop](img/environment_auto_stop_v13_10.png) - -The `auto_stop_in` setting is overwritten and the environment remains active until it's stopped manually. - ### Delete an environment Delete an environment when you want to remove it and all its deployments. @@ -730,7 +727,7 @@ Delete an environment when you want to remove it and all its deployments. Prerequisites: - You must have at least the Developer role. -- You must [stop](#stop-an-environment) the environment before it can be deleted. +- You must [stop](#stopping-an-environment) the environment before it can be deleted. To delete an environment: @@ -866,7 +863,7 @@ It may take a minute or two for data to appear after initial deployment. Metric charts can be embedded in GitLab Flavored Markdown. See [Embedding Metrics in GitLab Flavored Markdown](../../operations/metrics/embed.md) for more details. -### Web terminals (DEPRECATED) +### Web terminals (deprecated) > [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. @@ -991,19 +988,6 @@ like [Review Apps](../review_apps/index.md) (`review/*`). The most specific spec takes precedence over the other wildcard matching. In this case, the `review/feature-1` spec takes precedence over `review/*` and `*` specs. -### Rename an environment - -> Renaming environments through the UI was [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68550) in GitLab 14.3. Renaming environments through the API was deprecated and [is planned to be removed](https://gitlab.com/gitlab-org/gitlab/-/issues/338897) in GitLab 15.0. - -Renaming an environment through the UI is not possible. -Instead, you need to delete the old environment and create a new one: - -1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Deployments > Environments**. -1. Find the environment and stop it. -1. Delete the environment. -1. Create a new environment with your preferred name. - ## Related topics - [Use GitLab CI to deploy to multiple environments (blog post)](https://about.gitlab.com/blog/2021/02/05/ci-deployment-and-environments/) diff --git a/doc/ci/environments/protected_environments.md b/doc/ci/environments/protected_environments.md index fc49be798ec..f752e2179df 100644 --- a/doc/ci/environments/protected_environments.md +++ b/doc/ci/environments/protected_environments.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -42,9 +42,22 @@ To protect an environment: - You can select groups that are already associated with the project only. - Users must have at least the Developer role to appear in the **Allowed to deploy** list. -1. In the **Required approvals** list, select the number of approvals required to deploy to this environment. - - Ensure that this number is less than the number of users allowed to deploy. +1. In the **Approvers** list, select the role, users, or groups you + want to give deploy access to. Keep in mind that: + + - There are two roles to choose from: + - **Maintainers**: Allows access to all of the project's users with the Maintainer role. + - **Developers**: Allows access to all of the project's users with the Maintainer and Developer role. + - You can select groups that are already associated with the project only. + - Users must have at least the Developer role to appear in + the **Allowed to deploy** list. + +1. In the **Approval rules** section: + + - Ensure that this number is less than or equal to the number of members in + the rule. - See [Deployment Approvals](deployment_approvals.md) for more information about this feature. + 1. Select **Protect**. The protected environment now appears in the list of protected environments. @@ -124,7 +137,7 @@ access to a protected environment through any of these methods: If the user also has push or merge access to the branch deployed on production, they have the following privileges: -- [Stop an environment](index.md#stop-an-environment). +- [Stop an environment](index.md#stopping-an-environment). - [Delete an environment](index.md#delete-an-environment). - [Create an environment terminal](index.md#web-terminals-deprecated). diff --git a/doc/ci/examples/authenticating-with-hashicorp-vault/index.md b/doc/ci/examples/authenticating-with-hashicorp-vault/index.md index 7bb14c4c900..f45c60bdd1f 100644 --- a/doc/ci/examples/authenticating-with-hashicorp-vault/index.md +++ b/doc/ci/examples/authenticating-with-hashicorp-vault/index.md @@ -1,21 +1,21 @@ --- stage: Verify -group: Pipeline Authoring +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: tutorial --- -# Authenticating and reading secrets with HashiCorp Vault **(PREMIUM)** +# Authenticating and reading secrets with HashiCorp Vault (Deprecated) **(PREMIUM)** -This tutorial demonstrates how to authenticate, configure, and read secrets with HashiCorp's Vault from GitLab CI/CD. +WARNING: +Authenticating with HashiCorp Vault by using `CI_JOB_JWT` was [deprecated in GitLab 15.9](../../../update/deprecations.md#old-versions-of-json-web-tokens-are-deprecated) +and the token is scheduled to be removed in GitLab 16.5. This change is a breaking change. +Use [ID tokens to authenticate with HashiCorp Vault](../../secrets/id_token_authentication.md#automatic-id-token-authentication-with-hashicorp-vault) +instead. -NOTE: -[GitLab Premium](https://about.gitlab.com/pricing/) supports read access to a -HashiCorp Vault, and enables you to -[use Vault secrets in a CI job](../../secrets/index.md#use-vault-secrets-in-a-ci-job). -For more information, see [Using external secrets in CI](../../secrets/index.md). +This tutorial demonstrates how to authenticate, configure, and read secrets with HashiCorp's Vault from GitLab CI/CD. -## Requirements +## Prerequisites This tutorial assumes you are familiar with GitLab CI/CD and Vault. @@ -29,7 +29,7 @@ You must replace the `vault.example.com` URL below with the URL of your Vault se ## How it works -Each job has JSON Web Token (JWT) provided as CI/CD variable named `CI_JOB_JWT`. This JWT can be used to authenticate with Vault using the [JWT Auth](https://developer.hashicorp.com/vault/docs/auth/jwt#jwt-authentication) method. +ID tokens are JSON Web Tokens (JWTs) used for OIDC authentication with third-party services. If a job has at least one ID token defined, the `secrets` keyword automatically uses that token to authenticate with Vault. The following fields are included in the JWT: @@ -53,6 +53,7 @@ The following fields are included in the JWT: | `job_id` | Always | ID of this job | | `ref` | Always | Git ref for this job | | `ref_type` | Always | Git ref type, either `branch` or `tag` | +| `ref_path` | Always | Fully qualified ref for the job. For example, `refs/heads/main`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/119075) in GitLab 16.0. | | `ref_protected` | Always | `true` if this Git ref is protected, `false` otherwise | | `environment` | Job specifies an environment | Environment this job specifies ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/294440) in GitLab 13.9) | | `environment_protected` | Job specifies an environment | `true` if specified environment is protected, `false` otherwise ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/294440) in GitLab 13.9) | @@ -80,6 +81,7 @@ Example JWT payload: "job_id": "1212", "ref": "auto-deploy-2020-04-01", "ref_type": "branch", + "ref_path": "refs/heads/auto-deploy-2020-04-01", "ref_protected": "true", "environment": "production", "environment_protected": "true" @@ -254,61 +256,36 @@ $ vault write auth/jwt/config \ For the full list of available configuration options, see Vault's [API documentation](https://developer.hashicorp.com/vault/api-docs/auth/jwt#configure). -The following job, when run for the default branch, is able to read secrets under `secret/myproject/staging/`, but not the secrets under `secret/myproject/production/`: +In GitLab, create the following [CI/CD variables](../../variables/index.md#for-a-project) to provide details about your Vault server: -```yaml -read_secrets: - image: vault:latest - script: - # Check job's ref name - - echo $CI_COMMIT_REF_NAME - # and is this ref protected - - echo $CI_COMMIT_REF_PROTECTED - # Vault's address can be provided here or as CI/CD variable - - export VAULT_ADDR=http://vault.example.com:8200 - # Authenticate and get token. Token expiry time and other properties can be configured - # when configuring JWT Auth - https://developer.hashicorp.com/vault/api-docs/auth/jwt#parameters-1 - - export VAULT_TOKEN="$(vault write -field=token auth/jwt/login role=myproject-staging jwt=$CI_JOB_JWT)" - # Now use the VAULT_TOKEN to read the secret and store it in an environment variable - - export PASSWORD="$(vault kv get -field=password secret/myproject/staging/db)" - # Use the secret - - echo $PASSWORD - # This will fail because the role myproject-staging can not read secrets from secret/myproject/production/* - - export PASSWORD="$(vault kv get -field=password secret/myproject/production/db)" -``` - -NOTE: -If you're using a Vault instance provided by HashiCorp Cloud Platform, -you need to export the `VAULT_NAMESPACE` variable. Its default value is `admin`. +- `VAULT_SERVER_URL` - The URL of your Vault server, for example `https://vault.example.com:8200`. +- `VAULT_AUTH_ROLE` - Optional. The role to use when attempting to authenticate. If no role is specified, Vault uses the [default role](https://developer.hashicorp.com/vault/api-docs/auth/jwt#default_role) specified when the authentication method was configured. +- `VAULT_AUTH_PATH` - Optional. The path where the authentication method is mounted. Default is `jwt`. +- `VAULT_NAMESPACE` - Optional. The [Vault Enterprise namespace](https://developer.hashicorp.com/vault/docs/enterprise/namespaces) to use for reading secrets and authentication. If no namespace is specified, Vault uses the root (`/`) namespace. The setting is ignored by Vault Open Source. -![read secrets staging example](img/vault-read-secrets-staging.png) - -The following job is able to authenticate using the `myproject-production` role and read secrets under `/secret/myproject/production/`: +The following job, when run for the default branch, can read secrets under `secret/myproject/staging/`, but not the secrets under `secret/myproject/production/`: ```yaml -read_secrets: - image: vault:latest +job_with_secrets: + id_tokens: + VAULT_ID_TOKEN: + aud: https://example.vault.com + secrets: + STAGING_DB_PASSWORD: + vault: secret/myproject/staging/db/password@secrets # authenticates using $VAULT_ID_TOKEN script: - # Check job's ref name - - echo $CI_COMMIT_REF_NAME - # and is this ref protected - - echo $CI_COMMIT_REF_PROTECTED - # Vault's address can be provided here or as CI/CD variable - - export VAULT_ADDR=http://vault.example.com:8200 - # Authenticate and get token. Token expiry time and other properties can be configured - # when configuring JWT Auth - https://developer.hashicorp.com/vault/api-docs/auth/jwt#parameters-1 - - export VAULT_TOKEN="$(vault write -field=token auth/jwt/login role=myproject-production jwt=$CI_JOB_JWT)" - # Now use the VAULT_TOKEN to read the secret and store it in environment variable - - export PASSWORD="$(vault kv get -field=password secret/myproject/production/db)" - # Use the secret - - echo $PASSWORD + - access-staging-db.sh --token $STAGING_DB_PASSWORD ``` -![read secrets production example](img/vault-read-secrets-production.png) +In this example: + +- `@secrets` - The vault name, where your Secrets Engines are enabled. +- `secret/myproject/staging/db` - The path location of the secret in Vault. +- `password` The field to be fetched within the referenced secret. ### Limit token access to Vault secrets -You can control `CI_JOB_JWT` access to Vault secrets by using Vault protections +You can control ID token access to Vault secrets by using Vault protections and GitLab features. For example, restrict the token by: - Using Vault [bound claims](https://developer.hashicorp.com/vault/docs/auth/jwt#bound-claims) diff --git a/doc/ci/examples/deployment/composer-npm-deploy.md b/doc/ci/examples/deployment/composer-npm-deploy.md index 533b6519d9a..20083413f66 100644 --- a/doc/ci/examples/deployment/composer-npm-deploy.md +++ b/doc/ci/examples/deployment/composer-npm-deploy.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: tutorial --- diff --git a/doc/ci/examples/deployment/index.md b/doc/ci/examples/deployment/index.md index 67b4ec6b279..664ce84c488 100644 --- a/doc/ci/examples/deployment/index.md +++ b/doc/ci/examples/deployment/index.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: tutorial --- @@ -13,7 +13,7 @@ used with GitLab CI/CD. Dpl can be used to deploy to any of the [supported providers](https://github.com/travis-ci/dpl#supported-providers). -## Requirements +## Prerequisite To use Dpl you need at least Ruby 1.9.3 with ability to install gems. diff --git a/doc/ci/examples/end_to_end_testing_webdriverio/index.md b/doc/ci/examples/end_to_end_testing_webdriverio/index.md index 7b5690e2d3d..3385aca1ef2 100644 --- a/doc/ci/examples/end_to_end_testing_webdriverio/index.md +++ b/doc/ci/examples/end_to_end_testing_webdriverio/index.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Insights +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments author: Vincent Tunru author_gitlab: Vinnl diff --git a/doc/ci/examples/index.md b/doc/ci/examples/index.md index 6738c55b6fa..f2871e50617 100644 --- a/doc/ci/examples/index.md +++ b/doc/ci/examples/index.md @@ -2,7 +2,6 @@ stage: Verify group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false type: index --- @@ -76,7 +75,7 @@ choose one of these templates: - [dotNET Core (`dotNET-Core.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml) - [Elixir (`Elixir.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml) - [Flutter (`Flutter.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml) -- [Golang (`Go.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Go.gitlab-ci.yml) +- [Go (`Go.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Go.gitlab-ci.yml) - [Gradle (`Gradle.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml) - [Grails (`Grails.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Grails.gitlab-ci.yml) - [iOS with fastlane (`iOS-Fastlane.gitlab-ci.yml`)](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml) diff --git a/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md index 062bd602c29..cf12943d279 100644 --- a/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md @@ -2,7 +2,6 @@ stage: Verify group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -disqus_identifier: 'https://docs.gitlab.com/ee/articles/laravel_with_gitlab_and_envoy/index.html' author: Mehran Rasulian author_gitlab: mehranrasulian --- @@ -126,7 +125,7 @@ We'll use this variable in the `.gitlab-ci.yml` later, to easily connect to our ![variables page](img/variables_page.png) -We also need to add the public key to **Project** > **Settings** > **Repository** as a [Deploy Key](../../../user/project/deploy_keys/index.md), which gives us the ability to access our repository from the server through [SSH protocol](../../../gitlab-basics/command-line-commands.md#start-working-on-your-project). +We also need to add the public key to **Project** > **Settings** > **Repository** as a [Deploy Key](../../../user/project/deploy_keys/index.md), which gives us the ability to access our repository from the server through the SSH protocol. ```shell # As the deployer user on the server diff --git a/doc/ci/examples/semantic-release.md b/doc/ci/examples/semantic-release.md index 9f299448d52..2146d76e4d6 100644 --- a/doc/ci/examples/semantic-release.md +++ b/doc/ci/examples/semantic-release.md @@ -90,7 +90,7 @@ As part of publishing a package, semantic-release increases the version number i -1. On the top bar, in the upper right, select your avatar. +1. On the top bar, in the upper-right corner, select your avatar. 1. On the left sidebar, select **Access Tokens**. 1. In the **Token name** box, enter a token name. 1. Under **select scopes**, select the **api** checkbox. @@ -168,3 +168,18 @@ Then, install the module: ```shell npm install --save @gitlab-examples/semantic-release-npm ``` + +## Troubleshooting + +### Deleted Git tags reappear + +A [Git tag](../../user/project/repository/tags/index.md) deleted from the repository +can sometimes be recreated by `semantic-release` when GitLab runners use a cached +version of the repository. If the job runs on a runner with a cached repository that +still has the tag, `semantic-release` recreates the tag in the main repository. + +To avoid this behavior, you can either: + +- Configure the runner with [`GIT_STRATEGY: clone`](../runners/configure_runners.md#git-strategy). +- Include the [`git fetch --prune-tags` command](https://git-scm.com/docs/git-fetch#Documentation/git-fetch.txt---prune-tags) + in your CI/CD script. diff --git a/doc/ci/git_submodules.md b/doc/ci/git_submodules.md index 07ba3d8f916..31cb6bc9946 100644 --- a/doc/ci/git_submodules.md +++ b/doc/ci/git_submodules.md @@ -14,9 +14,13 @@ repository into your project and keep your commits separate. ## Configure the `.gitmodules` file When you use Git submodules, your project should have a file named `.gitmodules`. -You might need to modify it to work in a GitLab CI/CD job. +You have multiple options to configure it to work in a GitLab CI/CD job. -For example, your `.gitmodules` configuration might look like the following if: +### Using absolute URLs + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3198) in GitLab Runner 15.11. + +For example, your generated `.gitmodules` configuration might look like the following if: - Your project is located at `https://gitlab.com/secret-group/my-project`. - Your project depends on `https://gitlab.com/group/project`, which you want @@ -26,19 +30,43 @@ For example, your `.gitmodules` configuration might look like the following if: ```ini [submodule "project"] path = project - url = ../../group/project.git + url = git@gitlab.com:secret-group/project.git +``` + +In this case, use the [`GIT_SUBMODULE_FORCE_HTTPS`](runners/configure_runners.md#rewrite-submodule-urls-to-https) variable +to instruct GitLab Runner to convert the URL to HTTPS before it clones the submodules. + +Alternatively, if you also use HTTPS locally, you can configure an HTTPS URL: + +```ini +[submodule "project"] + path = project + url = https://gitlab.com/secret-group/project.git ``` -When your submodule is on the same GitLab server, you should use relative URLs in -your `.gitmodules` file. Then you can clone with HTTPS in all your CI/CD jobs. You -can also use SSH for all your local checkouts. +You do not need to configure additional variables in this case, but you need to use a +[personal access token](../user/profile/personal_access_tokens.md) to clone it locally. + +### Using relative URLs + +WARNING: +If you use relative URLs, submodules may resolve incorrectly in forking workflows. +Use absolute URLs instead if you expect your project to have forks. + +When your submodule is on the same GitLab server, you can also use relative URLs in +your `.gitmodules` file: + +```ini +[submodule "project"] + path = project + url = ../../project.git +``` The above configuration instructs Git to automatically deduce the URL to -use when cloning sources. Git uses the same configuration for both HTTPS and SSH. -GitLab CI/CD uses HTTPS for cloning your sources, and you can continue to use SSH -to clone locally. +use when cloning sources. You can clone with HTTPS in all your CI/CD jobs, and you +can continue to use SSH to clone locally. -For submodules not located on the same GitLab server, use the full URL: +For submodules not located on the same GitLab server, always use the full URL: ```ini [submodule "project-x"] @@ -50,8 +78,6 @@ For submodules not located on the same GitLab server, use the full URL: To make submodules work correctly in CI/CD jobs: -1. Make sure you use [relative URLs](#configure-the-gitmodules-file) - for submodules located in the same GitLab server. 1. You can set the `GIT_SUBMODULE_STRATEGY` variable to either `normal` or `recursive` to tell the runner to [fetch your submodules before the job](runners/configure_runners.md#git-submodule-strategy): @@ -60,6 +86,9 @@ To make submodules work correctly in CI/CD jobs: GIT_SUBMODULE_STRATEGY: recursive ``` +1. For submodules located on the same GitLab server and configured with a Git or SSH URL, make sure + you set the [`GIT_SUBMODULE_FORCE_HTTPS`](runners/configure_runners.md#rewrite-submodule-urls-to-https) variable. + 1. Use `GIT_SUBMODULE_DEPTH` to configure the cloning depth of submodules independently of the [`GIT_DEPTH`](runners/configure_runners.md#shallow-cloning) variable: ```yaml diff --git a/doc/ci/index.md b/doc/ci/index.md index ea48a5e461d..65e6394b439 100644 --- a/doc/ci/index.md +++ b/doc/ci/index.md @@ -2,7 +2,6 @@ stage: Verify group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false description: "Learn how to use GitLab CI/CD, the GitLab built-in Continuous Integration, Continuous Deployment, and Continuous Delivery toolset to build, test, and deploy your application." type: index --- @@ -47,7 +46,7 @@ GitLab CI/CD uses a number of concepts to describe and run your build and deploy | [Pipelines](pipelines/index.md) | Structure your CI/CD process through pipelines. | | [CI/CD variables](variables/index.md) | Reuse values based on a variable/value key pair. | | [Environments](environments/index.md) | Deploy your application to different environments (for example, staging, production). | -| [Job artifacts](pipelines/job_artifacts.md) | Output, use, and reuse job artifacts. | +| [Job artifacts](jobs/job_artifacts.md) | Output, use, and reuse job artifacts. | | [Cache dependencies](caching/index.md) | Cache your dependencies for a faster execution. | | [GitLab Runner](https://docs.gitlab.com/runner/) | Configure your own runners to execute your scripts. | | [Pipeline efficiency](pipelines/pipeline_efficiency.md) | Configure your pipelines to run quickly and efficiently. | @@ -96,7 +95,7 @@ GitLab CI/CD features, grouped by DevOps stage, include: | [Building Docker images](docker/using_docker_build.md) | Maintain Docker-based projects using GitLab CI/CD. | | [Canary Deployments](../user/project/canary_deployments.md) | Ship features to only a portion of your pods and let a percentage of your user base to visit the temporarily deployed feature. | | [Deploy boards](../user/project/deploy_boards.md) | Check the current health and status of each CI/CD environment running on Kubernetes. | -| [Feature Flags](../operations/feature_flags.md) | Deploy your features behind Feature Flags. | +| [Feature flags](../operations/feature_flags.md) | Deploy your features behind Feature flags. | | [GitLab Pages](../user/project/pages/index.md) | Deploy static websites. | | [GitLab Releases](../user/project/releases/index.md) | Add release notes to Git tags. | | [Cloud deployment](cloud_deployment/index.md) | Deploy your application to a main cloud provider. | @@ -120,7 +119,7 @@ using GitLab CI/CD with various: You can change the default behavior of GitLab CI/CD for: -- An entire GitLab instance in the [CI/CD administration settings](../administration/index.md#cicd-settings). +- An entire GitLab instance in the [CI/CD administration settings](../administration/cicd.md). - Specific projects in the [pipelines settings](pipelines/settings.md). See also: @@ -129,13 +128,13 @@ See also: ## Related topics -- [Why you might choose GitLab CI/CD](https://about.gitlab.com/blog/2016/10/17/gitlab-ci-oohlala/). -- [Reasons you might migrate from another platform](https://about.gitlab.com/blog/2016/07/22/building-our-web-app-on-gitlab-ci/). -- [Five teams that made the switch to GitLab CI/CD](https://about.gitlab.com/blog/2019/04/25/5-teams-that-made-the-switch-to-gitlab-ci-cd/). +- [Why you might choose GitLab CI/CD](https://about.gitlab.com/blog/2016/10/17/gitlab-ci-oohlala/) +- [Reasons you might migrate from another platform](https://about.gitlab.com/blog/2016/07/22/building-our-web-app-on-gitlab-ci/) +- [Five teams that made the switch to GitLab CI/CD](https://about.gitlab.com/blog/2019/04/25/5-teams-that-made-the-switch-to-gitlab-ci-cd/) - If you use VS Code to edit your GitLab CI/CD configuration, the [GitLab Workflow VS Code extension](../user/project/repository/vscode.md) helps you [validate your configuration](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow#validate-gitlab-ci-configuration) - and [view your pipeline status](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow#information-about-your-branch-pipelines-mr-closing-issue). + and [view your pipeline status](https://marketplace.visualstudio.com/items?itemName=GitLab.gitlab-workflow#information-about-your-branch-pipelines-mr-closing-issue) See also the [Why CI/CD?](https://docs.google.com/presentation/d/1OGgk2Tcxbpl7DJaIOzCX4Vqg3dlwfELC3u2jEeCBbDk) presentation. diff --git a/doc/ci/interactive_web_terminal/index.md b/doc/ci/interactive_web_terminal/index.md index c7fb94535ff..a7923cb84a0 100644 --- a/doc/ci/interactive_web_terminal/index.md +++ b/doc/ci/interactive_web_terminal/index.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Interactive Web Terminals **(FREE)** +# Interactive web terminals **(FREE)** > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/50144) in GitLab 11.3. @@ -77,6 +77,6 @@ close the terminal window. ![finished job with terminal open](img/finished_job_with_terminal_open.png) -## Interactive Web Terminals for the Web IDE +## Interactive web terminals for the Web IDE -Read the Web IDE docs to learn how to run [Interactive Terminals through the Web IDE](../../user/project/web_ide/index.md#interactive-web-terminals-for-the-web-ide). +To run interactive web terminals for the Web IDE, see [Web IDE](../../user/project/web_ide/index.md). diff --git a/doc/ci/introduction/index.md b/doc/ci/introduction/index.md index e62c38fc1ec..bf07af5e761 100644 --- a/doc/ci/introduction/index.md +++ b/doc/ci/introduction/index.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Execution +group: Pipeline Authoring info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments description: "An overview of Continuous Integration, Continuous Delivery, and Continuous Deployment, as well as an introduction to GitLab CI/CD." type: concepts diff --git a/doc/ci/jobs/ci_job_token.md b/doc/ci/jobs/ci_job_token.md index d9cfbdf124e..9cbf45a16e7 100644 --- a/doc/ci/jobs/ci_job_token.md +++ b/doc/ci/jobs/ci_job_token.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Execution +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -12,7 +12,7 @@ When a pipeline job is about to run, GitLab generates a unique token and injects You can use a GitLab CI/CD job token to authenticate with specific API endpoints: - Packages: - - [Package Registry](../../user/packages/package_registry/index.md#use-gitlab-cicd-to-build-packages). + - [Package Registry](../../user/packages/package_registry/index.md#to-build-packages). - [Packages API](../../api/packages.md) (project-level). - [Container Registry](../../user/packages/container_registry/build_and_push_images.md#use-gitlab-cicd) (the `$CI_REGISTRY_PASSWORD` is `$CI_JOB_TOKEN`). @@ -30,7 +30,7 @@ job to run. A user can cause a job to run by taking action like pushing a commit triggering a manual job, or being the owner of a scheduled pipeline. Therefore, this user must be assigned to [a role that has the required privileges](../../user/permissions.md#gitlab-cicd-permissions). -The token is valid only while the pipeline job runs. After the job finishes, you can't +The token is valid only while the pipeline job runs. After the job finishes, you cannot use the token anymore. A job token can access a project's resources without any configuration, but it might @@ -65,7 +65,9 @@ tries to steal tokens from other jobs. You can control what projects a CI/CD job token can access to increase the job token's security. A job token might give extra permissions that aren't necessary -to access specific private resources. +to access specific private resources. The job token scope only controls access +to private projects. If an accessed project is public or internal, token scoping does +not apply. If a job token is leaked, it could potentially be used to access private data to the job token's user. By limiting the job token access scope, private data cannot @@ -74,38 +76,44 @@ be accessed unless projects are explicitly authorized. There is a proposal to add more strategic control of the access permissions, see [epic 3559](https://gitlab.com/groups/gitlab-org/-/epics/3559). +NOTE: +Because `CI_REGISTRY_TOKEN` uses `CI_JOB_TOKEN` to authenticate, the access configuration +also applies to `CI_REGISTRY_TOKEN`. + ### Allow access to your project with a job token -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/346298/) in GitLab 15.9. [Deployed behind the `:inbound_ci_scoped_job_token` feature flag](../../user/feature_flags.md), enabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/346298/) in GitLab 15.9. [Deployed behind the `:inbound_ci_scoped_job_token` feature flag](../../user/feature_flags.md), enabled by default. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/346298/) in GitLab 15.10. -Control your project's job token scope by creating an **inbound** allowlist of projects which can -access your project through its `CI_JOB_TOKEN`. +Create an allowlist of projects which can access your project through +their `CI_JOB_TOKEN`. -For example, you can add project `B` to the inbound allowlist for project `A`. Jobs -in the pipeline for "allowed project" `B` can now use the CI/CD job token to authenticate -API calls to access project `A`. +For example, project `A` can add project `B` to the allowlist. CI/CD jobs +in project `B` (the "allowed project") can now use their CI/CD job token to +authenticate API calls to access project `A`. If project `A` is public or internal, +the project can be accessed by project `B` without adding it to the allowlist. -By default the allowlist includes your current project. +By default, the allowlist of any project only includes itself. It is a security risk to disable this feature, so project maintainers or owners should keep this setting enabled at all times. Add projects to the allowlist only when cross-project access is needed. -### Disable the inbound job token scope allowlist +### Disable the job token scope allowlist WARNING: It is a security risk to disable the allowlist. A malicious user could try to compromise a pipeline created in an unauthorized project. If the pipeline was created by one of your maintainers, the job token could be used in an attempt to access your project. -You can disable the inbound job token scope allowlist for testing or a similar reason, +You can disable the job token scope allowlist for testing or a similar reason, but you should enable it again as soon as possible. Prerequisite: - You must have at least the Maintainer role for the project. -To disable the inbound job token scope allowlist: +To disable the job token scope allowlist: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Settings > CI/CD**. @@ -113,9 +121,11 @@ To disable the inbound job token scope allowlist: 1. Toggle **Allow access to this project with a CI_JOB_TOKEN** to disabled. Enabled by default in new projects. -### Add a project to the inbound job token scope allowlist +You can also disable the allowlist [with the API](../../api/graphql/reference/index.md#mutationprojectcicdsettingsupdate). -You can add projects to the inbound allowlist for a project. Projects added to the allowlist +### Add a project to the job token scope allowlist + +You can add projects to the allowlist for a project. Projects added to the allowlist can make API calls from running pipelines by using the CI/CD job token. Prerequisite: @@ -133,6 +143,8 @@ To add a project: 1. Under **Allow CI job tokens from the following projects to access this project**, add projects to the allowlist. +You can also add a target project to the allowlist [with the API](../../api/graphql/reference/index.md#mutationcijobtokenscopeaddproject). + ### Limit your project's job token access > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/328553) in GitLab 14.1. [Deployed behind the `:ci_scoped_job_token` feature flag](../../user/feature_flags.md), disabled by default. @@ -141,9 +153,9 @@ To add a project: NOTE: This feature is disabled by default for all new projects and is [scheduled for removal](https://gitlab.com/gitlab-org/gitlab/-/issues/383084) -in GitLab 16.0. Project maintainers or owners should enable the **inbound** access control instead. +in GitLab 17.0. Project maintainers or owners should enable the access control instead. -Control your project's job token scope by creating an **outbound** allowlist of projects which +Control your project's job token scope by creating an allowlist of projects which can be accessed by your project's job token. By default, the allowlist includes your current project. @@ -155,16 +167,16 @@ limited only by the user's access permissions. For example, when the setting is enabled, jobs in a pipeline in project `A` have a `CI_JOB_TOKEN` scope limited to project `A`. If the job needs to use the token to make an API request to a private project `B`, then `B` must be added to the allowlist for `A`. -If project `B` is public or internal, it's not required to be added to the allowlist. -The job token scope is only for controlling access to private projects. +If project `B` is public or internal, you do not need to add +`B` to the allowlist to grant access. -### Configure the outbound job token scope +### Configure the job token scope Prerequisite: - You must not have more than 100 projects added to the token's scope. -To configure the outbound job token scope: +To configure the job token scope: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Settings > CI/CD**. @@ -230,10 +242,17 @@ CI job token failures are usually shown as responses like `404 Not Found` or sim While troubleshooting CI/CD job token authentication issues, be aware that: +- A [GraphQL example mutation](../../api/graphql/getting_started.md#update-project-settings) + is available to toggle the scope settings per project. +- [This comment](https://gitlab.com/gitlab-org/gitlab/-/issues/351740#note_1335673157) + demonstrates how to use graphQL with Bash and cURL to: + - Enable the inbound token access scope. + - Give access to project B from project A, or add B to A's allowlist. + - To remove project access. - When the [CI/CD job token scopes](#configure-cicd-job-token-access) are enabled, and the job token is being used to access a different project: - The user that executes the job must be a member of the project that is being accessed. - The user must have the [permissions](../../user/permissions.md) to perform the action. - - The accessed project must have the project attempting to access it [added to the inbound allowlist](#add-a-project-to-the-inbound-job-token-scope-allowlist). + - The accessed project must have the project attempting to access it [added to the allowlist](#add-a-project-to-the-job-token-scope-allowlist). - The CI job token becomes invalid if the job is no longer running, has been erased, or if the project is in the process of being deleted. diff --git a/doc/ci/jobs/index.md b/doc/ci/jobs/index.md index 5e69ecff7b8..b9c2ee409b8 100644 --- a/doc/ci/jobs/index.md +++ b/doc/ci/jobs/index.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Execution +group: Pipeline Authoring info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -339,8 +339,8 @@ In the example above: - `date +%s`: The Unix timestamp (for example `1560896352`). - `my_first_section`: The name given to the section. - `\r\e[0K`: Prevents the section markers from displaying in the rendered (colored) - job log, but they are displayed in the raw job log. To see them, in the upper right - of the job log, select **{doc-text}** (**Show complete raw**). + job log, but they are displayed in the raw job log. To see them, in the upper-right corner + of the job log, select **Show complete raw** (**{doc-text}**). - `\r`: carriage return. - `\e[0K`: clear line ANSI escape code. diff --git a/doc/ci/jobs/job_artifacts.md b/doc/ci/jobs/job_artifacts.md new file mode 100644 index 00000000000..0c9dd658ce2 --- /dev/null +++ b/doc/ci/jobs/job_artifacts.md @@ -0,0 +1,358 @@ +--- +stage: Verify +group: Pipeline Security +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Job artifacts **(FREE)** + +Jobs can output an archive of files and directories. This output is known as a job artifact. + +You can download job artifacts by using the GitLab UI or the [API](../../api/job_artifacts.md#get-job-artifacts). + + +For an overview of job artifacts, watch the video [GitLab CI pipelines, artifacts, and environments](https://www.youtube.com/watch?v=PCKDICEe10s). +Or, for an introduction, watch [GitLab CI pipeline tutorial for beginners](https://www.youtube.com/watch?v=Jav4vbUrqII). + +For administrator information about job artifact storage, see [administering job artifacts](../../administration/job_artifacts.md). + +## Create job artifacts + +To create job artifacts, use the [`artifacts`](../yaml/index.md#artifacts) keyword in your `.gitlab-ci.yml` file: + +```yaml +pdf: + script: xelatex mycv.tex + artifacts: + paths: + - mycv.pdf +``` + +In this example, a job named `pdf` calls the `xelatex` command to build a PDF file from the +LaTeX source file, `mycv.tex`. + +The [`paths`](../yaml/index.md#artifactspaths) keyword determines which files to add to the job artifacts. +All paths to files and directories are relative to the repository where the job was created. + +### With wildcards + +You can use wildcards for paths and directories. For example, to create an artifact +with all the files inside the directories that end with `xyz`: + +```yaml +job: + script: echo "build xyz project" + artifacts: + paths: + - path/*xyz/* +``` + +### With an expiry + +The [`expire_in`](../yaml/index.md#artifactsexpire_in) keyword determines how long +GitLab keeps the job artifacts. For example: + +```yaml +pdf: + script: xelatex mycv.tex + artifacts: + paths: + - mycv.pdf + expire_in: 1 week +``` + +If `expire_in` is not defined, the [instance-wide setting](../../user/admin_area/settings/continuous_integration.md#default-artifacts-expiration) +is used. + +To prevent artifacts from expiring, you can select **Keep** from the job details page. +The option is not available when an artifact has no expiry set. + +### With a dynamically defined name + +You can use [CI/CD variables](../variables/index.md) to dynamically define the +artifacts file's name. + +For example, to create an archive with a name of the current job: + +```yaml +job: + artifacts: + name: "$CI_JOB_NAME" + paths: + - binaries/ +``` + +To create an archive with a name of the current branch or tag including only +the binaries directory: + +```yaml +job: + artifacts: + name: "$CI_COMMIT_REF_NAME" + paths: + - binaries/ +``` + +If your branch-name contains forward slashes +(for example `feature/my-feature`) use `$CI_COMMIT_REF_SLUG` +instead of `$CI_COMMIT_REF_NAME` for proper naming of the artifact. + +### With a Windows runner or shell executor + +If you use Windows Batch to run your shell scripts you must replace `$` with `%`: + +```yaml +job: + artifacts: + name: "%CI_JOB_STAGE%-%CI_COMMIT_REF_NAME%" + paths: + - binaries/ +``` + +If you use Windows PowerShell to run your shell scripts you must replace `$` with `$env:`: + +```yaml +job: + artifacts: + name: "$env:CI_JOB_STAGE-$env:CI_COMMIT_REF_NAME" + paths: + - binaries/ +``` + +### Without excluded files + +Use [`artifacts:exclude`](../yaml/index.md#artifactsexclude) to prevent files from +being added to an artifacts archive. + +For example, to store all files in `binaries/`, but not `*.o` files located in +subdirectories of `binaries/`. + +```yaml +artifacts: + paths: + - binaries/ + exclude: + - binaries/**/*.o +``` + +Unlike [`artifacts:paths`](../yaml/index.md#artifactspaths), `exclude` paths are not recursive. +To exclude all of the contents of a directory, match them explicitly rather +than matching the directory itself. + +For example, to store all files in `binaries/` but nothing located in the `temp/` subdirectory: + +```yaml +artifacts: + paths: + - binaries/ + exclude: + - binaries/temp/**/* +``` + +### With untracked files + +Use [`artifacts:untracked`](../yaml/index.md#artifactsuntracked) to add all Git untracked +files as artifacts (along with the paths defined in [`artifacts:paths`](../yaml/index.md#artifactspaths)). Untracked +files are those that haven't been added to the repository but exist in the repository checkout. + +For example, to save all Git untracked files and files in `binaries`: + +```yaml +artifacts: + untracked: true + paths: + - binaries/ +``` + +For example, to save all untracked files but [exclude](../yaml/index.md#artifactsexclude) `*.txt` files: + +```yaml +artifacts: + untracked: true + exclude: + - "*.txt" +``` + +## Prevent a job from fetching artifacts + +Jobs downloads all artifacts from the completed jobs in previous stages by default. +To prevent a job from downloading any artifacts, set [`dependencies`](../yaml/index.md#dependencies) +to an empty array (`[]`): + +```yaml +job: + stage: test + script: make build + dependencies: [] +``` + +## View all job artifacts in a project + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31271) in GitLab 12.4 [with a flag](../../administration/feature_flags.md) named `artifacts_management_page`. Disabled by default. +> - [Improved look](https://gitlab.com/gitlab-org/gitlab/-/issues/33418) in GitLab 15.6. +> - [Improved performance](https://gitlab.com/gitlab-org/gitlab/-/issues/387765) in GitLab 15.9. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/407475) in GitLab 16.0. Feature flag `artifacts_management_page` removed. + +You can view all artifacts stored in a project from the **CI/CD > Artifacts** page. +This list displays all jobs and their associated artifacts. Expand an entry to access +all artifacts associated with a job, including: + +- Artifacts created with the `artifacts:` keyword. +- [Report artifacts](../yaml/artifacts_reports.md). +- Job logs and metadata, which are stored internally as separate artifacts. + +You can download or delete individual artifacts from this list. + +## Download job artifacts + +You can download job artifacts from: + +- Any **Pipelines** list. On the right of the pipeline, select **Download artifacts** (**{download}**). +- Any **Jobs** list. On the right of the job, select **Download artifacts** (**{download}**). +- A job's detail page. On the right of the page, select **Download**. +- A merge request **Overview** page. On the right of the latest pipeline, select **Artifacts** (**{download}**). +- The [**Artifacts**](#view-all-job-artifacts-in-a-project) page. On the right of the job, select **Download** (**{download}**). +- The [artifacts browser](#browse-the-contents-of-the-artifacts-archive). On the top of the page, + select **Download artifacts archive** (**{download}**). + +[Report artifacts](../yaml/artifacts_reports.md) can only be downloaded from the **Pipelines** list +or **Artifacts** page. + +You can download job artifacts from the latest successful pipeline by using [the job artifacts API](../../api/job_artifacts.md). +You cannot download [artifact reports](../yaml/artifacts_reports.md) with the job artifacts API, +unless the report is added as a regular artifact with `artifacts:paths`. + +### From a URL + +You can download the artifacts archive for a specific job with a publicly accessible +URL for the [job artifacts API](../../api/job_artifacts.md#download-the-artifacts-archive). + +For example, to download the latest artifacts of a job named `build` in the `main` branch of a project on GitLab.com: + +```plaintext +https://gitlab.com/api/v4/projects//jobs/artifacts/main/download?job=build +``` + +For example, to download the file `review/index.html` from the latest job named `build` in the `main` branch of a project on GitLab.com: + +```plaintext +https://gitlab.com/api/v4/projects//jobs/artifacts/main/raw/review/index.html?job=build +``` + +In both examples, replace `` with a valid project ID, found at the top of the project details page. + +Artifacts for [parent and child pipelines](../pipelines/downstream_pipelines.md#parent-child-pipelines) +are searched in hierarchical order from parent to child. For example, if both parent and +child pipelines have a job with the same name, the job artifacts from the parent pipeline are returned. + +## Browse the contents of the artifacts archive + +You can browse the contents of the artifacts from the UI without downloading the artifact locally, +from: + +- Any **Jobs** list. On the right of the job, select **Browse** (**{folder-open}**). +- A job's detail page. On the right of the page, select **Browse**. +- The **Artifacts** page. On the right of the job, select **Browse** (**{folder-open}**). + +If [GitLab Pages](../../administration/pages/index.md) is enabled in the project, you can preview +HTML files in the artifacts directly in your browser. If the project is internal or private, you must +enable [GitLab Pages access control](../../administration/pages/index.md#access-control) to preview +HTML files. + +### From a URL + +You can browse the job artifacts of the latest successful pipeline for a specific job +with a publicly accessible URL. + +For example, to browse the latest artifacts of a job named `build` in the `main` branch of a project on GitLab.com: + +```plaintext +https://gitlab.com//-/jobs/artifacts/main/browse?job=build +``` + +Replace `` with a valid project path, you can find it in the URL for your project. + +## Delete job log and artifacts + +WARNING: +Deleting the job log and artifacts is a destructive action that cannot be reverted. Use with caution. +Deleting certain files, including report artifacts, job logs, and metadata files, affects +GitLab features that use these files as data sources. + +You can delete a job's artifacts and log. + +Prerequisites: + +- You must be the owner of the job or a user with at least the Maintainer role for the project. + +To delete a job: + +1. Go to a job's detail page. +1. In the upper-right corner of the job's log, select **Erase job log and artifacts** (**{remove}**). + +You can also delete individual artifacts from the [**Artifacts** page](#bulk-delete-artifacts). + +### Bulk delete artifacts + +- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33348) in GitLab 15.10 [with a flag](../../administration/feature_flags.md) named `ci_job_artifact_bulk_destroy`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, +ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `ci_job_artifact_bulk_destroy`. +The feature is not ready for production use. + +You can delete multiple artifacts at the same time: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **CI/CD > Artifacts**. +1. Select the checkboxes next to the artifacts you want to delete. You can select up to 50 artifacts. +1. Select **Delete selected**. + +## Link to job artifacts in the merge request UI + +Use the [`artifacts:expose_as`](../yaml/index.md#artifactsexpose_as) keyword to display +a link to job artifacts in the [merge request](../../user/project/merge_requests/index.md) UI. + +For example, for an artifact with a single file: + +```yaml +test: + script: ["echo 'test' > file.txt"] + artifacts: + expose_as: 'artifact 1' + paths: ['file.txt'] +``` + +With this configuration, GitLab adds **artifact 1** as a link to `file.txt` to the +**View exposed artifact** section of the relevant merge request. + +## Keep artifacts from most recent successful jobs + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16267) in GitLab 13.0. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/229936) in GitLab 13.4. +> - [Made optional with a CI/CD setting](https://gitlab.com/gitlab-org/gitlab/-/issues/241026) in GitLab 13.8. + +By default artifacts are always kept for successful pipelines for the most recent commit on +each ref. This means that the latest artifacts do not immediately expire according +to the `expire_in` specification. + +If a pipeline for a new commit on the same ref completes successfully, the previous pipeline's +artifacts are deleted according to the `expire_in` configuration. The artifacts +of the new pipeline are kept automatically. If multiple pipelines run for the most +recent commit on the ref, all artifacts are kept. + +Keeping the latest artifacts can use a large amount of storage space in projects +with a lot of jobs or large artifacts. If the latest artifacts are not needed in +a project, you can disable this behavior to save space: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **Artifacts**. +1. Clear the **Keep artifacts from most recent successful jobs** checkbox. + +You can disable this behavior for all projects on a self-managed instance in the +[instance's CI/CD settings](../../user/admin_area/settings/continuous_integration.md#keep-the-latest-artifacts-for-all-jobs-in-the-latest-successful-pipelines). + +When **Keep artifacts from most recent successful jobs** is enabled, artifacts are always kept for [blocked](job_control.md#types-of-manual-jobs) +pipelines. These artifacts expire only after the blocking job is triggered and the pipeline completes. +[Issue 387087](https://gitlab.com/gitlab-org/gitlab/-/issues/387087) proposes to change this behavior. diff --git a/doc/ci/jobs/job_artifacts_troubleshooting.md b/doc/ci/jobs/job_artifacts_troubleshooting.md new file mode 100644 index 00000000000..f5951350085 --- /dev/null +++ b/doc/ci/jobs/job_artifacts_troubleshooting.md @@ -0,0 +1,149 @@ +--- +stage: Verify +group: Pipeline Security +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Troubleshooting job artifacts + +When working with [job artifacts](job_artifacts.md), you might encounter the following issues. + +## Job does not retrieve certain artifacts + +By default, jobs fetch all artifacts from previous stages, but jobs using `dependencies` +or `needs` do not fetch artifacts from all jobs by default. + +If you use these keywords, artifacts are fetched from only a subset of jobs. Review +the keyword reference for information on how to fetch artifacts with these keywords: + +- [`dependencies`](../yaml/index.md#dependencies) +- [`needs`](../yaml/index.md#needs) +- [`needs:artifacts`](../yaml/index.md#needsartifacts) + +## Job artifacts use too much disk space + +If job artifacts are using too much disk space, see the +[job artifacts administration documentation](../../administration/job_artifacts.md#job-artifacts-using-too-much-disk-space). + +## Error message `No files to upload` + +This message appears in job logs when a the runner can't find the file to upload. Either +the path to the file is incorrect, or the file was not created. You can check the job +log for other errors or warnings that specify the filename and why it wasn't +generated. + +For more detailed job logs, you can [enable CI/CD debug logging](../variables/index.md#enable-debug-logging) +and try the job again. This logging might provide more information about why the file +wasn't created. + +## Error message `Missing /usr/bin/gitlab-runner-helper. Uploading artifacts is disabled.` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3068) in GitLab 15.2, GitLab Runner uses `RUNNER_DEBUG` instead of `DEBUG`, fixing this issue. + +In GitLab 15.1 and earlier, setting a CI/CD variable named `DEBUG` can cause artifact uploads to fail. + +To work around this, you can: + +- Update to GitLab and GitLab Runner 15.2 +- Use a different variable name +- Set it as an environment variable in a `script` command: + + ```yaml + failing_test_job: # This job might fail due to issue gitlab-org/gitlab-runner#3068 + variables: + DEBUG: true + script: bin/mycommand + artifacts: + paths: + - bin/results + + successful_test_job: # This job does not define a CI/CD variable named `DEBUG` and is not affected by the issue + script: DEBUG=true bin/mycommand + artifacts: + paths: + - bin/results + ``` + +## Error message `FATAL: invalid argument` when uploading a dotenv artifact on a Windows runner + +The PowerShell `echo` command writes files with UCS-2 LE BOM (Byte Order Mark) encoding, +but only UTF-8 is supported. If you try to create a [`dotenv`](../yaml/artifacts_reports.md) +artifact with `echo`, it causes a `FATAL: invalid argument` error. + +Use PowerShell `Add-Content` instead, which uses UTF-8: + +```yaml +test-job: + stage: test + tags: + - windows + script: + - echo "test job" + - Add-Content -Path build.env -Value "MY_ENV_VAR=true" + artifacts: + reports: + dotenv: build.env +``` + +## Job artifacts do not expire + +If some job artifacts are not expiring as expected, check if the +[**Keep artifacts from most recent successful jobs**](job_artifacts.md#keep-artifacts-from-most-recent-successful-jobs) +setting is enabled. + +When this setting is enabled, job artifacts from the latest successful pipeline +of each ref do not expire and are not deleted. + +## Error message `This job could not start because it could not retrieve the needed artifacts.` + +A job configured with the [`needs:artifacts`](../yaml/index.md#needsartifacts) keyword +fails to start and returns this error message if: + +- The job's dependencies cannot be found. +- The job cannot access the relevant resources due to insufficient permissions. + +The troubleshooting steps to follow differ based on the syntax the job uses: + +- [`needs:project`](#for-a-job-configured-with-needsproject) +- [`needs:pipeline:job`](#for-a-job-configured-with-needspipelinejob) + +### For a job configured with `needs:project` + +The `could not retrieve the needed artifacts.` error can happen for a job using +[`needs:project`](../yaml/index.md#needsproject) with a configuration similar to: + +```yaml +rspec: + needs: + - project: my-group/my-project + job: dependency-job + ref: master + artifacts: true +``` + +To troubleshoot this error, verify that: + +- Project `my-group/my-project` is in a group with a Premium subscription plan. +- The user running the job can access resources in `my-group/my-project`. +- The `project`, `job`, and `ref` combination exists and results in the desired dependency. +- Any variables in use evaluate to the correct values. + +### For a job configured with `needs:pipeline:job` + +The `could not retrieve the needed artifacts.` error can happen for a job using +[`needs:pipeline:job`](../yaml/index.md#needspipelinejob) with a configuration similar to: + +```yaml +rspec: + needs: + - pipeline: $UPSTREAM_PIPELINE_ID + job: dependency-job + artifacts: true +``` + +To troubleshoot this error, verify that: + +- The `$UPSTREAM_PIPELINE_ID` CI/CD variable is available in the current pipeline's + parent-child pipeline hierarchy. +- The `pipeline` and `job` combination exists and resolves to an existing pipeline. +- `dependency-job` has run and finished successfully. diff --git a/doc/ci/jobs/job_control.md b/doc/ci/jobs/job_control.md index 3cd57ff6a6a..fa045a898fa 100644 --- a/doc/ci/jobs/job_control.md +++ b/doc/ci/jobs/job_control.md @@ -10,11 +10,7 @@ When a new pipeline starts, GitLab checks the pipeline configuration to determin which jobs should run in that pipeline. You can configure jobs to run depending on factors like the status of variables, or the pipeline type. -To configure a job to be included or excluded from certain pipelines, you can use: - -- [`rules`](../yaml/index.md#rules) -- [`only`](../yaml/index.md#only--except) -- [`except`](../yaml/index.md#only--except) +To configure a job to be included or excluded from certain pipelines, use [`rules`](../yaml/index.md#rules). Use [`needs`](../yaml/index.md#needs) to configure a job to run as soon as the earlier jobs it depends on finish running. diff --git a/doc/ci/large_repositories/index.md b/doc/ci/large_repositories/index.md index 5a553228ba1..f728e9b9e17 100644 --- a/doc/ci/large_repositories/index.md +++ b/doc/ci/large_repositories/index.md @@ -255,7 +255,7 @@ For very active repositories with a large number of references and files, you ca enabled on all Gitaly servers, we found that we no longer need a pre-clone step for `gitlab-org/gitlab` development. - Optimize your CI/CD jobs by seeding repository data in a pre-clone step with the [`pre_clone_script`](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section) of GitLab Runner. See - [SaaS runners on Linux](../runners/saas/linux_saas_runner.md#pre-clone-script) for details. + [SaaS runners on Linux](../runners/saas/linux_saas_runner.md#pre-clone-script-deprecated) for details. Besides speeding up pipelines in large and active projects, seeding the repository data also helps avoid `429 Too many requests` errors from Cloudflare. diff --git a/doc/ci/lint.md b/doc/ci/lint.md index 74e0f0cd5ef..119a0e5853e 100644 --- a/doc/ci/lint.md +++ b/doc/ci/lint.md @@ -26,7 +26,7 @@ To check CI/CD configuration with the CI lint tool: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **CI/CD > Pipelines**. -1. In the upper right, select **CI lint**. +1. In the upper-right corner, select **CI lint**. 1. Paste a copy of the CI/CD configuration you want to check into the text box. 1. Select **Validate**. @@ -47,7 +47,7 @@ To simulate a pipeline: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **CI/CD > Pipelines**. -1. In the upper right, select **CI lint**. +1. In the upper-right corner, select **CI lint**. 1. Paste a copy of the CI/CD configuration you want to check into the text box. 1. Select **Simulate pipeline creation for the default branch**. 1. Select **Validate**. diff --git a/doc/ci/migration/circleci.md b/doc/ci/migration/circleci.md index c1e9f97a169..b50695e8e50 100644 --- a/doc/ci/migration/circleci.md +++ b/doc/ci/migration/circleci.md @@ -2,7 +2,6 @@ stage: Verify group: Pipeline Authoring info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false type: index, howto --- @@ -292,8 +291,8 @@ Self-managed runners: GitLab.com shared runners: - Linux -- [Windows](../runners/saas/windows_saas_runner.md) ([Beta](../../policy/alpha-beta-support.md#beta-features)). -- [macOS](../runners/saas/macos_saas_runner.md) ([Beta](../../policy/alpha-beta-support.md#beta-features)). +- [Windows](../runners/saas/windows_saas_runner.md) ([Beta](../../policy/alpha-beta-support.md#beta)). +- [macOS](../runners/saas/macos_saas_runner.md) ([Beta](../../policy/alpha-beta-support.md#beta)). ### Machine and specific build environments @@ -321,16 +320,14 @@ Example of the same job using `tags` in GitLab CI/CD: ```yaml windows job: - stage: - - build + stage: build tags: - windows script: - echo Hello, %USERNAME%! osx job: - stage: - - build + stage: build tags: - osx script: diff --git a/doc/ci/migration/jenkins.md b/doc/ci/migration/jenkins.md index 71deaadf9ec..dc1944e65c6 100644 --- a/doc/ci/migration/jenkins.md +++ b/doc/ci/migration/jenkins.md @@ -2,7 +2,6 @@ stage: Verify group: Pipeline Authoring info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false type: index, howto --- @@ -179,7 +178,7 @@ rspec: Artifacts may work a bit differently than you've used them with Jenkins. In GitLab, any job can define a set of artifacts to be saved by using the `artifacts` keyword. This can be configured to point to a file or set of files that can then be persisted from job to job. Read more on our detailed -[artifacts documentation](../pipelines/job_artifacts.md): +[artifacts documentation](../jobs/job_artifacts.md): ```yaml pdf: diff --git a/doc/ci/mobile_devops.md b/doc/ci/mobile_devops.md index 1359191f6a6..175a63dc3b9 100644 --- a/doc/ci/mobile_devops.md +++ b/doc/ci/mobile_devops.md @@ -5,11 +5,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Mobile DevOps +# Mobile DevOps (Experimental) -GitLab Mobile DevOps is a collection of features and tools designed for mobile developers -and teams to automate their build and release process using GitLab CI/CD. Mobile DevOps -is an experimental feature developed by [GitLab Incubation Engineering](https://about.gitlab.com/handbook/engineering/incubation/). +Use GitLab Mobile DevOps to quickly build, sign, and release native and cross-platform mobile apps +for Android and iOS using GitLab CI/CD. Mobile DevOps is an experimental feature developed by +[GitLab Incubation Engineering](https://about.gitlab.com/handbook/engineering/incubation/). Mobile DevOps is still in development, but you can: @@ -17,22 +17,408 @@ Mobile DevOps is still in development, but you can: - [Report a bug](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=report_bug). - [Share feedback](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=general_feedback). -## Code Signing +## Build environments -[Project-level Secure Files](secure_files/index.md) makes it easier to manage key stores, provision profiles, -and signing certificates directly in a GitLab project. +Get started quickly by using [GitLab.com SaaS runners](../ci/runners/index.md), +or set up [self-managed runners](https://docs.gitlab.com/runner/#use-self-managed-runners) +for complete control over the build environment. -For a guided walkthrough of this feature, watch the [video demo](https://youtu.be/O7FbJu3H2YM). +### Android build environments -## Review Apps for Mobile +Set up an Android build environment by selecting an appropriate Docker image +and adding it to your `.gitlab-ci.yml` file. [Fabernovel](https://hub.docker.com/r/fabernovel/android/tags) +provides a variety of supported Android versions. -You can use [Review Apps](review_apps/index.md) to preview changes directly from a merge request. -Review Apps for Mobile brings that capability to mobile developers through an integration -with [Appetize](https://appetize.io/). +For example: -Watch a [video walkthrough](https://youtu.be/X15mI19TXa4) of this feature, or visit the -[setup instructions](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/readme/-/issues/15) -to get started. +```yaml +test: + image: fabernovel/android:api-33-v1.7.0 + stage: test + script: + - fastlane test +``` + +### iOS build environments + +GitLab SaaS runners on macOS are currently available in beta. Follow the [instructions to request access](../ci/runners/saas/macos_saas_runner.md#access-request-process) +for your project. + +After you are granted access to the beta macOS runners, [choose an image](../ci/runners/saas/macos/environment.md#available-images) +and add it to your `.gitlab-ci.yml` file. + +For example: + +```yaml +test: + image: macos-12-xcode-14 + stage: test + script: + - fastlane test + tags: + - saas-macos-medium-m1 +``` + +## Code signing + +All Android and iOS apps must be securely signed before being distributed through +the various app stores. Signing ensures that applications haven't been tampered with +before reaching a user's device. + +With [project-level secure files](secure_files/index.md), you can store the following +in GitLab, so that they can be used to securely sign apps in CI/CD builds: + +- Keystores +- Provision profiles +- Signing certificates + + +For an overview, see [Project-level secure files demo](https://youtu.be/O7FbJu3H2YM). + +### Code signing Android projects with fastlane & Gradle + +To set up code signing for Android: + +1. Upload your keystore and keystore properties files to project-level secure files. +1. Update the Gradle configuration to use those files in the build. + + +For an overview, see [How to build and release an Android app to Google Play with GitLab](https://youtu.be/u8yC8W2k85U). + +#### Create a keystore + +Run the following command to generate a keystore file if you don't already have one: + +```shell +keytool -genkey -v -keystore release-keystore.jks -storepass password -alias release -keypass password -keyalg RSA -keysize 2048 -validity 10000 +``` + +Next, put the keystore configuration in a file called `release-keystore.properties`, +which should look similar to this example: + +```plaintext +storeFile=.secure_files/release-keystore.jks +keyAlias=release +keyPassword=password +storePassword=password +``` + +After these files are created: + +- [Upload them as Secure Files](secure_files/index.md) in the GitLab project + so they can be used in CI/CD jobs. +- Add both files to your `.gitignore` file so they aren't committed to version control. + +#### Configure Gradle + +The next step is to configure Gradle to use the newly created keystore. In the app's `build.gradle` file: + +1. Immediately after the plugins section, add: + + ```gradle + def keystoreProperties = new Properties() + def keystorePropertiesFile = rootProject.file('.secure_files/release-keystore.properties') + if (keystorePropertiesFile.exists()) { + keystoreProperties.load(new FileInputStream(keystorePropertiesFile)) + } + ``` + +1. Anywhere within the `android` block, add: + + ```gradle + signingConfigs { + release { + keyAlias keystoreProperties['keyAlias'] + keyPassword keystoreProperties['keyPassword'] + storeFile keystoreProperties['storeFile'] ? file(keystoreProperties['storeFile']) : null + storePassword keystoreProperties['storePassword'] + } + } + ``` + +1. Add the `signingConfig` to the release build type: + + ```gradle + signingConfig signingConfigs.release + ``` + +With this configuration in place, you can use fastlane to build & sign the app +with the files stored in secure files. + +For example: + +- Sample `fastlane/Fastfile` file: + + ```ruby + default_platform(:android) + + platform :android do + desc "Create and sign a new build" + lane :build do + gradle(tasks: ["clean", "assembleRelease", "bundleRelease"]) + end + end + ``` + +- Sample `.gitlab-ci.yml` file: + + ```yaml + build: + image: fabernovel/android:api-33-v1.7.0 + stage: build + script: + - apt update -y && apt install -y curl + - curl --silent "https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/download-secure-files/-/raw/main/installer" | bash + - fastlane build + ``` + +### Code sign iOS projects with fastlane + +To set up code signing for iOS, you must: + +1. Install fastlane locally so you can upload your signing certificates to GitLab. +1. Configure the build to use those files. + + +For an overview, see [How to build and release an iOS app to Test Flight with GitLab](https://youtu.be/Ar8IsBgP1as). + +#### Initialize fastlane + +With fastlane installed, start by running: + +```shell +fastlane init +``` + +This command creates a `fastlane` folder in the project with an `Appfile` and a stubbed-out `fastfile`. +This process asks you for login credentials to App Store Connect +to generate an app identifier and App Store app if they don't already exist. + +The next step sets up fastlane match to manage code signing files for the project. +Run the following command to generate a `Matchfile` with the configuration: + +```shell +fastlane match init +``` + +This command prompts you to: + +- Choose which storage backend you want to use, you must select `gitlab_secure_files`. +- Input your project path, for example `gitlab-org/gitlab`. + +#### Generate and upload certificates + +Run the following command to generate certificates and profiles in the Apple Developer portal +and upload those files to GitLab: + +```shell +PRIVATE_TOKEN=YOUR-TOKEN bundle exec fastlane match development +``` + +In this example: + +- `YOUR-TOKEN` must be either a personal or project access token with Maintainer role for the GitLab project. +- Replace `development` with the type of build you want to sign, for example `appstore` or `ad-hoc`. + +You can view the files in your project's CI/CD settings as soon as the command completes. + +#### Upload-only + +If you have already created signing certificates and provisioning profiles for your project, +you can optionally use `fastlane match import` to load your existing files into GitLab: + +```shell +PRIVATE_TOKEN=YOUR-TOKEN bundle exec fastlane match import +``` + +You are prompted to input the path to your files. After you provide those details, +your files are uploaded and visible in your project's CI/CD settings. +If prompted for the `git_url` during the import, it is safe to leave it blank and press enter. + +With this configuration in place, you can use fastlane to build and sign the app with +the files stored in secure files. + +For example: + +- Sample `fastlane/Fastfile` file: + + ```ruby + default_platform(:ios) + + platform :ios do + desc "Build and sign the application for development" + lane :build do + setup_ci + + match(type: 'development', readonly: is_ci) + + build_app( + project: "ios demo.xcodeproj", + scheme: "ios demo", + configuration: "Debug", + export_method: "development" + ) + end + end + ``` + +- Sample `.gitlab-ci.yml` file: + + ```yaml + build_ios: + image: macos-12-xcode-14 + stage: build + script: + - fastlane build + tags: + - shared-macos-amd64 + ``` + +## Distribution + +Signed builds can be uploaded to the Google Play Store or Apple App Store by using +the Mobile DevOps Distribution integrations. + +### Android distribution with Google Play integration and fastlane + +To create an Android distribution with Google Play integration and fastlane, you must: + +1. [Create a Google service account](https://docs.fastlane.tools/actions/supply/#setup) + in Google Cloud Platform and grant that account access to the project in Google Play. +1. [Enable the Google Play integration](#enable-google-play-integration). +1. Add the release step to your pipeline. + + +For an overview, see [Google Play integration demo](https://youtu.be/Fxaj3hna4uk). + +#### Enable Google Play Integration + +Use the [Google Play integration](../user/project/integrations/google_play.md), +to configure your CI/CD pipelines to connect to the [Google Play Console](https://play.google.com/console) +to build and release Android apps. To enable the integration: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Integrations**. +1. Select **Google Play**. +1. In **Enable integration**, select the **Active** checkbox. +1. In **Package name**, enter the package name of the app. For example, `com.gitlab.app_name`. +1. In **Service account key (.JSON)** drag or upload your key file. +1. Select **Save changes**. + +With the integration enabled, you can use fastlane to distribute a build to Google Play. + +For example: + +- Sample `fastlane/Fastfile`: + + ```ruby + default_platform(:android) + + platform :android do + desc "Submit a new Beta build to the Google Play store" + lane :beta do + upload_to_play_store( + track: 'internal', + aab: 'app/build/outputs/bundle/release/app-release.aab', + release_status: 'draft' + ) + end + end + ``` + +- Sample `.gitlab-ci.yml`: + + ```yaml + beta: + image: fabernovel/android:api-33-v1.7.0 + stage: beta + script: + - fastlane beta + ``` + +### iOS distribution Apple Store integration and fastlane + +To create an iOS distribution with the Apple Store integration and fastlane, you must: + +1. Generate an API Key for App Store Connect API. In the Apple App Store Connect portal, + [generate a new private key for your project](https://developer.apple.com/documentation/appstoreconnectapi/creating_api_keys_for_app_store_connect_api). +1. [Enable the Apple App Store integration](#enable-apple-app-store-integration). +1. Add the release step to your pipeline and fastlane configuration. + + +For an overview, see [Apple App Store integration demo](https://youtu.be/CwzAWVgJeK8). + +#### Enable Apple App Store Integration + +Use the [Apple App Store integration](../user/project/integrations/apple_app_store.md) +to configure your CI/CD pipelines to connect to [App Store Connect](https://appstoreconnect.apple.com/) +to build and release apps for iOS, iPadOS, macOS, tvOS, and watchOS. To enable the integration: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Integrations**. +1. Select **Apple App Store**. +1. Turn on the **Active** toggle under **Enable Integration**. +1. Provide the Apple App Store Connect configuration information: + - **Issuer ID**: You can find the Apple App Store Connect Issuer ID in the **Keys** section under **Users and Access** in the Apple App Store Connect portal. + - **Key ID**: The key ID of the new private key that was just generated. + - **Private Key**: The private key that was just generated. You can only download this key one time. +1. Select **Save changes**. + +With the integration enabled, you can use fastlane to distribute a build to TestFlight +and the Apple App Store. + +For example: + +- Sample `fastlane/Fastfile`: + + ```ruby + default_platform(:ios) + + platform :ios do + desc "Build and sign the application for distribution, upload to TestFlight" + lane :beta do + setup_ci + + match(type: 'appstore', readonly: is_ci) + + app_store_connect_api_key + + increment_build_number( + build_number: latest_testflight_build_number(initial_build_number: 1) + 1, + xcodeproj: "ios demo.xcodeproj" + ) + + build_app( + project: "ios demo.xcodeproj", + scheme: "ios demo", + configuration: "Release", + export_method: "app-store" + ) + + upload_to_testflight + end + end + ``` + +- Sample `.gitlab-ci.yml`: + + ```yaml + beta_ios: + image: macos-12-xcode-14 + stage: beta + script: + - fastlane beta + ``` + +## Review apps for mobile + +You can use [review apps](review_apps/index.md) to preview changes directly from a merge request. +This feature is possible through an integration with [Appetize.io](https://appetize.io/). + + +For an overview, see [Review apps for mobile setup instructions](https://youtu.be/X15mI19TXa4). + +To get started, see the [setup instructions](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/readme/-/issues/15). ## Mobile SAST @@ -40,3 +426,15 @@ You can use [Static Application Security Testing (SAST)](../user/application_sec to run static analyzers on code to check for known security vulnerabilities. Mobile SAST expands this functionality for mobile teams with an [experimental SAST feature](../user/application_security/sast/index.md#experimental-features) based on [Mobile Security Framework (MobSF)](https://github.com/MobSF/Mobile-Security-Framework-MobSF). + +## Sample Reference Projects + +See the sample reference projects below for complete build, sign, and release pipeline examples for various platforms. A list of all available projects can be found in [the Mobile DevOps Demo Projects group](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/). + +- [Android Demo](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/android_demo) +- [iOS Demo](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/ios-demo) +- [Flutter Demo](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/demo-projects/flutter-demo) + +## Mobile DevOps Blog + +Additional reference material can be found in the [#mobile section](https://about.gitlab.com/blog/tags.html#mobile) of the GitLab blog. diff --git a/doc/ci/pipeline_editor/index.md b/doc/ci/pipeline_editor/index.md index 727977b3bc8..d920c34d90a 100644 --- a/doc/ci/pipeline_editor/index.md +++ b/doc/ci/pipeline_editor/index.md @@ -21,7 +21,7 @@ From the pipeline editor page you can: added with the [`include`](../yaml/index.md#include) keyword. - View a [list of the CI/CD configuration added with the `include` keyword](#view-included-cicd-configuration). - See a [visualization](#visualize-ci-configuration) of the current configuration. -- View an [expanded](#view-expanded-configuration) version of your configuration. +- View the [full configuration](#view-full-configuration), which displays the configuration with any configuration from `include` added. - [Commit](#commit-changes-to-ci-configuration) the changes to a specific branch. In GitLab 13.9 and earlier, you must already have [a `.gitlab-ci.yml` file](../quick_start/index.md#create-a-gitlab-ciyml-file) @@ -70,7 +70,7 @@ simulations in the [CI Lint tool](../lint.md#simulate-a-pipeline). > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/357219) in GitLab 15.1. You can review configuration added with the [`include`](../yaml/index.md#include) -keyword in the pipeline editor. In the upper right, select the file tree (**{file-tree}**) +keyword in the pipeline editor. In the upper-right corner, select the file tree (**{file-tree}**) to see a list of all included configuration files. Selected files open in a new tab for review. @@ -95,13 +95,14 @@ Hover over a job to highlight its `needs` relationships: If the configuration does not have any `needs` relationships, then no lines are drawn because each job depends only on the previous stage being completed successfully. -## View expanded configuration +## View full configuration > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/246801) in GitLab 13.9. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/301103) in GitLab 13.12. +> - **View merged YAML** tab [renamed to **Full configuration**](https://gitlab.com/gitlab-org/gitlab/-/issues/377404) in GitLab 16.0. To view the fully expanded CI/CD configuration as one combined file, go to the -pipeline editor's **View merged YAML** tab. This tab displays an expanded configuration +pipeline editor's **Full configuration** tab. This tab displays an expanded configuration where: - Configuration imported with [`include`](../yaml/index.md#include) is copied into the view. @@ -130,7 +131,7 @@ fully expanded version are both valid: - pyflakes python/ ``` -- Expanded configuration in **View merged YAML** tab: +- Expanded configuration in **Full configuration** tab: ```yaml ".python-req": @@ -169,7 +170,7 @@ It can happen when: - The syntax status on the **Edit** tab (valid or invalid). - The **Visualize** tab. - The **Lint** tab. - - The **View merged YAML** tab. + - The **Full configuration** tab. You can still work on your CI/CD configuration and commit the changes you made without any issues. As soon as the service becomes available again, the syntax validation diff --git a/doc/ci/pipelines/cicd_minutes.md b/doc/ci/pipelines/cicd_minutes.md index 4cbaa77b029..ee3f0d8c539 100644 --- a/doc/ci/pipelines/cicd_minutes.md +++ b/doc/ci/pipelines/cicd_minutes.md @@ -7,6 +7,9 @@ type: reference # CI/CD minutes quota **(PREMIUM)** +NOTE: +The term `CI/CD minutes` is being renamed to `units of compute`. During this transition, you might see references in the UI and documentation to `CI/CD minutes`, `CI minutes`, `pipeline minutes`, `CI pipeline minutes`, `pipeline minutes quota`, and `units of compute`. For more information, see [epic 2150](https://gitlab.com/groups/gitlab-com/-/epics/2150). + Administrators can limit the amount of time that projects can use to run jobs on [shared runners](../runners/runners_scope.md#shared-runners) each month. This limit is tracked with a quota of CI/CD minutes. @@ -19,7 +22,7 @@ end-to-end duration of a pipeline. On GitLab.com: -- CI/CD minutes quotas are enabled for both public and private projects, but public +- CI/CD minutes quotas are enabled for all projects, but certain projects [consume CI/CD minutes at a slower rate](#cost-factor). - The base monthly CI/CD minutes quota for a GitLab.com [namespace](../../user/namespace/index.md) is determined by its [license tier](https://about.gitlab.com/pricing/). @@ -83,11 +86,16 @@ NOTE: You can set a quota of CI/CD minutes for only top-level groups or user namespaces. If you set a quota for a subgroup, it is not used. -## View CI/CD minutes used by a group +## View CI/CD minutes -> Displaying shared runners duration per project [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355666) in GitLab 15.0. +Prerequisite: -You can view the number of CI/CD minutes being used by a group. +- You must have access to the build to view the total usage and quota summary for a namespace associated with a build. +- Access to **Usage Quotas** page is based on your role in the associated namespace or group. + +### View Usage Quota Reports for a group + +> Displaying shared runners duration per project [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355666) in GitLab 15.0. Prerequisite: @@ -105,10 +113,14 @@ The projects list shows projects with CI/CD minute usage or shared runners usage in the current month only. The list includes all projects in the namespace and its subgroups, sorted in descending order of CI/CD minute usage. -## View CI/CD minutes used by a personal namespace +### View Usage Quota reports for a personal namespace > Displaying shared runners duration [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345795) in GitLab 15.0. +Prerequisite: + +- The namespace must be your personal namespace. + You can view the number of CI/CD minutes being used by a personal namespace: 1. On the top bar, in the upper-right corner, select your avatar. @@ -144,6 +156,10 @@ You can find pricing for additional CI/CD minutes on the ### Purchase CI/CD minutes for a group **(FREE SAAS)** +Prerequisite: + +- You must have the Owner role for the group. + You can purchase additional CI/CD minutes for your group. You cannot transfer purchased CI/CD minutes from one group to another, so be sure to select the correct group. @@ -159,6 +175,10 @@ namespace. ### Purchase CI/CD minutes for a personal namespace **(FREE SAAS)** +Prerequisite: + +- The namespace must be your personal namespace. + To purchase additional minutes for your personal namespace: 1. On the top bar, in the upper-right corner, select your avatar. @@ -201,13 +221,12 @@ can be higher than the end-to-end duration of a pipeline. The cost factors for jobs running on shared runners on GitLab.com are: -- `1` for internal and private projects. -- `0.5` for public projects in the [GitLab for Open Source program](../../subscriptions/index.md#gitlab-for-open-source). -- `0.008` for public forks of public projects in the [GitLab for Open Source program](../../subscriptions/index.md#gitlab-for-open-source). For every 125 minutes of job execution time, +- `1` for internal, public, and private projects. +- Exceptions for public projects: + - `0.5` for projects in the [GitLab for Open Source program](../../subscriptions/community_programs.md#gitlab-for-open-source). + - `0.008` for forks of projects in the [GitLab for Open Source program](../../subscriptions/community_programs.md#gitlab-for-open-source). For every 125 minutes of job execution time, you use 1 CI/CD minute. -- `1` for other public projects, after October 1, 2022 (previously `0.04`). - For every 1 minute of job execution time, you use 1 CI/CD minute. -- Calculated differently for [community contributions to GitLab projects](#cost-factor-for-community-contributions-to-gitlab-projects). +- Discounted dynamically for [community contributions to GitLab projects](#cost-factor-for-community-contributions-to-gitlab-projects). The cost factors on self-managed instances are: @@ -244,9 +263,10 @@ GitLab SaaS runners have different cost factors, depending on the runner type (L | GitLab SaaS runner type | Machine Type | CI/CD minutes cost factor | | :--------- | :------------------- | :--------- | -| Linux OS + Docker executor| Small |1| -| Linux OS + Docker executor| Medium |2| -| Linux OS + Docker executor| Large |3| +| Linux OS | Small |1| +| Linux OS | Medium |2| +| Linux OS | Large |3| +| Linux OS + GPU-enabled | Medium, GPU Standard |7| ### Monthly reset of CI/CD minutes @@ -298,6 +318,13 @@ On GitLab SaaS an email notification is sent to the namespace owners when: - The available CI/CD minutes are below 5% of the quota. - All CI/CD minutes have been used. +### Special quota limits + +In some cases, the quota limit is replaced by one of the following labels: + +- **Unlimited minutes**: For namespaces with unlimited CI/CD minutes +- **Not supported minutes**: For namespaces where active shared runners are not enabled + ## Reduce consumption of CI/CD minutes If your project consumes too many CI/CD minutes, there are some strategies you can @@ -319,3 +346,19 @@ If you manage an open source project, these improvements can also reduce CI/CD m consumption for contributor fork projects, enabling more contributions. See our [pipeline efficiency guide](pipeline_efficiency.md) for more details. + +## Reset CI/CD minutes used **(PREMIUM SELF)** + +An administrator can reset the number of minutes used by a namespace for the current month. + +### Reset minutes for a personal namespace + +1. Find the [user in the admin area](../../user/admin_area/index.md#administering-users). +1. Select **Edit**. +1. In **Limits**, select **Reset pipeline minutes**. + +### Reset minutes for a group namespace + +1. Find the [group in the admin area](../../user/admin_area/index.md#administering-groups). +1. Select **Edit**. +1. In **Permissions and group features**, select **Reset pipeline minutes**. diff --git a/doc/ci/pipelines/downstream_pipelines.md b/doc/ci/pipelines/downstream_pipelines.md index e4560cd882d..ffcfee98f05 100644 --- a/doc/ci/pipelines/downstream_pipelines.md +++ b/doc/ci/pipelines/downstream_pipelines.md @@ -231,37 +231,43 @@ configuration for jobs that use the Windows runner, like scripts, use \ ### Run child pipelines with merge request pipelines -To trigger a child pipeline as a [merge request pipeline](merge_request_pipelines.md): +Pipelines, including child pipelines, run as branch pipelines by default when not using +[`rules`](../yaml/index.md#rules) or [`workflow:rules`](../yaml/index.md#workflowrules). +To configure child pipelines to run when triggered from a [merge request (parent) pipeline](merge_request_pipelines.md), use `rules` or `workflow:rules`. +For example, using `rules`: -1. Set the trigger job to run on merge requests in the parent pipeline's configuration file: +1. Set the parent pipeline's trigger job to run on merge requests: ```yaml - microservice_a: + trigger-child-pipeline-job: trigger: - include: path/to/microservice_a.yml + include: path/to/child-pipeline-configuration.yml rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" ``` -1. Configure the child pipeline jobs to run in merge request pipelines with [`rules`](../yaml/index.md#rules) - or [`workflow:rules`](../yaml/index.md#workflowrules). - For example, with `rules` in a child pipeline's configuration file: +1. Use `rules` to configure the child pipeline jobs to run when triggered by the parent pipeline: ```yaml job1: - script: echo "Child pipeline job 1" + script: echo "This child pipeline job runs any time the parent pipeline triggers it." rules: - - if: $CI_MERGE_REQUEST_ID + - if: $CI_PIPELINE_SOURCE == "parent_pipeline" job2: - script: echo "Child pipeline job 2" + script: echo "This child pipeline job runs only when the parent pipeline is a merge request pipeline" rules: - if: $CI_MERGE_REQUEST_ID ``` - In child pipelines, `$CI_PIPELINE_SOURCE` always has a value of `parent_pipeline` - and cannot be used to identify merge request pipelines. Use `$CI_MERGE_REQUEST_ID` - instead, which is always present in merge request pipelines. +In child pipelines, `$CI_PIPELINE_SOURCE` always has a value of `parent_pipeline`, so: + +- You can use `if: $CI_PIPELINE_SOURCE == "parent_pipeline"` to ensure child pipeline jobs always run. +- You _can't_ use `if: $CI_PIPELINE_SOURCE == "merge_request_event"` to configure child pipeline + jobs to run for merge request pipelines. Instead, use `if: $CI_MERGE_REQUEST_ID` + to set child pipeline jobs to run only when the parent pipeline is a merge request pipeline. The parent pipeline's + [`CI_MERGE_REQUEST_*` predefined variables](../variables/predefined_variables.md#predefined-variables-for-merge-request-pipelines) + are passed to the child pipeline jobs. ### Specify a branch for multi-project pipelines @@ -281,7 +287,7 @@ Use: - The `project` keyword to specify the full path to the downstream project. In [GitLab 15.3 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/367660), you can use [variable expansion](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). -- The `branch` keyword to specify the name of a branch or [tag](../../topics/git/tags.md) +- The `branch` keyword to specify the name of a branch or [tag](../../user/project/repository/tags/index.md) in the project specified by `project`. You can use variable expansion. ## Trigger a multi-project pipeline by using the API @@ -309,18 +315,33 @@ trigger_pipeline: > Hover behavior for pipeline cards [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/197140/) in GitLab 13.2. In the [pipeline graph view](index.md#view-full-pipeline-graph), downstream pipelines display -as a list of cards on the right of the graph. Hover over the pipeline's card to view -which job triggered the downstream pipeline. +as a list of cards on the right of the graph. From this view, you can: -### Retry a downstream pipeline +- Select a trigger job to see the triggered downstream pipeline's jobs. +- Select **Expand jobs** **{chevron-lg-right}** on a pipeline card to expand the view + with the downstream pipeline's jobs. You can view one downstream pipeline at a time. +- Hover over a pipeline card to have the job that triggered the downstream pipeline highlighted. + +### Retry failed and canceled jobs in a downstream pipeline > - Retry from graph view [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/354974) in GitLab 15.0 [with a flag](../../administration/feature_flags.md) named `downstream_retry_action`. Disabled by default. > - Retry from graph view [generally available and feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/357406) in GitLab 15.1. -To retry a completed downstream pipeline, select **Retry** (**{retry}**): +To retry failed and canceled jobs, select **Retry** (**{retry}**): - From the downstream pipeline's details page. -- On the pipeline's card in the [pipeline graph view](index.md#view-full-pipeline-graph). +- On the pipeline's card in the pipeline graph view. + +### Recreate a downstream pipeline + +> - Retry trigger job from graph view [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/367547) in GitLab 15.10 [with a flag](../../administration/feature_flags.md) named `ci_recreate_downstream_pipeline`. Disabled by default. +> - [Generally available](https://gitlab.com/groups/gitlab-org/-/epics/6947) in GitLab 15.11. Feature flag `ci_recreate_downstream_pipeline` removed. + +You can recreate a downstream pipeline by retrying its corresponding trigger job. The newly created downstream pipeline replaces the current downstream pipeline in the pipeline graph. + +To recreate a downstream pipeline: + +- Select **Run again** (**{retry}**) on the trigger job's card in the pipeline graph view. ### Cancel a downstream pipeline @@ -330,7 +351,7 @@ To retry a completed downstream pipeline, select **Retry** (**{retry}**): To cancel a downstream pipeline that is still running, select **Cancel** (**{cancel}**): - From the downstream pipeline's details page. -- On the pipeline's card in the [pipeline graph view](index.md#view-full-pipeline-graph). +- On the pipeline's card in the pipeline graph view. ### Mirror the status of a downstream pipeline in the trigger job @@ -365,13 +386,9 @@ trigger_job: After you trigger a multi-project pipeline, the downstream pipeline displays to the right of the [pipeline graph](index.md#visualize-pipelines). -![Multi-project pipeline graph](img/multi_project_pipeline_graph_v14_3.png) - In [pipeline mini graphs](index.md#pipeline-mini-graphs), the downstream pipeline displays to the right of the mini graph. -![Multi-project pipeline mini graph](img/pipeline_mini_graph_v15_0.png) - ## Fetch artifacts from an upstream pipeline Use [`needs:project`](../yaml/index.md#needsproject) to fetch artifacts from an @@ -604,8 +621,9 @@ the ones defined in the upstream project take precedence. ### Pass dotenv variables created in a job **(PREMIUM)** -You can pass variables to a downstream pipeline with [`dotenv` variable inheritance](../variables/index.md#pass-an-environment-variable-to-another-job) -and [`needs:project`](../yaml/index.md#needsproject). +You can pass variables to a downstream job with [`dotenv` variable inheritance](../variables/index.md#pass-an-environment-variable-to-another-job) +and [`needs:project`](../yaml/index.md#needsproject). These variables are only available in +the script of the job and can't be used to configure it, for example with `rules` or `artifact:paths`. For example, in a [multi-project pipeline](#multi-project-pipelines): @@ -656,6 +674,16 @@ With multi-project pipelines, the trigger job fails and does not create the down to run pipelines against the protected branch. See [pipeline security for protected branches](index.md#pipeline-security-on-protected-branches) for more information. +### Job in child pipeline is not created when the pipeline runs + +If the parent pipeline is a [merge request pipeline](merge_request_pipelines.md), +the child pipeline must [use `workflow:rules` or `rules` to ensure the jobs run](#run-child-pipelines-with-merge-request-pipelines). + +If no jobs in the child pipeline can run due to missing or incorrect `rules` configuration: + +- The child pipeline fails to start. +- The parent pipeline's trigger job fails with: `downstream pipeline can not be created, Pipeline will not run for the selected trigger. The rules configuration prevented any jobs from being added to the pipeline.` + ### `Ref is ambiguous` You cannot trigger a multi-project pipeline with a tag when a branch exists with the same diff --git a/doc/ci/pipelines/img/code_coverage_graph_v13_1.png b/doc/ci/pipelines/img/code_coverage_graph_v13_1.png deleted file mode 100644 index da92a1b4a86..00000000000 Binary files a/doc/ci/pipelines/img/code_coverage_graph_v13_1.png and /dev/null differ diff --git a/doc/ci/pipelines/img/coverage_check_approval_rule_14_1.png b/doc/ci/pipelines/img/coverage_check_approval_rule_14_1.png deleted file mode 100644 index 8c1d005e074..00000000000 Binary files a/doc/ci/pipelines/img/coverage_check_approval_rule_14_1.png and /dev/null differ diff --git a/doc/ci/pipelines/img/job_artifacts_browser_button_v13_11.png b/doc/ci/pipelines/img/job_artifacts_browser_button_v13_11.png deleted file mode 100644 index 710a82075ce..00000000000 Binary files a/doc/ci/pipelines/img/job_artifacts_browser_button_v13_11.png and /dev/null differ diff --git a/doc/ci/pipelines/img/job_artifacts_browser_v13_11.png b/doc/ci/pipelines/img/job_artifacts_browser_v13_11.png deleted file mode 100644 index 6a8ea49b833..00000000000 Binary files a/doc/ci/pipelines/img/job_artifacts_browser_v13_11.png and /dev/null differ diff --git a/doc/ci/pipelines/img/job_artifacts_jobs_page_v13_11.png b/doc/ci/pipelines/img/job_artifacts_jobs_page_v13_11.png deleted file mode 100644 index 9f09e36b927..00000000000 Binary files a/doc/ci/pipelines/img/job_artifacts_jobs_page_v13_11.png and /dev/null differ diff --git a/doc/ci/pipelines/img/job_artifacts_merge_request_v13_11.png b/doc/ci/pipelines/img/job_artifacts_merge_request_v13_11.png deleted file mode 100644 index f4b875de471..00000000000 Binary files a/doc/ci/pipelines/img/job_artifacts_merge_request_v13_11.png and /dev/null differ diff --git a/doc/ci/pipelines/img/job_artifacts_pipelines_page_v13_11.png b/doc/ci/pipelines/img/job_artifacts_pipelines_page_v13_11.png deleted file mode 100644 index c7b0b91d488..00000000000 Binary files a/doc/ci/pipelines/img/job_artifacts_pipelines_page_v13_11.png and /dev/null differ diff --git a/doc/ci/pipelines/img/job_latest_artifacts_browser.png b/doc/ci/pipelines/img/job_latest_artifacts_browser.png deleted file mode 100644 index c6d8856078b..00000000000 Binary files a/doc/ci/pipelines/img/job_latest_artifacts_browser.png and /dev/null differ diff --git a/doc/ci/pipelines/img/multi_project_pipeline_graph_v14_3.png b/doc/ci/pipelines/img/multi_project_pipeline_graph_v14_3.png deleted file mode 100644 index aadf8bb0979..00000000000 Binary files a/doc/ci/pipelines/img/multi_project_pipeline_graph_v14_3.png and /dev/null differ diff --git a/doc/ci/pipelines/img/pipeline_mini_graph_v15_0.png b/doc/ci/pipelines/img/pipeline_mini_graph_v15_0.png deleted file mode 100644 index 48a0ca9d84f..00000000000 Binary files a/doc/ci/pipelines/img/pipeline_mini_graph_v15_0.png and /dev/null differ diff --git a/doc/ci/pipelines/img/pipelines_test_coverage_build.png b/doc/ci/pipelines/img/pipelines_test_coverage_build.png deleted file mode 100644 index 7eaba1a256f..00000000000 Binary files a/doc/ci/pipelines/img/pipelines_test_coverage_build.png and /dev/null differ diff --git a/doc/ci/pipelines/img/pipelines_test_coverage_mr_widget.png b/doc/ci/pipelines/img/pipelines_test_coverage_mr_widget.png deleted file mode 100644 index fbcd612f3f2..00000000000 Binary files a/doc/ci/pipelines/img/pipelines_test_coverage_mr_widget.png and /dev/null differ diff --git a/doc/ci/pipelines/index.md b/doc/ci/pipelines/index.md index fa04cb6cb92..b0c5f3a6a69 100644 --- a/doc/ci/pipelines/index.md +++ b/doc/ci/pipelines/index.md @@ -2,7 +2,6 @@ stage: Verify group: Pipeline Authoring info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -disqus_identifier: 'https://docs.gitlab.com/ee/ci/pipelines.html' type: reference --- @@ -94,7 +93,7 @@ This table lists the refspecs injected for each pipeline type: The refs `refs/heads/` and `refs/tags/` exist in your project repository. GitLab generates the special ref `refs/pipelines/` during a running pipeline job. This ref can be created even after the associated branch or tag has been -deleted. It's therefore useful in some features such as [automatically stopping an environment](../environments/index.md#stop-an-environment), +deleted. It's therefore useful in some features such as [automatically stopping an environment](../environments/index.md#stopping-an-environment), and [merge trains](../pipelines/merge_trains.md) that might run pipelines after branch deletion. @@ -195,6 +194,7 @@ In this example: > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/363660) in GitLab 15.5 [with a flag](../../administration/feature_flags.md) named `run_pipeline_graphql`. Disabled by default. > - The `options` keyword was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/105502) in GitLab 15.7. > - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106038) in GitLab 15.7. Feature flag `run_pipeline_graphql` removed. +> - The variables list sometimes did not populate correctly due to [a bug](https://gitlab.com/gitlab-org/gitlab/-/issues/386245), which was resolved in GitLab 15.9. You can define an array of CI/CD variable values the user can select from when running a pipeline manually. These values are in a dropdown list in the **Run pipeline** page. Add the list of @@ -423,8 +423,7 @@ You can group the jobs by: - [Job dependencies](#view-job-dependencies-in-the-pipeline-graph), which arranges jobs based on their [`needs`](../yaml/index.md#needs) dependencies. -[Multi-project pipeline graphs](downstream_pipelines.md#view-multi-project-pipelines-in-pipeline-graphs) help -you visualize the entire pipeline, including all cross-project inter-dependencies. +Multi-project pipeline graphs help you visualize the entire pipeline, including all cross-project inter-dependencies. If a stage contains more than 100 jobs, only the first 100 jobs are listed in the pipeline graph. The remaining jobs still run as usual. To see the jobs: diff --git a/doc/ci/pipelines/job_artifacts.md b/doc/ci/pipelines/job_artifacts.md index 75c9852d3d0..c2630d6810d 100644 --- a/doc/ci/pipelines/job_artifacts.md +++ b/doc/ci/pipelines/job_artifacts.md @@ -1,523 +1,11 @@ --- -stage: Verify -group: Pipeline Insights -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -disqus_identifier: 'https://docs.gitlab.com/ee/user/project/pipelines/job_artifacts.html' +redirect_to: '../jobs/job_artifacts.md' +remove_date: '2023-07-04' --- -# Job artifacts **(FREE)** +This document was moved to [another location](../jobs/job_artifacts.md). -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16675) in GitLab 12.4, artifacts in internal and private projects can be previewed when [GitLab Pages access control](../../administration/pages/index.md#access-control) is enabled. - -Jobs can output an archive of files and directories. This output is known as a job artifact. - -You can download job artifacts by using the GitLab UI or the [API](../../api/job_artifacts.md#get-job-artifacts). - - -For an overview of job artifacts, watch the video [GitLab CI pipelines, artifacts, and environments](https://www.youtube.com/watch?v=PCKDICEe10s). -Or, for an introduction, watch [GitLab CI pipeline tutorial for beginners](https://www.youtube.com/watch?v=Jav4vbUrqII). - -For administrator information about job artifact storage, see [administering job artifacts](../../administration/job_artifacts.md). - -## Create job artifacts - -To create job artifacts, use the `artifacts` keyword in your `.gitlab-ci.yml` file: - -```yaml -pdf: - script: xelatex mycv.tex - artifacts: - paths: - - mycv.pdf - expire_in: 1 week -``` - -In this example, a job named `pdf` calls the `xelatex` command to build a PDF file from the -LaTeX source file, `mycv.tex`. - -The `paths` keyword determines which files to add to the job artifacts. -All paths to files and directories are relative to the repository where the job was created. - -The `expire_in` keyword determines how long GitLab keeps the job artifacts. -You can also [use the UI to keep job artifacts from expiring](#download-job-artifacts). -If `expire_in` is not defined, the -[instance-wide setting](../../user/admin_area/settings/continuous_integration.md#default-artifacts-expiration) -is used. - -If you run two types of pipelines (like branch and scheduled) for the same ref, -the pipeline that finishes later creates the job artifact. - -To disable artifact passing, define the job with empty [dependencies](../yaml/index.md#dependencies): - -```yaml -job: - stage: build - script: make build - dependencies: [] -``` - -You may want to create artifacts only for tagged releases to avoid filling the -build server storage with temporary build artifacts. For example, use [`rules`](../yaml/index.md#rules) -to create artifacts only for tags: - -```yaml -default-job: - script: - - mvn test -U - rules: - - if: $CI_COMMIT_BRANCH - -release-job: - script: - - mvn package -U - artifacts: - paths: - - target/*.war - rules: - - if: $CI_COMMIT_TAG -``` - -You can use wildcards for directories too. For example, if you want to get all the -files inside the directories that end with `xyz`: - -```yaml -job: - artifacts: - paths: - - path/*xyz/* -``` - -### Use CI/CD variables to define the artifacts name - -You can use [CI/CD variables](../variables/index.md) to dynamically define the -artifacts file's name. - -For example, to create an archive with a name of the current job: - -```yaml -job: - artifacts: - name: "$CI_JOB_NAME" - paths: - - binaries/ -``` - -To create an archive with a name of the current branch or tag including only -the binaries directory: - -```yaml -job: - artifacts: - name: "$CI_COMMIT_REF_NAME" - paths: - - binaries/ -``` - -If your branch-name contains forward slashes -(for example `feature/my-feature`) it's advised to use `$CI_COMMIT_REF_SLUG` -instead of `$CI_COMMIT_REF_NAME` for proper naming of the artifact. - -To create an archive with a name of the current job and the current branch or -tag including only the binaries directory: - -```yaml -job: - artifacts: - name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME" - paths: - - binaries/ -``` - -To create an archive with a name of the current [stage](../yaml/index.md#stages) and branch name: - -```yaml -job: - artifacts: - name: "$CI_JOB_STAGE-$CI_COMMIT_REF_NAME" - paths: - - binaries/ -``` - -If you use **Windows Batch** to run your shell scripts you must replace -`$` with `%`: - -```yaml -job: - artifacts: - name: "%CI_JOB_STAGE%-%CI_COMMIT_REF_NAME%" - paths: - - binaries/ -``` - -If you use **Windows PowerShell** to run your shell scripts you must replace -`$` with `$env:`: - -```yaml -job: - artifacts: - name: "$env:CI_JOB_STAGE-$env:CI_COMMIT_REF_NAME" - paths: - - binaries/ -``` - -### Exclude files from job artifacts - -Use [`artifacts:exclude`](../yaml/index.md#artifactsexclude) to prevent files from -being added to an artifacts archive. - -For example, to store all files in `binaries/`, but not `*.o` files located in -subdirectories of `binaries/`. - -```yaml -artifacts: - paths: - - binaries/ - exclude: - - binaries/**/*.o -``` - -Unlike [`artifacts:paths`](../yaml/index.md#artifactspaths), `exclude` paths are not recursive. -To exclude all of the contents of a directory, match them explicitly rather -than matching the directory itself. - -For example, to store all files in `binaries/` but nothing located in the `temp/` subdirectory: - -```yaml -artifacts: - paths: - - binaries/ - exclude: - - binaries/temp/**/* -``` - -### Add untracked files to artifacts - -Use [`artifacts:untracked`](../yaml/index.md#artifactsuntracked) to add all Git untracked -files as artifacts (along with the paths defined in [`artifacts:paths`](../yaml/index.md#artifactspaths)). Untracked -files are those that haven't been added to the repository but exist in the repository checkout. - -Save all Git untracked files and files in `binaries`: - -```yaml -artifacts: - untracked: true - paths: - - binaries/ -``` - -Save all untracked files but [exclude](../yaml/index.md#artifactsexclude) `*.txt`: - -```yaml -artifacts: - untracked: true - exclude: - - "*.txt" -``` - -## Download job artifacts - -You can download job artifacts or view the job archive: - -- On the **Pipelines** page, to the right of the pipeline: - - ![Job artifacts in Pipelines page](img/job_artifacts_pipelines_page_v13_11.png) - -- On the **Jobs** page, to the right of the job: - - ![Job artifacts in Jobs page](img/job_artifacts_jobs_page_v13_11.png) - -- On a job's detail page. The **Keep** button indicates an `expire_in` value was set: - - ![Job artifacts browser button](img/job_artifacts_browser_button_v13_11.png) - -- On a merge request, by the pipeline details: - - ![Job artifacts in merge request](img/job_artifacts_merge_request_v13_11.png) - -- When browsing an archive: - - ![Job artifacts browser](img/job_artifacts_browser_v13_11.png) - - If [GitLab Pages](../../administration/pages/index.md) is enabled in the project, you can preview - HTML files in the artifacts directly in your browser. If the project is internal or private, you must - enable [GitLab Pages access control](../../administration/pages/index.md#access-control) to preview - HTML files. - -## View failed job artifacts - -If the latest job has failed to upload the artifacts, you can see that -information in the UI. - -![Latest artifacts button](img/job_latest_artifacts_browser.png) - -## Delete job artifacts - -WARNING: -This is a destructive action that leads to data loss. Use with caution. - -You can delete a single job, which also removes the job's -artifacts and log. You must be: - -- The owner of the job. -- A user with at least the Maintainer role for the project. - -To delete a job: - -1. Go to a job's detail page. -1. In the upper right of the job's log, select **Erase job log** (**{remove}**). -1. On the confirmation dialog, select **OK**. - -## Expose job artifacts in the merge request UI - -Use the [`artifacts:expose_as`](../yaml/index.md#artifactsexpose_as) keyword to expose -[job artifacts](../pipelines/job_artifacts.md) in the [merge request](../../user/project/merge_requests/index.md) UI. - -For example, to match a single file: - -```yaml -test: - script: ["echo 'test' > file.txt"] - artifacts: - expose_as: 'artifact 1' - paths: ['file.txt'] -``` - -With this configuration, GitLab adds a link **artifact 1** to the relevant merge request -that points to `file.txt`. To access the link, select **View exposed artifact** -below the pipeline graph in the merge request overview. - -An example that matches an entire directory: - -```yaml -test: - script: ["mkdir test && echo 'test' > test/file.txt"] - artifacts: - expose_as: 'artifact 1' - paths: ['test/'] -``` - -## Retrieve job artifacts for other projects - -To retrieve a job artifact from a different project, you might need to use a -private token to [authenticate and download](../../api/job_artifacts.md#get-job-artifacts) -the artifact. - -## How searching for job artifacts works - -In [GitLab 13.5 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/201784), artifacts -for [parent and child pipelines](downstream_pipelines.md#parent-child-pipelines) are searched in hierarchical -order from parent to child. For example, if both parent and child pipelines have a -job with the same name, the job artifact from the parent pipeline is returned. - -## Access the latest job artifacts - -You can download job artifacts from the latest successful pipeline by using [the job artifacts API](../../api/job_artifacts.md). -You cannot download [artifact reports](../yaml/artifacts_reports.md) with the job artifacts API, -unless the report is added as a regular artifact with `artifacts:paths`. - -### Download the whole artifacts archive for a specific job - -You can download the artifacts archive for a specific job with [the job artifacts API](../../api/job_artifacts.md#download-the-artifacts-archive). - -For example, to download the latest artifacts of a job named `build` in the `main` branch of a project on GitLab.com: - -```plaintext -https://gitlab.com/api/v4/projects//jobs/artifacts/main/download?job=build -``` - -Replace `` with a valid project ID, found at the top of the project details page. - -### Download a single file from the artifacts - -You can download a specific file from the artifacts archive for a specific job with [the job artifacts API](../../api/job_artifacts.md#download-a-single-artifact-file-by-job-id). - -For example, to download the file `review/index.html` from the latest job named `build` in the `main` branch of the `gitlab` project in the `gitlab-org` namespace: - -```plaintext -https://gitlab.com/api/v4/projects/27456355/jobs/artifacts/main/raw/review/index.html?job=build -``` - -### Browse job artifacts - -To browse the job artifacts of the latest successful pipeline for a specific job you can use the following URL: - -```plaintext -https://example.com///-/jobs/artifacts//browse?job= -``` - -For example, to browse the latest artifacts of a job named `build` in the `main` branch of a project on GitLab.com: - -```plaintext -https://gitlab.com//-/jobs/artifacts/main/browse?job=build -``` - -Replace `` with a valid project path, you can find it in the URL for your project. - -## When job artifacts are deleted - -See the [`expire_in`](../yaml/index.md#artifactsexpire_in) documentation for information on when -job artifacts are deleted. - -### Keep artifacts from most recent successful jobs - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/16267) in GitLab 13.0. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/229936) in GitLab 13.4. -> - [Made optional with a CI/CD setting](https://gitlab.com/gitlab-org/gitlab/-/issues/241026) in GitLab 13.8. - -By default artifacts are always kept for successful pipelines for the most recent commit on -each ref. This means that the latest artifacts do not immediately expire according -to the `expire_in` specification. - -If a pipeline for a new commit on the same ref completes successfully, the previous pipeline's -artifacts are deleted according to the `expire_in` configuration. The artifacts -of the new pipeline are kept automatically. If multiple pipelines run for the most -recent commit on the ref, all artifacts are kept. - -Keeping the latest artifacts can use a large amount of storage space in projects -with a lot of jobs or large artifacts. If the latest artifacts are not needed in -a project, you can disable this behavior to save space: - -1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Settings > CI/CD**. -1. Expand **Artifacts**. -1. Clear the **Keep artifacts from most recent successful jobs** checkbox. - -You can disable this behavior for all projects on a self-managed instance in the -[instance's CI/CD settings](../../user/admin_area/settings/continuous_integration.md#keep-the-latest-artifacts-for-all-jobs-in-the-latest-successful-pipelines). - -When **Keep artifacts from most recent successful jobs** is enabled, artifacts are always kept for [blocked](../jobs/job_control.md#types-of-manual-jobs) -pipelines. These artifacts expire only after the blocking job is triggered and the pipeline completes. -For more information, see [issue 387087](https://gitlab.com/gitlab-org/gitlab/-/issues/387087). - -## Troubleshooting - -### Job does not retrieve certain artifacts - -By default, jobs fetch all artifacts from previous stages, but jobs using `dependencies` -or `needs` do not fetch artifacts from all jobs by default. - -If you use these keywords, artifacts are fetched from only a subset of jobs. Review -the keyword reference for information on how to fetch artifacts with these keywords: - -- [`dependencies`](../yaml/index.md#dependencies) -- [`needs`](../yaml/index.md#needs) -- [`needs:artifacts`](../yaml/index.md#needsartifacts) - -### Job artifacts using too much disk space - -There are a number of potential causes for this. -[Read more in the job artifacts administration documentation](../../administration/job_artifacts.md#job-artifacts-using-too-much-disk-space). - -### Error message `No files to upload` - -This message is often preceded by other errors or warnings that specify the filename and why it wasn't -generated. Check the job log for these messages. - -If you find no helpful messages, retry the failed job after activating -[CI/CD debug logging](../variables/index.md#enable-debug-logging). -This logging should provide information to help you investigate further. - -### Error message `Missing /usr/bin/gitlab-runner-helper. Uploading artifacts is disabled.` - -There is a [known issue](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3068) where setting a CI/CD variable named `DEBUG` can cause artifact uploads to fail. - -To work around this, either use a different variable name or set it inline with `script`: - -```yaml -# This job might fail due to issue gitlab-org/gitlab-runner#3068 -failing_test_job: - variables: - DEBUG: true - script: bin/mycommand - artifacts: - paths: - - bin/results - -# This job does not define a CI/CD variable named `DEBUG` and is not affected by the issue -successful_test_job: - script: DEBUG=true bin/mycommand - artifacts: - paths: - - bin/results -``` - -### Error message `FATAL: invalid argument` when uploading a dotenv artifact on a windows runner - -The PowerShell `echo` command writes files with UCS-2 LE BOM (Byte Order Mark) encoding, -but only UTF-8 is supported. If you try create the dotenv artifact with `echo`, it causes a -`FATAL: invalid argument` error. - -Use PowerShell `Add-Content` instead, which uses UTF-8: - -```yaml -test-job: - stage: test - tags: - - windows - script: - - echo "test job" - - Add-Content -Path build.env -Value "MY_ENV_VAR=true" - artifacts: - reports: - dotenv: build.env -``` - -### Job artifacts are not expired - -If some job artifacts are not expiring as expected, check if the -[**Keep artifacts from most recent successful jobs**](#keep-artifacts-from-most-recent-successful-jobs) -setting is enabled. - -When this setting is enabled, job artifacts from the latest successful pipeline -of each ref do not expire and are not deleted. - -### Error message `This job could not start because it could not retrieve the needed artifacts.` - -A job configured with [`needs:artifacts`](../yaml/index.md#needsartifacts) keyword -fails to start and returns this error message if: - -- The job's dependencies cannot be found. -- The job cannot access the relevant resources due to insufficient permissions. - -The troubleshooting steps to follow are determined by the syntax used in the job configuration. - -#### Job configured with `needs:project` - -The `could not retrieve the needed artifacts.` error can happen for a job using -[`needs:project`](../yaml/index.md#needsproject), with a configuration similar to: - -```yaml -rspec: - needs: - - project: org/another-project - job: dependency-job - ref: master - artifacts: true -``` - -To troubleshoot this job, verify that: - -- Project `org/another-project` is in a group with a Premium subscription plan. -- The user running the job has permissions to access resources in `org/another-project`. -- The `project`, `job`, and `ref` combination exists and results in the desired dependency. -- Any variables in use evaluate to the correct values. - -#### Job configured with `needs:pipeline:job` - -The `could not retrieve the needed artifacts.` error can happen for a job using -[`needs:pipeline:job`](../yaml/index.md#needspipelinejob), with a configuration similar to: - -```yaml -rspec: - needs: - - pipeline: $UPSTREAM_PIPELINE_ID - job: dependency-job - artifacts: true -``` - -To troubleshoot this job, verify that: - -- The `$UPSTREAM_PIPELINE_ID` CI/CD variable is available in the current pipeline's - parent-child pipeline hierarchy. -- The `pipeline` and `job` combination exists and resolves to an existing pipeline. -- `dependency-job` has run and finished successfully. + + + + diff --git a/doc/ci/pipelines/merge_request_pipelines.md b/doc/ci/pipelines/merge_request_pipelines.md index 7e18c11f234..d42f35627a2 100644 --- a/doc/ci/pipelines/merge_request_pipelines.md +++ b/doc/ci/pipelines/merge_request_pipelines.md @@ -140,14 +140,7 @@ the parent project. Additionally, if you do not trust the fork project's runner, running the pipeline in the parent project uses the parent project's trusted runners. WARNING: -Fork merge requests can contain malicious code that tries to steal secrets in the -parent project when the pipeline runs, even before merge. As a reviewer, carefully -check the changes in the merge request before triggering the pipeline. If you trigger -the pipeline by selecting **Run pipeline** or applying a suggestion, GitLab shows -a warning that you must accept before the pipeline runs. If you trigger the pipeline -by using any other method, including the API, [`/rebase` quick action](../../user/project/quick_actions.md#issues-merge-requests-and-epics), -or [**Rebase** option](../../user/project/merge_requests/methods/index.md#rebasing-in-semi-linear-merge-methods), -**no warning displays**. +Fork merge requests can contain malicious code that tries to steal secrets in the parent project when the pipeline runs, even before merge. As a reviewer, carefully check the changes in the merge request before triggering the pipeline. Unless you trigger the pipeline through the API or the [`/rebase` quick action](../../user/project/quick_actions.md#issues-merge-requests-and-epics), GitLab shows a warning that you must accept before the pipeline runs. Otherwise, **no warning displays**. Prerequisites: @@ -209,16 +202,20 @@ It's possible to have both branch pipelines and merge request pipelines in the **Pipelines** tab of a single merge request. This might be [by configuration](../yaml/workflow.md#switch-between-branch-pipelines-and-merge-request-pipelines), or [by accident](#two-pipelines-when-pushing-to-a-branch). -If both types of pipelines are in one merge request, the merge request's pipeline -is not considered successful if: - -- The branch pipeline succeeds. -- The merge request pipeline fails. - When using the [merge when pipeline succeeds](../../user/project/merge_requests/merge_when_pipeline_succeeds.md) feature and both pipelines types are present, the merge request pipelines are checked, not the branch pipelines. +Therefore, the MR pipeline result is marked as unsuccessful if the +**merge request pipeline** fails, independently of the **branch pipeline** result. + +However: + +- These conditions are not enforced. +- A race condition determines which pipeline's result is used to either block or pass merge requests. + +This bug is tracked on [issue 384927](https://gitlab.com/gitlab-org/gitlab/-/issues/384927). + ### `An error occurred while trying to run a new pipeline for this merge request.` This error can happen when you select **Run pipeline** in a merge request, but the diff --git a/doc/ci/pipelines/merge_trains.md b/doc/ci/pipelines/merge_trains.md index 370d81dacc4..af29c8105ee 100644 --- a/doc/ci/pipelines/merge_trains.md +++ b/doc/ci/pipelines/merge_trains.md @@ -6,6 +6,10 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Merge trains **(PREMIUM)** +FLAG: +In GitLab 15.11 and later, the **Start merge train** button is **Set to auto-merge** and the **Add to merge train** button is **Merge**. On self-managed GitLab, by default these changes are not available. To make them available, +ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `auto_merge_labels_mr_widget`. On GitLab.com, this feature is not available. + Use merge trains to queue merge requests and verify their changes work together before they are merged to the target branch. diff --git a/doc/ci/pipelines/merged_results_pipelines.md b/doc/ci/pipelines/merged_results_pipelines.md index 097f6bad87c..316d7819f55 100644 --- a/doc/ci/pipelines/merged_results_pipelines.md +++ b/doc/ci/pipelines/merged_results_pipelines.md @@ -74,3 +74,10 @@ is not found in the merge ref. This behavior was improved in GitLab 12.4 by introducing [persistent pipeline refs](../troubleshooting.md#fatal-reference-is-not-a-tree-error). Upgrade to GitLab 12.4 or later to resolve the problem. + +### Successful merged results pipeline overrides a failed branch pipeline + +A failed branch pipeline is sometimes ignored when the +[**Pipelines must succeed** setting](../../user/project/merge_requests/merge_when_pipeline_succeeds.md#require-a-successful-pipeline-for-merge) +is activated. +[Issue 385841](https://gitlab.com/gitlab-org/gitlab/-/issues/385841) is open to track this. diff --git a/doc/ci/pipelines/pipeline_architectures.md b/doc/ci/pipelines/pipeline_architectures.md index e4a3416f60b..dadf631e2bb 100644 --- a/doc/ci/pipelines/pipeline_architectures.md +++ b/doc/ci/pipelines/pipeline_architectures.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Execution +group: Pipeline Authoring info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: reference --- diff --git a/doc/ci/pipelines/pipeline_artifacts.md b/doc/ci/pipelines/pipeline_artifacts.md index aacaa110041..d8db79a54dc 100644 --- a/doc/ci/pipelines/pipeline_artifacts.md +++ b/doc/ci/pipelines/pipeline_artifacts.md @@ -1,13 +1,13 @@ --- stage: Verify -group: Pipeline Insights +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Pipeline artifacts **(FREE)** Pipeline artifacts are files created by GitLab after a pipeline finishes. Pipeline artifacts are -different to [job artifacts](job_artifacts.md) because they are not explicitly managed by +different to [job artifacts](../jobs/job_artifacts.md) because they are not explicitly managed by `.gitlab-ci.yml` definitions. Pipeline artifacts are used by the [test coverage visualization feature](../testing/test_coverage_visualization.md) diff --git a/doc/ci/pipelines/pipeline_efficiency.md b/doc/ci/pipelines/pipeline_efficiency.md index 0795005aa8e..c0c8d60b9d6 100644 --- a/doc/ci/pipelines/pipeline_efficiency.md +++ b/doc/ci/pipelines/pipeline_efficiency.md @@ -147,7 +147,7 @@ with embedded metric charts and all valuable details to analyze the problem. Review the storage use of the following to help analyze costs and efficiency: -- [Job artifacts](job_artifacts.md) and their [`expire_in`](../yaml/index.md#artifactsexpire_in) +- [Job artifacts](../jobs/job_artifacts.md) and their [`expire_in`](../yaml/index.md#artifactsexpire_in) configuration. If kept for too long, storage usage grows and could slow pipelines down. - [Container registry](../../user/packages/container_registry/index.md) usage. - [Package registry](../../user/packages/package_registry/index.md) usage. diff --git a/doc/ci/pipelines/schedules.md b/doc/ci/pipelines/schedules.md index 02728d5e1e0..3ff748644cf 100644 --- a/doc/ci/pipelines/schedules.md +++ b/doc/ci/pipelines/schedules.md @@ -2,7 +2,6 @@ stage: Verify group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -disqus_identifier: 'https://docs.gitlab.com/ee/user/project/pipelines/schedules.html' type: reference, howto --- @@ -84,8 +83,8 @@ To take ownership of a pipeline created by a different user: ## Related topics -- Pipeline schedules can be maintained by using the [Pipeline schedules API](../../api/pipeline_schedules.md). -- You can [control which jobs are added to scheduled pipelines](../jobs/job_control.md#run-jobs-for-scheduled-pipelines). +- [Pipeline schedules API](../../api/pipeline_schedules.md) +- [Adding jobs to scheduled pipelines](../jobs/job_control.md#run-jobs-for-scheduled-pipelines) - -- Simplecov (Ruby). Example: `/\(\d+.\d+\%\) covered/`. -- pytest-cov (Python). Example: `/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/`. -- Scoverage (Scala). Example: `/Statement coverage[A-Za-z\.*]\s*:\s*([^%]+)/`. -- `pest --coverage --colors=never` (PHP). Example: `/^\s*Cov:\s*\d+\.\d+?%$/`. -- `phpunit --coverage-text --colors=never` (PHP). Example: `/^\s*Lines:\s*\d+.\d+\%/`. -- gcovr (C/C++). Example: `/^TOTAL.*\s+(\d+\%)$/`. -- `tap --coverage-report=text-summary` (NodeJS). Example: `/^Statements\s*:\s*([^%]+)/`. -- `nyc npm test` (NodeJS). Example: `/All files[^|]*\|[^|]*\s+([\d\.]+)/`. -- `jest --ci --coverage` (NodeJS). Example: `/All files[^|]*\|[^|]*\s+([\d\.]+)/`. -- excoveralls (Elixir). Example: `/\[TOTAL\]\s+(\d+\.\d+)%/`. -- `mix test --cover` (Elixir). Example: `/\d+.\d+\%\s+\|\s+Total/`. -- JaCoCo (Java/Kotlin). Example: `/Total.*?([0-9]{1,3})%/`. -- `go test -cover` (Go). Example: `/coverage: \d+.\d+% of statements/`. -- .NET (OpenCover). Example: `/(Visited Points).*\((.*)\)/`. -- .NET (`dotnet test` line coverage). Example: `/Total\s*\|\s*(\d+(?:\.\d+)?)/`. -- tarpaulin (Rust). Example: `/^\d+.\d+% coverage/`. -- Pester (PowerShell). Example: `/Covered (\d+\.\d+%)/`. - - - -### View code coverage history - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/209121) the ability to download a `.csv` in GitLab 12.10. -> - Graph [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33743) in GitLab 13.1. - -To see the evolution of your project code coverage over time, -you can view a graph or download a CSV file with this data. - -1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Analytics > Repository**. - -The historic data for each job is listed in the dropdown list above the graph. - -To view a CSV file of the data, select **Download raw data (`.csv`)**. - -![Code coverage graph of a project over time](img/code_coverage_graph_v13_1.png) - -Code coverage data is also [available at the group level](../../user/group/repositories_analytics/index.md). - -### Coverage check approval rule **(PREMIUM)** - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15765) in GitLab 14.0. -> - [Made configurable in Project Settings](https://gitlab.com/gitlab-org/gitlab/-/issues/331001) in GitLab 14.1. - -You can implement merge request approvals to require approval by selected users or a group -when merging a merge request would cause the project's test coverage to decline. - -Follow these steps to enable the `Coverage-Check` MR approval rule: - -1. Set up a [`coverage`](../yaml/index.md#coverage) regular expression for all jobs you want to include in the overall coverage value. -1. Go to your project and select **Settings > Merge requests**. -1. Under **Merge request approvals**, select **Enable** next to the `Coverage-Check` approval rule. -1. Select the **Target branch**. -1. Set the number of **Approvals required** to greater than zero. -1. Select the users or groups to provide approval. -1. Select **Add approval rule**. - -![Coverage-Check approval rule](img/coverage_check_approval_rule_14_1.png) - -### Remove color codes from code coverage - -Some test coverage tools output with ANSI color codes that aren't -parsed correctly by the regular expression. This causes coverage -parsing to fail. - -Some coverage tools don't provide an option to disable color -codes in the output. If so, pipe the output of the coverage tool through a -small one line script that strips the color codes off. - -For example: - -```shell -lein cloverage | perl -pe 's/\e\[?.*?[\@-~]//g' -``` - ## Pipeline badges You can use [pipeline badges](../../user/project/badges.md) to indicate the pipeline status and test coverage of your projects. These badges are determined by the latest successful pipeline. - - diff --git a/doc/ci/quick_start/index.md b/doc/ci/quick_start/index.md index b9e0e39396c..ec58491e604 100644 --- a/doc/ci/quick_start/index.md +++ b/doc/ci/quick_start/index.md @@ -9,6 +9,9 @@ type: reference This tutorial shows you how to configure and run your first CI/CD pipeline in GitLab. +If you are already familiar with basic CI/CD concepts, you can learn about +common keywords in [Tutorial: Create a complex pipeline](tutorial.md). + ## Prerequisites Before you start, make sure you have: @@ -156,7 +159,7 @@ For the complete `.gitlab-ci.yml` syntax, see [the full `.gitlab-ci.yml` keyword ## Related topics -- [Follow this guide to migrate from CircleCI](../migration/circleci.md). -- [Follow this guide to migrate from Jenkins](../migration/jenkins.md). +- [Migrate from CircleCI](../migration/circleci.md) +- [Migrate from Jenkins](../migration/jenkins.md) -  Watch [First time GitLab & CI/CD](https://www.youtube.com/watch?v=kTNfi5z6Uvk&t=553s). This includes a quick introduction to GitLab, the first steps with CI/CD, building a Go project, running tests, using the CI/CD pipeline editor, detecting secrets and security vulnerabilities and offers more exercises for asynchronous practice. -  Watch [Intro to GitLab CI](https://www.youtube.com/watch?v=l5705U8s_nQ&t=358s). This workshop uses the Web IDE to quickly get going with building source code using CI/CD, and run unit tests. diff --git a/doc/ci/quick_start/tutorial.md b/doc/ci/quick_start/tutorial.md new file mode 100644 index 00000000000..88d35bf56b0 --- /dev/null +++ b/doc/ci/quick_start/tutorial.md @@ -0,0 +1,504 @@ +--- +stage: Verify +group: Pipeline Authoring +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Tutorial: Create a complex pipeline + +This tutorial walks you through configuring a progressively more complex CI/CD pipeline +through small, iterative steps. The pipeline is always fully functional, +but it gains more functionality with each step. + +When you finish this tutorial, you will have a new project on GitLab.com and a working documentation site on +[Docusaurus](https://docusaurus.io/). + +To complete this tutorial, you will: + +1. Create a project to hold the Docusaurus files +1. Create the initial pipeline configuration file +1. Add a job to build the site +1. Add a job to deploy the site +1. Add test jobs +1. Start using merge request pipelines +1. Reduce duplicated configuration + +## Prerequisites + +- You need an account on GitLab.com. +- You should be familiar with Git. +- Node.js must be installed on your local machine. For example, on macOS you can + [install node](https://formulae.brew.sh/formula/node) with `brew install node`. + +## Create a project to hold the Docusaurus files + +Before adding the pipeline configuration, you must first set up a Docusaurus project +on GitLab.com: + +1. Create a new project under your username (not a group): + 1. On the top bar, select **Main menu > Projects > View all projects**. + 1. On the right of the page, select **New project**. + 1. Select **Create blank project**. + 1. Enter the project details: + - In the **Project name** field, enter the name of your project, for example `My Pipeline Tutorial Project`. + - Select **Initialize repository with a README**. + 1. Select **Create project**. +1. On the right of the **Project Overview** page for your project, select **Clone** + to find the clone paths for your project. Copy the SSH or HTTP path and use the path + to clone the project locally. + + For example, to clone with SSH into a `pipeline-tutorial` directory on your computer: + + ```shell + git clone git@gitlab.com:my-username/my-pipeline-tutorial-project.git pipeline-tutorial + ``` + +1. Change to the project's directory, then generate a new Docusaurus site: + + ```shell + cd pipeline-tutorial + npm init docusaurus + ``` + + The Docusaurus initialization wizard prompts you with questions about the site. + Use all the default options. + +1. The initialization wizard sets up the site in `website/`, but the site should be in + the root of the project. Move the files up to the root and delete the old directory: + + ```shell + mv website/* . + rm -r website + ``` + +1. Update the Docusaurus configuration file with the details of your GitLab project. + In `docusaurus.config.js`: + + - Set `url:` to a path with this format: `https://.gitlab.io/`. + - Set `baseUrl:` to your project name, like `/my-pipeline-tutorial-project/`. + +1. Commit the changes, and push them to GitLab: + + ```shell + git add . + git commit -m "Add simple generated Docusaurus site" + git push origin + ``` + +## Create the initial CI/CD configuration file + +Start with the simplest possible pipeline configuration file to ensure CI/CD is enabled +in the project and runners are available to run jobs. + +This step introduces: + +- [Jobs](../jobs/index.md): These are self-contained parts of a pipeline that run your commands. + Jobs run on [runners](../runners/index.md), separate from the GitLab instance. +- [`script`](../yaml/index.md#script): This section of a job's configuration is + where you define the commands for jobs. If there are multiple commands (in an array), + they run in order. Each command executes as if it was run as a CLI command. + By default, if a command fails or returns an error, the job is flagged as failed + and no more commands run. + +In this step, create a `.gitlab-ci.yml` file in the root of the project with this configuration: + +```yaml +test-job: + script: + - echo "This is my first job!" + - date +``` + +Commit and push this change to GitLab, then: + +1. Go to **Build > Pipelines** and make sure a pipeline runs in GitLab with this single job. +1. Select the pipeline, then select the job to view the job's log and see the `This is my first job!` message + followed by the date. + +Now that you have a `.gitlab-ci.yml` file in your project, you can make all future changes +to pipeline configuration with the [pipeline editor](../pipeline_editor/index.md). + +## Add a job to build the site + +A common task for a CI/CD pipeline is to build the code in the project then deploy it. +Start by adding a job that builds the site. + +This step introduces: + +- [`image`](../yaml/index.md#image): Tell the runner which Docker + container to use to run the job in. The runner: + 1. Downloads the container image and starts it. + 1. Clones your GitLab project into the running container. + 1. Runs the `script` commands, one at a time. +- [`artifacts`](../yaml/index.md#artifacts): Jobs are self-contained and do not share + resources with each other. If you want files generated in one job to be used in + another job, you must save them as artifacts first. Then later jobs can retrieve the + artifacts and use the generated files. + +In this step, replace `test-job` with `build-job`: + +- Use `image` to configure the job to run with the latest `node` image. Docusaurus + is a Node.js project and the `node` image has the needed `npm` commands built in. +- Run `npm install` to install Docusaurus into the running `node` container, then run + `npm run build` to build the site. +- Docusaurus saves the built site in `build/`, so save these files with `artifacts`. + +```yaml +build-job: + image: node + script: + - npm install + - npm run build + artifacts: + paths: + - "build/" +``` + +Use the pipeline editor to commit this pipeline configuration to the default branch, +and check the job log. You can: + +- See the `npm` commands run and build the site. +- Verify that the artifacts are saved at the end. +- Browse the contents of the artifacts file by selecting **Browse** to the right of the job log + after the job completes. + +## Add a job to deploy the site + +After verifying the Docusaurus site builds in `build-job`, you can add a job that deploys it. + +This step introduces: + +- [`stage`](../yaml/index.md#stage) and [`stages`](../yaml/index.md#stage): The most common + pipeline configurations group jobs into stages. Jobs in the same stage can run in parallel, + while jobs in later stages wait for jobs in earlier stages to complete. If a job fails, + the whole stage is considered failed and jobs in later stages do not start running. +- [GitLab Pages](../../user/project/pages/index.md): To host your static site, you + will use GitLab Pages. + +In this step: + +- Add a job that fetches the built site and deploys it. When using GitLab Pages, + the job is always named `pages`. The artifacts from the `build-job` are fetched automatically + and extracted into the job. Pages looks for the site in the `public/` directory though, + so add a `script` command to move the site to that directory. +- Add a `stages` section, and define the stages for each job. `build-job` runs first + in the `build` stage, and `pages` runs after in the `deploy` stage. + +```yaml +stages: # List of stages for jobs and their order of execution + - build + - deploy + +build-job: + stage: build # Set this job to run in the `build` stage + image: node + script: + - npm install + - npm run build + artifacts: + paths: + - "build/" + +pages: + stage: deploy # Set this new job to run in the `deploy` stage + script: + - mv build/ public/ + artifacts: + paths: + - "public/" +``` + +Use the pipeline editor to commit this pipeline configuration to the default branch, +and view the pipeline details from the **Pipelines** list. Verify that: + +- The two jobs run in different stages, `build` and `deploy`. +- After the `pages` job completes a `pages-deploy` job appears, which is the GitLab process + that deploys the Pages site. When that job completes, you can visit your new Docusaurus + site. The Pages documentation explains [the URL formatting](../../user/project/pages/getting_started_part_one.md#gitlab-pages-default-domain-names), + which should be similar to `https://.gitlab.io//`. + +## Add test jobs + +Now that the site builds and deploys as expected, you can add tests and linting. +For example, a Ruby project might run RSpec test jobs. Docusaurus is a static site +that uses Markdown and generated HTML, so this tutorial adds jobs to test the Markdown and HTML. + +This step introduces: + +- [`allow_failure`](../yaml/index.md#allow_failure): Jobs that fail intermittently, + or are expected to fail, can slow down productivity or be difficult to troubleshoot. + Use `allow_failure` to let jobs fail without halting pipeline execution. +- [`dependencies`](../yaml/index.md#dependencies): Use `dependencies` to control + artifact downloads in individual jobs by listing which jobs to fetch artifacts from. + +In this step: + +- Add a new `test` stage that runs between `build` and `deploy`. These three stages + are the default stages when `stages` is undefined in the configuration. +- Add a `lint-markdown` job to run [markdownlint](https://github.com/DavidAnson/markdownlint) + and check the Markdown in your project. markdownlint is a static analysis tool that + checks that your Markdown files follow formatting standards. + - The sample Markdown files Docusaurus generates are in `blog/` and `docs/`. + - This tool scans the original Markdown files only, and does not need the generated HTML + saved in the `build-job` artifacts. Speed up the job with `dependencies: []` + so that it fetches no artifacts. + - A few of the sample Markdown files violate default markdownlint rules, so add + `allow_failure: true` to let the pipeline continue despite the rule violations. +- Add a `test-html` job to run [HTMLHint](https://htmlhint.com/) and check the generated HTML. + HTMLHint is a static analysis tool that scans generated HTML for known issues. +- Both `test-html` and `pages` need the generated HTML found in the `build-job` artifacts. + Jobs fetch artifacts from all jobs in earlier stages by default, but add `dependencies:` + to make sure the jobs don't accidentally download other artifacts after future pipeline changes. + +```yaml +stages: + - build + - test # Add a `test` stage for the test jobs + - deploy + +build-job: + stage: build + image: node + script: + - npm install + - npm run build + artifacts: + paths: + - "build/" + +lint-markdown: + stage: test + image: node + dependencies: [] # Don't fetch any artifacts + script: + - npm install markdownlint-cli2 --global # Install markdownlint into the container + - markdownlint-cli2 -v # Verify the version, useful for troubleshooting + - markdownlint-cli2 "blog/**/*.md" "docs/**/*.md" # Lint all markdown files in blog/ and docs/ + allow_failure: true # This job fails right now, but don't let it stop the pipeline. + +test-html: + stage: test + image: node + dependencies: + - build-job # Only fetch artifacts from `build-job` + script: + - npm install --save-dev htmlhint # Install HTMLHint into the container + - npx htmlhint --version # Verify the version, useful for troubleshooting + - npx htmlhint build/ # Lint all markdown files in blog/ and docs/ + +pages: + stage: deploy + dependencies: + - build-job # Only fetch artifacts from `build-job` + script: + - mv build/ public/ + artifacts: + paths: + - "public/" +``` + +Commit this pipeline configuration to the default branch, and view the pipeline details. + +- The `test-markdown` job fails because the sample Markdown violates the default + markdownlint rules, but is allowed to fail. You can: + - Ignore the violations for now. They do not need to be fixed as part of the tutorial. + - Fix the Markdown file violations. Then you can change `allow_failure` to `false`, + or remove `allow_failure` completely because `allow_failure: false` is the default behavior + when not defined. + - Add a markdownlint configuration file to limit which rule violations to alert on. +- You can also make changes to the Markdown file content and see the changes on the site + after the next deployment. + +## Start using merge request pipelines + +With the pipeline configurations above, the site deploys every time a pipeline completes +successfully, but this is not an ideal development workflow. It's better to work from +feature branches and merge requests, and only deploy the site when changes merge +to the default branch. + +This step introduces: + +- [`rules`](../yaml/index.md#rules): Add rules to each job to configure in which + pipelines they run. You can configure jobs to run in [merge request pipelines](../pipelines/merge_request_pipelines.md), + [scheduled pipelines](../pipelines/schedules.md), or other specific situations. + Rules are evaluated from top to bottom, and if a rule matches, the job is + added to the pipeline. +- [CI/CD variables](../variables/index.md): use these environment variables + to configure job behavior in the configuration file and in script commands. + [Predefined CI/CD variables](../variables/predefined_variables.md) are variables + that you do not need to manually define. They are automatically injected into pipelines + so you can use them to configure your pipeline. Variables are usually formatted as `$VARIABLE_NAME`. + and predefined variables are usually prefixed with `$CI_`. + +In this step: + +- Create a new feature branch and make the changes in the branch instead of the default branch. +- Add `rules` to each job: + - The site should only deploy for changes to the default branch. + - The other jobs should run for all changes in merge requests or the default branch. +- With this pipeline configuration, you can work from a feature branch without running any jobs, + which saves resources. When you are ready to validate your changes, create a merge request + and a pipeline runs with the jobs configured to run in merge requests. +- When your merge request is accepted and the changes merge to the default branch, + a new pipeline runs which also contains the `pages` deployment job. The site deploys + if no jobs fail. + +```yaml +stages: + - build + - test + - deploy + +build-job: + stage: build + image: node + script: + - npm install + - npm run build + artifacts: + paths: + - "build/" + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # Run for all changes to a merge request's source branch + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Run for all changes to the default branch + +lint-markdown: + stage: test + image: node + dependencies: [] + script: + - npm install markdownlint-cli2 --global + - markdownlint-cli2 -v + - markdownlint-cli2 "blog/**/*.md" "docs/**/*.md" + allow_failure: true + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # Run for all changes to a merge request's source branch + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Run for all changes to the default branch + +test-html: + stage: test + image: node + dependencies: + - build-job + script: + - npm install --save-dev htmlhint + - npx htmlhint --version + - npx htmlhint build/ + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # Run for all changes to a merge request's source branch + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Run for all changes to the default branch + +pages: + stage: deploy + dependencies: + - build-job + script: + - mv build/ public/ + artifacts: + paths: + - "public/" + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Run for all changes to the default branch only +``` + +Merge the changes in your merge request. This action updates the default branch. Verify that +the new pipeline contains the `pages` job that deploys the site. + +Be sure to use feature branches and merge requests for all future changes to pipeline configuration. +Other project changes, like creating a Git tag or adding a pipeline schedule, do not +trigger pipelines unless you add rules for those cases too. + +## Reduce duplicated configuration + +The pipeline now contains three jobs that all have identical `rules` and `image` +configuration. Instead of repeating these rules, use `extends` and `default` to create +single sources of truth. + +This step introduces: + +- [Hidden jobs](../jobs/index.md#hide-jobs): Jobs that start with `.` are never + added to a pipeline. Use them to hold configuration you want to reuse. +- [`extends`](../yaml/index.md#extends): Use extends to repeat configuration in + multiple places, often from hidden jobs. If you update the hidden job's configuration, + all jobs extending the hidden job use the updated configuration. +- [`default`](../yaml/index.md#default): Set keyword defaults that apply to all jobs + when not defined. +- YAML overriding: When reusing configuration with `extends` or `default`, you can explicitly + define a keyword in the job to override the `extends` or `default` configuration. + +In this step: + +- Add a `.standard-rules` hidden job to hold the rules that are repeated in `build-job`, + `lint-markdown`, and `test-html`. +- Use `extends` to reuse the `.standard-rules` configuration in the three jobs. +- Add a `default` section to define the `image` default as `node`. +- The `pages` deployment job does not need the default `node` image, so explicitly use + [`busybox`](https://hub.docker.com/_/busybox), an extremely tiny and fast image. + +```yaml +stages: + - build + - test + - deploy + +default: # Add a default section to define the `image` keyword's default value + image: node + +.standard-rules: # Make a hidden job to hold the common rules + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + +build-job: + extends: + - .standard-rules # Reuse the configuration in `.standard-rules` here + stage: build + script: + - npm install + - npm run build + artifacts: + paths: + - "build/" + +lint-markdown: + stage: test + extends: + - .standard-rules # Reuse the configuration in `.standard-rules` here + dependencies: [] + script: + - npm install markdownlint-cli2 --global + - markdownlint-cli2 -v + - markdownlint-cli2 "blog/**/*.md" "docs/**/*.md" + allow_failure: true + +test-html: + stage: test + extends: + - .standard-rules # Reuse the configuration in `.standard-rules` here + dependencies: + - build-job + script: + - npm install --save-dev htmlhint + - npx htmlhint --version + - npx htmlhint build/ + +pages: + stage: deploy + image: busybox # Override the default `image` value with `busybox` + dependencies: + - build-job + script: + - mv build/ public/ + artifacts: + paths: + - "public/" + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH +``` + +Use a merge request to commit this pipeline configuration to the default branch. +The file is simpler, but it should have the same behavior as the previous step. + +You've just created a full pipeline and streamlined it to be more efficient. Nice work! +Now you can take this knowledge, learn about [the rest of the `.gitlab-ci.yml` keywords](../yaml/index.md), +and build your own pipelines. diff --git a/doc/ci/resource_groups/index.md b/doc/ci/resource_groups/index.md index 7cf71f2a3ea..bae1bd9712b 100644 --- a/doc/ci/resource_groups/index.md +++ b/doc/ci/resource_groups/index.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments description: Control the job concurrency in GitLab CI/CD --- @@ -54,7 +54,7 @@ deploy: With this configuration, the safety on the deployments is assured while you can still run `build` jobs concurrently for maximizing the pipeline efficiency. -## Requirements +## Prerequisites - The basic knowledge of the [GitLab CI/CD pipelines](../pipelines/index.md) - The basic knowledge of the [GitLab Environments and Deployments](../environments/index.md) @@ -194,13 +194,11 @@ You must define [`strategy: depend`](../yaml/index.md#triggerstrategy) with the `trigger` keyword. This ensures that the lock isn't released until the downstream pipeline finishes. -## API +## Related topics -See the [API documentation](../../api/resource_groups.md). - -## Related features - -Read more how you can use GitLab for [safe deployments](../environments/deployment_safety.md). +- [API documentation](../../api/resource_groups.md) +- [Log documentation](../../administration/logs/index.md#ci_resource_groups_jsonlog) +- [GitLab for safe deployments](../environments/deployment_safety.md) ## Troubleshooting @@ -254,3 +252,66 @@ deploy: resource_group: production environment: production ``` + +### Jobs get stuck in "Waiting for resource" + +Sometimes, a job hangs with the message `Waiting for resource: `. To resolve, +first check that the resource group is working correctly: + +1. Go to the job details page. +1. Select **View job currently using resource**. +1. Check the job status: + - If the status is `running` or `pending`, the feature is working correctly. Wait until the job finishes and releases the resource. + - If the status is not `running` or `pending`, the feature might not be working correctly. + [Open a new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new) with the following information: + - The job ID. + - The job status. + - How often the problem occurs. + - Steps to reproduce the problem. + +You can also get job information from the GraphQL API. You should use the GraphQL API if you use pipeline-level concurrency control with cross-project/parent-child pipelines because the trigger jobs are not accessible from the UI. + +To get job information from the GraphQL API: + +1. Go to the pipeline details page. +1. Select the **Jobs** tab and find the ID of the stuck job. +1. Go to [GraphiQL explorer](../../api/graphql/index.md#graphiql). +1. Run the following query: + + ```graphql + { + project(fullPath: "") { + name + job(id: "gid://gitlab/Ci::Bridge/") { + name + detailedStatus { + action { + path + buttonTitle + } + } + } + } + } + ``` + + The `job.detailedStatus.action.path` field contains the job ID using the resource. + +1. Run the following query and check `job.status` field according to the criteria above. You can also visit the pipeline page from `pipeline.path` field. + + ```graphql + { + project(fullPath: "") { + name + job(id: "gid://gitlab/Ci::Bridge/") { + name + status + pipeline { + path + } + } + } + } + ``` + + If the status is not `running` or `pending`, open a new issue. diff --git a/doc/ci/review_apps/index.md b/doc/ci/review_apps/index.md index 7a0f43f9ec5..004bd9dfc5f 100644 --- a/doc/ci/review_apps/index.md +++ b/doc/ci/review_apps/index.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Insights +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -58,16 +58,16 @@ The process of configuring review apps is as follows: 1. [Install](https://docs.gitlab.com/runner/install/) and [configure](https://docs.gitlab.com/runner/commands/) a runner to do deployment. 1. Set up a job in `.gitlab-ci.yml` that uses the [predefined CI/CD variable](../variables/index.md) `${CI_COMMIT_REF_SLUG}` to create dynamic environments and restrict it to run only on branches. - Alternatively, you can get a YAML template for this job by [enabling review apps](#enable-review-app-button) for your project. -1. Optionally, set a job that [manually stops](../environments/index.md#stop-an-environment) the review apps. + Alternatively, you can get a YAML template for this job by [enabling review apps](#enable-review-apps-button) for your project. +1. Optionally, set a job that [manually stops](../environments/index.md#stopping-an-environment) the review apps. -### Enable review app button +### Enable review apps button > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118844) in GitLab 12.8. When configuring review apps for a project, you add a new job to the `.gitlab-ci.yml` file, as mentioned above. To facilitate this, and if you are using Kubernetes, you can select -**Enable review app** and GitLab prompts you with a template code block that +**Enable review apps** and GitLab prompts you with a template code block that you can copy and paste into `.gitlab-ci.yml` as a starting point. Prerequisite: @@ -78,7 +78,7 @@ To use the review apps template: 1. On the top bar, select **Main menu > Projects** and find the project you want to create a review app job for. 1. On the left sidebar, select **Deployments > Environments**. -1. Select **Enable review app**. +1. Select **Enable review apps**. 1. Copy the provided code snippet and paste it into your `.gitlab-ci.yml` file: @@ -193,13 +193,18 @@ After you have the route mapping set up, it takes effect in the following locati ![View on environment button in file view](img/view_on_env_blob.png) -## Visual Reviews **(PREMIUM)** + +## Visual Reviews (deprecated) **(PREMIUM)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10761) in GitLab 12.0. > - [Moved](https://about.gitlab.com/blog/2021/01/26/new-gitlab-product-subscription-model/) to GitLab Premium in 13.9. > - It's [deployed behind a feature flag](../../user/feature_flags.md), `anonymous_visual_review_feedback`, disabled by default. > - It's disabled on GitLab.com. +WARNING: +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387751) in GitLab 15.8 +and is planned for removal in 17.0. This change is a breaking change. + FLAG: On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `anonymous_visual_review_feedback`. @@ -305,3 +310,5 @@ the user must enter a [personal access token](../../user/profile/personal_access with `api` scope before submitting feedback. This same method can be used to require authentication for any public projects. + + diff --git a/doc/ci/runners/configure_runners.md b/doc/ci/runners/configure_runners.md index 0fd4bff1bff..cf4b95f511d 100644 --- a/doc/ci/runners/configure_runners.md +++ b/doc/ci/runners/configure_runners.md @@ -155,8 +155,11 @@ different places. ### Determine the IP address of a shared runner -To view the IP address of a shared runner you must have administrator access to -the GitLab instance. To determine this: +Prerequisite: + +- You must have administrator access to the instance. + +To determine the IP address of a shared runner: 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **CI/CD > Runners**. @@ -246,6 +249,29 @@ Example 2: 1. A job that has no tags defined is executed and run. 1. A second job that has a `docker` tag defined is stuck. +### A runner and a job have multiple tags + +The selection logic that matches the job and runner is based on the list of `tags` +defined in the job. + +The following examples illustrate the impact of a runner and a job having multiple tags. For a runner to be +selected to run a job, it must have all of the tags defined in the job script block. + +Example 1: + +1. The runner is configured with the tags `[docker, shell, gpu]`. +1. The job has the tags `[docker, shell, gpu]` and is executed and run. + +Example 2: + +1. The runner is configured with the tags `[docker, shell, gpu]`. +1. The job has the tags `[docker, shell,]` and is executed and run. + +Example 3: + +1. The runner is configured with the tags `[docker, shell]`. +1. The job has the tags `[docker, shell, gpu]` and is not executed. + ### Use tags to run jobs on different platforms You can use tags to run different jobs on different platforms. For @@ -254,16 +280,14 @@ example, if you have an OS X runner with tag `osx` and a Windows runner with tag ```yaml windows job: - stage: - - build + stage: build tags: - windows script: - echo Hello, %USERNAME%! osx job: - stage: - - build + stage: build tags: - osx script: @@ -310,6 +334,7 @@ globally or for individual jobs: - [`GIT_CLEAN_FLAGS`](#git-clean-flags) - [`GIT_FETCH_EXTRA_FLAGS`](#git-fetch-extra-flags) - [`GIT_SUBMODULE_UPDATE_FLAGS`](#git-submodule-update-flags) +- [`GIT_SUBMODULE_FORCE_HTTPS`](#rewrite-submodule-urls-to-https) - [`GIT_DEPTH`](#shallow-cloning) (shallow cloning) - [`GIT_SUBMODULE_DEPTH`](#git-submodule-depth) - [`GIT_CLONE_PATH`](#custom-build-directories) (custom build directories) @@ -566,6 +591,23 @@ You should be aware of the implications for the security, stability, and reprodu your builds when using the `--remote` flag. In most cases, it is better to explicitly track submodule commits as designed, and update them using an auto-remediation/dependency bot. +### Rewrite submodule URLs to HTTPS + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3198) in GitLab Runner 15.11. + +Use the `GIT_SUBMODULE_FORCE_HTTPS` variable to force a rewrite of all Git and SSH submodule URLs to HTTPS. +This allows you to clone submodules on the same GitLab instance that use absolute URLs, even if they were +configured with a Git or SSH protocol. + +```yaml +variables: + GIT_SUBMODULE_STRATEGY: recursive + GIT_SUBMODULE_FORCE_HTTPS: "true" +``` + +When enabled, GitLab Runner uses a [CI/CD job token](../jobs/ci_job_token.md) to clone the submodules with +the permissions of the user executing the job, and does not require SSH credentials. + ### Shallow cloning > Introduced in GitLab 8.9 as an experimental feature. @@ -779,7 +821,7 @@ variables: NOTE: Zip archives are the only supported artifact type. Follow [the issue for details](https://gitlab.com/gitlab-org/gitlab/-/issues/367203). -GitLab Runner can generate and produce attestation metadata for all build artifacts. To enable this feature, you must set the `RUNNER_GENERATE_ARTIFACTS_METADATA` environment variable to `true`. This variable can either be set globally or it can be set for individual jobs. The metadata is in rendered in a plain text `.json` file that's stored with the artifact. The filename is as follows: `{ARTIFACT_NAME}-metadata.json` where `ARTIFACT_NAME` is what was defined as the [name for the artifact](../pipelines/job_artifacts.md#use-cicd-variables-to-define-the-artifacts-name) in the CI file. The filename, however, defaults to `artifacts-metadata.json` if no name was given to the build artifacts. +GitLab Runner can generate and produce attestation metadata for all build artifacts. To enable this feature, you must set the `RUNNER_GENERATE_ARTIFACTS_METADATA` environment variable to `true`. This variable can either be set globally or it can be set for individual jobs. The metadata is in rendered in a plain text `.json` file that's stored with the artifact. The filename is as follows: `{ARTIFACT_NAME}-metadata.json` where `ARTIFACT_NAME` is what was defined as the [name for the artifact](../jobs/job_artifacts.md#with-a-dynamically-defined-name) in the CI file. The filename, however, defaults to `artifacts-metadata.json` if no name was given to the build artifacts. ### Attestation format diff --git a/doc/ci/runners/index.md b/doc/ci/runners/index.md index 41cde709143..6883f0727dd 100644 --- a/doc/ci/runners/index.md +++ b/doc/ci/runners/index.md @@ -11,8 +11,8 @@ If you use GitLab SaaS (GitLab.com), your [untagged](../../ci/runners/configure_ As long as shared runners are enabled for your project, no configuration is required. Your jobs can run on: - [Linux runners](saas/linux_saas_runner.md). -- [Windows runners](saas/windows_saas_runner.md) ([Beta](../../policy/alpha-beta-support.md#beta-features)). -- [macOS runners](saas/macos_saas_runner.md) ([Beta](../../policy/alpha-beta-support.md#beta-features)). +- [Windows runners](saas/windows_saas_runner.md) ([Beta](../../policy/alpha-beta-support.md#beta)). +- [macOS runners](saas/macos_saas_runner.md) ([Beta](../../policy/alpha-beta-support.md#beta)). The number of minutes you can use on these runners depends on the [maximum number of CI/CD minutes](../pipelines/cicd_minutes.md) @@ -20,9 +20,9 @@ in your [subscription plan](https://about.gitlab.com/pricing/). ## Security for GitLab SaaS runners -GitLab SaaS runners on Linux and Windows run on Google Compute Platform. The [Google Infrastructure Security Design Overview whitepaper](https://cloud.google.com/docs/security/infrastructure/design/resources/google_infrastructure_whitepaper_fa.pdf) provides an overview of how Google designs security into its technical infrastructure. The GitLab [Trust Center](https://about.gitlab.com/security/) and [GitLab Security Compliance Controls](https://about.staging.gitlab.com/handbook/engineering/security/security-assurance/security-compliance/sec-controls.html) pages provide an overview of the Security and Compliance controls that govern the GitLab SaaS runners. +GitLab SaaS runners on Linux and Windows run on Google Compute Platform. The [Google Infrastructure Security Design Overview whitepaper](https://cloud.google.com/docs/security/infrastructure/design/resources/google_infrastructure_whitepaper_fa.pdf) provides an overview of how Google designs security into its technical infrastructure. The GitLab [Trust Center](https://about.gitlab.com/security/) and [GitLab Security Compliance Controls](https://about.staging.gitlab.com/handbook/engineering/security/security-assurance/security-compliance/sec-controls.html) pages provide an overview of the security and compliance controls that govern the GitLab SaaS runners. -The runner that serves as a Runner Manager automatically initiates the creation and deletion of the virtual machines (VMs) used for CI jobs. When the Runner Manager picks up a GitLab SaaS CI job, it automatically executes that job on a new VM. There is no human or manual intervention in this process. The following section provides an overview of the additional built-in layers that harden the security of the GitLab Runner SaaS CI build environment. +The runner that serves as a runner manager automatically initiates the creation and deletion of the virtual machines (VMs) used for CI jobs. When the runner manager picks up a GitLab SaaS CI job, it automatically executes that job on a new VM. There is no human or manual intervention in this process. The following section provides an overview of the additional built-in layers that harden the security of the GitLab Runner SaaS CI build environment. ### Security of CI job execution on GitLab Runner SaaS (Linux, Windows) @@ -39,4 +39,4 @@ GitLab sends the command to remove the temporary runner VM to the Google Compute - Firewall rules only allow outbound communication from the temporary VM to the public internet. - Inbound communication from the public internet to the temporary VM is not allowed. - Firewall rules do not permit communication between VMs. -- The only internal communication allowed to the temporary VMs is from the Runner Manager. +- The only internal communication allowed to the temporary VMs is from the runner manager. diff --git a/doc/ci/runners/register_runner.md b/doc/ci/runners/register_runner.md new file mode 100644 index 00000000000..f53297099d7 --- /dev/null +++ b/doc/ci/runners/register_runner.md @@ -0,0 +1,117 @@ +--- +stage: Verify +group: Runner +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Generate runner tokens **(FREE)** + +To register a runner, you can use either: + +- An authentication token assigned to the runner when you create the runner in the UI. The runner uses the token to authenticate with GitLab when picking up jobs from the job queue. +- A registration token (deprecated). + +## Generate an authentication token + +Registration with an authentication token is available for all runners. + +NOTE: +The token only displays in the UI for a short period of time during registration. + +### For a shared runner + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/383139) in GitLab 15.10. Deployed behind the `create_runner_workflow_for_admin` [flag](../../administration/feature_flags.md) +> - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/389269) in GitLab 16.0. + +FLAG: +On self-managed GitLab, by default this feature is available. To hide the feature, +ask an administrator to [disable the feature flag](../../administration/feature_flags.md) named `create_runner_workflow_for_admin`. + +Prerequisites: + +- You must be an administrator. + +To generate an authentication token for a shared runner: + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **CI/CD > Runners**. +1. Select **New instance runner**. +1. Select a platform. +1. Optional. Enter configurations for the runner. +1. Select **Submit**. +1. Follow the instructions to register the runner from the command line. + +### For a group runner + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/383143) in GitLab 15.10. Deployed behind the `create_runner_workflow_for_namespace` [flag](../../administration/feature_flags.md). Disabled by default. +> - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/393919) in GitLab 16.0. + +FLAG: +On self-managed GitLab, by default this feature is available. To hide the feature, +ask an administrator to [disable the feature flag](../../administration/feature_flags.md) named `create_runner_workflow_for_namespace`. + +Prerequisites: + +- You must have the Owner role for the group. + +To generate an authentication token for a group runner: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **CI/CD > Runners**. +1. Select **New group runner**. +1. Select a platform. +1. Optional. Enter configurations for the runner. +1. Select **Submit**. +1. Follow the instructions to register the runner from the command line. + +### For a project runner + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/383143) in GitLab 15.10. Deployed behind the `create_runner_workflow_for_namespace` [flag](../../administration/feature_flags.md). Disabled by default. +> - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/393919) in GitLab 16.0. + +FLAG: +On self-managed GitLab, by default this feature is available. To hide the feature, +ask an administrator to [disable the feature flag](../../administration/feature_flags.md) named `create_runner_workflow_for_namespace`. + +Prerequisites: + +- You must have the Maintainer role for the project. + +To generate an authentication token for a project runner: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > CI/CD**. +1. Select **New project runner**. +1. Select a platform. +1. Optional. Enter configurations for the runner. +1. Select **Submit**. +1. Follow the instructions to register the runner from the command line. + +## Generate a registration token (deprecated) + +WARNING: +The ability to pass a runner registration token, and support for certain configuration arguments was +[deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/380872) in GitLab 15.6. Authentication tokens +should be used instead to register runners. Registration tokens, and support for certain configuration arguments +will be disabled behind a feature flag in GitLab 16.6 and removed in GitLab 17.0. The configuration arguments disabled for `glrt-` tokens are `--locked`, `--access-level`, `--run-untagged`, `--maximum-timeout`, `--paused`, `--tag-list`, and `--maintenance-note`. This change is a breaking +change. + +### For a shared runner + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **CI/CD > Runners**. +1. Select **Register an instance runner**. +1. Copy the registration token. + +### For a group runner + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **CI/CD > Runners**. +1. Copy the registration token. + +### For a project runner + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand the **Runners** section. +1. Copy the registration token. diff --git a/doc/ci/runners/runners_scope.md b/doc/ci/runners/runners_scope.md index d20ef846df7..43204b463b3 100644 --- a/doc/ci/runners/runners_scope.md +++ b/doc/ci/runners/runners_scope.md @@ -95,10 +95,8 @@ To disable shared runners for a group: select **Allow projects and subgroups to override the group setting**. NOTE: -To re-enable the shared runners for a group, turn on the -**Enable shared runners for this group** toggle. -Then, a user with the Owner or Maintainer role must explicitly change this setting -for each project subgroup or project. +If you re-enable the shared runners for a group after you disable them, a user with the +Owner or Maintainer role must manually change this setting for each project subgroup or project. ### How shared runners pick jobs diff --git a/doc/ci/runners/saas/linux_saas_runner.md b/doc/ci/runners/saas/linux_saas_runner.md index 58c8a6d0815..3a45e056643 100644 --- a/doc/ci/runners/saas/linux_saas_runner.md +++ b/doc/ci/runners/saas/linux_saas_runner.md @@ -28,6 +28,15 @@ CI/CD jobs that run on `medium` and `large` machine types consumes CI minutes at Refer to the CI/CD minutes [cost factor](../../../ci/pipelines/cicd_minutes.md#cost-factor) for the cost factor applied to the machine type based on size. +## GPU-enabled SaaS runners on Linux **(PREMIUM SAAS)** + +We offer GPU-enabled SaaS runners for heavy compute including ModelOps or HPC workloads. Available to Premium and Ultimate plan customers, jobs on these instances consume the CI/CD minutes allocated to your namespace. + +| | Standard | +|-------------------|---------------------------| +| Specs | 4 vCPU, 16 GB RAM, 1 Nvidia Tesla T4 GPU (or similar) | +| GitLab CI/CD tags | `saas-linux-medium-gpu-standard` | + ## Example of how to tag a job To use a machine type other than `small`, add a `tags:` keyword to your job. @@ -88,12 +97,15 @@ Below are the settings for SaaS runners on Linux. and [#4070](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/4070). NOTE: -The final disk space your jobs can use is less than 25 GB. Some disk space -allocated to the instance is occupied by the operating system, the Docker image, -and a copy of your cloned repository. +SaaS runner instances are provisioned with a 25 GB storage volume. The underlying disk space of the storage volume +is shared by the operating system, the Docker image, and a copy of your cloned repository. +This means that the available free disk space that your jobs can use is **less than 25 GB**. -## Pre-clone script +## Pre-clone script (deprecated) +WARNING: +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/391896) in GitLab 15.9 +and is planned for removal in 16.0. Use [`pre_get_sources_script`](../../../ci/yaml/index.md#hookspre_get_sources_script) instead. This change is a breaking change. With SaaS runners on Linux, you can run commands in a CI/CD job before the runner attempts to run `git init` and `git fetch` to download a GitLab repository. The diff --git a/doc/ci/runners/saas/macos/codesigning.md b/doc/ci/runners/saas/macos/codesigning.md index 697f138eec6..809b8faf9df 100644 --- a/doc/ci/runners/saas/macos/codesigning.md +++ b/doc/ci/runners/saas/macos/codesigning.md @@ -8,112 +8,21 @@ info: To determine the technical writer assigned to the Stage/Group associated w Before you can integrate GitLab with Apple services, install to a device, or deploy to the Apple App Store, you must [code sign](https://developer.apple.com/support/code-signing/) your application. -To code sign an iOS project, you need the following files: - -- A certificate issued by Apple. -- A provisioning profile. - ## Code signing iOS Projects with fastlane When you use SaaS runners on macOS, each job runs on a VM. Included in each VM is [fastlane](https://fastlane.tools/), an open-source solution aimed at simplifying mobile app deployment. -These steps outline the minimal setup required to use fastlane to code sign your application. Refer to the fastlane [getting started guide](https://docs.fastlane.tools/), [best practices for integrating with GitLab CI](https://docs.fastlane.tools/best-practices/continuous-integration/gitlab/) and the [fastlane code signing getting started guide](https://docs.fastlane.tools/codesigning/getting-started/) for installation instructions, and an overview of how to use fastlane to handle code signing. - -To use fastlane to code sign your application: - -1. At the root of your project repository, on your local development system, run this command: - - ```plaintext - fastlane match init - ``` - - This command creates the `fastlane` directory and adds two files: `Fastfile` and `Appfile`. - -1. Open `Appfile` and edit it to include your Apple ID and app ID. - - ```plaintext - app_identifier("APP IDENTIFIER") # The bundle identifier of your app - - apple_id("APPLE ID") # Your Apple email address - ``` - -1. Open `Fastfile`, which includes the fastlane build steps. - In the following snippet, the steps `get_certificates`, `get_provisioning_profile,match`, `gym`, and - `upload_to_testflight` are fastlane [actions](https://docs.fastlane.tools/actions/). - - ```plaintext - # This file contains the fastlane.tools configuration - # You can find the documentation at https://docs.fastlane.tools - - default_platform(:ios) - - platform :ios do - desc "Build the application" - lane :beta do - increment_build_number( - build_number: latest_testflight_build_number + 1, - xcodeproj: "${PROJECT_NAME}.xcodeproj" - ) - get_certificates - get_provisioning_profile - # match(type: "appstore",read_only: true) - gym - upload_to_testflight - end - end - ``` - -The example configuration also includes an optional `Gymfile`. This file stores configuration -parameters and is used by the fastlane [`gym`](https://docs.fastlane.tools/actions/gym/) action. - -## Using fastlane match - -To simplify the code signing process and implement the -[Code Signing Best Practices Guide](https://codesigning.guide/) recommendations, -use [fastlane match](https://docs.fastlane.tools/actions/match/). - -- Use one code signing identity shared across your team. -- Store the required certificates and provisioning profiles in a separate GitLab project repository. - -Match automatically syncs iOS and macOS keys and provisioning profiles across all team members with access to the GitLab project. Each team member with access to the project can use the credentials for code signing. - -To use fastlane match: - -1. Initialize match in the project repository: - - ```shell - bundle exec fastlane match init - ``` - -1. Select `git` as your storage node. -1. Enter the URL of the GitLab project you plan to use to store your code signing identities. -1. Optional. To create a new certificate and provisioning profile, run: - - ```shell - bundle exec fastlane match development - ``` - -For different code signing identities' storage options, and for a complete step-by-step guide for using match, -refer to the [match documentation](https://docs.fastlane.tools/actions/match/#usage). - -### Environment variables and authentication - -To complete the setup, you must configure environment variables to use with fastlane. The required variables are outlined in the [fastlane documentation](https://docs.fastlane.tools/best-practices/continuous-integration/#environment-variables-to-set). - -To support Apple's two factor authentication requirement, configure these variables: - -- `FASTLANE_APPLE_APPLICATION_SPECIFIC_PASSWORD` and -- `FASTLANE_SESSION` - -To authenticate fastlane with the App Store for the TestFlight upload, configure these variables: +For information about how to set up code signing for your application, see the instructions in the [Mobile DevOps documentation](../../../../ci/mobile_devops.md#code-sign-ios-projects-with-fastlane). -- `FASTLANE_USER` and -- `FASTLANE_PASSWORD` +These instructions provide the minimal setup to use fastlane to code sign your application. For more information about using fastlane to handle code signing, see the following resources: -View the [fastlane authentication with Apple Services guide](https://docs.fastlane.tools/getting-started/ios/authentication/) for an overview of authentication options. +- [fastlane getting started guide](https://docs.fastlane.tools/) +- [Best practices for integrating with GitLab CI](https://docs.fastlane.tools/best-practices/continuous-integration/gitlab/) +- [fastlane code signing getting started guide](https://docs.fastlane.tools/codesigning/getting-started/) ## Related topics - [Apple Developer Support - Code Signing](https://developer.apple.com/support/code-signing/) - [Code Signing Best Practice Guide](https://codesigning.guide/) +- [fastlane authentication with Apple Services guide](https://docs.fastlane.tools/getting-started/ios/authentication/) diff --git a/doc/ci/runners/saas/macos/environment.md b/doc/ci/runners/saas/macos/environment.md index 1fc7afea7e6..7aa0f33fc59 100644 --- a/doc/ci/runners/saas/macos/environment.md +++ b/doc/ci/runners/saas/macos/environment.md @@ -20,11 +20,11 @@ Each time you run a job that requires tooling or dependencies not available in t GitLab SaaS provides macOS build machines on Apple servers with Intel x86-64 processors. The expectation is that virtual machines running on the Apple M1 chip will be available in the second half of 2022. -At this time there is only one available machine type offered, `gbc-macos-large`. +At this time there is only one available machine type offered, `shared-macos-amd64`. | Instance type | vCPUS | Memory (GB) | | --------- | --- | ------- | -| `gbc-macos-large` | 4 | 10 | +| `shared-macos-amd64` | 4 | 10 | ## VM images diff --git a/doc/ci/runners/saas/macos_saas_runner.md b/doc/ci/runners/saas/macos_saas_runner.md index 96e294ff828..20be2f2a147 100644 --- a/doc/ci/runners/saas/macos_saas_runner.md +++ b/doc/ci/runners/saas/macos_saas_runner.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # SaaS runners on macOS (Beta) **(PREMIUM SAAS)** -SaaS runners on macOS are in [Beta](../../../policy/alpha-beta-support.md#beta-features) for approved open source programs and customers in Premium and Ultimate plans. +SaaS runners on macOS are in [Beta](../../../policy/alpha-beta-support.md#beta) for approved open source programs and customers in Premium and Ultimate plans. SaaS runners on macOS provide an on-demand macOS build environment integrated with GitLab SaaS [CI/CD](../../../ci/index.md). @@ -84,4 +84,4 @@ In SaaS runners on macOS, the objective is to make 90% of CI jobs start executin - If the VM image does not include the specific software version you need for your job, then the job execution time will increase as the required software needs to be fetched and installed. - At this time, it is not possible to bring your own OS image. -- The keychain for user `gitlab` is not publicly available. You must create a keychain instead. +- The keychain for user `gitlab` is not publicly available. You must create a keychain instead. diff --git a/doc/ci/runners/saas/windows_saas_runner.md b/doc/ci/runners/saas/windows_saas_runner.md index 2c6540833b0..6e4ffc036b0 100644 --- a/doc/ci/runners/saas/windows_saas_runner.md +++ b/doc/ci/runners/saas/windows_saas_runner.md @@ -4,9 +4,9 @@ group: Runner info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# SaaS runners on Windows (beta) **(FREE SAAS)** +# SaaS runners on Windows (Beta) **(FREE SAAS)** -SaaS runners on Windows are in [beta](../../../policy/alpha-beta-support.md#beta-features) +SaaS runners on Windows are in [Beta](../../../policy/alpha-beta-support.md#beta) and shouldn't be used for production workloads. During this beta period, the [shared runner quota for CI/CD minutes](../../pipelines/cicd_minutes.md) @@ -126,7 +126,7 @@ test: ## Limitations and known issues -- All the limitations mentioned in our [beta definition](../../../policy/alpha-beta-support.md#beta-features). +- All the limitations mentioned in our [Beta definition](../../../policy/alpha-beta-support.md#beta). - The average provisioning time for a new Windows VM is 5 minutes. This means that you may notice slower build start times on the Windows runner fleet during the beta. In a future diff --git a/doc/ci/secrets/id_token_authentication.md b/doc/ci/secrets/id_token_authentication.md index f622bc2a7b1..1ff2a6efbcf 100644 --- a/doc/ci/secrets/id_token_authentication.md +++ b/doc/ci/secrets/id_token_authentication.md @@ -1,12 +1,14 @@ --- stage: Verify -group: Pipeline Authoring +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: tutorial --- # OpenID Connect (OIDC) Authentication Using ID Tokens **(FREE)** +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356986) in GitLab 15.7. + You can authenticate with third party services using GitLab CI/CD's [ID tokens](../yaml/index.md#id_tokens). @@ -35,60 +37,72 @@ services with which a token can authenticate. This reduces the severity of havin ### Token payload -The following fields are included in each ID token: +The following standard claims are included in each ID token: + +| Field | Description | +|--------------------------------------------------------------------|-------------| +| [`iss`](https://www.rfc-editor.org/rfc/rfc7519.html#section-4.1.1) | Issuer of the token, which is the domain of the GitLab instance ("issuer" claim). | +| [`sub`](https://www.rfc-editor.org/rfc/rfc7519.html#section-4.1.2) | `project_path:{group}/{project}:ref_type:{type}:ref:{branch_name}` ("subject" claim). | +| [`aud`](https://www.rfc-editor.org/rfc/rfc7519.html#section-4.1.3) | Intended audience for the token ("audience" claim). Specified in the [ID tokens](../yaml/index.md#id_tokens) configuration. The domain of the GitLab instance by default. | +| [`exp`](https://www.rfc-editor.org/rfc/rfc7519.html#section-4.1.4) | The expiration time ("expiration time" claim). | +| [`nbf`](https://www.rfc-editor.org/rfc/rfc7519.html#section-4.1.5) | The time after which the token becomes valid ("not before" claim). | +| [`iat`](https://www.rfc-editor.org/rfc/rfc7519.html#section-4.1.6) | The time the JWT was issued ("issued at" claim). | +| [`jti`](https://www.rfc-editor.org/rfc/rfc7519.html#section-4.1.7) | Unique identifier for the token ("JWT ID" claim). | + +The token also includes custom claims provided by GitLab: | Field | When | Description | |-------------------------|------------------------------|-------------| -| [`aud`](https://www.rfc-editor.org/rfc/rfc7519.html#section-4.1.3) | Always | Intended audience for the token ("audience" claim). Configured in GitLab the CI/CD configuration. The domain of the GitLab instance by default. | -| [`exp`](https://www.rfc-editor.org/rfc/rfc7519.html#section-4.1.4) | Always | The expiration time ("expiration time" claim). | -| [`iat`](https://www.rfc-editor.org/rfc/rfc7519.html#section-4.1.6) | Always | The time the JWT was issued ("issued at" claim). | -| [`iss`](https://www.rfc-editor.org/rfc/rfc7519.html#section-4.1.1) | Always | Issuer of the token, which is the domain of the GitLab instance ("issuer" claim). | -| [`jti`](https://www.rfc-editor.org/rfc/rfc7519.html#section-4.1.7) | Always | Unique identifier for the token ("JWT ID" claim). | -| [`nbf`](https://www.rfc-editor.org/rfc/rfc7519.html#section-4.1.5) | Always | The time after which the token becomes valid ("not before" claim). | -| [`sub`](https://www.rfc-editor.org/rfc/rfc7519.html#section-4.1.2) | Always | The job ID ("subject" claim). | -| `deployment_tier` | Job specifies an environment | [Deployment tier](../environments/index.md#deployment-tier-of-environments) of the environment the job specifies. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/363590) in GitLab 15.2. | -| `environment_protected` | Job specifies an environment | `true` if specified environment is protected, `false` otherwise. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/294440) in GitLab 13.9. | -| `environment` | Job specifies an environment | Environment the job specifies. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/294440) in GitLab 13.9. | -| `job_id` | Always | ID of the job. | -| `namespace_id` | Always | Use to scope to group or user level namespace by ID. | -| `namespace_path` | Always | Use to scope to group or user level namespace by path. | +| `namespace_id` | Always | Use this to scope to group or user level namespace by ID. | +| `namespace_path` | Always | Use this to scope to group or user level namespace by path. | +| `project_id` | Always | Use this to scope to project by ID. | +| `project_path` | Always | Use this to scope to project by path. | +| `user_id` | Always | ID of the user executing the job. | +| `user_login` | Always | Username of the user executing the job. | +| `user_email` | Always | Email of the user executing the job. | | `pipeline_id` | Always | ID of the pipeline. | | `pipeline_source` | Always | [Pipeline source](../jobs/job_control.md#common-if-clauses-for-rules). | -| `project_id` | Always | Use to scope to project by ID. | -| `project_path` | Always | Use to scope to project by path. | -| `ref_protected` | Always | `true` if the Git ref is protected, `false` otherwise. | -| `ref_type` | Always | Git ref type, either `branch` or `tag`. | +| `job_id` | Always | ID of the job. | | `ref` | Always | Git ref for the job. | -| `user_email` | Always | Email of the user executing the job. | -| `user_id` | Always | ID of the user executing the job. | -| `user_login` | Always | Username of the user executing the job. | - -Example ID token payload: +| `ref_type` | Always | Git ref type, either `branch` or `tag`. | +| `ref_path` | Always | Fully qualified ref for the job. For example, `refs/heads/main`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/119075) in GitLab 16.0. | +| `ref_protected` | Always | `true` if the Git ref is protected, `false` otherwise. | +| `environment` | Job specifies an environment | Environment this job deploys to ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/294440) in GitLab 13.9). | +| `environment_protected` | Job specifies an environment | `true` if deployed environment is protected, `false` otherwise ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/294440) in GitLab 13.9). | +| `deployment_tier` | Job specifies an environment | [Deployment tier](../environments/index.md#deployment-tier-of-environments) of the environment the job specifies. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/363590) in GitLab 15.2. | +| `runner_id` | Always | ID of the runner executing the job. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/404722) in GitLab 16.0. | +| `runner_environment` | Always | The type of runner used by the job. Can be either `gitlab-hosted` or `self-hosted`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/404722) in GitLab 16.0. | +| `sha` | Always | The commit SHA for the job. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/404722) in GitLab 16.0. | ```json { - "jti": "c82eeb0c-5c6f-4a33-abf5-4c474b92b558", - "aud": "hashicorp.example.com", - "iss": "gitlab.example.com", - "iat": 1585710286, - "nbf": 1585798372, - "exp": 1585713886, - "sub": "job_1212", - "namespace_id": "1", - "namespace_path": "mygroup", - "project_id": "22", - "project_path": "mygroup/myproject", - "user_id": "42", - "user_login": "myuser", - "user_email": "myuser@example.com", - "pipeline_id": "1212", - "pipeline_source": "web", - "job_id": "1212", - "ref": "auto-deploy-2020-04-01", + "namespace_id": "72", + "namespace_path": "my-group", + "project_id": "20", + "project_path": "my-group/my-project", + "user_id": "1", + "user_login": "sample-user", + "user_email": "sample-user@example.com", + "pipeline_id": "574", + "pipeline_source": "push", + "job_id": "302", + "ref": "feature-branch-1", "ref_type": "branch", - "ref_protected": "true", - "environment": "production", - "environment_protected": "true" + "ref_path": "refs/heads/feature-branch-1", + "ref_protected": "false", + "environment": "test-environment2", + "environment_protected": "false", + "deployment_tier": "testing", + "runner_id": 1, + "runner_environment": "self-hosted", + "sha": "714a629c0b401fdce83e847fc9589983fc6f46bc", + "jti": "235b3a54-b797-45c7-ae9a-f72d7bc6ef5b", + "iss": "https://gitlab.example.com", + "iat": 1681395193, + "nbf": 1681395188, + "exp": 1681398793, + "sub": "project_path:my-group/my-project:ref_type:branch:ref:feature-branch-1", + "aud": "https://vault.example.com" } ``` @@ -118,15 +132,6 @@ manual_authentication: You can use ID tokens to automatically fetch secrets from HashiCorp Vault with the [`secrets`](../yaml/index.md#secrets) keyword. -### Enable automatic ID token authentication - -To enable automatic ID token authentication: - -1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Settings > CI/CD**. -1. Expand **Token Access**. -1. Toggle **Limit JSON Web Token (JWT) access** to enabled. - ### Configure automatic ID Token authentication If one ID token is defined, the `secrets` keyword automatically uses it to authenticate with Vault. For example: @@ -163,3 +168,20 @@ job_with_secrets: - access-first-db.sh --token $FIRST_DB_PASSWORD - access-second-db.sh --token $SECOND_DB_PASSWORD ``` + + + +### Enable automatic ID token authentication (deprecated) + +WARNING: +This setting was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/391886) in GitLab 16.0. +ID token authentication is now always available, and JSON Web Token access is always limited. + +To enable automatic ID token authentication: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **Token Access**. +1. Toggle **Limit JSON Web Token (JWT) access** to enabled. + + diff --git a/doc/ci/secrets/index.md b/doc/ci/secrets/index.md index 14f771431e5..db9f36f295a 100644 --- a/doc/ci/secrets/index.md +++ b/doc/ci/secrets/index.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Authoring +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: concepts, howto --- @@ -23,14 +23,7 @@ GitLab has selected [Vault by HashiCorp](https://www.vaultproject.io) as the first supported provider, and [KV-V2](https://developer.hashicorp.com/vault/docs/secrets/kv/kv-v2) as the first supported secrets engine. -By default, GitLab authenticates using Vault's -[JSON Web Token (JWT) authentication method](https://developer.hashicorp.com/vault/docs/auth/jwt#jwt-authentication), using -the [JSON Web Token](https://gitlab.com/gitlab-org/gitlab/-/issues/207125) (`CI_JOB_JWT`). - -[ID tokens](../yaml/index.md#id_tokens) is the preferred secure way to authenticate with Vault, -because ID tokens are defined per-job. GitLab can also authenticate with Vault by using the `CI_JOB_JWT`, -but that token is provided to every job, which can be a security risk. - +Use [ID tokens](../yaml/index.md#id_tokens) to [authenticate with Vault](https://developer.hashicorp.com/vault/docs/auth/jwt#jwt-authentication). The [Authenticating and Reading Secrets With HashiCorp Vault](../examples/authenticating-with-hashicorp-vault/index.md) tutorial has more details about authenticating with ID tokens. @@ -40,7 +33,7 @@ can use [use Vault secrets in a CI job](#use-vault-secrets-in-a-ci-job). The flow for using GitLab with HashiCorp Vault is summarized by this diagram: -![Flow between GitLab and HashiCorp](../img/gitlab_vault_workflow_v13_4.png "How GitLab CI_JOB_JWT works with HashiCorp Vault") +![Flow between GitLab and HashiCorp](../img/gitlab_vault_workflow_v13_4.png "How GitLab authenticates with HashiCorp Vault") 1. Configure your vault and secrets. 1. Generate your JWT and provide it to your CI job. @@ -60,7 +53,7 @@ and supports multiple secrets engines. To configure your Vault server: -1. Ensure your Vault server is running on version 1.2.0 or higher. +1. Ensure your Vault server is running on version 1.2.0 or later. 1. Enable the authentication method by running these commands. They provide your Vault server the [JSON Web Key Set](https://www.rfc-editor.org/rfc/rfc7517) (JWKS) endpoint for your GitLab instance, so Vault can fetch the public signing key and verify the JSON Web Token (JWT) when authenticating: diff --git a/doc/ci/secure_files/index.md b/doc/ci/secure_files/index.md index 10baa57cabb..41e6fe06f84 100644 --- a/doc/ci/secure_files/index.md +++ b/doc/ci/secure_files/index.md @@ -1,26 +1,23 @@ --- stage: Verify -group: Pipeline Authoring +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: reference --- # Project-level Secure Files **(FREE)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78227) in GitLab 14.8. [Deployed behind the `ci_secure_files` flag](../../administration/feature_flags.md), disabled by default. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/350748) in GitLab 15.7. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78227) in GitLab 14.8 [with a flag](../../administration/feature_flags.md) named `ci_secure_files`. Disabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/350748) in GitLab 15.7. Feature flag `ci_secure_files` removed. -Project-level Secure Files is an experimental feature developed by [GitLab Incubation Engineering](https://about.gitlab.com/handbook/engineering/incubation/). +This feature is part of [Mobile DevOps](../mobile_devops.md) developed by [GitLab Incubation Engineering](https://about.gitlab.com/handbook/engineering/incubation/). The feature is still in development, but you can: - [Request a feature](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=feature_request). - [Report a bug](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=report_bug). - [Share feedback](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=general_feedback). -You can securely store up to 100 files for use in CI/CD pipelines as "secure files". These files -are stored securely outside of your project's repository, and are not version controlled. -It is safe to store sensitive information in these files. Secure files support both -plain text and binary file types, but must be 5 MB or less. +You can securely store up to 100 files for use in CI/CD pipelines as secure files. These files are stored securely outside of your project's repository and are not version controlled. It is safe to store sensitive information in these files. Secure files support both plain text and binary file types but must be 5 MB or less. You can manage secure files in the project settings, or with the [secure files API](../../api/secure_files.md). diff --git a/doc/ci/services/index.md b/doc/ci/services/index.md index f26751c56e7..2e7e89fc601 100644 --- a/doc/ci/services/index.md +++ b/doc/ci/services/index.md @@ -2,7 +2,6 @@ stage: Verify group: Runner info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false type: index --- @@ -27,7 +26,7 @@ It's easier and faster to use an existing image and run it as an additional cont than to install `mysql`, for example, every time the project is built. You're not limited to only database services. You can add as many -services you need to `.gitlab-ci.yml` or manually modify `config.toml`. +services you need to `.gitlab-ci.yml` or manually modify the [`config.toml`](https://docs.gitlab.com/runner/configuration/advanced-configuration.html). Any image found at [Docker Hub](https://hub.docker.com/) or your private Container Registry can be used as a service. @@ -190,7 +189,7 @@ following these rules: - Slash (`/`) is replaced with double underscores (`__`) and the primary alias is created. - Slash (`/`) is replaced with a single dash (`-`) and the secondary alias is - created (requires GitLab Runner v1.1.0 or higher). + created (requires GitLab Runner v1.1.0 or later). To override the default behavior, you can [specify a service alias](#available-settings-for-services). @@ -269,7 +268,7 @@ test: | `entrypoint` | no | 9.4 |Command or script to execute as the container's entrypoint. It's translated to the Docker `--entrypoint` option while creating the container. The syntax is similar to [`Dockerfile`'s `ENTRYPOINT`](https://docs.docker.com/engine/reference/builder/#entrypoint) directive, where each shell token is a separate string in the array. | | `command` | no | 9.4 |Command or script that should be used as the container's command. It's translated to arguments passed to Docker after the image's name. The syntax is similar to [`Dockerfile`'s `CMD`](https://docs.docker.com/engine/reference/builder/#cmd) directive, where each shell token is a separate string in the array. | | `alias` (1) | no | 9.4 | Additional alias that can be used to access the service from the job's container. Read [Accessing the services](#accessing-the-services) for more information. | -| `variables` (2) | no | 14.5 | Additional environment variables that are passed exclusively to the service. The syntax is the same as [Job Variables](../variables/index.md). | +| `variables` (2) | no | 14.5 | Additional environment variables that are passed exclusively to the service. The syntax is the same as [Job Variables](../variables/index.md). Service variables cannot reference themselves. | (1) Alias support for the Kubernetes executor was [introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2229) in GitLab Runner 12.8, and is only available for Kubernetes version 1.7 or later. diff --git a/doc/ci/ssh_keys/index.md b/doc/ci/ssh_keys/index.md index c1154485018..ab767697cbc 100644 --- a/doc/ci/ssh_keys/index.md +++ b/doc/ci/ssh_keys/index.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Execution +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: tutorial --- diff --git a/doc/ci/test_cases/index.md b/doc/ci/test_cases/index.md index 4088e5e82c6..edc962c9921 100644 --- a/doc/ci/test_cases/index.md +++ b/doc/ci/test_cases/index.md @@ -1,12 +1,12 @@ --- stage: Plan -group: Certify +group: Product Planning info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments description: Test cases in GitLab can help your teams create testing scenarios in their existing development platform. type: reference --- -# Test Cases **(ULTIMATE)** +# Test cases **(ULTIMATE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/233479) in GitLab 13.6. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/241983) in GitLab 13.7. @@ -16,6 +16,12 @@ Test cases in GitLab can help your teams create testing scenarios in their exist Now your Implementation and Testing teams can collaborate better, as they no longer have to use external test planning tools, which require additional overhead, context switching, and expense. +NOTE: +[Requirements](../../user/project/requirements/index.md) and test cases are being +[migrated to work items](https://gitlab.com/groups/gitlab-org/-/epics/5171). +[Issue 323790](https://gitlab.com/gitlab-org/gitlab/-/issues/323790) proposes to link requirements to test cases. +For more information, see [Product Stage Direction - Plan](https://about.gitlab.com/direction/plan/). + ## Create a test case Prerequisite: @@ -24,14 +30,14 @@ Prerequisite: To create a test case in a GitLab project: -1. Go to **CI/CD > Test Cases**. +1. Go to **CI/CD > Test cases**. 1. Select **New test case**. You are taken to the new test case form. Here you can enter the new case's title, [description](../../user/markdown.md), attach a file, and assign [labels](../../user/project/labels.md). 1. Select **Submit test case**. You are taken to view the new test case. ## View a test case -You can view all test cases in the project in the Test Cases list. Filter the +You can view all test cases in the project in the test cases list. Filter the issue list with a search query, including labels or the test case's title. Prerequisite: @@ -43,7 +49,7 @@ Whether you can view an test case depends on the [project visibility level](../. To view a test case: -1. In a project, go to **CI/CD > Test Cases**. +1. In a project, go to **CI/CD > Test cases**. 1. Select the title of the test case you want to view. You are taken to the test case page. ![An example test case page](img/test_case_show_v13_10.png) @@ -77,7 +83,7 @@ To archive a test case, on the test case's page, select **Archive test case**. To view archived test cases: -1. Go to **CI/CD > Test Cases**. +1. Go to **CI/CD > Test cases**. 1. Select **Archived**. ## Reopen an archived test case diff --git a/doc/ci/testing/accessibility_testing.md b/doc/ci/testing/accessibility_testing.md index fa57371a7d5..b03e4a23153 100644 --- a/doc/ci/testing/accessibility_testing.md +++ b/doc/ci/testing/accessibility_testing.md @@ -1,13 +1,18 @@ --- stage: Verify -group: Pipeline Insights +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Accessibility testing **(FREE)** + +# Accessibility testing (deprecated) **(FREE)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25144) in GitLab 12.8. +WARNING: +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/390424) in GitLab 15.9 +and is planned for removal in 17.0. This change is a breaking change. + If your application offers a web interface, you can use [GitLab CI/CD](../index.md) to determine the accessibility impact of pending code changes. @@ -25,7 +30,7 @@ As of [GitLab 14.5](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73309) ## Accessibility merge request widget > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/39425) in GitLab 13.0 behind the disabled [feature flag](../../administration/feature_flags.md) `:accessibility_report_view`. -> - [Feature Flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/217372) in GitLab 13.1. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/217372) in GitLab 13.1. GitLab displays an **Accessibility Report** in the merge request widget area: @@ -63,7 +68,7 @@ The `a11y` job in your CI/CD pipeline generates these files: file is named `gl-accessibility.json`. In GitLab versions 12.10 and earlier, this file is named [`accessibility.json`](https://gitlab.com/gitlab-org/ci-cd/accessibility/-/merge_requests/9). -You can [view job artifacts in your browser](../pipelines/job_artifacts.md#download-job-artifacts). +You can [view job artifacts in your browser](../jobs/job_artifacts.md#download-job-artifacts). NOTE: For GitLab versions earlier than 12.9, use `include:remote` and @@ -74,3 +79,5 @@ The job definition provided by the template does not support Kubernetes. You cannot pass configurations into Pa11y via CI configuration. To change the configuration, edit a copy of the template in your CI file. + + \ No newline at end of file diff --git a/doc/ci/testing/browser_performance_testing.md b/doc/ci/testing/browser_performance_testing.md index ff013f0037e..600b1a2cf4b 100644 --- a/doc/ci/testing/browser_performance_testing.md +++ b/doc/ci/testing/browser_performance_testing.md @@ -1,13 +1,18 @@ --- stage: Verify -group: Pipeline Insights +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- + # Browser Performance Testing **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3507) in GitLab 10.3. +WARNING: +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/388719) in GitLab 15.9 +and is planned for removal in 17.0. This change is a breaking change. + If your application offers a web interface and you're using [GitLab CI/CD](../index.md), you can quickly determine the rendering performance impact of pending code changes in the browser. diff --git a/doc/ci/testing/code_coverage.md b/doc/ci/testing/code_coverage.md new file mode 100644 index 00000000000..bd1246d2f78 --- /dev/null +++ b/doc/ci/testing/code_coverage.md @@ -0,0 +1,129 @@ +--- +stage: Verify +group: Pipeline Execution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Code coverage **(FREE)** + +Use code coverage to provide insights on what source code is being validated by a test suite. Code coverage is one of many test metrics that can determine software performance and quality. + +## View Code Coverage results + +Code Coverage results are shown in: + +- Merge request widget +- Project repository analytics +- Group repository analytics +- Repository badge + +For more information on test coverage visualization in the file diff of the MR, see [Test Coverage Visualization](test_coverage_visualization.md). + +### View code coverage results in the MR + +If you use test coverage in your code, you can use a regular expression to +find coverage results in the job log. You can then include these results +in the merge request in GitLab. + +If the pipeline succeeds, the coverage is shown in the merge request widget and +in the jobs table. If multiple jobs in the pipeline have coverage reports, they are +averaged. + +![MR widget coverage](img/pipelines_test_coverage_mr_widget.png) + +![Build status coverage](img/pipelines_test_coverage_build.png) + +#### Add test coverage results using `coverage` keyword + +To add test coverage results to a merge request using the project's `.gitlab-ci.yml` file, provide a regular expression +using the [`coverage`](../yaml/index.md#coverage) keyword. + +#### Test coverage examples + +Use this regex for commonly used test tools. + + + +- Simplecov (Ruby). Example: `/\(\d+.\d+\%\) covered/`. +- pytest-cov (Python). Example: `/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/`. +- Scoverage (Scala). Example: `/Statement coverage[A-Za-z\.*]\s*:\s*([^%]+)/`. +- `pest --coverage --colors=never` (PHP). Example: `/^\s*Cov:\s*\d+\.\d+?%$/`. +- `phpunit --coverage-text --colors=never` (PHP). Example: `/^\s*Lines:\s*\d+.\d+\%/`. +- gcovr (C/C++). Example: `/^TOTAL.*\s+(\d+\%)$/`. +- `tap --coverage-report=text-summary` (NodeJS). Example: `/^Statements\s*:\s*([^%]+)/`. +- `nyc npm test` (NodeJS). Example: `/All files[^|]*\|[^|]*\s+([\d\.]+)/`. +- `jest --ci --coverage` (NodeJS). Example: `/All files[^|]*\|[^|]*\s+([\d\.]+)/`. +- excoveralls (Elixir). Example: `/\[TOTAL\]\s+(\d+\.\d+)%/`. +- `mix test --cover` (Elixir). Example: `/\d+.\d+\%\s+\|\s+Total/`. +- JaCoCo (Java/Kotlin). Example: `/Total.*?([0-9]{1,3})%/`. +- `go test -cover` (Go). Example: `/coverage: \d+.\d+% of statements/`. +- .NET (OpenCover). Example: `/(Visited Points).*\((.*)\)/`. +- .NET (`dotnet test` line coverage). Example: `/Total\s*\|\s*(\d+(?:\.\d+)?)/`. +- tarpaulin (Rust). Example: `/^\d+.\d+% coverage/`. +- Pester (PowerShell). Example: `/Covered (\d+\.\d+%)/`. + + + +### View history of project code coverage + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/209121) the ability to download a `.csv` in GitLab 12.10. +> - Graph [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33743) in GitLab 13.1. + +To see the evolution of your project code coverage over time, +you can view a graph or download a CSV file with this data. + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Analytics > Repository**. + +The historic data for each job is listed in the dropdown list above the graph. + +To view a CSV file of the data, select **Download raw data (`.csv`)**. + +![Code coverage graph of a project over time](img/code_coverage_graph_v13_1.png) + +### View history of group code coverage **(PREMIUM)** + +To see the all the project's code coverage under a group over time, you can find view [group repository analytics](../../user/group/repositories_analytics/index.md). + +![Code coverage graph of a group over time](img/code_coverage_group_report.png) + +### Pipeline badges + +You can use [pipeline badges](../../user/project/badges.md#test-coverage-report-badges) to indicate the pipeline status and +test coverage of your projects. These badges are determined by the latest successful pipeline. + +## Coverage check approval rule **(PREMIUM)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15765) in GitLab 14.0. +> - [Made configurable in Project Settings](https://gitlab.com/gitlab-org/gitlab/-/issues/331001) in GitLab 14.1. + +When merging a request that would cause the project's test coverage to decline, you can stipulate that such merge requests require approval by selected users or a group. + +Follow these steps to enable the `Coverage-Check` MR approval rule: + +1. Set up a [`coverage`](../yaml/index.md#coverage) regular expression for all jobs you want to include in the overall coverage value. +1. Go to your project and select **Settings > Merge requests**. +1. Under **Merge request approvals**, select **Enable** next to the `Coverage-Check` approval rule. +1. Select the **Target branch**. +1. Set the number of **Approvals required** to greater than zero. +1. Select the users or groups to provide approval. +1. Select **Add approval rule**. + +![Coverage-Check approval rule](img/coverage_check_approval_rule_14_1.png) + +## Troubleshooting + +### Remove color codes from code coverage + +Some test coverage tools output with ANSI color codes that aren't +parsed correctly by the regular expression. This causes coverage +parsing to fail. + +Some coverage tools do not provide an option to disable color +codes in the output. If so, pipe the output of the coverage tool through a one-line script that strips the color codes. + +For example: + +```shell +lein cloverage | perl -pe 's/\e\[?.*?[\@-~]//g' +``` diff --git a/doc/ci/testing/code_quality.md b/doc/ci/testing/code_quality.md index 4b826991bb5..367777960b5 100644 --- a/doc/ci/testing/code_quality.md +++ b/doc/ci/testing/code_quality.md @@ -44,6 +44,7 @@ Code Quality results are shown in the: - Merge request widget - Merge request changes view - Pipeline details view +- Project quality view ### Merge request widget @@ -73,6 +74,12 @@ tab of the pipeline's details page. ![Code Quality Report](img/code_quality_report_13_11.png) +### Project quality view **(ULTIMATE)** + +The project quality view displays an overview of the code quality findings. The view can be found under **Analytics > CI/CD**, and requires [`project_quality_summary_page`](../../user/feature_flags.md) feature flag to be enabled for this particular project. + +![Code Quality Summary](img/code_quality_summary_15_9.png) + ## Enable Code Quality Prerequisites: @@ -119,7 +126,7 @@ Quality because: This alternative configuration uses socket binding to share the Runner's Docker daemon with the job environment. Before implementing this configuration, consider its -[limitations](../docker/using_docker_build.md#use-docker-socket-binding). +[limitations](../docker/using_docker_build.md#use-the-docker-executor-with-docker-socket-binding). To use private runners: @@ -253,7 +260,7 @@ code_quality: ``` The full JSON file is available as a -[downloadable artifact](../pipelines/job_artifacts.md#download-job-artifacts) of the `code_quality` +[downloadable artifact](../jobs/job_artifacts.md#download-job-artifacts) of the `code_quality` job. ### Download output in JSON and HTML format @@ -280,7 +287,7 @@ code_quality_html: ``` Both the JSON and HTML files are available as -[downloadable artifacts](../pipelines/job_artifacts.md#download-job-artifacts) of the `code_quality` +[downloadable artifacts](../jobs/job_artifacts.md#download-job-artifacts) of the `code_quality` job. ### Download output in only HTML format @@ -304,7 +311,7 @@ code_quality: ``` The HTML file is available as a -[downloadable artifact](../pipelines/job_artifacts.md#download-job-artifacts) of the `code_quality` +[downloadable artifact](../jobs/job_artifacts.md#download-job-artifacts) of the `code_quality` job. ## Use Code Quality with merge request pipelines @@ -448,6 +455,7 @@ properties: | Name | Description | | ---------------------- | ----------------------------------------------------------------------------------------- | | `description` | A description of the code quality violation. | +| `check_name` | A unique name representing the static analysis check that emitted this issue. | | `fingerprint` | A unique fingerprint to identify the code quality violation. For example, an MD5 hash. | | `severity` | A severity string (can be `info`, `minor`, `major`, `critical`, or `blocker`). | | `location.path` | The relative path to the file containing the code quality violation. | @@ -471,6 +479,7 @@ Example: [ { "description": "'unused' is assigned a value but never used.", + "check_name": "no-unused-vars", "fingerprint": "7815696ecbf1c96e6894b779456d330e", "severity": "minor", "location": { diff --git a/doc/ci/testing/fail_fast_testing.md b/doc/ci/testing/fail_fast_testing.md index 4ad1656c8d6..6b1ed4a594d 100644 --- a/doc/ci/testing/fail_fast_testing.md +++ b/doc/ci/testing/fail_fast_testing.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Insights +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -34,7 +34,7 @@ Fail fast testing gives you a faster feedback loop from the pipeline. It lets yo know quickly that the new tests are passing and the new functionality did not break other tests. -## Requirements +## Prerequisites This template requires: diff --git a/doc/ci/testing/img/code_coverage_graph_v13_1.png b/doc/ci/testing/img/code_coverage_graph_v13_1.png new file mode 100644 index 00000000000..da92a1b4a86 Binary files /dev/null and b/doc/ci/testing/img/code_coverage_graph_v13_1.png differ diff --git a/doc/ci/testing/img/code_coverage_group_report.png b/doc/ci/testing/img/code_coverage_group_report.png new file mode 100644 index 00000000000..3e28a9b4cdd Binary files /dev/null and b/doc/ci/testing/img/code_coverage_group_report.png differ diff --git a/doc/ci/testing/img/code_quality_summary_15_9.png b/doc/ci/testing/img/code_quality_summary_15_9.png new file mode 100644 index 00000000000..60077123488 Binary files /dev/null and b/doc/ci/testing/img/code_quality_summary_15_9.png differ diff --git a/doc/ci/testing/img/coverage_check_approval_rule_14_1.png b/doc/ci/testing/img/coverage_check_approval_rule_14_1.png new file mode 100644 index 00000000000..8c1d005e074 Binary files /dev/null and b/doc/ci/testing/img/coverage_check_approval_rule_14_1.png differ diff --git a/doc/ci/testing/img/pipelines_test_coverage_build.png b/doc/ci/testing/img/pipelines_test_coverage_build.png new file mode 100644 index 00000000000..7eaba1a256f Binary files /dev/null and b/doc/ci/testing/img/pipelines_test_coverage_build.png differ diff --git a/doc/ci/testing/img/pipelines_test_coverage_mr_widget.png b/doc/ci/testing/img/pipelines_test_coverage_mr_widget.png new file mode 100644 index 00000000000..fbcd612f3f2 Binary files /dev/null and b/doc/ci/testing/img/pipelines_test_coverage_mr_widget.png differ diff --git a/doc/ci/testing/index.md b/doc/ci/testing/index.md index 41d474f0e60..a8fb6d688d7 100644 --- a/doc/ci/testing/index.md +++ b/doc/ci/testing/index.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Insights +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -14,8 +14,9 @@ display reports or link to important information directly from [merge requests]( | [Accessibility Testing](accessibility_testing.md) | Automatically report A11y violations for changed pages in merge requests. | | [Browser Performance Testing](browser_performance_testing.md) | Quickly determine the browser performance impact of pending code changes. | | [Load Performance Testing](load_performance_testing.md) | Quickly determine the server performance impact of pending code changes. | +| [Code Coverage](code_coverage.md) | See code coverage results in the MR, project or group. | | [Code Quality](code_quality.md) | Analyze your source code quality using the [Code Climate](https://codeclimate.com/) analyzer and show the Code Climate report right in the merge request widget area. | -| [Display arbitrary job artifacts](../yaml/index.md#artifactsexpose_as) | Configure CI pipelines with the `artifacts:expose_as` parameter to directly link to selected [artifacts](../pipelines/job_artifacts.md) in merge requests. | +| [Display arbitrary job artifacts](../yaml/index.md#artifactsexpose_as) | Configure CI pipelines with the `artifacts:expose_as` parameter to directly link to selected [artifacts](../jobs/job_artifacts.md) in merge requests. | | [Unit test reports](unit_test_reports.md) | Configure your CI jobs to use Unit test reports, and let GitLab display a report on the merge request so that it's easier and faster to identify the failure without having to check the entire job log. | | [License Compliance](../../user/compliance/license_compliance/index.md) | Manage the licenses of your dependencies. | | [Metrics Reports](metrics_reports.md) | Display the Metrics Report on the merge request so that it's fast and easier to identify changes to important metrics. | diff --git a/doc/ci/testing/load_performance_testing.md b/doc/ci/testing/load_performance_testing.md index 7e527fae562..2897d7fe0ab 100644 --- a/doc/ci/testing/load_performance_testing.md +++ b/doc/ci/testing/load_performance_testing.md @@ -1,13 +1,18 @@ --- stage: Verify -group: Pipeline Insights +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Load Performance Testing **(PREMIUM)** + +# Load Performance Testing (deprecated) **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10683) in GitLab 13.2. +WARNING: +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/388723) in GitLab 15.9 +and is planned for removal in 17.0. This change is a breaking change. + With Load Performance Testing, you can test the impact of any pending code changes to your application's backend in [GitLab CI/CD](../index.md). @@ -161,7 +166,7 @@ For example: 1. In the `review` job: 1. Capture the dynamic URL and save it into a `.env` file, for example, `echo "ENVIRONMENT_URL=$CI_ENVIRONMENT_URL" >> review.env`. - 1. Set the `.env` file to be a [job artifact](../pipelines/job_artifacts.md). + 1. Set the `.env` file to be a [job artifact](../jobs/job_artifacts.md). 1. In the `load_performance` job: 1. Set it to depend on the review job, so it inherits the environment file. 1. Set the `K6_DOCKER_OPTIONS` variable with the [Docker CLI option for environment files](https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file), for example `--env-file review.env`. @@ -199,3 +204,5 @@ load_performance: rules: - if: $CI_COMMIT_BRANCH # Modify to match your pipeline rules, or use `only/except` if needed. ``` + + diff --git a/doc/ci/testing/metrics_reports.md b/doc/ci/testing/metrics_reports.md index e084e4d3bc7..9257ba8990e 100644 --- a/doc/ci/testing/metrics_reports.md +++ b/doc/ci/testing/metrics_reports.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Insights +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/ci/testing/test_coverage_visualization.md b/doc/ci/testing/test_coverage_visualization.md index e2a1a4a2c5e..31d1b7f3337 100644 --- a/doc/ci/testing/test_coverage_visualization.md +++ b/doc/ci/testing/test_coverage_visualization.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Insights +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -33,7 +33,7 @@ This format was originally developed for Java, but most coverage analysis framew for other languages have plugins to add support for it, like: - [simplecov-cobertura](https://rubygems.org/gems/simplecov-cobertura) (Ruby) -- [gocover-cobertura](https://github.com/boumenot/gocover-cobertura) (Golang) +- [gocover-cobertura](https://github.com/boumenot/gocover-cobertura) (Go) Other coverage analysis frameworks support the format out of the box, for example: @@ -54,8 +54,8 @@ of times the line was checked by tests. Uploading a test coverage report does not enable: -- [Test coverage results in merge requests](../pipelines/settings.md#merge-request-test-coverage-results). -- [Code coverage history](../pipelines/settings.md#view-code-coverage-history). +- [Test coverage results in merge requests](code_coverage.md#view-code-coverage-results-in-the-mr). +- [Code coverage history](code_coverage.md#view-history-of-project-code-coverage). You must configure these separately. @@ -112,24 +112,14 @@ attempts to build the full path by: #### Path correction example -As an example, a project with: +As an example, a C# project with: -- A full path of `test-org/test-project`. +- A full path of `test-org/test-cs-project`. - The following files relative to the project root: ```shell Auth/User.cs Lib/Utils/User.cs - src/main/java - ``` - -In the: - -- Cobertura XML, the `filename` attribute in the `class` element assumes the value is a relative - path to the project's root: - - ```xml - ``` - `sources` from Cobertura XML, the following paths in the format @@ -137,8 +127,8 @@ In the: ```xml - /builds/test-org/test-project/Auth - /builds/test-org/test-project/Lib/Utils + /builds/test-org/test-cs-project/Auth + /builds/test-org/test-cs-project/Lib/Utils ``` @@ -153,6 +143,29 @@ The parser: 100 iterations. If it reaches this limit without finding a matching path in the file tree, the class is not included in the final coverage report. +Automatic class path correction also works for a Java project with: + +- A full path of `test-org/test-java-project`. +- The following files relative to the project root: + + ```shell + src/main/java/com/gitlab/security_products/tests/App.java + ``` + +- `sources` from Cobertura XML: + + ```xml + + /builds/test-org/test-java-project/src/main/java/ + + ``` + +- `class` element with the `filename` value of `com/gitlab/security_products/tests/App.java`: + + ```xml + + ``` + NOTE: Automatic class path correction only works on `source` paths in the format `//...`. The `source` is ignored if the path does not follow this pattern. The parser assumes that the @@ -261,10 +274,7 @@ coverage-jdk11: ### Python example -The following [`.gitlab-ci.yml`](../yaml/index.md) example for Python uses [pytest-cov](https://pytest-cov.readthedocs.io/) to collect test coverage data and [coverage.py](https://coverage.readthedocs.io/) to convert the report to use full relative paths. -The information isn't displayed without the conversion. - -This example assumes that the code for your package is in `src/` and your tests are in `tests.py`: +The following [`.gitlab-ci.yml`](../yaml/index.md) example uses [pytest-cov](https://pytest-cov.readthedocs.io/) to collect test coverage data: ```yaml run tests: @@ -272,9 +282,7 @@ run tests: image: python:3 script: - pip install pytest pytest-cov - - coverage run -m pytest - - coverage report - - coverage xml + - pytest --cov --cov-report term --cov-report xml:coverage.xml coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/' artifacts: reports: diff --git a/doc/ci/testing/unit_test_report_examples.md b/doc/ci/testing/unit_test_report_examples.md index 87426fc8496..c63e225a2a7 100644 --- a/doc/ci/testing/unit_test_report_examples.md +++ b/doc/ci/testing/unit_test_report_examples.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Insights +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -274,3 +274,24 @@ phpunit: reports: junit: report.xml ``` + +## Rust + +This example uses [cargo2junit](https://crates.io/crates/cargo2junit), +which is installed in the current directory. +To retrieve JSON output from `cargo test`, you must enable the nightly compiler. + +```yaml +run unittests: + image: rust:latest + stage: test + before_script: + - cargo install --root . cargo2junit + script: + - cargo test -- -Z unstable-options --format json --report-time | bin/cargo2junit > report.xml + artifacts: + when: always + reports: + junit: + - report.xml +``` diff --git a/doc/ci/testing/unit_test_reports.md b/doc/ci/testing/unit_test_reports.md index 0fe9b2b6d64..88a5d90d30e 100644 --- a/doc/ci/testing/unit_test_reports.md +++ b/doc/ci/testing/unit_test_reports.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Insights +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/ci/triggers/index.md b/doc/ci/triggers/index.md index 16530ea20b6..b9e5dd87b24 100644 --- a/doc/ci/triggers/index.md +++ b/doc/ci/triggers/index.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Execution +group: Pipeline Authoring info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: tutorial --- @@ -187,7 +187,12 @@ In pipelines triggered with a trigger token, jobs are labeled as `triggered` in ## Troubleshooting -### `404 not found` when triggering a pipeline +### `403 Forbidden` when you trigger a pipeline with a webhook + +When you trigger a pipeline with a webhook, the API might return a `{"message":"403 Forbidden"}` response. +To avoid trigger loops, do not use [pipeline events](../../user/project/integrations/webhook_events.md#pipeline-events) to trigger pipelines. + +### `404 Not Found` when triggering a pipeline A response of `{"message":"404 Not Found"}` when triggering a pipeline might be caused by using a [personal access token](../../user/profile/personal_access_tokens.md) @@ -201,3 +206,13 @@ doesn't exist, GitLab returns `The requested URL returned error: 400`. For example, you might accidentally use `main` for the branch name in a project that uses a different branch name for its default branch. + +Another possible cause for this error is a rule that prevents creation of the pipelines when `CI_PIPELINE_SOURCE` value is `trigger`, such as: + +```yaml +rules: + - if: $CI_PIPELINE_SOURCE == "trigger" + when: never +``` + +Review your [`workflow:rules`](../yaml/index.md#workflowrules) to ensure a pipeline can be created when `CI_PIPELINE_SOURCE` value is `trigger`. diff --git a/doc/ci/troubleshooting.md b/doc/ci/troubleshooting.md index 17ce184ee28..973c6b90fc5 100644 --- a/doc/ci/troubleshooting.md +++ b/doc/ci/troubleshooting.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Execution +group: Pipeline Authoring info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: reference --- @@ -37,7 +37,7 @@ If you need to link to the schema directly, it is at: ```plaintext -https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/editor/schema/ci.json. +https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/editor/schema/ci.json ``` To see the full list of custom tags covered by the CI/CD schema, check the @@ -74,7 +74,8 @@ and [templates](examples/index.md#cicd-templates). ### Documentation for pipeline types -Some pipeline types have their own detailed usage guides that you should read +Branch pipelines are the most basic type. +Other pipeline types have their own detailed usage guides that you should read if you are using that type: - [Multi-project pipelines](pipelines/downstream_pipelines.md#multi-project-pipelines): Have your pipeline trigger @@ -269,6 +270,7 @@ This message is shown if the [merge request pipeline](pipelines/merge_request_pi [merged results pipeline](pipelines/merged_results_pipelines.md), or [merge train pipeline](pipelines/merge_trains.md) has failed or been canceled. +This does not happen when a basic branch pipeline fails. If a merge request pipeline or merged result pipeline was canceled or failed, you can: @@ -311,7 +313,7 @@ likely to hit the default memory limit. To reduce the configuration size, you can: - Check the length of the expanded CI/CD configuration in the pipeline editor's - [merged YAML](pipeline_editor/index.md#view-expanded-configuration) tab. Look for + [Full configuration](pipeline_editor/index.md#view-full-configuration) tab. Look for duplicated configuration that can be removed or simplified. - Move long or repeated `script` sections into standalone scripts in the project. - Use [parent and child pipelines](pipelines/downstream_pipelines.md#parent-child-pipelines) to move some diff --git a/doc/ci/variables/index.md b/doc/ci/variables/index.md index ec5c6c240a6..64a0f1038ad 100644 --- a/doc/ci/variables/index.md +++ b/doc/ci/variables/index.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Authoring +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: reference --- @@ -19,12 +19,13 @@ or have them [prefilled in manual pipelines](../pipelines/index.md#prefill-varia Variable names are limited by the [shell the runner uses](https://docs.gitlab.com/runner/shells/index.html) to execute scripts. Each shell has its own set of reserved variable names. -> For more information about advanced use of GitLab CI/CD: -> -> -  Get to productivity faster with these [7 advanced GitLab CI workflow hacks](https://about.gitlab.com/webcast/7cicd-hacks/) -> shared by GitLab engineers. -> -  Learn how the Cloud Native Computing Foundation (CNCF) [eliminates the complexity](https://about.gitlab.com/customers/cncf/) -> of managing projects across many cloud providers with GitLab CI/CD. +To ensure consistent behavior, you should always put variable values in single or double quotes. +Variables are internally parsed by the [Psych YAML parser](https://docs.ruby-lang.org/en/master/Psych.html), +so quoted and unquoted variables might be parsed differently. For example, `VAR1: 012345` +is interpreted as an octal value, so the value becomes `5349`, but `VAR1: "012345"` is parsed +as a string with a value of `012345`. + +For more information about advanced use of GitLab CI/CD, see [7 advanced GitLab CI workflow hacks](https://about.gitlab.com/webcast/7cicd-hacks/) shared by GitLab engineers. ## Predefined CI/CD variables @@ -189,7 +190,7 @@ You can make a CI/CD variable available to all projects and groups in a GitLab i Prerequisite: -- You must have administrator access. +- You must have administrator access to the instance. To add an instance variable: @@ -206,9 +207,6 @@ To add an instance variable: - **Mask variable** Optional. If selected, the variable's **Value** is not shown in job logs. The variable is not saved if the value does not meet the [masking requirements](#mask-a-cicd-variable). -The instance variables that are available in a project are listed in the project's -**Settings > CI/CD > Variables** section. - ## CI/CD variable security Code pushed to the `.gitlab-ci.yml` file could compromise your variables. Variables could @@ -471,7 +469,8 @@ To pass a job-created environment variable to other jobs: - Values can be wrapped in quotes, but cannot contain newline characters. 1. Save the `.env` file as an [`artifacts:reports:dotenv`](../yaml/artifacts_reports.md#artifactsreportsdotenv) artifact. -1. Jobs in later stages can then [use the variable in scripts](#use-cicd-variables-in-job-scripts). +1. Jobs in later stages can then [use the variable in scripts](#use-cicd-variables-in-job-scripts), + unless [jobs are configured not to receive `dotenv` variables](#control-which-jobs-receive-dotenv-variables). For example: @@ -743,7 +742,7 @@ export CI_JOB_ID="50" export CI_COMMIT_SHA="1ecfd275763eff1d6b4844ea3168962458c9f27a" export CI_COMMIT_SHORT_SHA="1ecfd275" export CI_COMMIT_REF_NAME="main" -export CI_REPOSITORY_URL="https://gitlab-ci-token:[masked]@example.com/gitlab-org/gitlab-foss.git" +export CI_REPOSITORY_URL="https://gitlab-ci-token:[masked]@example.com/gitlab-org/gitlab.git" export CI_COMMIT_TAG="1.0.0" export CI_JOB_NAME="spec:other" export CI_JOB_STAGE="test" @@ -753,11 +752,11 @@ export CI_JOB_TOKEN="[masked]" export CI_PIPELINE_ID="1000" export CI_PIPELINE_IID="10" export CI_PAGES_DOMAIN="gitlab.io" -export CI_PAGES_URL="https://gitlab-org.gitlab.io/gitlab-foss" +export CI_PAGES_URL="https://gitlab-org.gitlab.io/gitlab" export CI_PROJECT_ID="34" -export CI_PROJECT_DIR="/builds/gitlab-org/gitlab-foss" -export CI_PROJECT_NAME="gitlab-foss" -export CI_PROJECT_TITLE="GitLab FOSS" +export CI_PROJECT_DIR="/builds/gitlab-org/gitlab" +export CI_PROJECT_NAME="gitlab" +export CI_PROJECT_TITLE="GitLab" ... ``` @@ -845,6 +844,10 @@ if [[ -d "/builds/gitlab-examples/ci-debug-trace/.git" ]]; then ++ CI_SERVER_HOST=gitlab.com ++ export CI_SERVER_PORT=3000 ++ CI_SERVER_PORT=3000 +++ export CI_SERVER_SHELL_SSH_HOST=gitlab.com +++ CI_SERVER_SHELL_SSH_HOST=gitlab.com +++ export CI_SERVER_SHELL_SSH_PORT=22 +++ CI_SERVER_SHELL_SSH_PORT=22 ++ export CI_SERVER_PROTOCOL=https ++ CI_SERVER_PROTOCOL=https ++ export CI_SERVER_NAME=GitLab diff --git a/doc/ci/variables/predefined_variables.md b/doc/ci/variables/predefined_variables.md index b9557b98066..001a599776a 100644 --- a/doc/ci/variables/predefined_variables.md +++ b/doc/ci/variables/predefined_variables.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Authoring +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: reference --- @@ -27,6 +27,7 @@ as it can cause the pipeline to behave unexpectedly. | `CHAT_USER_ID` | 14.4 | all | The chat service's user ID of the user who triggered the [ChatOps](../chatops/index.md) command. | | `CI` | all | 0.4 | Available for all jobs executed in CI/CD. `true` when available. | | `CI_API_V4_URL` | 11.7 | all | The GitLab API v4 root URL. | +| `CI_API_GRAPHQL_URL` | 15.11 | all | The GitLab API GraphQL root URL. | | `CI_BUILDS_DIR` | all | 11.10 | The top-level directory where builds are executed. | | `CI_COMMIT_AUTHOR` | 13.11 | all | The author of the commit in `Name ` format. | | `CI_COMMIT_BEFORE_SHA` | 11.2 | all | The previous latest commit present on a branch or tag. Is always `0000000000000000000000000000000000000000` in merge request pipelines and for the first commit in pipelines for branches or tags. | @@ -67,9 +68,9 @@ as it can cause the pipeline to behave unexpectedly. | `CI_HAS_OPEN_REQUIREMENTS` | 13.1 | all | Only available if the pipeline's project has an open [requirement](../../user/project/requirements/index.md). `true` when available. | | `CI_JOB_ID` | 9.0 | all | The internal ID of the job, unique across all jobs in the GitLab instance. | | `CI_JOB_IMAGE` | 12.9 | 12.9 | The name of the Docker image running the job. | -| `CI_JOB_JWT` | 12.10 | all | A RS256 JSON web token to authenticate with third party systems that support JWT authentication, for example [HashiCorp's Vault](../secrets/index.md). | -| `CI_JOB_JWT_V1` | 14.6 | all | The same value as `CI_JOB_JWT`. | -| `CI_JOB_JWT_V2` | 14.6 | all | A newly formatted RS256 JSON web token to increase compatibility. Similar to `CI_JOB_JWT`, except the issuer (`iss`) claim is changed from `gitlab.com` to `https://gitlab.com`, `sub` has changed from `job_id` to a string that contains the project path, and an `aud` claim is added. Format is subject to change. Be aware, the `aud` field is a constant value. Trusting JWTs in multiple relying parties can lead to [one RP sending a JWT to another one and acting maliciously as a job](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/72555#note_769112331). **Note:** The `CI_JOB_JWT_V2` variable is available for testing, but the full feature is planned to be generally available when [issue 360657](https://gitlab.com/gitlab-org/gitlab/-/issues/360657) is complete.| +| `CI_JOB_JWT` (Deprecated) | 12.10 | all | A RS256 JSON web token to authenticate with third party systems that support JWT authentication, for example [HashiCorp's Vault](../secrets/index.md). [Deprecated in GitLab 15.9](../../update/deprecations.md#old-versions-of-json-web-tokens-are-deprecated) and scheduled to be removed in GitLab 16.5. Use [ID tokens](../yaml/index.md#id_tokens) instead. | +| `CI_JOB_JWT_V1` (Deprecated) | 14.6 | all | The same value as `CI_JOB_JWT`. [Deprecated in GitLab 15.9](../../update/deprecations.md#old-versions-of-json-web-tokens-are-deprecated) and scheduled to be removed in GitLab 16.5. Use [ID tokens](../yaml/index.md#id_tokens) instead. | +| `CI_JOB_JWT_V2` (Deprecated) | 14.6 | all | A newly formatted RS256 JSON web token to increase compatibility. Similar to `CI_JOB_JWT`, except the issuer (`iss`) claim is changed from `gitlab.com` to `https://gitlab.com`, `sub` has changed from `job_id` to a string that contains the project path, and an `aud` claim is added. The `aud` field is a constant value. Trusting JWTs in multiple relying parties can lead to [one RP sending a JWT to another one and acting maliciously as a job](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/72555#note_769112331). [Deprecated in GitLab 15.9](../../update/deprecations.md#old-versions-of-json-web-tokens-are-deprecated) and scheduled to be removed in GitLab 16.5. Use [ID tokens](../yaml/index.md#id_tokens) instead. | | `CI_JOB_MANUAL` | 8.12 | all | Only available if the job was started manually. `true` when available. | | `CI_JOB_NAME` | 9.0 | 0.5 | The name of the job. | | `CI_JOB_NAME_SLUG` | 15.4 | all | `CI_JOB_NAME_SLUG` in lowercase, shortened to 63 bytes, and with everything except `0-9` and `a-z` replaced with `-`. No leading / trailing `-`. Use in paths. | @@ -121,6 +122,8 @@ as it can cause the pipeline to behave unexpectedly. | `CI_SERVER_NAME` | all | all | The name of CI/CD server that coordinates jobs. | | `CI_SERVER_PORT` | 12.8 | all | The port of the GitLab instance URL, without host or protocol. For example `8080`. | | `CI_SERVER_PROTOCOL` | 12.8 | all | The protocol of the GitLab instance URL, without host or port. For example `https`. | +| `CI_SERVER_SHELL_SSH_HOST` | 15.11 | all | The SSH host of the GitLab instance, used for access to Git repositories via SSH. For example `gitlab.com`. | +| `CI_SERVER_SHELL_SSH_PORT` | 15.11 | all | The SSH port of the GitLab instance, used for access to Git repositories via SSH. For example `22`. | | `CI_SERVER_REVISION` | all | all | GitLab revision that schedules jobs. | | `CI_SERVER_TLS_CA_FILE` | all | all | File containing the TLS CA certificate to verify the GitLab server when `tls-ca-file` set in [runner settings](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section). | | `CI_SERVER_TLS_CERT_FILE` | all | all | File containing the TLS certificate to verify the GitLab server when `tls-cert-file` set in [runner settings](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section). | @@ -139,6 +142,7 @@ as it can cause the pipeline to behave unexpectedly. | `GITLAB_USER_ID` | 8.12 | all | The ID of the user who started the pipeline, unless the job is a manual job. In manual jobs, the value is the ID of the user who started the job. | | `GITLAB_USER_LOGIN` | 10.0 | all | The username of the user who started the pipeline, unless the job is a manual job. In manual jobs, the value is the username of the user who started the job. | | `GITLAB_USER_NAME` | 10.0 | all | The name of the user who started the pipeline, unless the job is a manual job. In manual jobs, the value is the name of the user who started the job. | +| `KUBECONFIG` | 14.2 | all | The path to the `kubeconfig` file with contexts for every shared agent connection. Only available when a [GitLab agent is authorized to access the project](../../user/clusters/agent/ci_cd_workflow.md#authorize-the-agent). | | `TRIGGER_PAYLOAD` | 13.9 | all | The webhook payload. Only available when a pipeline is [triggered with a webhook](../triggers/index.md#use-a-webhook-payload). | ## Predefined variables for merge request pipelines diff --git a/doc/ci/variables/where_variables_can_be_used.md b/doc/ci/variables/where_variables_can_be_used.md index 6ea1f454467..c20d9be51e7 100644 --- a/doc/ci/variables/where_variables_can_be_used.md +++ b/doc/ci/variables/where_variables_can_be_used.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Authoring +group: Pipeline Security info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: reference --- diff --git a/doc/ci/yaml/artifacts_reports.md b/doc/ci/yaml/artifacts_reports.md index 151a043da5e..78c9e98c33f 100644 --- a/doc/ci/yaml/artifacts_reports.md +++ b/doc/ci/yaml/artifacts_reports.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Insights +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -22,7 +22,7 @@ date for the artifacts. Some `artifacts:reports` types can be generated by multiple jobs in the same pipeline, and used by merge request or pipeline features from each job. -To be able to browse the report output files, include the [`artifacts:paths`](index.md#artifactspaths) keyword. +To browse the report output files, ensure you include the [`artifacts:paths`](index.md#artifactspaths) keyword in your job definition. NOTE: Combined reports in parent pipelines using [artifacts from child pipelines](index.md#needspipelinejob) is @@ -142,9 +142,10 @@ following the [CycloneDX](https://cyclonedx.org/docs/1.4) protocol format. You can specify multiple CycloneDX reports per job. These can be either supplied as a list of filenames, a filename pattern, or both: -- List of filenames: `cyclonedx: [gl-sbom-npm-npm.cdx.json, gl-sbom-bundler-gem.cdx.json]`. -- A filename pattern: `cyclonedx: gl-sbom-*.json`. -- Combination of both of the above: `cyclonedx: [gl-sbom-*.json, my-cyclonedx.json]`. +- A filename pattern (`cyclonedx: gl-sbom-*.json`, `junit: test-results/**/*.json`). +- An array of filenames (`cyclonedx: [gl-sbom-npm-npm.cdx.json, gl-sbom-bundler-gem.cdx.json]`). +- A combination of both (`cyclonedx: [gl-sbom-*.json, my-cyclonedx.json]`). +- Directories are not supported(`cyclonedx: test-results`, `cyclonedx: test-results/**`). Below is an example of a job exposing CycloneDX artifacts: @@ -209,7 +210,7 @@ The exceptions to the [original dotenv rules](https://github.com/motdotla/dotenv - The `.env` file can't have empty lines or comments (starting with `#`). - Key values in the `env` file cannot have spaces or newline characters (`\n`), including when using single or double quotes. - Quote escaping during parsing (`key = 'value'` -> `{key: "value"}`) is not supported. -- Only UTF-8 encoding is [supported](../pipelines/job_artifacts.md#error-message-fatal-invalid-argument-when-uploading-a-dotenv-artifact-on-a-windows-runner). +- Only UTF-8 encoding is [supported](../jobs/job_artifacts_troubleshooting.md#error-message-fatal-invalid-argument-when-uploading-a-dotenv-artifact-on-a-windows-runner). ## `artifacts:reports:junit` @@ -239,9 +240,10 @@ GitLab can display the results of one or more reports in: Some JUnit tools export to multiple XML files. You can specify multiple test report paths in a single job to concatenate them into a single file. Use either: -- A filename pattern (`junit: rspec-*.xml`). -- an array of filenames (`junit: [rspec-1.xml, rspec-2.xml, rspec-3.xml]`). -- A Combination of both (`junit: [rspec.xml, test-results/TEST-*.xml]`). +- A filename pattern (`junit: rspec-*.xml`, `junit: test-results/**/*.xml`). +- An array of filenames (`junit: [rspec-1.xml, rspec-2.xml, rspec-3.xml]`). +- A combination of both (`junit: [rspec.xml, test-results/TEST-*.xml]`). +- Directories are not supported(`junit: test-results`, `junit: test-results/**`). ## `artifacts:reports:license_scanning` **(ULTIMATE)** diff --git a/doc/ci/yaml/includes.md b/doc/ci/yaml/includes.md index bf0b7444e78..35b302d0373 100644 --- a/doc/ci/yaml/includes.md +++ b/doc/ci/yaml/includes.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# GitLab CI/CD include examples **(FREE)** +# Use CI/CD configuration from other files **(FREE)** You can use [`include`](index.md#include) to include external YAML files in your CI/CD jobs. @@ -20,13 +20,6 @@ To include a single configuration file, use either of these syntax options: include: '/templates/.after-script-template.yml' ``` -- `include` with a single file, and you specify the `include` type: - - ```yaml - include: - remote: 'https://gitlab.com/awesome-project/raw/main/.before-script-template.yml' - ``` - ## Include an array of configuration files You can include an array of configuration files: @@ -159,12 +152,18 @@ do not change. This method is called *merging*. ### Merge method for `include` -For a file containing `include` directives, the included files are read in order (possibly -recursively), and the configuration in these files is likewise merged in order. If the parameters overlap, the last included file takes precedence. Finally, the directives in the -file itself are merged with the configuration from the included files. +The `include` configuration merges with the main configuration file with this process: + +- Included files are read in the order defined in the configuration file, and + the included configuration is merged together in the same order. +- If an included file also uses `include`, that nested `include` configuration is merged first (recursively). +- If parameters overlap, the last included file takes precedence when merging the configuration + from the included files. +- After all configuration added with `include` is merged together, the main configuration + is merged with the included configuration. This merge method is a _deep merge_, where hash maps are merged at any depth in the -configuration. To merge hash map A (containing the configuration merged so far) and B (the next piece +configuration. To merge hash map "A" (that contains the configuration merged so far) and "B" (the next piece of configuration), the keys and values are processed as follows: - When the key only exists in A, use the key and value from A. @@ -172,9 +171,7 @@ of configuration), the keys and values are processed as follows: - When the key exists in both A and B, and one of the values is not a hash map, use the value from B. - Otherwise, use the key and value from B. -For example: - -We have a configuration consisting of two files. +For example, with a configuration that consists of two files: - The `.gitlab-ci.yml` file: @@ -211,7 +208,7 @@ We have a configuration consisting of two files. dotenv: deploy.env ``` -The merged result: +The merged result is: ```yaml variables: @@ -374,7 +371,7 @@ In `include` sections in your `.gitlab-ci.yml` file, you can use: - [Project variables](../variables/index.md#for-a-project). - [Group variables](../variables/index.md#for-a-group). - [Instance variables](../variables/index.md#for-an-instance). -- Project [predefined variables](../variables/predefined_variables.md). +- Project [predefined variables](../variables/predefined_variables.md) (`CI_PROJECT_*`). - In GitLab 14.2 and later, the `$CI_COMMIT_REF_NAME` [predefined variable](../variables/predefined_variables.md). When used in `include`, the `CI_COMMIT_REF_NAME` variable returns the full @@ -386,15 +383,7 @@ In GitLab 14.5 and later, you can also use: - [Trigger variables](../triggers/index.md#pass-cicd-variables-in-the-api-call). - [Scheduled pipeline variables](../pipelines/schedules.md#add-a-pipeline-schedule). - [Manual pipeline run variables](../pipelines/index.md#run-a-pipeline-manually). -- Pipeline [predefined variables](../variables/predefined_variables.md). - - YAML files are parsed before the pipeline is created, so the following pipeline predefined variables - are **not** available: - - - `CI_PIPELINE_ID` - - `CI_PIPELINE_URL` - - `CI_PIPELINE_IID` - - `CI_PIPELINE_CREATED_AT` +- The `CI_PIPELINE_SOURCE` and `CI_PIPELINE_TRIGGERED` [predefined variables](../variables/predefined_variables.md). For example: @@ -416,7 +405,8 @@ see this [CI/CD variable demo](https://youtu.be/4XR8gw3Pkos). > - Introduced in GitLab 14.2 [with a flag](../../administration/feature_flags.md) named `ci_include_rules`. Disabled by default. > - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/337507) in GitLab 14.3. > - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/337507) in GitLab 14.4. Feature flag `ci_include_rules` removed. -> - [Support for `exists` keyword added](https://gitlab.com/gitlab-org/gitlab/-/issues/341511) in GitLab 14.5. +> - Support for `exists` keyword [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/341511) in GitLab 14.5. +> - Support for `needs` job dependency [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345377) in GitLab 15.11. You can use [`rules`](index.md#rules) with `include` to conditionally include other configuration files. @@ -426,11 +416,6 @@ these keywords: - [`rules:if`](index.md#rulesif). - [`rules:exists`](index.md#rulesexists). -You cannot use [`needs:`](index.md#needs) to create a job dependency that points to -a job added with `include:local:rules`. When the configuration is validated, -GitLab returns `undefined need: `. [Issue 345377](https://gitlab.com/gitlab-org/gitlab/-/issues/345377) -proposes improving this behavior. - ### `include` with `rules:if` Use [`rules:if`](index.md#rulesif) to conditionally include other configuration files @@ -517,3 +502,107 @@ When the pipeline runs, GitLab: # This matches all `.yml` files only in subfolders of `configs`. include: 'configs/**/*.yml' ``` + +## Define inputs for configuration added with `include` (Beta) + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/391331) in GitLab 15.11 as a Beta feature. + +FLAG: +`spec` and `with` are experimental [Open Beta features](../../policy/alpha-beta-support.md#beta) +and subject to change without notice. + +### Define input parameters with `spec:inputs` + +Use `spec:inputs` to define input parameters for CI/CD configuration intended to be added +to a pipeline with `include`. Use [`include:inputs`](#set-input-parameter-values-with-includeinputs) +to define the values to use when the pipeline runs. + +The specs must be declared at the top of the configuration file, in a header section. +Separate the header from the rest of the configuration with `---`. + +Use the interpolation format `$[[ input.input-id ]]` to reference the values outside of the header section. +The inputs are evaluated and interpolated once, when the configuration is fetched +during pipeline creation, but before the configuration is merged with the contents of the `.gitlab-ci.yml`. + +```yaml +spec: + inputs: + environment: + job-stage: +--- + +scan-website: + stage: $[[ inputs.job-stage ]] + script: ./scan-website $[[ inputs.environment ]] +``` + +When using `spec:inputs`: + +- Defined inputs are mandatory by default. +- Inputs can be made optional by specifying a `default`. Use `default: null` to have no default value. +- A string containing an interpolation block must not exceed 1 MB. +- The string inside an interpolation block must not exceed 1 KB. + +For example, a `custom_configuration.yml`: + +```yaml +spec: + inputs: + website: + user: + default: 'test-user' + flags: + default: null +--- + +# The pipeline configuration would follow... +``` + +In this example: + +- `website` is mandatory and must be defined. +- `user` is optional. If not defined, the value is `test-user`. +- `flags` is optional. If not defined, it has no value. + +### Set input parameter values with `include:inputs` + +> `include:with` [renamed to `include:inputs`](https://gitlab.com/gitlab-org/gitlab/-/issues/406780) in GitLab 16.0. + +Use `include:inputs` to set the values for the parameters when the included configuration +is added to the pipeline. + +For example, to include a `custom_configuration.yml` that has the same specs +as the [example above](#define-input-parameters-with-specinputs): + +```yaml +include: + - local: 'custom_configuration.yml' + inputs: + website: "My website" +``` + +In this example: + +- `website` has a value of `My website` for the included configuration. +- `user` has a value of `test-user`, because that is the default when not specified. +- `flags` has no value, because it is optional and has no default when not specified. + +## Troubleshooting + +### `Maximum of 150 nested includes are allowed!` error + +The maximum number of [nested included files](#use-nested-includes) for a pipeline is 150. +If you receive the `Maximum 150 includes are allowed` error message in your pipeline, +it's likely that either: + +- Some of the nested configuration includes an overly large number of additional nested `include` configuration. +- There is an accidental loop in the nested includes. For example, `include1.yml` includes + `include2.yml` which includes `include1.yml`, creating a recursive loop. + +To help reduce the risk of this happening, edit the pipeline configuration file +with the [pipeline editor](../pipeline_editor/index.md), which validates if the +limit is reached. You can remove one included file at a time to try to narrow down +which configuration file is the source of the loop or excessive included files. + +In [GitLab 16.0 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/207270) self-managed users can +change the [maximum includes](../../user/admin_area/settings/continuous_integration.md#maximum-includes) value. diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md index d5c0fe1d41d..ab5226c1c30 100644 --- a/doc/ci/yaml/index.md +++ b/doc/ci/yaml/index.md @@ -136,11 +136,7 @@ The `include` files are: - Always evaluated first and then merged with the content of the `.gitlab-ci.yml` file, regardless of the position of the `include` keyword. -You can [nest](includes.md#use-nested-includes) up to 100 includes. In [GitLab 14.9 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/28987), -the same file can be included multiple times in nested includes, but duplicates are ignored. - -In [GitLab 12.4 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/28212), -the time limit to resolve all files is 30 seconds. +The time limit to resolve all files is 30 seconds. **Keyword type**: Global keyword. @@ -153,6 +149,8 @@ the time limit to resolve all files is 30 seconds. **Additional details**: +- Only [certain CI/CD variables](includes.md#use-variables-with-include) can be used + with `include` keywords. - Use merging to customize and override included CI/CD configurations with local - You can override included configuration by having the same job name or global keyword in the `.gitlab-ci.yml` file. The two configurations are merged together, and the @@ -163,6 +161,16 @@ the time limit to resolve all files is 30 seconds. do not affect job reruns. - Pipeline, the `include` files are fetched again. If they changed after the last pipeline run, the new pipeline uses the changed configuration. +- You can have up to 150 includes per pipeline by default, including [nested](includes.md#use-nested-includes). Additionally: + - In [GitLab 16.0 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/207270) self-managed users can + change the [maximum includes](../../user/admin_area/settings/continuous_integration.md#maximum-includes) value. + - In [GitLab 15.10 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/367150) you can have up to 150 includes. + In nested includes, the same file can be included multiple times, but duplicated includes + count towards the limit. + - From [GitLab 14.9 to GitLab 15.9](https://gitlab.com/gitlab-org/gitlab/-/issues/28987), you can have up to 100 includes. + The same file can be included multiple times in nested includes, but duplicates are ignored. + - In GitLab 14.9 and earlier you can have up to 100 includes, but the same file can not + be included multiple times. **Related topics**: @@ -171,7 +179,7 @@ the time limit to resolve all files is 30 seconds. #### `include:local` -Use `include:local` to include a file that is in the same repository as the project running the pipeline. +Use `include:local` to include a file that is in the same repository as the configuration file containing the `include` keyword. Use `include:local` instead of symbolic links. **Keyword type**: Global keyword. @@ -201,8 +209,8 @@ include: '.gitlab-ci-production.yml' - The `.gitlab-ci.yml` file and the local file must be on the same branch. - You can't include local files through Git submodules paths. -- All [nested includes](includes.md#use-nested-includes) are executed in the scope of the same project, - so you can use local, project, remote, or template includes. +- All [nested includes](includes.md#use-nested-includes) are executed in the scope of the project containing the configuration file with the `include` keyword, not the project running the pipeline. + You can use local, project, remote, or template includes. #### `include:project` @@ -220,8 +228,7 @@ use `include:project` and `include:file`. The YAML files must have the `.yml` or `.yaml` extension. - `include:ref`: Optional. The ref to retrieve the file from. Defaults to the `HEAD` of the project when not specified. - -You can use [certain CI/CD variables](includes.md#use-variables-with-include). +- You can use [certain CI/CD variables](includes.md#use-variables-with-include). **Example of `include:project`**: @@ -252,8 +259,8 @@ include: **Additional details**: -- All [nested includes](includes.md#use-nested-includes) are executed in the scope of the target project. - You can use `local` (relative to the target project), `project`, `remote`, or `template` includes. +- All [nested includes](includes.md#use-nested-includes) are executed in the scope of the project containing the configuration file with the nested `include` keyword. + You can use `local` (relative to the project containing the configuration file with the `include` keyword), `project`, `remote`, or `template` includes. - When the pipeline starts, the `.gitlab-ci.yml` file configuration included by all methods is evaluated. The configuration is a snapshot in time and persists in the database. GitLab does not reflect any changes to the referenced `.gitlab-ci.yml` file configuration until the next pipeline starts. @@ -422,23 +429,30 @@ A configuration with different pipeline names depending on the pipeline conditio ```yaml variables: - PIPELINE_NAME: 'Default pipeline name' # A default is not required. + PROJECT1_PIPELINE_NAME: 'Default pipeline name' # A default is not required. workflow: - name: '$PIPELINE_NAME' + name: '$PROJECT1_PIPELINE_NAME' rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' variables: - PIPELINE_NAME: 'MR pipeline: $CI_COMMIT_BRANCH' + PROJECT1_PIPELINE_NAME: 'MR pipeline: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME' - if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-in-ruby3/' variables: - PIPELINE_NAME: 'Ruby 3 pipeline' + PROJECT1_PIPELINE_NAME: 'Ruby 3 pipeline' ``` **Additional details**: - If the name is an empty string, the pipeline is not assigned a name. A name consisting of only CI/CD variables could evaluate to an empty string if all the variables are also empty. +- `workflow:rules:variables` become [global variables](#variables) available in all jobs, + including [`trigger`](#trigger) jobs which forward variables to downstream pipelines by default. + If the downstream pipeline uses the same variable, the [variable is overwritten](../variables/index.md#cicd-variable-precedence) + by the upstream variable value. Be sure to either: + - Use a unique variable name in every project's pipeline configuration, like `PROJECT1_PIPELINE_NAME`. + - Use [`inherit:variables`](#inheritvariables) in the trigger job and list the + exact variables you want to forward to the downstream pipeline. #### `workflow:rules` @@ -552,6 +566,16 @@ When the branch is something else: - job1's `DEPLOY_VARIABLE` is `job1-default-deploy`. - job2's `DEPLOY_VARIABLE` is `default-deploy`. +**Additional details**: + +- `workflow:rules:variables` become [global variables](#variables) available in all jobs, + including [`trigger`](#trigger) jobs which forward variables to downstream pipelines by default. + If the downstream pipeline uses the same variable, the [variable is overwritten](../variables/index.md#cicd-variable-precedence) + by the upstream variable value. Be sure to either: + - Use unique variable names in every project's pipeline configuration, like `PROJECT1_VARIABLE_NAME`. + - Use [`inherit:variables`](#inheritvariables) in the trigger job and list the + exact variables you want to forward to the downstream pipeline. + ## Job keywords The following topics explain how to use keywords to configure CI/CD pipelines. @@ -709,7 +733,7 @@ test_job_2: ### `artifacts` -Use `artifacts` to specify which files to save as [job artifacts](../pipelines/job_artifacts.md). +Use `artifacts` to specify which files to save as [job artifacts](../jobs/job_artifacts.md). Job artifacts are a list of files and directories that are attached to the job when it [succeeds, fails, or always](#artifactswhen). @@ -727,7 +751,7 @@ artifacts from the jobs defined in the `needs` configuration. Job artifacts are only collected for successful jobs by default, and artifacts are restored after [caches](#cache). -[Read more about artifacts](../pipelines/job_artifacts.md). +[Read more about artifacts](../jobs/job_artifacts.md). #### `artifacts:paths` @@ -766,7 +790,7 @@ This example creates an artifact with `.config` and all the files in the `binari **Related topics**: - To restrict which jobs a specific job fetches artifacts from, see [`dependencies`](#dependencies). -- [Create job artifacts](../pipelines/job_artifacts.md#create-job-artifacts). +- [Create job artifacts](../jobs/job_artifacts.md#create-job-artifacts). #### `artifacts:exclude` @@ -805,7 +829,7 @@ subdirectories of `binaries/`. **Related topics**: -- [Exclude files from job artifacts](../pipelines/job_artifacts.md#exclude-files-from-job-artifacts). +- [Exclude files from job artifacts](../jobs/job_artifacts.md#without-excluded-files). #### `artifacts:expire_in` @@ -815,12 +839,12 @@ subdirectories of `binaries/`. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/276583) in GitLab 13.9, keeping latest job artifacts can be disabled instance-wide. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/321323) in GitLab 13.12, the latest pipeline artifacts are kept regardless of expiry time. -Use `expire_in` to specify how long [job artifacts](../pipelines/job_artifacts.md) are stored before +Use `expire_in` to specify how long [job artifacts](../jobs/job_artifacts.md) are stored before they expire and are deleted. The `expire_in` setting does not affect: -- Artifacts from the latest job, unless keeping the latest job artifacts is: - - [Disabled at the project level](../pipelines/job_artifacts.md#keep-artifacts-from-most-recent-successful-jobs). - - [Disabled instance-wide](../../user/admin_area/settings/continuous_integration.md#keep-the-latest-artifacts-for-all-jobs-in-the-latest-successful-pipelines). +- Artifacts from the latest job, unless keeping the latest job artifacts is disabled + [at the project level](../jobs/job_artifacts.md#keep-artifacts-from-most-recent-successful-jobs). + or [instance-wide](../../user/admin_area/settings/continuous_integration.md#keep-the-latest-artifacts-for-all-jobs-in-the-latest-successful-pipelines). - [Pipeline artifacts](../pipelines/pipeline_artifacts.md). You can't specify an expiration date for pipeline artifacts. See [When pipeline artifacts are deleted](../pipelines/pipeline_artifacts.md#when-pipeline-artifacts-are-deleted) for more information. @@ -866,7 +890,7 @@ job: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15018) in GitLab 12.5. Use the `artifacts:expose_as` keyword to -[expose job artifacts in the merge request UI](../pipelines/job_artifacts.md#expose-job-artifacts-in-the-merge-request-ui). +[expose job artifacts in the merge request UI](../jobs/job_artifacts.md#link-to-job-artifacts-in-the-merge-request-ui). **Keyword type**: Job keyword. You can use it only as part of a job or in the [`default` section](#default). @@ -892,7 +916,7 @@ test: - A maximum of 10 job artifacts per merge request can be exposed. - Glob patterns are unsupported. - If a directory is specified and there is more than one file in the directory, - the link is to the job [artifacts browser](../pipelines/job_artifacts.md#download-job-artifacts). + the link is to the job [artifacts browser](../jobs/job_artifacts.md#download-job-artifacts). - If [GitLab Pages](../../administration/pages/index.md) is enabled, GitLab automatically renders the artifacts when the artifacts is a single file with one of these extensions: - `.html` or `.htm` @@ -903,7 +927,7 @@ test: **Related topics**: -- [Expose job artifacts in the merge request UI](../pipelines/job_artifacts.md#expose-job-artifacts-in-the-merge-request-ui). +- [Expose job artifacts in the merge request UI](../jobs/job_artifacts.md#link-to-job-artifacts-in-the-merge-request-ui). #### `artifacts:name` @@ -934,14 +958,12 @@ job: **Related topics**: -- [Use CI/CD variables to define the artifacts name](../pipelines/job_artifacts.md#use-cicd-variables-to-define-the-artifacts-name). +- [Use CI/CD variables to define the artifacts name](../jobs/job_artifacts.md#with-a-dynamically-defined-name). #### `artifacts:public` -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49775) in GitLab 13.8 -> - It's [deployed behind a feature flag](../../user/feature_flags.md), disabled by default. -> - It's disabled on GitLab.com. -> - It's recommended for production use. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/223273) in GitLab 13.8 [with a flag](../../user/feature_flags.md) named `non_public_artifacts`, disabled by default. +> - [Updated](https://gitlab.com/gitlab-org/gitlab/-/issues/322454) in GitLab 15.10. Artifacts created with `artifacts:public` before 15.10 are not guaranteed to remain private after this update. FLAG: On self-managed GitLab, by default this feature is not available. To make it available, @@ -1031,7 +1053,7 @@ job: **Related topics**: -- [Add untracked files to artifacts](../pipelines/job_artifacts.md#add-untracked-files-to-artifacts). +- [Add untracked files to artifacts](../jobs/job_artifacts.md#with-untracked-files). #### `artifacts:when` @@ -1458,6 +1480,33 @@ faster-test-job: - echo "Running tests..." ``` +#### `cache:fallback_keys` + +Use `cache:fallback_keys` to specify a list of keys to try to restore cache from +if there is no cache found for the `cache:key`. Caches are retrieved in the order specified +in the `fallback_keys` section. + +**Keyword type**: Job keyword. You can use it only as part of a job or in the +[`default` section](#default). + +**Possible inputs**: + +- An array of cache keys + +**Example of `cache:fallback_keys`**: + +```yaml +rspec: + script: rspec + cache: + key: gems-$CI_COMMIT_REF_SLUG + paths: + - rspec/ + fallback_keys: + - gems + when: 'always' +``` + ### `coverage` Use `coverage` with a custom regular expression to configure how code coverage @@ -1490,6 +1539,7 @@ In this example: **Additional details**: +- You can find parse examples in [Code Coverage](../testing/code_coverage.md#test-coverage-examples). - If there is more than one matched line in the job output, the last line is used (the first result of reverse search). - If there are multiple matches in a single line, the last match is searched @@ -1604,7 +1654,7 @@ the [stage](#stages) precedence. - The job status does not matter. If a job fails or it's a manual job that isn't triggered, no error occurs. - If the artifacts of a dependent job are [expired](#artifactsexpire_in) or - [deleted](../pipelines/job_artifacts.md#delete-job-artifacts), then the job fails. + [deleted](../jobs/job_artifacts.md#delete-job-log-and-artifacts), then the job fails. ### `environment` @@ -1712,7 +1762,7 @@ Use the `action` keyword to specify how the job interacts with the environment. |:----------|:----------------| | `start` | Default value. Indicates that the job starts the environment. The deployment is created after the job starts. | | `prepare` | Indicates that the job is only preparing the environment. It does not trigger deployments. [Read more about preparing environments](../environments/index.md#access-an-environment-for-preparation-or-verification-purposes). | -| `stop` | Indicates that the job stops an environment. [Read more about stopping an environment](../environments/index.md#stop-an-environment). | +| `stop` | Indicates that the job stops an environment. [Read more about stopping an environment](../environments/index.md#stopping-an-environment). | | `verify` | Indicates that the job is only verifying the environment. It does not trigger deployments. [Read more about verifying environments](../environments/index.md#access-an-environment-for-preparation-or-verification-purposes). | | `access` | Indicates that the job is only accessing the environment. It does not trigger deployments. [Read more about accessing environments](../environments/index.md#access-an-environment-for-preparation-or-verification-purposes). | @@ -1924,12 +1974,8 @@ rspec: ### `hooks` -> Introduced in GitLab 15.6 [with a flag](../../administration/feature_flags.md) named `ci_hooks_pre_get_sources_script`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, -ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `ci_hooks_pre_get_sources_script`. -The feature is not ready for production use. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356850) in GitLab 15.6 [with a flag](../../administration/feature_flags.md) named `ci_hooks_pre_get_sources_script`. Disabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/381840) in GitLab 15.10. Feature flag `ci_hooks_pre_get_sources_script` removed. Use `hooks` to specify lists of commands to execute on the runner at certain stages of job execution, like before retrieving the Git repository. @@ -1943,7 +1989,8 @@ at certain stages of job execution, like before retrieving the Git repository. #### `hooks:pre_get_sources_script` -> Introduced in GitLab 15.6 [with a flag](../../administration/feature_flags.md) named `ci_hooks_pre_get_sources_script`. Disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356850) in GitLab 15.6 [with a flag](../../administration/feature_flags.md) named `ci_hooks_pre_get_sources_script`. Disabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/381840) in GitLab 15.10. Feature flag `ci_hooks_pre_get_sources_script` removed. Use `hooks:pre_get_sources_script` to specify a list of commands to execute on the runner before retrieving the Git repository and any submodules. You can use it @@ -2537,11 +2584,10 @@ can use that variable in `needs:pipeline` to download artifacts from the parent To need a job that sometimes does not exist in the pipeline, add `optional: true` to the `needs` configuration. If not defined, `optional: false` is the default. -Jobs that use [`rules`](#rules), [`only`, or `except`](#only--except) might not always -be added to a pipeline. GitLab checks the `needs` relationships before starting a -pipeline: +Jobs that use [`rules`](#rules), [`only`, or `except`](#only--except) and that are added with [`include`](#include) +might not always be added to a pipeline. GitLab checks the `needs` relationships before starting a pipeline: -- If the needs entry has `optional: true` and the needed job is present in the pipeline, +- If the `needs` entry has `optional: true` and the needed job is present in the pipeline, the job waits for it to complete before starting. - If the needed job is not present, the job can start when all other needs requirements are met. - If the `needs` section contains only optional jobs, and none are added to the pipeline, @@ -2594,9 +2640,9 @@ In this example: #### `needs:pipeline` -You can mirror the pipeline status from an upstream pipeline to a bridge job by +You can mirror the pipeline status from an upstream pipeline to a job by using the `needs:pipeline` keyword. The latest pipeline status from the default branch is -replicated to the bridge job. +replicated to the job. **Keyword type**: Job keyword. You can use it only as part of a job. @@ -2609,7 +2655,7 @@ replicated to the bridge job. **Example of `needs:pipeline`**: ```yaml -upstream_bridge: +upstream_status: stage: test needs: pipeline: other/project @@ -2886,7 +2932,7 @@ Parallel jobs are named sequentially from `job_name 1/N` to `job_name N/N`. **Possible inputs**: -- A numeric value from `2` to `200`. +- A numeric value from `1` to `200`. **Example of `parallel`**: @@ -3159,6 +3205,14 @@ job: description: './path/to/CHANGELOG.md' ``` +**Additional details**: + +- The `description` is evaluated by the shell that runs `release-cli`. + You can use CI/CD variables to define the description, but some shells + [use different syntax](../variables/index.md#use-cicd-variables-in-job-scripts) + to reference variables. Similarly, some shells might require special characters + to be escaped. For example, backticks (`` ` ``) might need to be escaped with a backslash (\). + #### `release:ref` The `ref` for the release, if the `release: tag_name` doesn't exist yet. @@ -3482,6 +3536,7 @@ docker build: - If the pipeline is a merge request pipeline, check `Dockerfile` for changes. - If `Dockerfile` has changed, add the job to the pipeline as a manual job, and the pipeline continues running even if the job is not triggered (`allow_failure: true`). +- A maximum of 50 patterns or file paths can be defined per `rules:changes` section. - If `Dockerfile` has not changed, do not add job to any pipeline (same as `when: never`). - [`rules:changes:paths`](#ruleschangespaths) is the same as `rules:changes` without any subkeys. @@ -3594,11 +3649,12 @@ job: - Glob patterns are interpreted with Ruby [`File.fnmatch`](https://docs.ruby-lang.org/en/2.7.0/File.html#method-c-fnmatch) with the flags `File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB`. -- For performance reasons, GitLab matches a maximum of 10,000 `exists` patterns or - file paths. After the 10,000th check, rules with patterned globs always match. - In other words, `exists` always reports `true` if more than 10,000 checks - run. Repositories with less than 10,000 files might still be impacted if the `exists` - rules are checked more than 10,000 times. +- For performance reasons, GitLab performs a maximum of 10,000 checks against + `exists` patterns or file paths. After the 10,000th check, rules with patterned + globs always match. In other words, the `exists` rule always assumes a match in + projects with more than 10,000 files, or if there are fewer than 10,000 files but + the `exists` rules are checked more than 10,000 times. +- A maximum of 50 patterns or file paths can be defined per `rules:exists` section. - `exists` resolves to `true` if any of the listed files are found (an `OR` operation). #### `rules:allow_failure` @@ -3637,6 +3693,55 @@ If the rule matches, then the job is a manual job with `allow_failure: true`. - The rule-level `rules:allow_failure` overrides the job-level [`allow_failure`](#allow_failure), and only applies when the specific rule triggers the job. +#### `rules:needs` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31581) in GitLab 16.0 [with a flag](../../user/feature_flags.md) named `introduce_rules_with_needs`. Disabled by default. + +Use `needs` in rules to update a job's [`needs`](#needs) for specific conditions. When a condition matches a rule, the job's `needs` configuration is completely replaced with the `needs` in the rule. + +**Keyword type**: Job-specific. You can use it only as part of a job. + +**Possible inputs**: + +- An array of job names as strings. +- A hash with a job name, optionally with additional attributes. +- An empty array (`[]`), to set the job needs to none when the specific condition is met. + +**Example of `rules:needs`**: + +```yaml +build-dev: + stage: build + rules: + - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH + script: echo "Feature branch, so building dev version..." + +build-prod: + stage: build + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + script: echo "Default branch, so building prod version..." + +specs: + stage: test + needs: ['build-dev'] + rules: + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + needs: ['build-prod'] + - when: on_success # Run the job in other cases + script: echo "Running dev specs by default, or prod specs when default branch..." +``` + +In this example: + +- If the pipeline runs on a branch that is not the default branch, the `specs` job needs the `build-dev` job (default behavior). +- If the pipeline runs on the default branch, and therefore the rule matches the condition, the `specs` job needs the `build-prod` job instead. + +**Additional details**: + +- `needs` in rules override any `needs` defined at the job-level. When overridden, the behavior is same as [job-level `needs`](#needs). +- `needs` in rules can accept [`artifacts`](#needsartifacts) and [`optional`](#needsoptional). + #### `rules:variables` > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/209864) in GitLab 13.7. @@ -3810,9 +3915,6 @@ job: Use `secrets:token` to explicitly select a token to use when authenticating with Vault by referencing the token's CI/CD variable. -This keyword has no effect if [**Limit JSON Web Token (JWT) access**](../secrets/id_token_authentication.md#enable-automatic-id-token-authentication) -is disabled. - **Keyword type**: Job keyword. You can use it only as part of a job. **Possible inputs**: @@ -3836,8 +3938,8 @@ job: **Additional details**: -- When the `token` keyword is not set and **Limit JSON Web Token (JWT) access** enabled, the first ID token - is used to authenticate. +- When the `token` keyword is not set, the first ID token is used to authenticate. +- In GitLab 15.8 to 15.11, you must enable [**Limit JSON Web Token (JWT) access**](../secrets/id_token_authentication.md#enable-automatic-id-token-authentication-deprecated) for this keyword to be available. - When **Limit JSON Web Token (JWT) access** is disabled, the `token` keyword is ignored and the `CI_JOB_JWT` CI/CD variable is used to authenticate. @@ -3944,9 +4046,9 @@ If `stage` is not defined, the job uses the `test` stage by default. **Keyword type**: Job keyword. You can use it only as part of a job. -**Possible inputs**: An array including any number of stage names. Stage names can be: +**Possible inputs**: A string, which can be a: -- The [default stages](#stages). +- [Default stage](#stages). - User-defined stages. **Example of `stage`**: @@ -4348,18 +4450,19 @@ child3: ### `variables` -Use `variables` to define [CI/CD variables](../variables/index.md#define-a-cicd-variable-in-the-gitlab-ciyml-file), -which are configurable values that are passed to jobs. - -Variables are always available in `script`, `before_script`, and `after_script` commands. -You can also use variables as inputs in some job keywords. +Use `variables` to define [CI/CD variables](../variables/index.md#define-a-cicd-variable-in-the-gitlab-ciyml-file) for jobs. **Keyword type**: Global and job keyword. You can use it at the global level, and also at the job level. -If you define `variables` at the global level, each variable is copied to -every job configuration when the pipeline is created. If the job already has that -variable defined, the [job-level variable takes precedence](../variables/index.md#cicd-variable-precedence). +If you define `variables` as a [global keyword](#keywords), it behaves like default variables +for all jobs. Each variable is copied to every job configuration when the pipeline is created. +If the job already has that variable defined, the [job-level variable takes precedence](../variables/index.md#cicd-variable-precedence). + +Variables defined at the global-level cannot be used as inputs for other global keywords +like [`include`](includes.md#use-variables-with-include). These variables can only +be used at the job-level, in `script`, `before_script`, and `after_script` sections, +as well as inputs in some job keywords like [`rules`](../jobs/job_control.md#cicd-variable-expressions). **Possible inputs**: Variable name and value pairs: @@ -4536,14 +4639,16 @@ the default value is `when: on_success`. **Possible inputs**: -- `on_success` (default): Run the job only when all jobs in earlier stages succeed +- `on_success` (default): Run the job only when no jobs in earlier stages fail or have `allow_failure: true`. -- `manual`: Run the job only when [triggered manually](../jobs/job_control.md#create-a-job-that-must-be-run-manually). +- `on_failure`: Run the job only when at least one job in an earlier stage fails. A job in an earlier stage + with `allow_failure: true` is always considered successful. +- `never`: Don't run the job regardless of the status of jobs in earlier stages. + Can only be used in a [`rules`](#rules) section or `workflow: rules`. - `always`: Run the job regardless of the status of jobs in earlier stages. Can also be used in `workflow:rules`. -- `on_failure`: Run the job only when at least one job in an earlier stage fails. A job with `allow_failure: true` is always considered successful. +- `manual`: Run the job only when [triggered manually](../jobs/job_control.md#create-a-job-that-must-be-run-manually). - `delayed`: [Delay the execution of a job](../jobs/job_control.md#run-a-job-after-a-delay) for a specified duration. -- `never`: Don't run the job. Can only be used in a [`rules`](#rules) section or `workflow: rules`. **Example of `when`**: diff --git a/doc/ci/yaml/yaml_optimization.md b/doc/ci/yaml/yaml_optimization.md index 07019a2776f..2cfda1116fe 100644 --- a/doc/ci/yaml/yaml_optimization.md +++ b/doc/ci/yaml/yaml_optimization.md @@ -65,7 +65,7 @@ test2: `&` sets up the name of the anchor (`job_configuration`), `<<` means "merge the given hash into the current one," and `*` includes the named anchor -(`job_configuration` again). The expanded version of this example is: +(`job_configuration` again). The [expanded](../pipeline_editor/index.md#view-full-configuration) version of this example is: ```yaml .job_template: @@ -123,7 +123,7 @@ test:mysql: services: *mysql_configuration ``` -The expanded version is: +The [expanded](../pipeline_editor/index.md#view-full-configuration) version is: ```yaml .job_template: diff --git a/doc/cloud_seed/index.md b/doc/cloud_seed/index.md index 1021c7f7700..7ac38ef2c15 100644 --- a/doc/cloud_seed/index.md +++ b/doc/cloud_seed/index.md @@ -1,5 +1,5 @@ --- -stage: Release +stage: Deploy group: Incubation info: Cloud Seed (formerly 5mp) is a GitLab Incubation Engineering program. No technical writer assigned to this group. --- diff --git a/doc/development/advanced_search.md b/doc/development/advanced_search.md new file mode 100644 index 00000000000..73a8191b789 --- /dev/null +++ b/doc/development/advanced_search.md @@ -0,0 +1,336 @@ +--- +stage: Data Stores +group: Global Search +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Advanced search development guidelines + +This page includes information about developing and working with Elasticsearch. + +Information on how to enable Elasticsearch and perform the initial indexing is in +the [Elasticsearch integration documentation](../integration/advanced_search/elasticsearch.md#enable-advanced-search). + +## Deep Dive + +In June 2019, Mario de la Ossa hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/-/issues/1`) on the GitLab [Elasticsearch integration](../integration/advanced_search/elasticsearch.md) to share his domain specific knowledge with anyone who may work in this part of the codebase in the future. You can find the [recording on YouTube](https://www.youtube.com/watch?v=vrvl-tN2EaA), and the slides on [Google Slides](https://docs.google.com/presentation/d/1H-pCzI_LNrgrL5pJAIQgvLX8Ji0-jIKOg1QeJQzChug/edit) and in [PDF](https://gitlab.com/gitlab-org/create-stage/uploads/c5aa32b6b07476fa8b597004899ec538/Elasticsearch_Deep_Dive.pdf). Everything covered in this deep dive was accurate as of GitLab 12.0, and while specific details might have changed, it should still serve as a good introduction. + +In August 2020, a second Deep Dive was hosted, focusing on [GitLab-specific architecture for multi-indices support](#zero-downtime-reindexing-with-multiple-indices). The [recording on YouTube](https://www.youtube.com/watch?v=0WdPR9oB2fg) and the [slides](https://lulalala.gitlab.io/gitlab-elasticsearch-deepdive/) are available. Everything covered in this deep dive was accurate as of GitLab 13.3. + +## Supported Versions + +See [Version Requirements](../integration/advanced_search/elasticsearch.md#version-requirements). + +Developers making significant changes to Elasticsearch queries should test their features against all our supported versions. + +## Setting up development environment + +See the [Elasticsearch GDK setup instructions](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/doc/howto/elasticsearch.md) + +## Helpful Rake tasks + +- `gitlab:elastic:test:index_size`: Tells you how much space the current index is using, as well as how many documents are in the index. +- `gitlab:elastic:test:index_size_change`: Outputs index size, reindexes, and outputs index size again. Useful when testing improvements to indexing size. + +Additionally, if you need large repositories or multiple forks for testing, please consider [following these instructions](rake_tasks.md#extra-project-seed-options) + +## How does it work? + +The Elasticsearch integration depends on an external indexer. We ship an [indexer written in Go](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). The user must trigger the initial indexing via a Rake task but, after this is done, GitLab itself will trigger reindexing when required via `after_` callbacks on create, update, and destroy that are inherited from [`/ee/app/models/concerns/elastic/application_versioned_search.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/models/concerns/elastic/application_versioned_search.rb). + +After initial indexing is complete, create, update, and delete operations for all models except projects (see [#207494](https://gitlab.com/gitlab-org/gitlab/-/issues/207494)) are tracked in a Redis [`ZSET`](https://redis.io/docs/manual/data-types/#sorted-sets). A regular `sidekiq-cron` `ElasticIndexBulkCronWorker` processes this queue, updating many Elasticsearch documents at a time with the [Bulk Request API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html). + +Search queries are generated by the concerns found in [`ee/app/models/concerns/elastic`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/app/models/concerns/elastic). These concerns are also in charge of access control, and have been a historic source of security bugs so please pay close attention to them! + +### Custom routing + +[Custom routing](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-routing-field.html#_searching_with_custom_routing) +is used in Elasticsearch for document types that are associated with a project. The routing format is `project_`. Routing is set +during indexing and searching operations. Some of the benefits and tradeoffs to using custom routing are: + +- Project scoped searches are much faster. +- Routing is not used if too many shards would be hit for global and group scoped searches. +- Shard size imbalance might occur. + +## Existing Analyzers/Tokenizers/Filters + +These are all defined in [`ee/lib/elastic/latest/config.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/elastic/latest/config.rb) + +### Analyzers + +#### `path_analyzer` + +Used when indexing blobs' paths. Uses the `path_tokenizer` and the `lowercase` and `asciifolding` filters. + +Please see the `path_tokenizer` explanation below for an example. + +#### `sha_analyzer` + +Used in blobs and commits. Uses the `sha_tokenizer` and the `lowercase` and `asciifolding` filters. + +Please see the `sha_tokenizer` explanation later below for an example. + +#### `code_analyzer` + +Used when indexing a blob's filename and content. Uses the `whitespace` tokenizer and the filters: [`code`](#code), `lowercase`, and `asciifolding` + +The `whitespace` tokenizer was selected to have more control over how tokens are split. For example the string `Foo::bar(4)` needs to generate tokens like `Foo` and `bar(4)` to be properly searched. + +Please see the `code` filter for an explanation on how tokens are split. + +NOTE: +The [Elasticsearch `code_analyzer` doesn't account for all code cases](../integration/advanced_search/elasticsearch_troubleshooting.md#elasticsearch-code_analyzer-doesnt-account-for-all-code-cases). + +#### `code_search_analyzer` + +Not directly used for indexing, but rather used to transform a search input. Uses the `whitespace` tokenizer and the `lowercase` and `asciifolding` filters. + +### Tokenizers + +#### `sha_tokenizer` + +This is a custom tokenizer that uses the [`edgeNGram` tokenizer](https://www.elastic.co/guide/en/elasticsearch/reference/5.5/analysis-edgengram-tokenizer.html) to allow SHAs to be searchable by any sub-set of it (minimum of 5 chars). + +Example: + +`240c29dc7e` becomes: + +- `240c2` +- `240c29` +- `240c29d` +- `240c29dc` +- `240c29dc7` +- `240c29dc7e` + +#### `path_tokenizer` + +This is a custom tokenizer that uses the [`path_hierarchy` tokenizer](https://www.elastic.co/guide/en/elasticsearch/reference/5.5/analysis-pathhierarchy-tokenizer.html) with `reverse: true` to allow searches to find paths no matter how much or how little of the path is given as input. + +Example: + +`'/some/path/application.js'` becomes: + +- `'/some/path/application.js'` +- `'some/path/application.js'` +- `'path/application.js'` +- `'application.js'` + +### Filters + +#### `code` + +Uses a [Pattern Capture token filter](https://www.elastic.co/guide/en/elasticsearch/reference/5.5/analysis-pattern-capture-tokenfilter.html) to split tokens into more easily searched versions of themselves. + +Patterns: + +- `"(\\p{Ll}+|\\p{Lu}\\p{Ll}+|\\p{Lu}+)"`: captures CamelCase and lowerCamelCase strings as separate tokens +- `"(\\d+)"`: extracts digits +- `"(?=([\\p{Lu}]+[\\p{L}]+))"`: captures CamelCase strings recursively. For example: `ThisIsATest` => `[ThisIsATest, IsATest, ATest, Test]` +- `'"((?:\\"|[^"]|\\")*)"'`: captures terms inside quotes, removing the quotes +- `"'((?:\\'|[^']|\\')*)'"`: same as above, for single-quotes +- `'\.([^.]+)(?=\.|\s|\Z)'`: separate terms with periods in-between +- `'([\p{L}_.-]+)'`: some common chars in file names to keep the whole filename intact (for example `my_file-ñame.txt`) +- `'([\p{L}\d_]+)'`: letters, numbers and underscores are the most common tokens in programming. Always capture them greedily regardless of context. + +## Gotchas + +- Searches can have their own analyzers. Remember to check when editing analyzers +- `Character` filters (as opposed to token filters) always replace the original character, so they're not a good choice as they can hinder exact searches + +## Zero downtime reindexing with multiple indices + +NOTE: +This is not applicable yet as multiple indices functionality is not fully implemented. + +Currently GitLab can only handle a single version of setting. Any setting/schema changes would require reindexing everything from scratch. Since reindexing can take a long time, this can cause search functionality downtime. + +To avoid downtime, GitLab is working to support multiple indices that +can function at the same time. Whenever the schema changes, the administrator +will be able to create a new index and reindex to it, while searches +continue to go to the older, stable index. Any data updates will be +forwarded to both indices. Once the new index is ready, an administrator can +mark it active, which will direct all searches to it, and remove the old +index. + +This is also helpful for migrating to new servers, for example, moving to/from AWS. + +Currently we are on the process of migrating to this new design. Everything is hardwired to work with one single version for now. + +### Architecture + +The traditional setup, provided by `elasticsearch-rails`, is to communicate through its internal proxy classes. Developers would write model-specific logic in a module for the model to include in (for example, `SnippetsSearch`). The `__elasticsearch__` methods would return a proxy object, for example: + +- `Issue.__elasticsearch__` returns an instance of `Elasticsearch::Model::Proxy::ClassMethodsProxy` +- `Issue.first.__elasticsearch__` returns an instance of `Elasticsearch::Model::Proxy::InstanceMethodsProxy`. + +These proxy objects would talk to Elasticsearch server directly (see top half of the diagram). + +![Elasticsearch Architecture](img/elasticsearch_architecture.svg) + +In the planned new design, each model would have a pair of corresponding sub-classed proxy objects, in which model-specific logic is located. For example, `Snippet` would have `SnippetClassProxy` and `SnippetInstanceProxy` (being subclass of `Elasticsearch::Model::Proxy::ClassMethodsProxy` and `Elasticsearch::Model::Proxy::InstanceMethodsProxy`, respectively). + +`__elasticsearch__` would represent another layer of proxy object, keeping track of multiple actual proxy objects. It would forward method calls to the appropriate index. For example: + +- `model.__elasticsearch__.search` would be forwarded to the one stable index, since it is a read operation. +- `model.__elasticsearch__.update_document` would be forwarded to all indices, to keep all indices up-to-date. + +The global configurations per version are now in the `Elastic::(Version)::Config` class. You can change mappings there. + +### Creating new version of schema + +NOTE: +This is not applicable yet as multiple indices functionality is not fully implemented. + +Folders like `ee/lib/elastic/v12p1` contain snapshots of search logic from different versions. To keep a continuous Git history, the latest version lives under `ee/lib/elastic/latest`, but its classes are aliased under an actual version (for example, `ee/lib/elastic/v12p3`). When referencing these classes, never use the `Latest` namespace directly, but use the actual version (for example, `V12p3`). + +The version name basically follows the GitLab release version. If setting is changed in 12.3, we will create a new namespace called `V12p3` (p stands for "point"). Raise an issue if there is a need to name a version differently. + +If the current version is `v12p1`, and we need to create a new version for `v12p3`, the steps are as follows: + +1. Copy the entire folder of `v12p1` as `v12p3` +1. Change the namespace for files under `v12p3` folder from `V12p1` to `V12p3` (which are still aliased to `Latest`) +1. Delete `v12p1` folder +1. Copy the entire folder of `latest` as `v12p1` +1. Change the namespace for files under `v12p1` folder from `Latest` to `V12p1` +1. Make changes to files under the `latest` folder as needed + +## Performance Monitoring + +### Prometheus + +GitLab exports [Prometheus metrics](../administration/monitoring/prometheus/gitlab_metrics.md) +relating to the number of requests and timing for all web/API requests and Sidekiq jobs, +which can help diagnose performance trends and compare how Elasticsearch timing +is impacting overall performance relative to the time spent doing other things. + +#### Indexing queues + +GitLab also exports [Prometheus metrics](../administration/monitoring/prometheus/gitlab_metrics.md) +for indexing queues, which can help diagnose performance bottlenecks and determine +whether or not your GitLab instance or Elasticsearch server can keep up with +the volume of updates. + +### Logs + +All of the indexing happens in Sidekiq, so much of the relevant logs for the +Elasticsearch integration can be found in +[`sidekiq.log`](../administration/logs/index.md#sidekiqlog). In particular, all +Sidekiq workers that make requests to Elasticsearch in any way will log the +number of requests and time taken querying/writing to Elasticsearch. This can +be useful to understand whether or not your cluster is keeping up with +indexing. + +Searching Elasticsearch is done via ordinary web workers handling requests. Any +requests to load a page or make an API request, which then make requests to +Elasticsearch, will log the number of requests and the time taken to +[`production_json.log`](../administration/logs/index.md#production_jsonlog). These +logs will also include the time spent on Database and Gitaly requests, which +may help to diagnose which part of the search is performing poorly. + +There are additional logs specific to Elasticsearch that are sent to +[`elasticsearch.log`](../administration/logs/index.md#elasticsearchlog) +that may contain information to help diagnose performance issues. + +### Performance Bar + +Elasticsearch requests will be displayed in the +[`Performance Bar`](../administration/monitoring/performance/performance_bar.md), which can +be used both locally in development and on any deployed GitLab instance to +diagnose poor search performance. This will show the exact queries being made, +which is useful to diagnose why a search might be slow. + +### Correlation ID and `X-Opaque-Id` + +Our [correlation ID](distributed_tracing.md#developer-guidelines-for-working-with-correlation-ids) +is forwarded by all requests from Rails to Elasticsearch as the +[`X-Opaque-Id`](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html#_identifying_running_tasks) +header which allows us to track any +[tasks](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html) +in the cluster back the request in GitLab. + +## Troubleshooting + +### Getting `flood stage disk watermark [95%] exceeded` + +You might get an error such as + +```plaintext +[2018-10-31T15:54:19,762][WARN ][o.e.c.r.a.DiskThresholdMonitor] [pval5Ct] + flood stage disk watermark [95%] exceeded on + [pval5Ct7SieH90t5MykM5w][pval5Ct][/usr/local/var/lib/elasticsearch/nodes/0] free: 56.2gb[3%], + all indices on this node will be marked read-only +``` + +This is because you've exceeded the disk space threshold - it thinks you don't have enough disk space left, based on the default 95% threshold. + +In addition, the `read_only_allow_delete` setting will be set to `true`. It will block indexing, `forcemerge`, etc + +```shell +curl "http://localhost:9200/gitlab-development/_settings?pretty" +``` + +Add this to your `elasticsearch.yml` file: + +```yaml +# turn off the disk allocator +cluster.routing.allocation.disk.threshold_enabled: false +``` + +_or_ + +```yaml +# set your own limits +cluster.routing.allocation.disk.threshold_enabled: true +cluster.routing.allocation.disk.watermark.flood_stage: 5gb # ES 6.x only +cluster.routing.allocation.disk.watermark.low: 15gb +cluster.routing.allocation.disk.watermark.high: 10gb +``` + +Restart Elasticsearch, and the `read_only_allow_delete` will clear on its own. + +_from "Disk-based Shard Allocation | Elasticsearch Reference" [5.6](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/disk-allocator.html#disk-allocator) and [6.x](https://www.elastic.co/guide/en/elasticsearch/reference/6.7/disk-allocator.html)_ + +### Disaster recovery/data loss/backups + +The use of Elasticsearch in GitLab is only ever as a secondary data store. +This means that all of the data stored in Elasticsearch can always be derived +again from other data sources, specifically PostgreSQL and Gitaly. Therefore if +the Elasticsearch data store is ever corrupted for whatever reason you can reindex +everything from scratch. + +If your Elasticsearch index is incredibly large it may be too time consuming or +cause too much downtime to reindex from scratch. There aren't any built in +mechanisms for automatically finding discrepancies and resyncing an +Elasticsearch index if it gets out of sync but one tool that may be useful is +looking at the logs for all the updates that occurred in a time range you +believe may have been missed. This information is very low level and only +useful for operators that are familiar with the GitLab codebase. It is +documented here in case it is useful for others. The relevant logs that could +theoretically be used to figure out what needs to be replayed are: + +1. All non-repository updates that were synced can be found in + [`elasticsearch.log`](../administration/logs/index.md#elasticsearchlog) by + searching for + [`track_items`](https://gitlab.com/gitlab-org/gitlab/-/blob/1e60ea99bd8110a97d8fc481e2f41cab14e63d31/ee/app/services/elastic/process_bookkeeping_service.rb#L25) + and these can be replayed by sending these items again through + `::Elastic::ProcessBookkeepingService.track!` +1. All repository updates that occurred can be found in + [`elasticsearch.log`](../administration/logs/index.md#elasticsearchlog) by + searching for + [`indexing_commit_range`](https://gitlab.com/gitlab-org/gitlab/-/blob/6f9d75dd3898536b9ec2fb206e0bd677ab59bd6d/ee/lib/gitlab/elastic/indexer.rb#L41). + Replaying these requires resetting the + [`IndexStatus#last_commit/last_wiki_commit`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/models/index_status.rb) + to the oldest `from_sha` in the logs and then triggering another index of + the project using + [`ElasticCommitIndexerWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic_commit_indexer_worker.rb) +1. All project deletes that occurred can be found in + [`sidekiq.log`](../administration/logs/index.md#sidekiqlog) by searching for + [`ElasticDeleteProjectWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic_delete_project_worker.rb). + These updates can be replayed by triggering another + `ElasticDeleteProjectWorker`. + +With the above methods and taking regular +[Elasticsearch snapshots](https://www.elastic.co/guide/en/elasticsearch/reference/current/snapshot-restore.html) +we should be able to recover from different kinds of data loss issues in a +relatively short period of time compared to indexing everything from +scratch. diff --git a/doc/development/ai_architecture.md b/doc/development/ai_architecture.md new file mode 100644 index 00000000000..e9994c8a6f4 --- /dev/null +++ b/doc/development/ai_architecture.md @@ -0,0 +1,108 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# AI Architecture (Experiment) + +GitLab has created a common set of tools to support our product groups and their utilization of AI. Our goals with this common architecture are: + +1. Increase the velocity of feature teams by providing a set of high quality, ready to use tools +1. Ability to switch underlying technologies quickly and easily + +AI is moving very quickly, and we need to be able to keep pace with changes in the area. We have built an [abstraction layer](../../ee/development/ai_features.md) to do this, allowing us to take a more "pluggable" approach to the underlying models, data stores, and other technologies. + +The following diagram shows a simplified view of how the different components in GitLab interact. The abstraction layer helps avoid code duplication within the REST APIs within the `AI API` block. + +```plantuml +@startuml +skin rose + +package "Code Suggestions" { + node "Model Gateway" + node "Triton Inference Server" as Triton +} + +package "Code Suggestions Models" as CSM { + node "codegen" + node "PaLM" +} + +package "Suggested Reviewers" { + node "Model Gateway (SR)" + node "Extractor" + node "Serving Model" +} + +package "AI API" as AIF { + node "OpenAI" + node "Vertex AI" +} + +package GitLab { + node "Web IDE" + + package "Web" { + node "REST API" + node "GraphQL" + } + + package "Jobs" { + node "Sidekiq" + } +} + +package Databases { + node "Vector Database" + node "PostgreSQL" +} + +node "VSCode" + +"Model Gateway" --> Triton +Triton --> CSM +GitLab --> Databases +VSCode --> "Model Gateway" +"Web IDE" --> "Model Gateway" +"Web IDE" --> "GraphQL" +"Web IDE" --> "REST API" +"Model Gateway" -[#blue]--> "REST API": user authorized? + +"Sidekiq" --> AIF +Web --> AIF + +"Model Gateway (SR)" --> "REST API" +"Model Gateway (SR)" --> "Serving Model" +"Extractor" --> "GraphQL" +"Sidekiq" --> "Model Gateway (SR)" + +@enduml +``` + +## SaaS-based AI abstraction layer + +GitLab currently operates a cloud-hosted AI architecture. We are exploring how self-managed instances integrate with it. + +There are two primary reasons for this: the best AI models are cloud-based as they often depend on specialized hardware designed for this purpose, and operating self-managed infrastructure capable of AI at-scale and with appropriate performance is a significant undertaking. We are actively [tracking self-managed customers interested in AI](https://gitlab.com/gitlab-org/gitlab/-/issues/409183). + +## Supported technologies + +As part of the AI working group, we have been investigating various technologies and vetting them. Below is a list of the tools which have been reviewed and already approved for use within the GitLab application. + +It is possible to utilize other models or technologies, however they will need to go through a review process prior to use. Use the [AI Project Proposal template](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=AI%20Project%20Proposal) as part of your idea and include the new tools required to support it. + +### Models + +The following models have been approved for use: + +- [OpenAI models](https://platform.openai.com/docs/models) +- Google's [Vertex AI](https://cloud.google.com/vertex-ai) and [model garden](https://cloud.google.com/model-garden) +- [AI Code Suggestions](https://gitlab.com/gitlab-org/modelops/applied-ml/code-suggestions/ai-assist/-/tree/main) +- [Suggested reviewer](https://gitlab.com/gitlab-org/modelops/applied-ml/applied-ml-updates/-/issues/10) + +### Vector stores + +The following vector stores have been approved for use: + +- [`pgvector`](https://github.com/pgvector/pgvector) is a Postgres extension adding support for storing vector embeddings and calculating ANN (approximate nearest neighbor). diff --git a/doc/development/ai_features.md b/doc/development/ai_features.md new file mode 100644 index 00000000000..6ed1d59c3e0 --- /dev/null +++ b/doc/development/ai_features.md @@ -0,0 +1,453 @@ +--- +stage: none +group: none +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# AI features based on 3rd-party integrations + +[Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/117296) in GitLab 15.11. + +## Features + +- Async execution of the long running API requests + - GraphQL Action starts the request + - Background workers execute + - GraphQL subscriptions deliver results back in real time +- Abstraction for + - OpenAI + - Google Vertex AI +- Rate Limiting +- Circuit Breaker +- Multi-Level feature flags +- License checks on group level +- Snowplow execution tracking +- Tracking of Token Spent on Prometheus +- Configuration for Moderation check of inputs +- Automatic Markdown Rendering of responses +- Centralised Group Level settings for experiment and 3rd party +- Experimental API endpoints for exploration of AI API’s by GitLab team members without the need for credentials + - OpenAI + - Google Vertex AI + +## Feature flags + +Apply the following two feature flags to any AI feature work: + +- A general that applies to all AI features. +- A flag specific to that feature. The feature flag name [must be different](feature_flags/index.md#feature-flags-for-licensed-features) than the licensed feature name. + +See the [feature flag tracker](https://gitlab.com/gitlab-org/gitlab/-/issues/405161) for the list of all feature flags and how to use them. + +## Implement a new AI action + +To implement a new AI action, connect to the OpenAI API. You can connect to this API using either the: + +- Experimental REST API. +- Abstraction layer. + +All AI features are experimental. + +## Test AI features locally + +1. Enable the required general feature flags: + + ```ruby + Feature.enable(:ai_related_settings) + Feature.enable(:openai_experimentation) + Feature.enable(:tofa_experimentation_main_flag) + Feature.enable(:anthropic_experimentation) + ``` + +1. Simulate the GDK to [simulate SaaS](ee_features.md#simulate-a-saas-instance) and ensure the group you want to test has an Ultimate license +1. Enable `Experimental features` and `Third-party AI services` + 1. Go to the group with the Ultimate license + 1. **Group Settings** > **General** -> **Permissions and group features** + 1. Enable **Experiment features** + 1. Enable **Third-party AI services** +1. Enable the specific feature flag for the feature you want to test +1. Set either the required access token `OpenAi` or `Vertex`. Ask in [`#ai_enablement_team`](https://gitlab.slack.com/archives/C051K31F30R) to receive an access token. + +### Set up the embedding database + +For features that use the embedding database, additional setup is needed. + +1. Enable [pgvector](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/pgvector.md#enable-pgvector-in-the-gdk) in GDK +1. Enable the embedding database in GDK + + ```shell + gdk config set gitlab.rails.databases.embedding.enabled true + ``` + +1. Run `gdk reconfigure` +1. Run database migrations to create the embedding database + +### Setup for GitLab chat + +To populate the embedding database for GitLab chat: + +1. Open a rails console +1. Run [this script](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/10588#note_1373586079) to populate the embedding database + +### Configure GCP Vertex access + +In order to obtain a GCP service key for local development, please follow the steps below: + +- Create a sandbox GCP environment by visiting [this page](https://about.gitlab.com/handbook/infrastructure-standards/#individual-environment) and following the instructions +- In the GCP console, go to `IAM & Admin` > `Service Accounts` and click on the "Create new service account" button +- Name the service account something specific to what you're using it for. Select Create and Continue. Under `Grant this service account access to project`, select the role `Vertex AI User`. Select `Continue` then `Done` +- Select your new service account and `Manage keys` > `Add Key` > `Create new key`. This will download the **private** JSON credentials for your service account. Your full settings should then be: + +```ruby +Gitlab::CurrentSettings.update(tofa_credentials: File.read('/YOUR_FILE.json')) + +# Note: These credential examples will not work locally for all models +Gitlab::CurrentSettings.update(tofa_host: "") # Example: us-central1-aiplatform.googleapis.com +Gitlab::CurrentSettings.update(tofa_url: "") # Example: https://ROOT-DOMAIN/v1/projects/MY-COOL-PROJECT/locations/us-central1/publishers/google/models/MY-SPECIAL-MODEL:predict +``` + +Internal team members can [use this snippet](https://gitlab.com/gitlab-com/gl-infra/production/-/snippets/2541742) for help configuring these endpoints. + +### Configure OpenAI access + +```ruby +Gitlab::CurrentSettings.update(openai_api_key: "") +``` + +### Configure Anthropic access + +```ruby +Feature.enable(:anthropic_experimentation) +Gitlab::CurrentSettings.update!(anthropic_api_key: ) +``` + +## Experimental REST API + +Use the [experimental REST API endpoints](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/api/ai/experimentation) to quickly experiment and prototype AI features. + +The endpoints are: + +- `https://gitlab.example.com/api/v4/ai/experimentation/openai/completions` +- `https://gitlab.example.com/api/v4/ai/experimentation/openai/embeddings` +- `https://gitlab.example.com/api/v4/ai/experimentation/openai/chat/completions` +- `https://gitlab.example.com/api/v4/ai/experimentation/anthropic/complete` +- `https://gitlab.example.com/api/v4/ai/experimentation/tofa/chat` + +These endpoints are only for prototyping, not for rolling features out to customers. +The experimental endpoint is only available to GitLab team members on production. Use the +[GitLab API token](../user/profile/personal_access_tokens.md) to authenticate. + +## Abstraction layer + +### GraphQL API + +To connect to the OpenAI API using the Abstraction Layer, use an extendable GraphQL API called +[`aiAction`](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/app/graphql/mutations/ai/action.rb). +The `input` accepts key/value pairs, where the `key` is the action that needs to be performed. +We only allow one AI action per mutation request. + +Example of a mutation: + +```graphql +mutation { + aiAction(input: {summarizeComments: {resourceId: "gid://gitlab/Issue/52"}}) { + clientMutationId + } +} +``` + +As an example, assume we want to build an "explain code" action. To do this, we extend the `input` with a new key, +`explainCode`. The mutation would look like this: + +```graphql +mutation { + aiAction(input: {explainCode: {resourceId: "gid://gitlab/MergeRequest/52", code: "foo() { console.log()" }}) { + clientMutationId + } +} +``` + +The GraphQL API then uses the [OpenAI Client](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/lib/gitlab/llm/open_ai/client.rb) +to send the response. + +#### How to receive a response + +As the OpenAI API requests are handled in a background job, we do not keep the request alive and +the response is sent through the `aiCompletionResponse` subscription: + +```mutation +subscription aiCompletionResponse($userId: UserID, $resourceId: AiModelID!) { + aiCompletionResponse(userId: $userId, resourceId: $resourceId) { + responseBody + errors + } +} +``` + +WARNING: +You should only subscribe to the subscription once the mutation is sent. If multiple subscriptions are active on the same page, they currently all receive updates as our identifier is the user and the resource. To mitigate this, you should only subscribe when the mutation is sent. You can use [`skip()`](You can use [`skip()`](https://apollo.vuejs.org/guide/apollo/subscriptions.html#skipping-the-subscription)) for this case. To prevent this problem in the future, we implement a [request identifier](https://gitlab.com/gitlab-org/gitlab/-/issues/408196). + +#### Current abstraction layer flow + +```mermaid +flowchart TD +A[GitLab frontend] -->B[AiAction GraphQL mutation] +B --> C[Llm::ExecuteMethodService] +C --> D[One of services, for example: Llm::GenerateSummaryService] +D -->|scheduled| E[AI worker:Llm::CompletionWorker] +E -->F[::Gitlab::Llm::Completions::Factory] +F -->G[`::Gitlab::Llm::OpenAi::Completions::...` class using `::Gitlab::Llm::OpenAi::Templates::...` class] +G -->|calling| H[Gitlab::Llm::OpenAi::Client] +H --> |response| I[::Gitlab::Llm::OpenAi::ResponseService] +I --> J[GraphqlTriggers.ai_completion_response] +J --> K[::GitlabSchema.subscriptions.trigger] +``` + +## CircuitBreaker + +The CircuitBreaker concern is a reusable module that you can include in any class that needs to run code with circuit breaker protection. The concern provides a `run_with_circuit` method that wraps a code block with circuit breaker functionality, which helps prevent cascading failures and improves system resilience. For more information about the circuit breaker pattern, see: + +- [What is Circuit breaker](https://martinfowler.com/bliki/CircuitBreaker.html). +- [The Hystrix documentation on CircuitBreaker](https://github.com/Netflix/Hystrix/wiki/How-it-Works#circuit-breaker). + +### Use CircuitBreaker + +To use the CircuitBreaker concern, you need to include it in a class. For example: + +```ruby +class MyService + include Gitlab::Llm::Concerns::CircuitBreaker + + def call_external_service + run_with_circuit do + # Code that interacts with external service goes here + + raise InternalServerError + end + end +end +``` + +The `call_external_service` method is an example method that interacts with an external service. +By wrapping the code that interacts with the external service with `run_with_circuit`, the method is executed within the circuit breaker. +The circuit breaker is created and configured by the `circuit` method, which is called automatically when the `CircuitBreaker` module is included. +The method should raise `InternalServerError` error which will be counted towards the error threshold if raised during the execution of the code block. + +The circuit breaker tracks the number of errors and the rate of requests, +and opens the circuit if it reaches the configured error threshold or volume threshold. +If the circuit is open, subsequent requests fail fast without executing the code block, and the circuit breaker periodically allows a small number of requests through to test the service's availability before closing the circuit again. + +### Configuration + +The circuit breaker is configured with two constants which control the number of errors and requests at which the circuit will open: + +- `ERROR_THRESHOLD` +- `VOLUME_THRESHOLD` + +You can adjust these values as needed for the specific service and usage pattern. +The `InternalServerError` is the exception class counted towards the error threshold if raised during the execution of the code block. +This is the exception class that triggers the circuit breaker when raised by the code that interacts with the external service. + +NOTE: +The `CircuitBreaker` module depends on the `Circuitbox` gem to provide the circuit breaker implementation. By default, the service name is inferred from the class name where the concern module is included. Override the `service_name` method if the name needs to be different. + +### Testing + +To test code that uses the `CircuitBreaker` concern, you can use `RSpec` shared examples and pass the `service` and `subject` variables: + +```ruby +it_behaves_like 'has circuit breaker' do + let(:service) { dummy_class.new } + let(:subject) { service.dummy_method } +end +``` + +## How to implement a new action + +### Register a new method + +Go to the `Llm::ExecuteMethodService` and add a new method with the new service class you will create. + +```ruby +class ExecuteMethodService < BaseService + METHODS = { + # ... + amazing_new_ai_feature: Llm::AmazingNewAiFeatureService + }.freeze +``` + +### Create a Service + +1. Create a new service under `ee/app/services/llm/` and inherit it from the `BaseService`. +1. The `resource` is the object we want to act on. It can be any object that includes the `Ai::Model` concern. For example it could be a `Project`, `MergeRequest`, or `Issue`. + +```ruby +# ee/app/services/llm/amazing_new_ai_feature_service.rb + +module Llm + class AmazingNewAiFeatureService < BaseService + private + + def perform + ::Llm::CompletionWorker.perform_async(user.id, resource.id, resource.class.name, :amazing_new_ai_feature) + success + end + + def valid? + super && Ability.allowed?(user, :amazing_new_ai_feature, resource) + end + end +end +``` + +### Authorization + +We recommend to use [policies](policies.md) to deal with authorization for a feature. Currently we need to make sure to cover the following checks: + +1. General AI feature flag is enabled +1. Feature specific feature flag is enabled +1. The namespace has the required license for the feature +1. User is a member of the group/project +1. Resource is allowed to be sent (see `send_to_ai?` method) +1. `experiment_features_enabled` and `third_party_ai_features_enabled` flags are set on the `Namespace` + +For our example, we need to implement the `allowed?(:amazing_new_ai_feature)` call. As an example, you can look at the [Issue Policy for the summarize comments feature](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/policies/ee/issue_policy.rb). In our example case, we want to implement the feature for Issues as well: + +```ruby +# ee/app/policies/ee/issue_policy.rb + +module EE + module IssuePolicy + extend ActiveSupport::Concern + prepended do + with_scope :subject + condition(:ai_available) do + ::Feature.enabled?(:openai_experimentation) && + @subject.send_to_ai? + end + + with_scope :subject + condition(:amazing_new_ai_feature_enabled) do + ::Feature.enabled?(:amazing_new_ai_feature, subject_container) && + subject_container.licensed_feature_available?(:amazing_new_ai_feature) + end + + rule do + ai_available & amazing_new_ai_feature_enabled & is_project_member + end.enable :amazing_new_ai_feature + end + end +end +``` + +### Implement `send_to_ai?` + +To make sure we only send data that is allowed to be sent, we have the `send_to_ai?` method. It checks if the resource is not confidential and public data. +Some resources already implement `send_to_ai?`. Make sure yours does as well. In our case, `Issue` is already covered with the `Issuable` concern. This is an example how it could look like: + +```ruby +# ee/app/models/concerns/ee + +def send_to_ai? + !try(:confidential) && resource_parent.public? +end +``` + +### Check if feature is allowed for this resource based on namespace settings + +There are two settings allowed on root namespace level that restrict the use of AI features: + +- `experiment_features_enabled` +- `third_party_ai_features_enabled`. + +To check if that feature is allowed for a given namespace, call: + +```ruby +Gitlab::Llm::StageCheck.available?(namespace, :name_of_the_feature) +``` + +Add the name of the feature to the `Gitlab::Llm::StageCheck` class. There are arrays there that differentiate +between experimental and beta features. + +This way we are ready for the following different cases: + +- If the feature is not in any array, the check will return `true`. For example, the feature was moved to GA and does not use a third-party setting. +- If feature is in GA, but uses a third-party setting, the class will return a proper answer based on the namespace third-party setting. + +To move the feature from the experimental phase to the beta phase, move the name of the feature from the `EXPERIMENTAL_FEATURES` array to the `BETA_FEATURES` array. + +### Implement calls to AI APIs and the prompts + +The `CompletionWorker` will call the `Completions::Factory` which will initialize the Service and execute the actual call to the API. +In our example, we will use OpenAI and implement two new classes: + +```ruby +# /ee/lib/gitlab/llm/open_ai/completions/amazing_new_ai_feature.rb + +module Gitlab + module Llm + module OpenAi + module Completions + class AmazingNewAiFeature + def initialize(ai_prompt_class) + @ai_prompt_class = ai_prompt_class + end + + def execute(user, issue, options) + options = ai_prompt_class.get_options(options[:messages]) + + ai_response = Gitlab::Llm::OpenAi::Client.new(user).chat(content: nil, **options) + + ::Gitlab::Llm::OpenAi::ResponseService.new(user, issue, ai_response, options: {}).execute( + Gitlab::Llm::OpenAi::ResponseModifiers::Chat.new + ) + end + + private + + attr_reader :ai_prompt_class + end + end + end + end +end +``` + +```ruby +# /ee/lib/gitlab/llm/open_ai/templates/amazing_new_ai_feature.rb + +module Gitlab + module Llm + module OpenAi + module Templates + class AmazingNewAiFeature + TEMPERATURE = 0.3 + + def self.get_options(messages) + system_content = <<-TEMPLATE + You are an assistant that writes code for the following input: + """ + TEMPLATE + + { + messages: [ + { role: "system", content: system_content }, + { role: "user", content: messages }, + ], + temperature: TEMPERATURE + } + end + end + end + end + end +end +``` + +### Add Ai Action to GraphQL + +TODO + +## Security + +Refer to the [secure coding guidelines for Artificial Intelligence (AI) features](secure_coding_guidelines.md#artificial-intelligence-ai-features). diff --git a/doc/development/api_graphql_styleguide.md b/doc/development/api_graphql_styleguide.md index 94abbda9671..15b587e3b1e 100644 --- a/doc/development/api_graphql_styleguide.md +++ b/doc/development/api_graphql_styleguide.md @@ -40,6 +40,13 @@ GraphiQL is an interactive GraphQL API explorer where you can play around with e You can access it in any GitLab environment on `https:///-/graphql-explorer`. For example, the one for [GitLab.com](https://gitlab.com/-/graphql-explorer). +## Reviewing merge requests with GraphQL changes + +The GraphQL framework has some specific gotchas to be aware of, and domain expertise is required to ensure they are satisfied. + +If you are asked to review a merge request that modifies any GraphQL files or adds an endpoint, please have a look at +[our GraphQL review guide](graphql_guide/reviewing.md). + ## Authentication Authentication happens through the `GraphqlController`, right now this @@ -287,13 +294,13 @@ or by calling `#to_global_id` on an object that has mixed in the `GlobalID::Identification` module. Using an example from -[`Types::Notes::DiscussionType`](https://gitlab.com/gitlab-org/gitlab/-/blob/3c95bd9/app/graphql/types/notes/discussion_type.rb#L24-26): +[`Types::Notes::DiscussionType`](https://gitlab.com/gitlab-org/gitlab/-/blob/af48df44/app/graphql/types/notes/discussion_type.rb#L22-30): ```ruby -field :reply_id, GraphQL::Types::ID +field :reply_id, Types::GlobalIDType[Discussion] def reply_id - ::Gitlab::GlobalId.build(object, id: object.reply_id) + Gitlab::GlobalId.build(object, id: object.reply_id) end ``` @@ -597,7 +604,7 @@ description 'Mutates an object. Does not mutate the object if ' \ def resolve(id: ) object = authorized_find!(id: id) - raise Gitlab::Graphql::Errors::ResourceNotAvailable, '`my_feature_flag` feature flag is disabled.' \ + raise_resource_not_available_error! '`my_feature_flag` feature flag is disabled.' \ if Feature.disabled?(:my_feature_flag, object) # ... end @@ -785,7 +792,7 @@ The documentation mentions that the old Global ID style is now deprecated. ## Mark schema items as Alpha You can mark GraphQL schema items (fields, arguments, enum values, and mutations) as -[Alpha](../policy/alpha-beta-support.md#alpha-features). +[Alpha](../policy/alpha-beta-support.md#experiment). An item marked as Alpha is [exempt from the deprecation process](#breaking-change-exemptions) and can be removed at any time without notice. Mark an item as Alpha when it is @@ -875,7 +882,7 @@ module Types graphql_name 'IssuableSeverity' description 'Incident severity' - ::IssuableSeverity.severities.keys.each do |severity| + ::IssuableSeverity.severities.each_key do |severity| value severity.upcase, value: severity, description: "#{severity.titleize} severity." end end @@ -1132,7 +1139,9 @@ you need to do something more custom however, remember, if you encounter an object the `current_user` does not have access to when resolving a field, then the entire field should resolve to `null`. -### Deriving resolvers (`BaseResolver.single` and `BaseResolver.last`) +### Deriving resolvers + +(including `BaseResolver.single` and `BaseResolver.last`) For some use cases, we can derive resolvers from others. The main use case for this is one resolver to find all items, and another to @@ -1704,8 +1713,8 @@ object on the mutation. This would allow you to use the When a user is not allowed to perform the action, or an object is not found, we should raise a -`Gitlab::Graphql::Errors::ResourceNotAvailable` error which is -correctly rendered to the clients. +`Gitlab::Graphql::Errors::ResourceNotAvailable` by calling `raise_resource_not_available_error!` +from in the `resolve` method. ### Errors in mutations diff --git a/doc/development/api_styleguide.md b/doc/development/api_styleguide.md index 006d0a01abb..0652b88617b 100644 --- a/doc/development/api_styleguide.md +++ b/doc/development/api_styleguide.md @@ -57,6 +57,59 @@ get do end ``` +## Breaking changes + +We must not make breaking changes to our REST API v4, even in major GitLab releases. + +Our REST API maintains its own versioning independent of GitLab versioning. +The current REST API version is `4`. [We commit to follow semantic versioning for our REST API](../api/rest/index.md#compatibility-guidelines), +which means we cannot make breaking changes until a major version change (most likely, `5`). + +Because version `5` is not scheduled, we allow rare [exceptions](#exceptions). + +### Accommodating backward compatibility instead of breaking changes + +Backward compatibility can often be accommodated in the API by continuing to adapt a changed feature to +the old API schema. For example, our REST API +[exposes](https://gitlab.com/gitlab-org/gitlab/-/blob/c104f6b8/lib/api/entities/merge_request_basic.rb#L43-47) both +`work_in_progress` and `draft` fields. + +### Exceptions + +The exception is only when: + +- A feature must be removed in a major GitLab release. +- Backward compatibility cannot be maintained + [in any form](#accommodating-backward-compatibility-instead-of-breaking-changes). + +This exception should be rare. + +Even in this exception, rather than removing a field or argument, we must always do the following: + +- Return an empty response for a field (for example, `"null"` or `[]`). +- Turn an argument into a no-op. + +## What is a breaking change + +Some examples of breaking changes are: + +- Removing or renaming fields, arguments, or enum values. +- Removing endpoints. +- Adding new redirects (not all clients follow redirects). +- Changing the type of fields in the response (for example, from `String` to `Integer`). +- Adding a new **required** argument. +- Changing authentication, authorization, or other header requirements. +- Changing [any status code](../api/rest/index.md#status-codes) other than `500`. + +## What is not a breaking change + +Some examples of non-breaking changes: + +- Any additive change, such as adding endpoints, non-required arguments, fields, or enum values. +- Changes to error messages. +- Changes from a `500` status code to [any supported status code](../api/rest/index.md#status-codes) (this is a bugfix). +- Changes to the order of fields returned in a response. + ## Declared parameters Grape allows you to access only the parameters that have been declared by your @@ -333,6 +386,18 @@ expect(response).to match_response_schema('merge_requests') Also see [verifying N+1 performance](#verifying-with-tests) in tests. +## Error handling + +When throwing an error with a message that is meant to be user-facing, you should +use the error message utility function contained in `lib/gitlab/utils/error_message.rb`. +It adds a prefix to the error message, making it distinguishable from non-user-facing error messages. + +```ruby +Gitlab::Utils::ErrorMessage.to_user_facing('Example user-facing error-message') +``` + +Please make sure that the Frontend is aware of the prefix usage and is using the according utils. See [Error handling](fe_guide/style/javascript.md#error-handling) in JavaScript style guide for more information. + ## Include a changelog entry All client-facing changes **must** include a [changelog entry](changelog.md). diff --git a/doc/development/application_secrets.md b/doc/development/application_secrets.md index 526cc6c3f61..5b0755e97e3 100644 --- a/doc/development/application_secrets.md +++ b/doc/development/application_secrets.md @@ -17,7 +17,7 @@ This page is a development guide for application secrets. |`db_key_base` | The base key to encrypt the data for `attr_encrypted` columns | |`openid_connect_signing_key` | The signing key for OpenID Connect | | `encrypted_settings_key_base` | The base key to encrypt settings files with | -| `ci_jwt_signing_key` | The base key for encrypting the `CI_JOB_JWT` and `CI_JOB_JWT_V2` predefined CI/CD variables | +| `ci_jwt_signing_key` | The base key for encrypting the `CI_JOB_JWT` and `CI_JOB_JWT_V2` predefined CI/CD variables. `CI_JOB_JWT` and `CI_JOB_JWT_V2` were [deprecated in GitLab 15.9](../update/deprecations.md#old-versions-of-json-web-tokens-are-deprecated) and are scheduled to be removed in GitLab 16.5. | ## Where the secrets are stored diff --git a/doc/development/application_slis/index.md b/doc/development/application_slis/index.md index bd4587333e0..f48088a6e08 100644 --- a/doc/development/application_slis/index.md +++ b/doc/development/application_slis/index.md @@ -24,7 +24,7 @@ to be emitted from the rails application: ## Existing SLIs -1. [`rails_request_apdex`](rails_request_apdex.md) +1. [`rails_request`](rails_request.md) 1. `global_search_apdex` 1. `global_search_error_rate` 1. `global_search_indexing_apdex` diff --git a/doc/development/application_slis/rails_request.md b/doc/development/application_slis/rails_request.md new file mode 100644 index 00000000000..b3ee326aa87 --- /dev/null +++ b/doc/development/application_slis/rails_request.md @@ -0,0 +1,282 @@ +--- +stage: Platforms +group: Scalability +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Rails request SLIs (service level indicators) + +> [Introduced](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/525) in GitLab 14.4 + +NOTE: +This SLI is used for service monitoring. But not for [error budgets for stage groups](../stage_group_observability/index.md#error-budget) +by default. You can [opt in](#error-budget-attribution-and-ownership). + +The request Apdex SLI and the error rate SLI are [SLIs defined in the application](index.md). + +The request Apdex measures the duration of successful requests as an indicator for +application performance. This includes the REST and GraphQL API, and the +regular controller endpoints. + +The error rate measures unsuccessful requests as an indicator for +server misbehavior. This includes the REST API, and the +regular controller endpoints. + +1. `gitlab_sli_rails_request_apdex_total`: This counter gets + incremented for every request that did not result in a response + with a `5xx` status code. It ensures slow failures are not + counted twice, because the request is already counted in the error SLI. + +1. `gitlab_sli_rails_request_apdex_success_total`: This counter gets + incremented for every successful request that performed faster than + the [defined target duration depending on the endpoint's urgency](#adjusting-request-urgency). + +1. `gitlab_sli_rails_request_error_total`: This counter gets + incremented for every request that resulted in a response + with a `5xx` status code. + +1. `gitlab_sli_rails_request_total`: This counter gets + incremented for every request. + +These counters are labeled with: + +1. `endpoint_id`: The identification of the Rails Controller or the + Grape-API endpoint. + +1. `feature_category`: The feature category specified for that + controller or API endpoint. + +## Request Apdex SLO + +These counters can be combined into a success ratio. The objective for +this ratio is defined in the service catalog per service. For this SLI to meet SLO, +the ratio recorded must be higher than: + +- [Web: 0.998](https://gitlab.com/gitlab-com/runbooks/blob/master/metrics-catalog/services/web.jsonnet#L19) +- [API: 0.995](https://gitlab.com/gitlab-com/runbooks/blob/master/metrics-catalog/services/api.jsonnet#L19) +- [Git: 0.998](https://gitlab.com/gitlab-com/runbooks/blob/master/metrics-catalog/services/git.jsonnet#L22) + +For example: for the web-service, we want at least 99.8% of requests +to be faster than their target duration. + +We use these targets for alerting and service monitoring. Set durations taking +these targets into account, so we don't cause alerts. The goal, however, is to +set the urgency to a target that satisfies our users. + +Both successful measurements and unsuccessful ones affect the +error budget for stage groups. + +## Adjusting request urgency + +Not all endpoints perform the same type of work, so it is possible to +define different urgency levels for different endpoints. An endpoint with a +lower urgency can have a longer request duration than endpoints with high urgency. + +Long-running requests are more expensive for our infrastructure. While serving +one request, the thread remains occupied for the duration of that request. The thread +can handle nothing else. Due to Ruby's Global VM Lock, the thread might keep the +lock and stall other requests handled by the same Puma worker +process. The request is, in fact, a noisy neighbor for other requests +handled by the worker. We cap the upper bound for a target duration at 5 seconds +for this reason. + +## Decreasing the urgency (setting a higher target duration) + +You can decrease the urgency on an existing endpoint on +a case-by-case basis. Take the following into account: + +1. Apdex is about perceived performance. If a user is actively waiting + for the result of a request, waiting 5 seconds might not be + acceptable. However, if the endpoint is used by an automation + requiring a lot of data, 5 seconds could be acceptable. + + A product manager can help to identify how an endpoint is used. + +1. The workload for some endpoints can sometimes differ greatly + depending on the parameters specified by the caller. The urgency + needs to accommodate those differences. In some cases, you could + define a separate [application SLI](index.md#defining-a-new-sli) + for what the endpoint is doing. + + When the endpoints in certain cases turn into no-ops, making them + very fast, we should ignore these fast requests when setting the + target. For example, if the `MergeRequests::DraftsController` is + hit for every merge request being viewed, but rarely renders + anything, then we should pick the target that + would still accommodate the endpoint performing work. + +1. Consider the dependent resources consumed by the endpoint. If the endpoint + loads a lot of data from Gitaly or the database, and this causes + unsatisfactory performance, consider optimizing the + way the data is loaded rather than increasing the target duration + by lowering the urgency. + + In these cases, it might be appropriate to temporarily decrease + urgency to make the endpoint meet SLO, if this is bearable for the + infrastructure. In such cases, create a code comment linking to an issue. + + If the endpoint consumes a lot of CPU time, we should also consider + this: these kinds of requests are the kind of noisy neighbors we + should try to keep as short as possible. + +1. Traffic characteristics should also be taken into account. If the + traffic to the endpoint sometimes bursts, like CI traffic spinning up a + big batch of jobs hitting the same endpoint, then having these + endpoints take five seconds is unacceptable from an infrastructure point of + view. We cannot scale up the fleet fast enough to accommodate for + the incoming slow requests alongside the regular traffic. + +When lowering the urgency for an existing endpoint, please involve a +[Scalability team member](https://about.gitlab.com/handbook/engineering/infrastructure/team/scalability/#team-members) +in the review. We can use request rates and durations available in the +logs to come up with a recommendation. You can pick a threshold +using the same process as for +[increasing urgency](#increasing-urgency-setting-a-lower-target-duration), +picking a duration that is higher than the SLO for the service. + +We shouldn't set the longest durations on endpoints in the merge +requests that introduces them, because we don't yet have data to support +the decision. + +## Increasing urgency (setting a lower target duration) + +When increasing the urgency, we must make sure the endpoint +still meets SLO for the fleet that handles the request. You can use the +information in the logs to check: + +1. Open [this table in Kibana](https://log.gprd.gitlab.net/goto/bbb6465c68eb83642269e64a467df3df) + +1. The table loads information for the busiest endpoints by + default. To speed the response, add both: + + - A filter for `json.meta.caller_id.keyword`. + - The identifier you're interested in, for example: + + ```ruby + Projects::RawController#show + ``` + + or: + + ```plaintext + GET /api/:version/projects/:id/snippets/:snippet_id/raw + ``` + +1. Check the [appropriate percentile duration](#request-apdex-slo) for + the service handling the endpoint. The overall duration should + be lower than your intended target. + +1. If the overall duration is below the intended target, check the peaks over time + in [this graph](https://log.gprd.gitlab.net/goto/9319c4a402461d204d13f3a4924a89fc) + in Kibana. Here, the percentile in question should not peak above + the target duration we want to set. + +As decreasing a threshold too much could result in alerts for the +Apdex degradation, please also involve a Scalability team member in +the merge request. + +## How to adjust the urgency + +You can specify urgency similar to how endpoints +[get a feature category](../feature_categorization/index.md). Endpoints without a +specific target use the default urgency: 1s duration. These configurations +are available: + +| Urgency | Duration in seconds | Notes | +|------------|---------------------|-----------------------------------------------| +| `:high` | 0.25s | | +| `:medium` | 0.5s | | +| `:default` | 1s | The default when nothing is specified. | +| `:low` | 5s | | + +### Rails controller + +An urgency can be specified for all actions in a controller: + +```ruby +class Boards::ListsController < ApplicationController + urgency :high +end +``` + +To also specify the urgency for certain actions in a controller: + +```ruby +class Boards::ListsController < ApplicationController + urgency :high, [:index, :show] +end +``` + +A custom RSpec matcher is available to check endpoint's request urgency in the controller specs: + +```ruby +specify do + expect(get(:index, params: request_params)).to have_request_urgency(:medium) +end +``` + +### Grape endpoints + +To specify the urgency for an entire API class: + +```ruby +module API + class Issues < ::API::Base + urgency :low + end +end +``` + +To specify the urgency also for certain actions in a API class: + +```ruby +module API + class Issues < ::API::Base + urgency :medium, [ + '/groups/:id/issues', + '/groups/:id/issues_statistics' + ] + end +end +``` + +Or, we can specify the urgency per endpoint: + +```ruby +get 'client/features', urgency: :low do + # endpoint logic +end +``` + +A custom RSpec matcher is also compatible with grape endpoints' specs: + +```ruby + +specify do + expect(get(api('/avatar'), params: { email: 'public@example.com' })).to have_request_urgency(:medium) +end +``` + +WARNING: +We can't specify the urgency at the namespace level. The directive is ignored when doing so. + +### Error budget attribution and ownership + +This SLI is used for service level monitoring. It feeds into the +[error budget for stage groups](../stage_group_observability/index.md#error-budget). + +For more information, read the epic for +[defining custom SLIs and incorporating them into error budgets](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/525)). +The endpoints for the SLI feed into a group's error budget based on the +[feature category declared on it](../feature_categorization/index.md). + +To know which endpoints are included for your group, you can see the +request rates on the +[group dashboard for your group](https://dashboards.gitlab.net/dashboards/f/stage-groups/stage-groups). +In the **Budget Attribution** row, the **Puma Apdex** log link shows you +how many requests are not meeting a 1s or 5s target. + +For more information about the content of the dashboard, see +[Dashboards for stage groups](../stage_group_observability/index.md). For more information +about our exploration of the error budget itself, see +[issue 1365](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1365). diff --git a/doc/development/application_slis/rails_request_apdex.md b/doc/development/application_slis/rails_request_apdex.md deleted file mode 100644 index dc9d67b0a2b..00000000000 --- a/doc/development/application_slis/rails_request_apdex.md +++ /dev/null @@ -1,253 +0,0 @@ ---- -stage: Platforms -group: Scalability -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments ---- - -# Rails request Apdex SLI - -> [Introduced](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/525) in GitLab 14.4 - -NOTE: -This SLI is used for service monitoring. But not for [error budgets for stage groups](../stage_group_observability/index.md#error-budget) -by default. You can [opt in](#error-budget-attribution-and-ownership). - -The request Apdex SLI (Service Level Indicator) is [an SLI defined in the application](index.md). -It measures the duration of successful requests as an indicator for -application performance. This includes the REST and GraphQL API, and the -regular controller endpoints. It consists of these counters: - -1. `gitlab_sli:rails_request_apdex:total`: This counter gets - incremented for every request that did not result in a response - with a `5xx` status code. It ensures slow failures are not - counted twice, because the request is already counted in the error SLI. - -1. `gitlab_sli:rails_request_apdex:success_total`: This counter gets - incremented for every successful request that performed faster than - the [defined target duration depending on the endpoint's urgency](#adjusting-request-urgency). - -Both these counters are labeled with: - -1. `endpoint_id`: The identification of the Rails Controller or the - Grape-API endpoint. - -1. `feature_category`: The feature category specified for that - controller or API endpoint. - -## Request Apdex SLO - -These counters can be combined into a success ratio. The objective for -this ratio is defined in the service catalog per service. For this SLI to meet SLO, -the ratio recorded must be higher than: - -- [Web: 0.998](https://gitlab.com/gitlab-com/runbooks/blob/master/metrics-catalog/services/web.jsonnet#L19) -- [API: 0.995](https://gitlab.com/gitlab-com/runbooks/blob/master/metrics-catalog/services/api.jsonnet#L19) -- [Git: 0.998](https://gitlab.com/gitlab-com/runbooks/blob/master/metrics-catalog/services/git.jsonnet#L22) - -For example: for the web-service, we want at least 99.8% of requests -to be faster than their target duration. - -We use these targets for alerting and service monitoring. Set durations taking -these targets into account, so we don't cause alerts. The goal, however, is to -set the urgency to a target that satisfies our users. - -Both successful measurements and unsuccessful ones affect the -error budget for stage groups. - -## Adjusting request urgency - -Not all endpoints perform the same type of work, so it is possible to -define different urgency levels for different endpoints. An endpoint with a -lower urgency can have a longer request duration than endpoints with high urgency. - -Long-running requests are more expensive for our infrastructure. While serving -one request, the thread remains occupied for the duration of that request. The thread -can handle nothing else. Due to Ruby's Global VM Lock, the thread might keep the -lock and stall other requests handled by the same Puma worker -process. The request is, in fact, a noisy neighbor for other requests -handled by the worker. We cap the upper bound for a target duration at 5 seconds -for this reason. - -## Decreasing the urgency (setting a higher target duration) - -You can decrease the urgency on an existing endpoint on -a case-by-case basis. Take the following into account: - -1. Apdex is about perceived performance. If a user is actively waiting - for the result of a request, waiting 5 seconds might not be - acceptable. However, if the endpoint is used by an automation - requiring a lot of data, 5 seconds could be acceptable. - - A product manager can help to identify how an endpoint is used. - -1. The workload for some endpoints can sometimes differ greatly - depending on the parameters specified by the caller. The urgency - needs to accommodate those differences. In some cases, you could - define a separate [application SLI](index.md#defining-a-new-sli) - for what the endpoint is doing. - - When the endpoints in certain cases turn into no-ops, making them - very fast, we should ignore these fast requests when setting the - target. For example, if the `MergeRequests::DraftsController` is - hit for every merge request being viewed, but rarely renders - anything, then we should pick the target that - would still accommodate the endpoint performing work. - -1. Consider the dependent resources consumed by the endpoint. If the endpoint - loads a lot of data from Gitaly or the database, and this causes - unsatisfactory performance, consider optimizing the - way the data is loaded rather than increasing the target duration - by lowering the urgency. - - In these cases, it might be appropriate to temporarily decrease - urgency to make the endpoint meet SLO, if this is bearable for the - infrastructure. In such cases, create a code comment linking to an issue. - - If the endpoint consumes a lot of CPU time, we should also consider - this: these kinds of requests are the kind of noisy neighbors we - should try to keep as short as possible. - -1. Traffic characteristics should also be taken into account. If the - traffic to the endpoint sometimes bursts, like CI traffic spinning up a - big batch of jobs hitting the same endpoint, then having these - endpoints take five seconds is unacceptable from an infrastructure point of - view. We cannot scale up the fleet fast enough to accommodate for - the incoming slow requests alongside the regular traffic. - -When lowering the urgency for an existing endpoint, please involve a -[Scalability team member](https://about.gitlab.com/handbook/engineering/infrastructure/team/scalability/#team-members) -in the review. We can use request rates and durations available in the -logs to come up with a recommendation. You can pick a threshold -using the same process as for -[increasing urgency](#increasing-urgency-setting-a-lower-target-duration), -picking a duration that is higher than the SLO for the service. - -We shouldn't set the longest durations on endpoints in the merge -requests that introduces them, because we don't yet have data to support -the decision. - -## Increasing urgency (setting a lower target duration) - -When increasing the urgency, we must make sure the endpoint -still meets SLO for the fleet that handles the request. You can use the -information in the logs to check: - -1. Open [this table in Kibana](https://log.gprd.gitlab.net/goto/bbb6465c68eb83642269e64a467df3df) - -1. The table loads information for the busiest endpoints by - default. To speed the response, add both: - - - A filter for `json.meta.caller_id.keyword`. - - The identifier you're interested in, for example: - - ```ruby - Projects::RawController#show - ``` - - or: - - ```plaintext - GET /api/:version/projects/:id/snippets/:snippet_id/raw - ``` - -1. Check the [appropriate percentile duration](#request-apdex-slo) for - the service handling the endpoint. The overall duration should - be lower than your intended target. - -1. If the overall duration is below the intended target, check the peaks over time - in [this graph](https://log.gprd.gitlab.net/goto/9319c4a402461d204d13f3a4924a89fc) - in Kibana. Here, the percentile in question should not peak above - the target duration we want to set. - -As decreasing a threshold too much could result in alerts for the -Apdex degradation, please also involve a Scalability team member in -the merge request. - -## How to adjust the urgency - -You can specify urgency similar to how endpoints -[get a feature category](../feature_categorization/index.md). Endpoints without a -specific target use the default urgency: 1s duration. These configurations -are available: - -| Urgency | Duration in seconds | Notes | -|------------|---------------------|-----------------------------------------------| -| `:high` | 0.25s | | -| `:medium` | 0.5s | | -| `:default` | 1s | The default when nothing is specified. | -| `:low` | 5s | | - -### Rails controller - -An urgency can be specified for all actions in a controller: - -```ruby -class Boards::ListsController < ApplicationController - urgency :high -end -``` - -To also specify the urgency for certain actions in a controller: - -```ruby -class Boards::ListsController < ApplicationController - urgency :high, [:index, :show] -end -``` - -### Grape endpoints - -To specify the urgency for an entire API class: - -```ruby -module API - class Issues < ::API::Base - urgency :low - end -end -``` - -To specify the urgency also for certain actions in a API class: - -```ruby -module API - class Issues < ::API::Base - urgency :medium, [ - '/groups/:id/issues', - '/groups/:id/issues_statistics' - ] - end -end -``` - -Or, we can specify the urgency per endpoint: - -```ruby -get 'client/features', urgency: :low do - # endpoint logic -end -``` - -WARNING: -We can't specify the urgency at the namespace level. The directive is ignored when doing so. - -### Error budget attribution and ownership - -This SLI is used for service level monitoring. It feeds into the -[error budget for stage groups](../stage_group_observability/index.md#error-budget). - -For more information, read the epic for -[defining custom SLIs and incorporating them into error budgets](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/525)). -The endpoints for the SLI feed into a group's error budget based on the -[feature category declared on it](../feature_categorization/index.md). - -To know which endpoints are included for your group, you can see the -request rates on the -[group dashboard for your group](https://dashboards.gitlab.net/dashboards/f/stage-groups/stage-groups). -In the **Budget Attribution** row, the **Puma Apdex** log link shows you -how many requests are not meeting a 1s or 5s target. - -For more information about the content of the dashboard, see -[Dashboards for stage groups](../stage_group_observability/index.md). For more information -about our exploration of the error budget itself, see -[issue 1365](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1365). diff --git a/doc/development/approval_rules.md b/doc/development/approval_rules.md deleted file mode 100644 index 2e36be1231d..00000000000 --- a/doc/development/approval_rules.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'merge_request_concepts/approval_rules.md' -remove_date: '2023-04-23' ---- - -This document was moved to [another location](merge_request_concepts/approval_rules.md). - - - - - diff --git a/doc/development/architecture.md b/doc/development/architecture.md index 96b70e2fbd8..133cf8a2998 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -413,7 +413,7 @@ GitLab can be considered to have two layers from a process perspective: - [Omnibus](https://github.com/certbot/certbot/blob/master/README.rst) - [Charts](https://github.com/jetstack/cert-manager/blob/master/README.md) - Configuration: - - [Omnibus](https://docs.gitlab.com/omnibus/settings/ssl.html) + - [Omnibus](https://docs.gitlab.com/omnibus/settings/ssl/index.html) - [Charts](https://docs.gitlab.com/charts/installation/tls.html) - [Source](../install/installation.md#using-https) - [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/doc/howto/https.md) @@ -448,7 +448,7 @@ Consul is a tool for service discovery and configuration. Consul is distributed, - [Source](../integration/advanced_search/elasticsearch.md) - [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/doc/howto/elasticsearch.md) - Layer: Core Service (Data) -- GitLab.com: [Get Advanced Search working on GitLab.com (Closed)](https://gitlab.com/groups/gitlab-org/-/epics/153) epic. +- GitLab.com: [Get advanced search working on GitLab.com (Closed)](https://gitlab.com/groups/gitlab-org/-/epics/153) epic. Elasticsearch is a distributed RESTful search engine built for the cloud. @@ -558,7 +558,7 @@ GitLab CI/CD is the open-source continuous integration service included with Git #### GitLab Workhorse -- [Project page](https://gitlab.com/gitlab-org/gitlab-workhorse/blob/master/README.md) +- [Project page](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/workhorse/index.md) - Configuration: - [Omnibus](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-config-template/gitlab.rb.template) - [Charts](https://docs.gitlab.com/charts/charts/gitlab/webservice/) @@ -567,7 +567,7 @@ GitLab CI/CD is the open-source continuous integration service included with Git - Process: `gitlab-workhorse` - GitLab.com: [Service Architecture](https://about.gitlab.com/handbook/engineering/infrastructure/production/architecture/#service-architecture) -[GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse) is a program designed at GitLab to help alleviate pressure from Puma. You can read more about the [historical reasons for developing](https://about.gitlab.com/blog/2016/04/12/a-brief-history-of-gitlab-workhorse/). It's designed to act as a smart reverse proxy to help speed up GitLab as a whole. +[GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/workhorse) is a program designed at GitLab to help alleviate pressure from Puma. You can read more about the [historical reasons for developing](https://about.gitlab.com/blog/2016/04/12/a-brief-history-of-gitlab-workhorse/). It's designed to act as a smart reverse proxy to help speed up GitLab as a whole. #### Grafana @@ -636,7 +636,7 @@ MinIO is an object storage server released under the GNU AGPL v3.0. It is compat - Configuration: - [Omnibus](https://docs.gitlab.com/omnibus/settings/) - [Charts](https://docs.gitlab.com/charts/charts/nginx/) - - [Source](../install/installation.md#9-nginx) + - [Source](../install/installation.md#10-nginx) - Layer: Core Service (Processor) - Process: `nginx` - GitLab.com: [Service Architecture](https://about.gitlab.com/handbook/engineering/infrastructure/production/architecture/#service-architecture) @@ -692,10 +692,10 @@ Prometheus exporter for PgBouncer. Exports metrics at 9127/metrics. - Configuration: - [Omnibus](https://docs.gitlab.com/omnibus/settings/database.html) - [Charts](https://docs.gitlab.com/charts/installation/deployment.html#postgresql) - - [Source](../install/installation.md#6-database) + - [Source](../install/installation.md#7-database) - Layer: Core Service (Data) - Process: `postgresql` -- GitLab.com: [PostgreSQL](../user/gitlab_com/index.md#postgresql) +- GitLab.com: [PostgreSQL](https://about.gitlab.com/handbook/engineering/infrastructure/database/) GitLab packages the popular Database to provide storage for Application meta data and user information. @@ -729,7 +729,7 @@ Prometheus is a time-series tool that helps GitLab administrators expose metrics - Configuration: - [Omnibus](https://docs.gitlab.com/omnibus/settings/redis.html) - [Charts](https://docs.gitlab.com/charts/installation/deployment.html#redis) - - [Source](../install/installation.md#7-redis) + - [Source](../install/installation.md#8-redis) - Layer: Core Service (Data) - Process: `redis` - GitLab.com: [Service Architecture](https://about.gitlab.com/handbook/engineering/infrastructure/production/architecture/#service-architecture) @@ -1101,12 +1101,29 @@ PostgreSQL: GitLab has configuration files located in `/home/git/gitlab/config/*`. Commonly referenced configuration files include: -- `gitlab.yml`: GitLab configuration +- `gitlab.yml`: GitLab Rails configuration - `puma.rb`: Puma web server settings - `database.yml`: Database connection settings GitLab Shell has a configuration file at `/home/git/gitlab-shell/config.yml`. +#### Adding a new setting in GitLab Rails + +Settings which belong in `gitlab.yml` include those related to: + +- How the application is wired together across multiple services. For example, Gitaly addresses, Redis addresses, Postgres addresses, and Consul addresses. +- Distributed Tracing configuration, and some observability configurations. For example, histogram bucket boundaries. +- Anything that needs to be configured during Rails initialization, possibly before the Postgres connection has been established. + +Many other settings are better placed in the app itself, in `ApplicationSetting`. Managing settings in UI is usually a better user experience compared to managing configuration files. With respect to development cost, modifying `gitlab.yml` often seems like a faster iteration, but when you consider all the deployment methods below, it may be a poor tradeoff. + +When adding a setting to `gitlab.yml`: + +1. Ensure that it is also + [added to Omnibus](https://docs.gitlab.com/omnibus/settings/gitlab.yml#adding-a-new-setting-to-gitlabyml). +1. Ensure that it is also [added to Charts](https://docs.gitlab.com/charts/development/style_guide.html), if needed. +1. Ensure that it is also [added to GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/support/templates/gitlab/config/gitlab.yml.erb). + ### Maintenance tasks [GitLab](https://gitlab.com/gitlab-org/gitlab/-/tree/master) provides Rake tasks with which you see version information and run a quick check on your configuration to ensure it is configured properly within the application. See [maintenance Rake tasks](../administration/raketasks/maintenance.md). @@ -1128,3 +1145,7 @@ they don't always work in RHEL. The [GitLab.com architecture](https://about.gitlab.com/handbook/engineering/infrastructure/production/architecture/) is detailed for your reference, but this architecture is only useful if you have millions of users. + +### AI architecture + +A [SaaS model gateway](ai_architecture.md) is available to enable AI-powered features. diff --git a/doc/development/audit_event_guide/index.md b/doc/development/audit_event_guide/index.md index 8df5121a2f7..c7c723d1686 100644 --- a/doc/development/audit_event_guide/index.md +++ b/doc/development/audit_event_guide/index.md @@ -4,7 +4,7 @@ group: Compliance info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Audit Event Guide +# Audit event development guidelines This guide provides an overview of how Audit Events work, and how to instrument new audit events. diff --git a/doc/development/auto_devops.md b/doc/development/auto_devops.md index 53033cd19ff..1f98a37ac9d 100644 --- a/doc/development/auto_devops.md +++ b/doc/development/auto_devops.md @@ -1,10 +1,10 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Auto DevOps development guide +# Auto DevOps development guidelines This document provides a development guide for contributors to [Auto DevOps](../topics/autodevops/index.md). diff --git a/doc/development/backend/create_source_code_be/gitaly_touch_points.md b/doc/development/backend/create_source_code_be/gitaly_touch_points.md index 6ee2c7f0e51..c689af2f150 100644 --- a/doc/development/backend/create_source_code_be/gitaly_touch_points.md +++ b/doc/development/backend/create_source_code_be/gitaly_touch_points.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## RPCs -Gitaly is a wrapper around the `git` binary, running in a [Gitaly Cluster](../../../administration/gitaly/index.md). It provides managed access to the file system housing the `git` repositories, via Golang Remote Procedure Calls (RPCs). Other functions are access optimization, caching, and a form of pagination against the file system. +Gitaly is a wrapper around the `git` binary, running in a [Gitaly Cluster](../../../administration/gitaly/index.md). It provides managed access to the file system housing the `git` repositories, using Go Remote Procedure Calls (RPCs). Other functions are access optimization, caching, and a form of pagination against the file system. The comprehensive [Beginner's guide to Gitaly contributions](https://gitlab.com/gitlab-org/gitaly/-/blob/master/doc/beginners_guide.md) is focused on making updates to Gitaly, and offers many insights into how to understand the Gitaly code. diff --git a/doc/development/backend/create_source_code_be/index.md b/doc/development/backend/create_source_code_be/index.md index 77c98982210..d894a14b006 100644 --- a/doc/development/backend/create_source_code_be/index.md +++ b/doc/development/backend/create_source_code_be/index.md @@ -4,16 +4,16 @@ group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Create: Source Code Backend +# Create: Source Code backend -The Create:Source Code BE team focuses on the GitLab suite of Source Code Management +The Create: Source Code backend (BE) team focuses on the GitLab suite of Source Code Management (SCM) tools. It is responsible for all backend aspects of the product categories that fall under the [Source Code group](https://about.gitlab.com/handbook/product/categories/#source-code-group) of the [Create stage](https://about.gitlab.com/handbook/product/categories/#create-stage) of the [DevOps lifecycle](https://about.gitlab.com/handbook/product/categories/#devops-stages). We interface with the Gitaly and Code Review teams, and work closely with the -[Create:Source Code Frontend team](https://about.gitlab.com/handbook/engineering/development/dev/create/create-source-code-fe/). The features +[Create: Source Code frontend team](https://about.gitlab.com/handbook/engineering/development/dev/create/create-source-code-fe/). The features we work with are listed on the [Features by Group Page](https://about.gitlab.com/handbook/product/categories/features/#createsource-code-group). @@ -39,7 +39,7 @@ To learn about the reasoning behind our creation of `gitlab-sshd`, read the blog ### Gitaly touch points -Gitaly is a Golang RPC service which handles all the `git` calls made by GitLab. +Gitaly is a Go RPC service which handles all the `git` calls made by GitLab. GitLab is not exposed directly, and all traffic comes through Create: Source Code. For more information, read [Gitaly touch points](gitaly_touch_points.md). diff --git a/doc/development/bulk_import.md b/doc/development/bulk_import.md index 0d9c7348915..b1fa4726b65 100644 --- a/doc/development/bulk_import.md +++ b/doc/development/bulk_import.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -17,8 +17,6 @@ including projects, from one GitLab instance to another. ## Design decisions -### Overview - The following architectural diagram illustrates how the Group Migration works with a set of [ETL](#etl) Pipelines leveraging from the current [GitLab APIs](#api). @@ -37,13 +35,14 @@ idea is to have one ETL pipeline for each relation to be imported. ### API -The current [Project](../user/project/settings/import_export.md) and [Group](../user/group/settings/import_export.md) Import are file based, so they require an export -step to generate the file to be imported. +The current [project](../user/project/settings/import_export.md) and +[group](../user/group/import/index.md#migrate-groups-by-uploading-an-export-file-deprecated) imports are file based, so +they require an export step to generate the file to be imported. -GitLab Group migration leverages on [GitLab API](../api/rest/index.md) to speed the migration. +Group migration by direct transfer leverages the [GitLab API](../api/rest/index.md) to speed the migration. And, because we're on the road to [GraphQL](../api/graphql/index.md), -GitLab Group Migration will be contributing towards to expand the GraphQL API coverage, which benefits both GitLab +Group migration by direct transfer can contribute to expanding GraphQL API coverage, which benefits both GitLab and its users. ### Namespace diff --git a/doc/development/caching.md b/doc/development/caching.md index 9b3f9a4215e..dff94b68ca0 100644 --- a/doc/development/caching.md +++ b/doc/development/caching.md @@ -332,11 +332,11 @@ views/projects/_home_panel:462ad2485d7d6957e03ceba2c6717c29/projects/16-20210316 ``` 1. The view name and template tree digest - `views/projects/_home_panel:462ad2485d7d6957e03ceba2c6717c29` + `views/projects/_home_panel:462ad2485d7d6957e03ceba2c6717c29` 1. The model name, ID, and `updated_at` values - `projects/16-20210316142425469452` + `projects/16-20210316142425469452` 1. The symbol we passed in, converted to a string - `tag_list` + `tag_list` ### Look for diff --git a/doc/development/cascading_settings.md b/doc/development/cascading_settings.md index 389623e68d8..42f4b5dd6f3 100644 --- a/doc/development/cascading_settings.md +++ b/doc/development/cascading_settings.md @@ -23,13 +23,13 @@ Settings are not cascading by default. To define a cascading setting, take the f 1. In the `NamespaceSetting` model, define the new attribute using the `cascading_attr` helper method. You can use an array to define multiple attributes on a single line. - ```ruby - class NamespaceSetting - include CascadingNamespaceSettingAttribute + ```ruby + class NamespaceSetting + include CascadingNamespaceSettingAttribute - cascading_attr :delayed_project_removal - end - ``` + cascading_attr :delayed_project_removal + end + ``` 1. Create the database columns. @@ -37,21 +37,21 @@ Settings are not cascading by default. To define a cascading setting, take the f The helper creates four columns, two each in `namespace_settings` and `application_settings`. - ```ruby - class AddDelayedProjectRemovalCascadingSetting < Gitlab::Database::Migration[2.1] - include Gitlab::Database::MigrationHelpers::CascadingNamespaceSettings + ```ruby + class AddDelayedProjectRemovalCascadingSetting < Gitlab::Database::Migration[2.1] + include Gitlab::Database::MigrationHelpers::CascadingNamespaceSettings - enable_lock_retries! + enable_lock_retries! - def up - add_cascading_namespace_setting :delayed_project_removal, :boolean, default: false, null: false - end + def up + add_cascading_namespace_setting :delayed_project_removal, :boolean, default: false, null: false + end - def down - remove_cascading_namespace_setting :delayed_project_removal - end - end - ``` + def down + remove_cascading_namespace_setting :delayed_project_removal + end + end + ``` Existing settings being converted to a cascading setting will require individual migrations to add columns and change existing columns. Use the specifications diff --git a/doc/development/changelog.md b/doc/development/changelog.md index 27bcffe7560..56b5fa8c976 100644 --- a/doc/development/changelog.md +++ b/doc/development/changelog.md @@ -94,7 +94,7 @@ EE: true uses system fonts for all text." - Any client-facing change to our REST and GraphQL APIs **must** have a changelog entry. See the [complete list what comprises a GraphQL breaking change](api_graphql_styleguide.md#breaking-changes). -- Any change that introduces an [Advanced Search migration](elasticsearch.md#creating-a-new-advanced-search-migration) +- Any change that introduces an [advanced search migration](search/advanced_search_migration_styleguide.md#creating-a-new-advanced-search-migration) **must** have a changelog entry. - A fix for a regression introduced and then fixed in the same release (such as fixing a bug introduced during a monthly release candidate) **should not** diff --git a/doc/development/chatops_on_gitlabcom.md b/doc/development/chatops_on_gitlabcom.md index 378639c4a43..22f57e8ba43 100644 --- a/doc/development/chatops_on_gitlabcom.md +++ b/doc/development/chatops_on_gitlabcom.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Manage +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/development/cicd/cicd_tables.md b/doc/development/cicd/cicd_tables.md new file mode 100644 index 00000000000..b246328a817 --- /dev/null +++ b/doc/development/cicd/cicd_tables.md @@ -0,0 +1,121 @@ +--- +stage: Verify +group: Pipeline Execution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Add new tables to the CI database + +The [pipeline data partitioning](../../architecture/blueprints/ci_data_decay/pipeline_partitioning.md) +design blueprint describes how to partition existing tables in the CI domain. However, +you still need to add tables for new features. Sometimes these tables hold +references to larger tables that need to be partitioned. To reduce future +work, all tables that use a `belongs_to` association to partitionable tables +should be partitioned from the start. + +## Create a new routing table + +Here is an example on how to use database helpers to create a new table and foreign keys: + +```ruby + include Gitlab::Database::PartitioningMigrationHelpers + disable_ddl_transaction! + + def up + create_table(:p_ci_examples, primary_key: [:id, :partition_id], options: 'PARTITION BY LIST (partition_id)', if_not_exists: true) do |t| + t.bigserial :id, null: false + t.bigint :partition_id, null: false + t.bigint :build_id, null: false + end + + add_concurrent_partitioned_foreign_key( + :p_ci_examples, :ci_builds, + column: [:partition_id, :build_id], + target_column: [:partition_id, :id], + on_update: :cascade, + on_delete: :cascade, + reverse_lock_order: true + ) + end + + def down + drop_table :p_ci_examples + end +``` + +This table is called a routing table and it does not hold any data. The +data is stored in partitions. + +When creating the routing table: + +- The table name must start with the `p_` prefix. There are analyzers in place to ensure that all queries go + through the routing tables and do not access the partitions directly. +- Each new table needs a `partition_id` column and its value must equal + the value from the related association. In this example, that is `ci_builds`. All resources + belonging to a pipeline share the same `partition_id` value. +- The primary key must have the columns ordered this way to allow efficient + search only by `id`. +- The foreign key constraint must include the `ON UPDATE CASCADE` option because + the `partition_id` value should be able to update it for re-balancing the + partitions. + +## Create the first partition + +Usually, you rely on the application to create the initial partition at boot time. +However, due to the high traffic on the CI tables and the large number of nodes, +it can be difficult to acquire a lock on the referenced table. +Consequently, during deployment, a node may fail to start. +To prevent this failure, you must ensure that the partition is already in place before +the application runs: + +```ruby + disable_ddl_transaction! + + def up + with_lock_retries do + connection.execute(<<~SQL) + LOCK TABLE ci_builds IN SHARE UPDATE EXCLUSIVE MODE; + LOCK TABLE ONLY p_ci_examples IN ACCESS EXCLUSIVE MODE; + SQL + + connection.execute(<<~SQL) + CREATE TABLE IF NOT EXISTS gitlab_partitions_dynamic.ci_examples_100 + PARTITION OF p_ci_examples + FOR VALUES IN (100); + SQL + end + end +``` + +Partitions are created in `gitlab_partitions_dynamic` schema. + +When creating a partition, remember: + +- Partition names do not use the `p_` prefix. +- The default value for `partition_id` is `100`. + +## Cascade the partition value + +To cascade the partition value, the module should use the `Ci::Partitionable` module: + +```ruby +class Ci::Example < Ci::ApplicationRecord + include Ci::Partitionable + + self.table_name = :p_ci_examples + self.primary_key = :id + + belongs_to :build, class_name: 'Ci::Build' + partitionable scope: :build, partitioned: true +end +``` + +## Manage partitions + +The model must be included in the [`PARTITIONABLE_MODELS`](https://gitlab.com/gitlab-org/gitlab/-/blob/920147293ae304639915f66b260dc14e4f629850/app/models/concerns/ci/partitionable.rb#L25-44) +list because it is used to test that the `partition_id` is +propagated correctly. + +If it's missing, specifying `partitioned: true` creates the first partition. The model also needs to be registered in the +[`postgres_partitioning.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/920147293ae304639915f66b260dc14e4f629850/config/initializers/postgres_partitioning.rb) +initializer. diff --git a/doc/development/cicd/index.md b/doc/development/cicd/index.md index 641d0ea4f6f..2cc8fca02dd 100644 --- a/doc/development/cicd/index.md +++ b/doc/development/cicd/index.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: index, concepts, howto --- -# CI/CD development documentation +# CI/CD development guidelines Development guides that are specific to CI/CD are listed here: @@ -147,6 +147,7 @@ This API endpoint runs [`Ci::RegisterJobService`](https://gitlab.com/gitlab-org/ There are 3 top level queries that this service uses to gather the majority of the jobs and they are selected based on the level where the runner is registered to: - Select jobs for shared runner (instance level) + - Utilizes a fair scheduling algorithm which prioritizes projects with fewer running builds - Select jobs for group runner - Select jobs for project runner diff --git a/doc/development/cicd/templates.md b/doc/development/cicd/templates.md index 4f6799d12d8..1bf4a780e26 100644 --- a/doc/development/cicd/templates.md +++ b/doc/development/cicd/templates.md @@ -436,7 +436,6 @@ To add a metric definition for a new template: ```yaml - name: p_ci_templates_my_template_name category: ci_templates - redis_slot: ci_templates aggregation: weekly ``` diff --git a/doc/development/code_intelligence/index.md b/doc/development/code_intelligence/index.md index 107a1588e18..2034f57d995 100644 --- a/doc/development/code_intelligence/index.md +++ b/doc/development/code_intelligence/index.md @@ -4,7 +4,7 @@ group: Code Review info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Code Intelligence +# Code intelligence development guidelines > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/1576) in GitLab 13.1. diff --git a/doc/development/code_owners/index.md b/doc/development/code_owners/index.md new file mode 100644 index 00000000000..d962a36b497 --- /dev/null +++ b/doc/development/code_owners/index.md @@ -0,0 +1,135 @@ +--- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Code Owners development guidelines + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/219916) in GitLab 15.10. + +This document was created to help contributors understand the code design of +[Code Owners](../../user/project/codeowners/index.md). You should read this +document before making changes to the code for this feature. + +This document is intentionally limited to an overview of how the code is +designed, as code can change often. To understand how a specific part of the +feature works, view the code and the specs. The details here explain how the +major components of the Code Owners feature work. + +NOTE: +This document should be updated when parts of the codebase referenced in this +document are updated, removed, or new parts are added. + +## Business logic + +All of the business logic for code owners is located in the `Gitlab::CodeOwners` +namespace. Code Owners is an EE-only feature, so the files only exist in the `./ee` directory. + +- `Gitlab::CodeOwners`: the main module used to interact with the code owner rules. + - Defined in `./ee/lib/gitlab/code_owners.rb`. +- `Gitlab::CodeOwners::File`: wraps a `CODEOWNERS` file and exposes the data through + the class' public methods. + - Defined in `./ee/lib/gitlab/code_owners/file.rb`. +- `Gitlab::CodeOwners::Section`: wraps a section heading from a + `CODEOWNERS` file and parses the different parts. + - Defined in `./ee/lib/gitlab/code_owners/section.rb`. +- `Gitlab::CodeOwners::Entry`: wraps an entry (a pattern and owners line) in a + `CODEOWNERS` file and exposes the data through the class' public methods. + - Defined in `./ee/lib/gitlab/code_owners/entry.rb`. +- `Gitlab::CodeOwners::Loader`: finds the correct `CODEOWNER` file and loads the + content into a `Gitlab::CodeOwners::File` instance. + - Defined in `./ee/lib/gitlab/code_owners/loader.rb`. +- `Gitlab::CodeOwners::ReferenceExtractor`: extracts `CODEOWNER` user, group, + and email references from texts. + - Defined in `./ee/lib/gitlab/code_owners/reference_extractor.rb`. +- `Gitlab::CodeOwners::UsersLoader`: the correct `CODEOWNER` file and loads the + content into a `Gitlab::CodeOwners::File` instance. + - Defined in `./ee/lib/gitlab/code_owners/users_loader.rb`. +- `Gitlab::CodeOwners::GroupsLoader`: finds the correct `CODEOWNER` file and loads + the content into a `Gitlab::CodeOwners::File` instance. + - Defined in `./ee/lib/gitlab/code_owners/groups_loader.rb`. +- `Gitlab::CodeOwners::Validator`: validates no files in the `CODEOWNERS` entries + have been changed when a user pushes to a protected branch with `require_code_owner_approval` enabled. + - Defined in `./ee/lib/gitlab/code_owners/validator.rb`. + +## Related models + +### `ProtectedBranch` + +The `ProtectedBranch` model is defined in `app/models/protected_branch.rb` and +extended in `ee/app/ee/models/protected_branch.rb`. The EE version includes a column +named `require_code_owner_approval` which prevents changes from being pushed directly +to the branch being protected if the file is listed in `CODEOWNERS`. + +### `ApprovalMergeRequestRule` + +The `ApprovalMergeRequestRule` model is defined in `ee/app/models/approval_merge_request_rule.rb`. +The model stores approval rules for a merge request. We use multiple rule types, +including a `code_owner` type rule. + +## Controllers and Services + +The following controllers and services below are being used for the approval +rules feature to work: + +### `Api::Internal::Base` + +This `/internal/allowed` endpoint is called when pushing to GitLab to ensure the +user is allowed to push. The `/internal/allowed` endpoint performs a `Gitlab::Checks::DiffCheck`. +In EE, this includes code owner checks. + +Defined in `lib/api/internal/base.rb`. + +### `Repositories::GitHttpController` + +When changes are pushed to GitLab over HTTP, the controller performs an access check +to ensure the user is allowed to push. The checks perform a `Gitlab::Checks::DiffCheck`. +In EE, this includes Code Owner checks. + +Defined in `app/controllers/repositories/git_http_controller.rb`. + +### `EE::Gitlab::Checks::DiffCheck` + +This module extends the CE `Gitlab::Checks::DiffChecks` class and adds code owner +validation. It uses the `Gitlab::CodeOwner::Validator` class to verify users are +not pushing files listed in `CODEOWNER` directly to a protected branch while the +branch requires code owner approval. + +### `MergeRequests::SyncCodeOwnerApprovalRules` + +This service is defined in `services/merge_requests/sync_code_owner_approval_rules.rb` and used for: + +- Deleting outdated code owner approval rules when new changes are pushed to a merge request. +- Creating code owner approval rules for each changed file in a merge request that is also listed in the `CODEOWNER` file. + +## Flow + +These flowcharts should help explain the flow from the controllers down to the +models for different features. + +### Push changes to a protected branch with `require_code_owner_approval` enabled + +```mermaid +graph TD + Api::Internal::Base --> Gitlab::GitAccess + Gitlab::GitAccess --> Gitlab::Checks::DiffCheck + Gitlab::Checks::DiffCheck --> Gitlab::CodeOwners::Validator + Gitlab::CodeOwners::Validator --> ProtectedBranch + Gitlab::CodeOwners::Validator --> Gitlab::CodeOwners::Loader + Gitlab::CodeOwners::Loader --> Gitlab::CodeOwners::Entry +``` + +### Sync code owner rules to merge request approval rules + +```mermaid +graph TD + EE::ProtectedBranches::CreateService --> MergeRequest::SyncCodeOwnerApprovalRules + EE::MergeRequestRefreshService --> MergeRequest::SyncCodeOwnerApprovalRules + EE::MergeRequests::ReloadMergeHeadDiffService --> MergeRequest::SyncCodeOwnerApprovalRules + EE::MergeRequests::CreateService --> MergeRequests::SyncCodeOwnerApprovalRulesWorker + EE::MergeRequests::UpdateService --> MergeRequests::SyncCodeOwnerApprovalRulesWorker + MergeRequests::SyncCodeOwnerApprovalRulesWorker --> MergeRequest::SyncCodeOwnerApprovalRules + MergeRequest::SyncCodeOwnerApprovalRules --> id1{delete outdated code owner rules} + MergeRequest::SyncCodeOwnerApprovalRules --> id2{create rule for each code owner entry} +``` diff --git a/doc/development/code_review.md b/doc/development/code_review.md index 245fb2152cd..f2edc882d91 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -68,6 +68,9 @@ When a suitable [domain expert](#domain-experts) isn't available, you can choose To find a domain expert: +- In the Merge Request approvals widget, select [View eligible approvers](../user/project/merge_requests/approvals/rules.md#eligible-approvers). + This widget shows recommended and required approvals per area of the codebase. + These rules are defined in [Code Owners](../user/project/merge_requests/approvals/rules.md#code-owners-as-eligible-approvers). - View the list of team members who work in the [stage or group](https://about.gitlab.com/handbook/product/categories/#devops-stages) related to the merge request. - View team members' domain expertise on the [engineering projects](https://about.gitlab.com/handbook/engineering/projects/) page or on the [GitLab team page](https://about.gitlab.com/company/team/). Domains are self-identified, so use your judgment to map the changes on your merge request to a domain. - Look for team members who have contributed to the files in the merge request. View the logs by running `git log `. @@ -163,7 +166,7 @@ with [domain expertise](#domain-experts). | End-to-end **and** non-end-to-end changes (*4*) | [Software Engineer in Test](https://about.gitlab.com/handbook/engineering/quality/#individual-contributors). | | Only End-to-end changes (*4*) **or** if the MR author is a [Software Engineer in Test](https://about.gitlab.com/handbook/engineering/quality/#individual-contributors) | [Quality maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_maintainers_qa). | | A new or updated [application limit](https://about.gitlab.com/handbook/product/product-processes/#introducing-application-limits) | [Product manager](https://about.gitlab.com/company/team/). | -| Product Intelligence (telemetry or analytics) changes | [Product Intelligence engineer](https://gitlab.com/gitlab-org/analytics-section/product-intelligence/engineers). | +| Analytics Instrumentation (telemetry or analytics) changes | [Analytics Instrumentation engineer](https://gitlab.com/gitlab-org/analytics-section/product-intelligence/engineers). | | An addition of, or changes to a [Feature spec](testing_guide/testing_levels.md#frontend-feature-tests) | [Quality maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_maintainers_qa) or [Quality reviewer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_reviewers_qa). | | A new service to GitLab (Puma, Sidekiq, Gitaly are examples) | [Product manager](https://about.gitlab.com/company/team/). See the [process for adding a service component to GitLab](adding_service_component.md) for details. | | Changes related to authentication or authorization | [Manage:Authentication and Authorization team member](https://about.gitlab.com/company/team/). Check the [code review section on the group page](https://about.gitlab.com/handbook/engineering/development/dev/manage/authentication-and-authorization/#additional-considerations) for more details. Patterns for files known to require review from the team are listed in the in the `Authentication and Authorization` section of the [`CODEOWNERS`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/CODEOWNERS) file, and the team will be listed in the approvers section of all merge requests that modify these files. | @@ -237,7 +240,7 @@ See the [test engineering process](https://about.gitlab.com/handbook/engineering ##### Compliance -1. You have confirmed that the correct [MR type label](contributing/issue_workflow.md#type-labels) has been applied. +1. You have confirmed that the correct [MR type label](labels/index.md) has been applied. ### The responsibility of the merge request author @@ -391,9 +394,8 @@ as a reviewer, it is recommended that they are not also picked as the maintainer Maintainers should check before merging if the merge request is approved by the required approvers. If still awaiting further approvals from others, remove yourself as a reviewer then `@` mention the author and explain why in a comment. Stay as reviewer if you're merging the code. -Note that certain merge requests may target a stable branch. These are rare -events. These types of merge requests cannot be merged by the Maintainer. -Instead, these should be sent to the [Release Manager](https://about.gitlab.com/community/release-managers/). +Certain merge requests may target a stable branch. For an overview of how to handle these requests, +see the [patch release runbook](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/patch/engineers.md). After merging, a maintainer should stay as the reviewer listed on the merge request. @@ -532,7 +534,7 @@ WARNING: Before taking the decision to merge: - Set the milestone. -- Confirm that the correct [MR type label](contributing/issue_workflow.md#type-labels) is applied. +- Confirm that the correct [MR type label](labels/index.md#type-labels) is applied. - Consider warnings and errors from danger bot, code quality, and other reports. Unless a strong case can be made for the violation, these should be resolved before merging. A comment must be posted if the MR is merged with any failed job. @@ -543,7 +545,9 @@ people who add commits to an MR are not authorized to approve or merge the MR an must seek a maintainer who has not contributed to the MR to approve and merge it. This policy is in place to satisfy the CHG-04 control of the GitLab -[Change Management Controls](https://about.gitlab.com/handbook/security/security-assurance/security-compliance/guidance/change-management.html). +[Change Management Controls](https://about.gitlab.com/handbook/security/change-management-policy.html). + + To implement this policy in `gitlab-org/gitlab`, we have enabled the following settings to ensure MRs get an approval from a top-level CODEOWNERS maintainer: @@ -718,12 +722,7 @@ Enterprise Edition instance. This has some implications: cached value returns (say, from a string or nil to an array), change the cache key at the same time. 1. **Settings** should be added as a - [last resort](https://about.gitlab.com/handbook/product/product-principles/#convention-over-configuration). - If you're adding a new setting in `gitlab.yml`: - 1. Try to avoid that, and add to `ApplicationSetting` instead. - 1. Ensure that it is also - [added to Omnibus](https://docs.gitlab.com/omnibus/settings/gitlab.yml#adding-a-new-setting-to-gitlabyml). - 1. Ensure that it is also [added to Charts](https://docs.gitlab.com/charts/development/style_guide.html), if needed. + [last resort](https://about.gitlab.com/handbook/product/product-principles/#convention-over-configuration). See [Adding a new setting to GitLab Rails](architecture.md#adding-a-new-setting-in-gitlab-rails). 1. **File system access** is not possible in a [cloud-native architecture](architecture.md#adapting-existing-and-introducing-new-components). Ensure that we support object storage for any file storage we need to perform. For more information, see the [uploads documentation](uploads/index.md). diff --git a/doc/development/contributing/community_roles.md b/doc/development/contributing/community_roles.md deleted file mode 100644 index 3c9362138c2..00000000000 --- a/doc/development/contributing/community_roles.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'index.md' -remove_date: '2023-05-08' ---- - -This document was moved to [another location](index.md). - - - - - diff --git a/doc/development/contributing/design.md b/doc/development/contributing/design.md index aec487ed184..d68bc194266 100644 --- a/doc/development/contributing/design.md +++ b/doc/development/contributing/design.md @@ -12,7 +12,7 @@ Follow these guidelines when contributing or reviewing design and user interface advice and best practices for code review in general. The basis for most of these guidelines is [Pajamas](https://design.gitlab.com/), -GitLab design system. We encourage you to [contribute to Pajamas](https://design.gitlab.com/get-started/contribute/) +GitLab design system. We encourage you to [contribute to Pajamas](https://design.gitlab.com/get-started/contributing/) with additions and improvements. ## Merge request reviews @@ -73,7 +73,7 @@ Check visual design properties using your browser's _elements inspector_ ([Chrom guidelines. - _Optionally_ consider [dark mode](../../user/profile/preferences.md#dark-mode). [^1] - [^1]: You're not required to design for [dark mode](../../user/profile/preferences.md#dark-mode) while the feature is in [alpha](../../policy/alpha-beta-support.md#alpha-features). The [UX Foundations team](https://about.gitlab.com/direction/manage/foundations/) plans to improve the dark mode in the future. Until we integrate [Pajamas](https://design.gitlab.com/) components into the product and the underlying design strategy is in place to support dark mode, we cannot guarantee that we won't introduce bugs and debt to this mode. At your discretion, evaluate the need to create dark mode patches. + [^1]: You're not required to design for [dark mode](../../user/profile/preferences.md#dark-mode) while the feature is an [Experiment](../../policy/alpha-beta-support.md#experiment). The [UX Foundations team](https://about.gitlab.com/direction/manage/foundations/) plans to improve the dark mode in the future. Until we integrate [Pajamas](https://design.gitlab.com/) components into the product and the underlying design strategy is in place to support dark mode, we cannot guarantee that we won't introduce bugs and debt to this mode. At your discretion, evaluate the need to create dark mode patches. ### States @@ -127,7 +127,7 @@ At any moment, but usually _during_ or _after_ the design's implementation: - Contribute [issues to Pajamas](https://design.gitlab.com/get-started/contributing#contribute-an-issue) for additions or enhancements to the design system. -- Create issues with the [`~UX debt`](issue_workflow.md#technical-and-ux-debt) +- Create issues with the [`~UX debt`](../labels/index.md#technical-and-ux-debt) label for intentional deviations from the agreed-upon UX requirements due to time or feasibility challenges, linking back to the corresponding issues or merge requests. diff --git a/doc/development/contributing/first_contribution.md b/doc/development/contributing/first_contribution.md new file mode 100644 index 00000000000..0c4b5b21171 --- /dev/null +++ b/doc/development/contributing/first_contribution.md @@ -0,0 +1,340 @@ +--- +stage: none +group: Development +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Tutorial: Make a GitLab contribution + +Anyone can contribute to the development of GitLab. + +Maybe you want to add functionality that you feel is missing. Or maybe +you noticed some UI text that you want to improve. + +This tutorial will walk you through the process of updating UI text +and related files by using the GitLab Development Kit and the GitLab community fork. +You can follow this example exactly to familiarize yourself with the process, +or you can choose other UI text to update. + +## Steps + +To make a contribution, you will: + +1. [Configure the GitLab Development Kit](#step-1-configure-the-gitlab-development-kit) +1. [Make your code updates](#step-2-change-the-code) +1. [Push your changes to the community fork](#step-3-push-your-changes-to-the-community-fork) +1. [Create a merge request](#step-4-create-a-merge-request) + +## Prerequisites + +On your local machine: + +- Ensure Git is installed. + (From the command line, type `git -v`. If you get a result, you have Git installed.) +- Install a source code editor, or decide which tool you're going to use to edit files. + +On GitLab.com: + +- Create an account. Ensure you can successfully sign in. +- Go to the [`gitlab-community/community-members` group](https://gitlab.com/gitlab-community/community-members) + and select **Request access**. This action will give you access to the GitLab + community fork, where you'll author your changes. + +## Step 1: Configure the GitLab Development Kit + +The GitLab Development Kit (GDK) is a local version of GitLab that's yours to play with. +It's just like an installation of self-managed GitLab. It includes sample projects you +can use to test functionality, and it gives you access to administrator functionality. +You can run it on your local machine, or use GitPod to run a remote version. + +![GDK](img/gdk_home.png) + +If you've never used the GDK before, and you think you might contribute +more than once, you should install it. +If you already have a working GDK, you should +[update it to use the community fork](#an-existing-gdk-installation). + +### A new GDK installation + +Set aside about two hours to install the GDK. If all goes smoothly, it +should take about an hour to install. + +Sometimes the installation needs some tweaks to make it work, so you should +also set aside some time for troubleshooting. +It might seem like a lot of work, but after you have the GDK running, +you'll be able to contribute much more often and more easily. + +To install the GDK: + +1. Ensure you're on + [one of the supported platforms](https://gitlab.com/gitlab-org/gitlab-development-kit/-/tree/main/#supported-platforms) + (macOS, Ubuntu, etc.). +1. Choose the directory where you want to install the GDK. + In this location, a repository called `gitlab-development-kit` will be created, + and the application will be installed. +1. From the command line, go to that directory. + In this example, we will use the `development` directory. + + ```shell + cd development + ``` + +1. Run the one-line installation command: + + ```shell + curl "https://gitlab.com/gitlab-org/gitlab-development-kit/-/raw/main/support/install" | bash + ``` + +1. For the message `Where would you like to install the GDK? [./gitlab-development-kit]`, + press Enter to accept the default location. + +1. For the message `Which GitLab repo URL would you like to clone?`, enter the GitLab community fork: + + ```shell + https://gitlab.com/gitlab-community/gitlab.git + ``` + +While the installation is running, copy any messages that are displayed. +If you have any problems with the installation, you can use this output as +part of troubleshooting. + +When the installation is complete: + +1. Go to the directory where the GDK was installed: + + ```shell + cd gitlab-development-kit + ``` + +1. Start the GDK: + + ```shell + gdk start + ``` + +1. Connect to the GDK by using the URL provided. It should be something like . +1. Use the username `root` and the password `5iveL!fe`. You will be prompted + to reset your password the first time you sign in. + +If you have any problems, try going to the `gitlab-development-kit/gitlab` +directory and running these commands: + +```shell +yarn install && bundle install +bundle exec rails db:migrate RAILS_ENV=development +``` + +From the `gitlab-development-kit` folder, you can also try running `gdk doctor`. + +For more advanced troubleshooting, see +[the troubleshooting docs](https://gitlab.com/gitlab-org/gitlab-development-kit/-/tree/main/doc/troubleshooting). + +### An existing GDK installation + +If you have an existing GDK installation, you should update it so it's +using the community fork. + +1. Delete the existing `gitlab-development-kit/gitlab` directory. +1. Clone the community fork into that location: + + ```shell + cd gitlab-development-kit + git clone https://gitlab.com/gitlab-community/gitlab.git + ``` + +To confirm it was successful: + +1. Ensure the `gitlab-development-kit/gitlab` directory exists. +1. Go to the top `gitlab-development-kit` directory and run `gdk stop` and `gdk start`. + +If you get errors, run `gdk doctor` to troubleshoot. For more advanced troubleshooting, see +[the troubleshooting docs](https://gitlab.com/gitlab-org/gitlab-development-kit/-/tree/main/doc/troubleshooting). + +## Step 2: Change the code + +Now for the fun part. Let's edit some code. + +In this example, I found some UI text I'd like to change. +In the upper-right corner in GitLab, I selected my avatar and then **Preferences**. +I want to change this text: + +![UI text](img/ui_text_before.png) + +Other settings on the page start with the word `Customize` and skip the `This setting allows you to` part. +I'll update this phrase to match the others. + +1. Search the `gitlab-development-kit/gitlab` directory for the string `This setting allows you to customize`. + + The results show one `.haml` file, two `.md` files, one `.pot` file, and + several `.po` files. + +1. Open the `.haml` file. This file is where the UI text resides. +1. Update the string. In this case, I'll remove the words before `customize` + and start the word `customize` with a capital `C`. +1. Save the file. + +You can check that you were successful: + +- In the `gitlab-development-kit/gitlab` directory, type `git status` + to show the file you modified: + + ```shell + modified: app/views/profiles/preferences/show.html.haml + ``` + +- Refresh the web browser where you're viewing the GDK. + The changes should be displayed. Take a screenshot. + + ![UI text](img/ui_text_after.png) + +### Update the translation files + +English UI strings are localized into many languages. +These strings are saved in a `.pot` file, which you must update +any time you update UI text. + +To generate the localization file: + +1. Ensure you are in the `gitlab-development-kit/gitlab` directory. +1. Run the following command: + + ```shell + bin/rake gettext:compile + ``` + + After several minutes, a `.pot` file is generated in the `/locale` directory. + +Now, in the `gitlab-development-kit/gitlab` directory, if you type `git status` +you should have both files listed: + +```shell + modified: app/views/profiles/preferences/show.html.haml + modified: locale/gitlab.pot +``` + +For more information about localization, see [internationalization](../i18n/externalization.md). + +### Update the documentation + +Documentation for GitLab is published on . +When you add or update a feature, you must update the docs as well. + +1. To find the documentation for a feature, the easiest thing is to search the + docs site. In this case, the setting is described on this documentation page: + + ```plaintext + https://docs.gitlab.com/ee/user/profile/preferences.html + ``` + +1. The URL shows you the location of the file in the `/doc` directory. + In this case, the location is: + + ```plaintext + doc/user/profile/preferences.md + ``` + +1. Go to this location in your local `gitlab` repository and update the `.md` file + and any related images. + +Now when you run `git status`, you should have something like: + +```plaintext + modified: app/views/profiles/preferences/show.html.haml + modified: doc/user/profile/img/profile-preferences-syntax-themes.png + modified: doc/user/profile/preferences.md + modified: locale/gitlab.pot +``` + +To view these changes in action, you can +[check out a merge request where these changes have already been made](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/116472). + +## Step 3: Push your changes to the community fork + +Now you're going to push your changes to the community fork. This is the next step +in getting your changes put into the main GitLab repository. + +1. Ensure you are in the `gitlab-development-kit/gitlab` directory. +1. Create a branch. You don't want to work in the `master` branch. + Instead, you want to create a branch for your work. In this example, + we're going to call the branch `ui-updates`. + + ```shell + git checkout -b ui-updates + ``` + +1. Add the files to the staging area. + + ```shell + git add . + ``` + +1. Provide a commit message. GitLab has somewhat strict + [commit message guidelines](merge_request_workflow.md#commit-messages-guidelines). + To be safe, a general rule is to use three to five words, start with a capital letter, + and do **not** end with a period. + + ```shell + git commit -m "Updating UI text + + Standardizing the text on this page so + that each area uses consistent language. + + Changelog: changed" + ``` + + The `Changelog: changed` is because we're changing an existing feature. If we were adding a feature, we'd + use `Changelog: added`. For details, see [changelog entries](../changelog.md). + +1. Push the changes to the community fork. At the same time, set the fork as your upstream, + so that it will be in sync for any future contributions. + + ```shell + git push --set-upstream origin ui-updates + ``` + +## Step 4: Create a merge request + +Now you're ready to push changes from the community fork to the main GitLab repository! + +1. Go to [the community fork on GitLab.com](https://gitlab.com/gitlab-community/gitlab). + You should see a message like this one: + + ![Create merge request banner](img/mr_button.png) + + Select **Create merge request**. + If you don't see this message, on the left sidebar, select **Merge requests > New merge request**. + +1. Take a look at the branch names. You should be merging from your branch + in the community fork to the `master` branch in the GitLab repository. + + ![New merge request](img/new_merge_request.png) + +1. Fill out the information and then select **Save changes**. + Don't worry if your merge request is not complete. If you don't want anyone + from GitLab to review it, you can select the **Mark as draft** checkbox. + If you're not happy with the merge request after you create it, you can + close it, no harm done. + +1. Select the **Changes** tab. It should look something like this: + + ![Changes tab](img/changes_tab.png) + + The red text shows the code before you made changes. The green shows what + the code looks like now. + +1. If you're happy with this merge request and want to start the review process, type + `@gitlab-bot ready` in a comment and then select **Comment**. + + ![GitLab bot ready comment](img/bot_ready.png) + +Someone from GitLab will look at your request and let you know what the next steps are. + +Now, any time you want to make a contribution to GitLab, you can just +go to the `gitlab-development-kit` folder and run `gdk update`. Then make +your changes in the `gitlab` directory and push them to the fork. + +If you need help at any point in the process, type `@gitlab-bot help` in a comment +or initiate a [mentor session](https://about.gitlab.com/community/contribute/mentor-sessions/) +on [Discord](https://discord.gg/gitlab). + +Congratulations on submitting your request, and thank you for your contribution! diff --git a/doc/development/contributing/img/bot_ready.png b/doc/development/contributing/img/bot_ready.png new file mode 100644 index 00000000000..85116c8957b Binary files /dev/null and b/doc/development/contributing/img/bot_ready.png differ diff --git a/doc/development/contributing/img/changes_tab.png b/doc/development/contributing/img/changes_tab.png new file mode 100644 index 00000000000..2158e9dd183 Binary files /dev/null and b/doc/development/contributing/img/changes_tab.png differ diff --git a/doc/development/contributing/img/gdk_home.png b/doc/development/contributing/img/gdk_home.png new file mode 100644 index 00000000000..7be4b9cd329 Binary files /dev/null and b/doc/development/contributing/img/gdk_home.png differ diff --git a/doc/development/contributing/img/mr_button.png b/doc/development/contributing/img/mr_button.png new file mode 100644 index 00000000000..5bf08b44412 Binary files /dev/null and b/doc/development/contributing/img/mr_button.png differ diff --git a/doc/development/contributing/img/new_merge_request.png b/doc/development/contributing/img/new_merge_request.png new file mode 100644 index 00000000000..34512d06dd2 Binary files /dev/null and b/doc/development/contributing/img/new_merge_request.png differ diff --git a/doc/development/contributing/img/ui_text_after.png b/doc/development/contributing/img/ui_text_after.png new file mode 100644 index 00000000000..3a54e7ef653 Binary files /dev/null and b/doc/development/contributing/img/ui_text_after.png differ diff --git a/doc/development/contributing/img/ui_text_before.png b/doc/development/contributing/img/ui_text_before.png new file mode 100644 index 00000000000..afd75a11ac2 Binary files /dev/null and b/doc/development/contributing/img/ui_text_before.png differ diff --git a/doc/development/contributing/index.md b/doc/development/contributing/index.md index 55827e00e43..82a08246503 100644 --- a/doc/development/contributing/index.md +++ b/doc/development/contributing/index.md @@ -5,25 +5,23 @@ group: Development info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Contribute to GitLab +# Contribute to GitLab development Thank you for your interest in contributing to GitLab. This guide details how -to contribute to GitLab in a way that is easy for everyone. +to contribute to the development of GitLab. -For a first-time step-by-step guide to the contribution process, see our -[Contributing to GitLab](https://about.gitlab.com/community/contribute/) page. +For a first-time step-by-step guide, see [Tutorial: Make a GitLab contribution](first_contribution.md). -Looking for something to work on? See the -[How to contribute](#how-to-contribute) section for more information. - -GitLab comes in two flavors: +## How to contribute -- GitLab Community Edition (CE), our free and open source edition. -- GitLab Enterprise Edition (EE), which is our commercial edition. +1. Read the code of conduct. +1. Choose or create an issue to work on. +1. Set up the GitLab Development Kit. +1. Open your merge request. -Throughout this guide you will see references to CE and EE for abbreviation. +Your merge request is triaged, reviewed, and can then be incorporated into the product. -## Code of conduct +### Code of conduct We want to create a welcoming environment for everyone who is interested in contributing. For more information about our commitment to an open and welcoming environment, see our [Code of Conduct page](https://about.gitlab.com/community/contribute/code-of-conduct/). @@ -31,37 +29,83 @@ For more information about our commitment to an open and welcoming environment, Issues and merge requests should be in English and contain appropriate language for audiences of all ages. -## How to contribute +### Choose or create an issue + +If you know what you're going to work on, see if an issue exists. If it doesn't, +open a [new issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issue%5Bmilestone_id%5D=). +Select the appropriate template, and add all the necessary information about the work you are planning on doing. +That way you can get more guidance and support from GitLab team members. + +If you're not sure what to work on, you can: + +- View issues with the + [`~Seeking community contributions` label](../labels/index.md#label-for-community-contributors). +- Optimize tests. Use [RSpec profiling statistics](https://gitlab-org.gitlab.io/rspec_profiling_stats/) + to identify the slowest tests. These tests are good candidates for improving and checking if any + [best practices](../testing_guide/best_practices.md) can speed them up. + +When you find an issue, leave a comment on the issue you want to work on. +This helps the GitLab team and members of the wider GitLab community know that you will be working on that issue. + +For details, see [the issues workflow](issue_workflow.md). + +### Set up the GitLab Development Kit + +To write and test your code, you will use the GitLab Development Kit. -If you would like to contribute to GitLab: - -- Issues with the - [`~Seeking community contributions` label](issue_workflow.md#label-for-community-contributors) - are a great place to start. -- Optimizing our tests is another great opportunity to contribute. You can use - [RSpec profiling statistics](https://gitlab-org.gitlab.io/rspec_profiling_stats/) to identify - slowest tests. These tests are good candidates for improving and checking if any of - [best practices](../testing_guide/best_practices.md) - could speed them up. -- Consult the [Contribution Flow](#contribution-flow) section to learn the process. - -### Contribution flow - -The general flow of contributing to GitLab is: - -1. [Create a fork](../../user/project/repository/forking_workflow.md#creating-a-fork) - of GitLab. In some cases, you will want to set up the - [GitLab Development Kit](https://gitlab.com/gitlab-org/gitlab-development-kit) to - [develop against your fork](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/index.md#develop-in-your-own-gitlab-fork). -1. Make your changes in your fork. -1. When you're ready, [create a new merge request](../../user/project/merge_requests/creating_merge_requests.md). -1. In the merge request's description: - - Ensure you provide complete and accurate information. - - Review the provided checklist. -1. Once you're ready, mark your MR as ready for review with `@gitlab-bot ready`. - - This will add the `~"workflow::ready for review"` label, and then automatically assign a merge request coach as reviewer. - - If you know a relevant reviewer (for example, someone that was involved a related issue), you can also - assign them directly with `@gitlab-bot ready @username`. +1. [Request access](https://gitlab.com/gitlab-community/meta#request-access-to-community-forks) to the [GitLab Community fork](https://gitlab.com/gitlab-community/meta). Alternatively, you can create your own public fork, but will miss out on the [benefits of the community forks](https://gitlab.com/gitlab-community/meta#why). +1. Some GitLab projects have a detailed contributing guide located in the README or CONTRIBUTING files in the repo. Reviewing these files before setting up your development environment will help ensure you get off to a good start. +1. Do one of the following: + - To run the development environment locally, download and set up the + [GitLab Development Kit](https://gitlab.com/gitlab-org/gitlab-development-kit). + See the [GDK README](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/README.md) for setup instructions + and [Troubleshooting](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/troubleshooting.md) if you get stuck. + + - GDK is heavy. If you need to build something fast, by trial and error, + consider doing so with an empty rails app and port it to GDK after. + + - To run a pre-configured GDK instance in the cloud, use [GDK with Gitpod](../../integration/gitpod.md). + From a project's repository, select the caret (angle-down) next to **Web IDE**, + and select **Gitpod** from the list. +1. If you want to contribute to the [website](https://about.gitlab.com/) or the [handbook](https://about.gitlab.com/handbook/), + go to the footer of any page and select **Edit in Web IDE** to open the [Web IDE](../../user/project/web_ide/index.md). + +### Open a merge request + +Now [Open a merge request](../../user/project/merge_requests/creating_merge_requests.md) +to merge your code and its documentation. The earlier you open a merge request, the sooner +you can get feedback. You can [mark it as a draft](../../user/project/merge_requests/drafts.md) +to signal that you’re not done yet. + +1. In the merge request, fill out all the information requested in the template, + like why you are introducing these changes and a link to the issue this merge request is attempting to close/fix. +1. [Add tests if needed](../testing_guide/best_practices.md), as well as [a changelog entry](../changelog.md). +1. If the change impacts users or admins, [update the documentation](../documentation/index.md). + +For details, see the [merge request workflow](merge_request_workflow.md). + +#### How community merge requests are triaged + +1. When you create a merge request, the [`@gitlab-bot`](https://gitlab.com/gitlab-bot) automatically applies + the ["~Community contribution"](https://about.gitlab.com/handbook/engineering/quality/triage-operations/#ensure-quick-feedback-for-community-contributions) label. +1. In the 24-48 hours after you create the merge request, a + [Merge Request Coach](https://about.gitlab.com/handbook/marketing/community-relations/contributor-success/merge-request-coach-lifecycle.html) + will review your merge request and apply stage, group, and type labels. +1. If a merge request was not automatically assigned, ask for a review by typing `@gitlab-bot ready` in a comment. + If your code has not been assigned a reviewer within two working days of its initial submission, you can ask + for help with `@gitlab-bot help`. +1. The Merge Request Coach will assign the relevant reviewers or tackle the review themselves if possible. + +The goal is to have a merge request reviewed within a week after a reviewer is assigned. At times this may take longer due to high workload, holidays, or other reasons. +If you need to, look at the [team page](https://about.gitlab.com/company/team/) for the merge request coach who specializes in +the type of code you have written and mention them in the merge request. For example, if you have +written some front-end code, you should mention the frontend merge request coach. If +your code has multiple disciplines, you can mention multiple merge request coaches. + +For details about timelines and how you can request help or escalate a merge request, +see the [Wider Community Merge Request guide](https://about.gitlab.com/handbook/engineering/quality/merge-request-triage/). + +After your merge request is reviewed and merged, your changes will be deployed to GitLab.com and included in the next release! #### Review process @@ -95,64 +139,33 @@ Lastly, keep the following in mind when submitting merge requests: be merged, as well as some guidance. The maintainers will be open to discussion about how to change the code so it can be approved and merged in the future. -#### Getting attention on your merge request - -GitLab will do its best to review community contributions as quickly as possible. Specially -appointed developers review community contributions daily. Look at the -[team page](https://about.gitlab.com/company/team/) for the merge request coach who specializes in -the type of code you have written and mention them in the merge request. For example, if you have -written some front-end code, you should mention the frontend merge request coach. If -your code has multiple disciplines, you may mention multiple merge request coaches. - -GitLab receives a lot of community contributions. If your code has not been reviewed within two -working days of its initial submission, you can ask for help with `@gitlab-bot help`. - -#### Addition of external libraries - -When submitting code to GitLab, you may feel that your contribution requires the aid of an external -library. If your code includes an external library, please provide a link to the library, as well as -reasons for including it. - -Mention a maintainer in merge requests that contain: - -- More than 500 changes. -- Any major [breaking changes](../deprecation_guidelines/index.md). -- External libraries. - -If you are not sure who to mention, the reviewer will do this for you early in the merge request process. - -#### Issues workflow - -This [documentation](issue_workflow.md) outlines the current issue workflow: - -- [Issue triaging](issue_workflow.md#issue-triaging) -- [Labels](issue_workflow.md#labels) -- [Feature proposals](issue_workflow.md#feature-proposals) -- [Issue weight](issue_workflow.md#issue-weight) -- [Regression issues](issue_workflow.md#regression-issues) -- [Technical and UX debt](issue_workflow.md#technical-and-ux-debt) -- [Technical debt in follow-up issues](issue_workflow.md#technical-debt-in-follow-up-issues) - -#### Merge requests workflow - -This [documentation](merge_request_workflow.md) outlines the current merge request process. - -- [Merge request guidelines](merge_request_workflow.md#merge-request-guidelines-for-contributors) -- [Contribution acceptance criteria](merge_request_workflow.md#contribution-acceptance-criteria) -- [Definition of done](merge_request_workflow.md#definition-of-done) -- [Dependencies](merge_request_workflow.md#dependencies) - ## Closing policy for issues and merge requests - For the criteria for closing issues, see [the Issue Triage handbook page](https://about.gitlab.com/handbook/engineering/quality/issue-triage/#outdated-issues). - For the criteria for closing merge requests, see [the Merge Request Workflow](merge_request_workflow.md). -## Getting an Enterprise Edition License +## Getting an Enterprise Edition license -If you need a license for contributing to an EE-feature, see -[relevant information](https://about.gitlab.com/handbook/marketing/community-relations/code-contributor-program/operations/#contributing-to-the-gitlab-enterprise-edition-ee). +GitLab has two development platforms: -## Finding help +- GitLab Community Edition (CE), our free and open source edition. +- GitLab Enterprise Edition (EE), which is our commercial edition. -- [Get help](https://about.gitlab.com/get-help/). -- Join the community-run [Discord server](https://discord.com/invite/gitlab) and find other contributors in the `#contribute` channel. +If you need a license for contributing to an EE-feature, see +[relevant information](https://about.gitlab.com/handbook/marketing/community-relations/contributor-success/community-contributors-workflows.html#contributing-to-the-gitlab-enterprise-edition-ee). + +## Get help + +If you need any help while contributing to GitLab: + +- If you need help with a merge request or need help finding a reviewer: + - Don't hesitate to ask for help by typing `@gitlab-bot help` in a comment. + - Find reviewers and maintainers of GitLab projects in our + [handbook](https://about.gitlab.com/handbook/engineering/projects/) and + [mention](../../user/group/subgroups/index.md#mention-subgroups) them in a comment. +- Join the community on the [GitLab Community Discord](https://discord.com/invite/gitlab) and find other + contributors in the `#contribute` channel or [initiate a mentor session](https://about.gitlab.com/community/contribute/mentor-sessions/). +- For any other questions or feedback on contributing: + - Ping `@gitlab-org/community-relations/contributor-success` in a comment on your merge request or issue. + - Feel free to [make a new issue with the Contributor Success team](https://gitlab.com/gitlab-org/community-relations/contributor-success/team-task/-/issues/) sharing your experience. +- Did you run out of compute credits for your GitLab merge requests? Join the [GitLab community forks](https://gitlab.com/gitlab-community/meta) project. diff --git a/doc/development/contributing/issue_workflow.md b/doc/development/contributing/issue_workflow.md index b55fef25302..50e87fc5341 100644 --- a/doc/development/contributing/issue_workflow.md +++ b/doc/development/contributing/issue_workflow.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w **Before you submit an issue, [search the issue tracker](https://gitlab.com/gitlab-org/gitlab/-/issues)** for similar entries. Someone else might have already had the same bug or feature proposal. -If you find an existing issue, show your support with an award emoji and add your notes to the discussion. +If you find an existing issue, show your support with an emoji reaction and add your notes to the discussion. To submit a bug: @@ -37,300 +37,7 @@ the affected files to find someone. We also have triage automation in place, described [in our handbook](https://about.gitlab.com/handbook/engineering/quality/triage-operations/). -## Labels - -To allow for asynchronous issue handling, we use [milestones](https://gitlab.com/groups/gitlab-org/-/milestones) -and [labels](https://gitlab.com/gitlab-org/gitlab/-/labels). Leads and product managers handle most of the -scheduling into milestones. Labeling is a task for everyone. (For some projects, labels can be set only by GitLab team members and not by community contributors). - -Most issues will have labels for at least one of the following: - -- Type. For example: `~"type::feature"`, `~"type::bug"`, or `~"type::maintenance"`. -- Stage. For example: `~"devops::plan"` or `~"devops::create"`. -- Group. For example: `~"group::source code"`, `~"group::knowledge"`, or `~"group::editor"`. -- Category. For example: `~"Category:Code Analytics"`, `~"Category:DevOps Reports"`, or `~"Category:Templates"`. -- Feature. For example: `~wiki`, `~ldap`, `~api`, `~issues`, or `~"merge requests"`. -- Department: `~UX`, `~Quality` -- Team: `~"Technical Writing"`, `~Delivery` -- Specialization: `~frontend`, `~backend`, `~documentation` -- Release Scoping: `~Deliverable`, `~Stretch`, `~"Next Patch Release"` -- Priority: `~"priority::1"`, `~"priority::2"`, `~"priority::3"`, `~"priority::4"` -- Severity: `~"severity::1"`, `~"severity::2"`, `~"severity::3"`, `~"severity::4"` - -Please add `~"breaking change"` label if the issue can be considered as a [breaking change](../deprecation_guidelines/index.md). - -Please add `~security` label if the issue is related to application security. - -All labels, their meaning and priority are defined on the -[labels page](https://gitlab.com/gitlab-org/gitlab/-/labels). - -If you come across an issue that has none of these, and you're allowed to set -labels, you can _always_ add the type, stage, group, and often the category/feature labels. - -### Type labels - -Type labels are very important. They define what kind of issue this is. Every -issue should have one and only one. - -The SSOT for type and subtype labels is [available in the handbook](https://about.gitlab.com/handbook/engineering/metrics/#work-type-classification). - -A number of type labels have a priority assigned to them, which automatically -makes them float to the top, depending on their importance. - -Type labels are always lowercase, and can have any color, besides blue (which is -already reserved for category labels). - -The descriptions on the [labels page](https://gitlab.com/groups/gitlab-org/-/labels) -explain what falls under each type label. - -The GitLab handbook documents [when something is a bug](https://about.gitlab.com/handbook/product/product-processes/#bug-issues) and [when it is a feature request](https://about.gitlab.com/handbook/product/product-processes/#feature-issues). - -### Stage labels - -Stage labels specify which [stage](https://about.gitlab.com/handbook/product/categories/#hierarchy) the issue belongs to. - -#### Naming and color convention - -Stage labels respects the `devops::` naming convention. -`` is the stage key as it is in the single source of truth for stages at - -with `_` replaced with a space. - -For instance, the "Manage" stage is represented by the `~"devops::manage"` label in -the `gitlab-org` group since its key under `stages` is `manage`. - -The current stage labels can be found by [searching the labels list for `devops::`](https://gitlab.com/groups/gitlab-org/-/labels?search=devops::). - -These labels are [scoped labels](../../user/project/labels.md#scoped-labels) -and thus are mutually exclusive. - -The Stage labels are used to generate the [direction pages](https://about.gitlab.com/direction/) automatically. - -### Group labels - -Group labels specify which [groups](https://about.gitlab.com/company/team/structure/#product-groups) the issue belongs to. - -It's highly recommended to add a group label, as it's used by our triage -automation to -[infer the correct stage label](https://about.gitlab.com/handbook/engineering/quality/triage-operations/#auto-labelling-of-issues-and-merge-requests). - -#### Naming and color convention - -Group labels respects the `group::` naming convention and -their color is `#A8D695`. -`` is the group key as it is in the single source of truth for groups at -, -with `_` replaced with a space. - -For instance, the "Pipeline Execution" group is represented by the -~"group::pipeline execution" label in the `gitlab-org` group since its key -under `stages.manage.groups` is `pipeline_execution`. - -The current group labels can be found by [searching the labels list for `group::`](https://gitlab.com/groups/gitlab-org/-/labels?search=group::). - -These labels are [scoped labels](../../user/project/labels.md#scoped-labels) -and thus are mutually exclusive. - -You can find the groups listed in the [Product Stages, Groups, and Categories](https://about.gitlab.com/handbook/product/categories/) page. - -We use the term group to map down product requirements from our product stages. -As a team needs some way to collect the work their members are planning to be assigned to, we use the `~group::` labels to do so. - -### Category labels - -From the handbook's -[Product stages, groups, and categories](https://about.gitlab.com/handbook/product/categories/#hierarchy) -page: - -> Categories are high-level capabilities that may be a standalone product at -another company, such as Portfolio Management, for example. - -It's highly recommended to add a category label, as it's used by our triage -automation to -[infer the correct group and stage labels](https://about.gitlab.com/handbook/engineering/quality/triage-operations/#auto-labelling-of-issues). - -If you are an expert in a particular area, it makes it easier to find issues to -work on. You can also subscribe to those labels to receive an email each time an -issue is labeled with a category label corresponding to your expertise. - -#### Naming and color convention - -Category labels respects the `Category:` naming convention and -their color is `#428BCA`. -`` is the category name as it is in the single source of truth for categories at -. - -For instance, the "DevOps Reports" category is represented by the -~"Category:DevOps Reports" label in the `gitlab-org` group since its -`devops_reports.name` value is "DevOps Reports". - -If a category's label doesn't respect this naming convention, it should be specified -with [the `label` attribute](https://about.gitlab.com/handbook/marketing/digital-experience/website/#category-attributes) -in . - -### Feature labels - -From the handbook's -[Product stages, groups, and categories](https://about.gitlab.com/handbook/product/categories/#hierarchy) -page: - -> Features: Small, discrete functionalities, for example Issue weights. Some common -features are listed within parentheses to facilitate finding responsible PMs by keyword. - -It's highly recommended to add a feature label if no category label applies, as -it's used by our triage automation to -[infer the correct group and stage labels](https://about.gitlab.com/handbook/engineering/quality/triage-operations/#auto-labelling-of-issues). - -If you are an expert in a particular area, it makes it easier to find issues to -work on. You can also subscribe to those labels to receive an email each time an -issue is labeled with a feature label corresponding to your expertise. - -Examples of feature labels are `~wiki`, `~ldap`, `~api`, `~issues`, and `~"merge requests"`. - -#### Naming and color convention - -Feature labels are all-lowercase. - -### Facet labels - -To track additional information or context about created issues, developers may -add _facet labels_. Facet labels are also sometimes used for issue prioritization -or for measurements (such as time to close). An example of a facet label is the -~customer label, which indicates customer interest. - -### Department labels - -The current department labels are: - -- ~UX -- ~Quality - -### Team labels - -**Important**: Most of the historical team labels (like Manage or Plan) are -now deprecated in favor of [Group labels](#group-labels) and [Stage labels](#stage-labels). - -Team labels specify what team is responsible for this issue. -Assigning a team label makes sure issues get the attention of the appropriate -people. - -The current team labels are: - -- ~Delivery -- ~"Technical Writing" - -#### Naming and color convention - -Team labels are always capitalized so that they show up as the first label for -any issue. - -### Specialization labels - -These labels narrow the [specialization](https://about.gitlab.com/company/team/structure/#specialist) on a unit of work. - -- ~frontend -- ~backend -- ~documentation - -### Release scoping labels - -Release Scoping labels help us clearly communicate expectations of the work for the -release. There are three levels of Release Scoping labels: - -- ~Deliverable: Issues that are expected to be delivered in the current - milestone. -- ~Stretch: Issues that are a stretch goal for delivering in the current - milestone. If these issues are not done in the current release, they will - strongly be considered for the next release. -- ~"Next Patch Release": Issues to put in the next patch release. Work on these - first, and add the `~"Pick into X.Y"` label to the merge request, along with the - appropriate milestone. - -Each issue scheduled for the current milestone should be labeled ~Deliverable -or ~"Stretch". Any open issue for a previous milestone should be labeled -~"Next Patch Release", or otherwise rescheduled to a different milestone. - -### Priority labels - -We have the following priority labels: - -- ~"priority::1" -- ~"priority::2" -- ~"priority::3" -- ~"priority::4" - -Please refer to the issue triage [priority label](https://about.gitlab.com/handbook/engineering/quality/issue-triage/#priority) section in our handbook to see how it's used. - -### Severity labels - -We have the following severity labels: - -- ~"severity::1" -- ~"severity::2" -- ~"severity::3" -- ~"severity::4" - -Please refer to the issue triage [severity label](https://about.gitlab.com/handbook/engineering/quality/issue-triage/#severity) section in our handbook to see how it's used. - -### Label for community contributors - -There are many issues that have a clear solution with uncontroversial benefit to GitLab users. -However, GitLab might not have the capacity for all these proposals in the current roadmap. -These issues are labeled ~"Seeking community contributions" because we welcome merge requests to resolve them. - -Community contributors can submit merge requests for any issue they want, but -the ~"Seeking community contributions" label has a special meaning. It points to -changes that: - -1. We already agreed on, -1. Are well-defined, -1. Are likely to get accepted by a maintainer. - -We want to avoid a situation when a contributor picks an -~"Seeking community contributions" issue and then their merge request gets closed, -because we realize that it does not fit our vision, or we want to solve it in a -different way. - -We manually add the ~"Seeking community contributions" label to issues -that fit the criteria described above. -We do not automatically add this label, because it requires human evaluation. - -We recommend people that have never contributed to any open source project to -look for issues labeled `~"Seeking community contributions"` with a -[weight of 1](https://gitlab.com/groups/gitlab-org/-/issues?sort=created_date&state=opened&label_name[]=Seeking+community+contributions&assignee_id=None&weight=1) or the `~"good for new contributors"` -[label](https://gitlab.com/gitlab-org/gitlab/-/issues?scope=all&state=opened&label_name[]=good%20for%20new%20contributors&assignee_id=None) -attached to it. -More experienced contributors are very welcome to tackle -[any of them](https://gitlab.com/groups/gitlab-org/-/issues?sort=created_date&state=opened&label_name[]=Seeking+community+contributions&assignee_id=None). - -For more complex features that have a weight of 2 or more and clear scope, we recommend looking at issues -with the [label `~"Community Challenge"`](https://gitlab.com/gitlab-org/gitlab/-/issues?sort=created_date&state=opened&label_name[]=Seeking+community+contributions&label_name[]=Community+challenge). -If your MR for the `~"Community Challenge"` issue gets merged, you will also have a chance to win a custom -GitLab merchandise. - -If you've decided that you would like to work on an issue, please @-mention -the [appropriate product manager](https://about.gitlab.com/handbook/product/#who-to-talk-to-for-what) -as soon as possible. The product manager will then pull in appropriate GitLab team -members to further discuss scope, design, and technical considerations. This will -ensure that your contribution is aligned with the GitLab product and minimize -any rework and delay in getting it merged into main. - -GitLab team members who apply the ~"Seeking community contributions" label to an issue -should update the issue description with a responsible product manager, inviting -any potential community contributor to @-mention per above. - -### Stewardship label - -For issues related to the open source stewardship of GitLab, -there is the ~"stewardship" label. - -This label is to be used for issues in which the stewardship of GitLab -is a topic of discussion. For instance if GitLab Inc. is planning to add -features from GitLab EE to GitLab CE, related issues would be labeled with -~"stewardship". - -A recent example of this was the issue for -[bringing the time tracking API to GitLab CE](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/25517#note_20019084). +For information about which labels to apply to issues, see [Labels](../labels/index.md). ## Feature proposals @@ -399,31 +106,6 @@ The release manager will [update the notes](https://gitlab.com/gitlab-org/release-tools/blob/master/doc/pro-tips.md#update-the-regression-issue) in the regression issue as fixes are addressed. -## Technical and UX debt - -In order to track things that can be improved in the GitLab codebase, -we use the ~"technical debt" label in the [GitLab issue tracker](https://gitlab.com/gitlab-org/gitlab/-/issues). -We use the ~"UX debt" label when we choose to deviate from the MVC, in a way that harms the user experience. - -These labels should be added to issues that describe things that can be improved, -shortcuts that have been taken, features that need additional attention, and all -other things that have been left behind due to high velocity of development. -For example, code that needs refactoring should use the ~"technical debt" label, -something that didn't ship according to our Design System guidelines should -use the ~"UX debt" label. - -Everyone can create an issue, though you may need to ask for adding a specific -label, if you do not have permissions to do it by yourself. Additional labels -can be combined with these labels, to make it easier to schedule -the improvements for a release. - -Issues tagged with these labels have the same priority like issues -that describe a new feature to be introduced in GitLab, and should be scheduled -for a release by the appropriate person. - -Make sure to mention the merge request that the ~"technical debt" issue or -~"UX debt" issue is associated with in the description of the issue. - ## Technical debt in follow-up issues It's common to discover technical debt during development of a new feature. In @@ -459,6 +141,6 @@ and assignee. The maintainer must always agree before an outstanding discussion is resolved in this manner, and will be the one to create the issue. The title and description should be of the same quality as those created -[in the usual manner](#technical-and-ux-debt) - in particular, the issue title +[in the usual manner](../labels/index.md#technical-and-ux-debt) - in particular, the issue title **must not** begin with `Follow-up`! The creating maintainer should also expect to be involved in some capacity when work begins on the follow-up issue. diff --git a/doc/development/contributing/merge_request_workflow.md b/doc/development/contributing/merge_request_workflow.md index 01bfdae5999..7a0269e551d 100644 --- a/doc/development/contributing/merge_request_workflow.md +++ b/doc/development/contributing/merge_request_workflow.md @@ -9,11 +9,33 @@ info: To determine the technical writer assigned to the Stage/Group associated w We welcome merge requests from everyone, with fixes and improvements to GitLab code, tests, and documentation. The issues that are specifically suitable -for community contributions have the [`Seeking community contributions`](issue_workflow.md#label-for-community-contributors) +for community contributions have the +[`Seeking community contributions`](../labels/index.md#label-for-community-contributors) label, but you are free to contribute to any issue you want. +## Working from issues + +If you find an issue, please submit a merge request with a fix or improvement, +if you can, and include tests. + +If you want to add a new feature that is not labeled, it is best to first create +an issue (if there isn't one already) and leave a comment asking for it +to be labeled as `Seeking community contributions`. See the [feature proposals](issue_workflow.md#feature-proposals) +section. + +If you don't know how to fix the issue but can write a test that exposes the +issue, we will accept that as well. In general, bug fixes that include a +regression test are merged quickly. New features without proper tests +might be slower to receive feedback. + +If you are new to GitLab development (or web development in general), see the +[how to contribute](index.md#how-to-contribute) section to get started with +some potentially easy issues. + +## Merge request ownership + If an issue is marked for the current milestone at any time, even -when you are working on it, a GitLab team member may take over the merge request to ensure the work is finished before the release date. +when you are working on it, a GitLab team member may take over the merge request to ensure the work is finished before the release date. If a contributor is no longer actively working on a submitted merge request, we can: @@ -31,79 +53,27 @@ we credit the original author by adding a changelog entry crediting the author and optionally include the original author on at least one of the commits within the MR. -If you want to add a new feature that is not labeled, it is best to first create -an issue (if there isn't one already) and leave a comment asking for it -to be labeled as `Seeking community contributions`. See the [feature proposals](issue_workflow.md#feature-proposals) -section. - -Merge requests should be submitted to the appropriate project at GitLab.com, for example -[GitLab](https://gitlab.com/gitlab-org/gitlab/-/merge_requests), -[GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests), or -[Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests). - -If you are new to GitLab development (or web development in general), see the -[how to contribute](index.md#how-to-contribute) section to get started with -some potentially easy issues. - -To start developing GitLab, download the [GitLab Development Kit](https://gitlab.com/gitlab-org/gitlab-development-kit) -and see the [Development section](../../index.md) for the required guidelines. - ## Merge request guidelines for contributors -If you find an issue, please submit a merge request with a fix or improvement, -if you can, and include tests. +For a walkthrough of the contribution process, see [Tutorial: Make a GitLab contribution](first_contribution.md). -NOTE: -Consider placing your code behind a feature flag if you think it might affect production availability. -Not sure? Read [When to use feature flags](https://about.gitlab.com/handbook/product-development-flow/feature-flag-lifecycle/#when-to-use-feature-flags). +### Best practices -If the change is non-trivial, we encourage you to -start a discussion with [a product manager or a member of the team](https://about.gitlab.com/handbook/product/categories/). -You can do -this by tagging them in an MR before submitting the code for review. Talking -to team members can be helpful when making design decisions. Communicating the -intent behind your changes can also help expedite merge request reviews. +- If the change is non-trivial, we encourage you to start a discussion with + [a product manager or a member of the team](https://about.gitlab.com/handbook/product/categories/). + You can do this by tagging them in an MR before submitting the code for review. Talking + to team members can be helpful when making design decisions. Communicating the + intent behind your changes can also help expedite merge request reviews. -If -you don't know how to fix the issue but can write a test that exposes the -issue, we will accept that as well. In general, bug fixes that include a -regression test are merged quickly. New features without proper tests -might be slower to receive feedback. +- Consider placing your code behind a feature flag if you think it might affect production availability. + Not sure? Read [When to use feature flags](https://about.gitlab.com/handbook/product-development-flow/feature-flag-lifecycle/#when-to-use-feature-flags). -To create a merge request: - -1. [Fork](../../user/project/repository/forking_workflow.md) the project into - your personal namespace (or group) on GitLab.com. -1. Create a feature branch in your fork (don't work off your [default branch](../../user/project/repository/branches/default.md)). -1. Follow the [commit messages guidelines](#commit-messages-guidelines). -1. If you have multiple commits, combine them into a few logically organized commits. -1. Push the commits to your working branch in your fork. -1. Submit a merge request (MR) against the default branch of the upstream project. -1. The MR title should describe the change you want to make. -1. The MR description should give a reason for your change. - 1. If you are contributing code, fill in the description according to the default - template already provided in the "Description" field. - 1. If you are contributing documentation, choose `Documentation` from the - "Choose a template" menu and fill in the description according to the template. - 1. Use the syntax `Solves #XXX`, `Closes #XXX`, or `Refs #XXX` to mention the issues your merge - request addresses. Referenced issues do not [close automatically](../../user/project/issues/managing_issues.md#closing-issues-automatically). - You must close them manually once the merge request is merged. -1. If you're allowed to, set a relevant milestone and [labels](issue_workflow.md). - MR labels should generally match the corresponding issue (if there is one). - The group label should reflect the group that executed or coached the work, - not necessarily the group that owns the feature. -1. Read and adhere to - [The responsibility of the merge request author](../code_review.md#the-responsibility-of-the-merge-request-author). -1. Read and follow - [Having your merge request reviewed](../code_review.md#having-your-merge-request-reviewed). -1. Make sure the merge request meets the [Definition of done](#definition-of-done). - -If you would like quick feedback on your merge request feel free to mention someone -from the [core team](https://about.gitlab.com/community/core-team/) or one of the -[merge request coaches](https://about.gitlab.com/company/team/). When having your code reviewed -and when reviewing merge requests, please keep the [code review guidelines](../code_review.md) -in mind. And if your code also makes changes to the database, or does expensive queries, -check the [database review guidelines](../database_review.md). +- If you would like quick feedback on your merge request feel free to mention someone + from the [core team](https://about.gitlab.com/community/core-team/) or one of the + [merge request coaches](https://about.gitlab.com/company/team/). When having your code reviewed + and when reviewing merge requests, please keep the [code review guidelines](../code_review.md) + in mind. And if your code also makes changes to the database, or does expensive queries, + check the [database review guidelines](../database_review.md). ### Keep it simple @@ -191,6 +161,10 @@ To make sure that your merge request can be approved, please ensure that it meet the contribution acceptance criteria below: 1. The change is as small as possible. +1. If the merge request contains more than 500 changes: + - Explain the reason + - Mention a maintainer +1. Mention any major [breaking changes](../deprecation_guidelines/index.md). 1. Include proper tests and make all tests pass (unless it contains a test exposing a bug in existing code). Every new class should have corresponding unit tests, even if the class is exercised at a higher level, such as a feature test. @@ -268,6 +242,15 @@ requirements. There isn't a way to know anything about our customers' data on their [self-managed instances](../../subscriptions/self_managed/index.md), so keep that in mind for any data implications with your merge request. +1. Consider self-managed functionality and upgrade paths. The change should consider both: + + - If additional work needs to be done for self-managed availability, and + - If the change requires a [required stop](../database/required_stops.md) when upgrading GitLab versions. + + Upgrade stops are sometimes requested when a GitLab code change is dependent + upon a background migration being already complete. Ideally, changes causing required + upgrade stops should be held for the next major release, or + [at least a 3 milestones notice in advance if unavoidable](../../update/index.md#upgrade-paths). ### Testing @@ -366,3 +349,8 @@ issue) that are incremental improvements, such as: Tag a merge request with ~"Stuff that should Just Work" to track work in this area. + +## Related topics + +- [The responsibility of the merge request author](../code_review.md#the-responsibility-of-the-merge-request-author) +- [Having your merge request reviewed](../code_review.md#having-your-merge-request-reviewed) diff --git a/doc/development/contributing/style_guides.md b/doc/development/contributing/style_guides.md index 28ce8e6ff4b..d24875e559a 100644 --- a/doc/development/contributing/style_guides.md +++ b/doc/development/contributing/style_guides.md @@ -5,7 +5,7 @@ group: Development info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Style guides +# Development style guides ## Editor/IDE styling standardization @@ -30,7 +30,9 @@ We were using Overcommit prior to Lefthook, so you may want to uninstall it firs ### Install Lefthook -1. Install the `lefthook` Ruby gem: +1. You can install lefthook in [different ways](https://github.com/evilmartians/lefthook/blob/master/docs/install.md#install-lefthook). + If you do not choose to install it globally (e.g. via Homebrew or package managers), and only want to use it for the GitLab project, + you can install the Ruby gem via: ```shell bundle install @@ -39,12 +41,18 @@ We were using Overcommit prior to Lefthook, so you may want to uninstall it firs 1. Install Lefthook managed Git hooks: ```shell + # If installed globally + lefthook install + # Or if installed via ruby gem bundle exec lefthook install ``` 1. Test Lefthook is working by running the Lefthook `pre-push` Git hook: ```shell + # If installed globally + lefthook run pre-push + # Or if installed via ruby gem bundle exec lefthook run pre-push ``` @@ -57,6 +65,18 @@ Lefthook is configured with a combination of: - Project configuration in [`lefthook.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lefthook.yml). - Any [local configuration](https://github.com/evilmartians/lefthook/blob/master/README.md#local-config). +### Lefthook auto-fixing files + +We have a custom lefthook target to run all the linters with auto-fix capabilities, +but just on the files which changed in your branch. + +```shell +# If installed globally +lefthook run auto-fix +# Or if installed via ruby gem +bundle exec lefthook run auto-fix +``` + ### Disable Lefthook temporarily To disable Lefthook temporarily, you can set the `LEFTHOOK` environment variable to `0`. For instance: diff --git a/doc/development/dangerbot.md b/doc/development/dangerbot.md index b08eaed2afa..ef1e563b668 100644 --- a/doc/development/dangerbot.md +++ b/doc/development/dangerbot.md @@ -141,27 +141,27 @@ To enable the Dangerfile on another existing GitLab project, complete the follow 1. Add [`gitlab-dangerfiles`](https://rubygems.org/gems/gitlab-dangerfiles) to your `Gemfile`. 1. Create a `Dangerfile` with the following content: - ```ruby - require "gitlab-dangerfiles" + ```ruby + require "gitlab-dangerfiles" - Gitlab::Dangerfiles.for_project(self, &:import_defaults) - ``` + Gitlab::Dangerfiles.for_project(self, &:import_defaults) + ``` 1. Add the following to your CI/CD configuration: - ```yaml - include: - - project: 'gitlab-org/quality/pipeline-common' - file: - - '/ci/danger-review.yml' - rules: - - if: $CI_SERVER_HOST == "gitlab.com" - ``` + ```yaml + include: + - project: 'gitlab-org/quality/pipeline-common' + file: + - '/ci/danger-review.yml' + rules: + - if: $CI_SERVER_HOST == "gitlab.com" + ``` 1. If your project is in the `gitlab-org` group, you don't need to set up any token as the `DANGER_GITLAB_API_TOKEN` variable is available at the group level. If not, follow these last steps: - 1. Create a [Project access tokens](../user/project/settings/project_access_tokens.md). - 1. Add the token as a CI/CD project variable named `DANGER_GITLAB_API_TOKEN`. + 1. Create a [Project access tokens](../user/project/settings/project_access_tokens.md). + 1. Add the token as a CI/CD project variable named `DANGER_GITLAB_API_TOKEN`. You should add the ~"Danger bot" label to the merge request before sending it for review. diff --git a/doc/development/database/add_foreign_key_to_existing_column.md b/doc/development/database/add_foreign_key_to_existing_column.md index 2c2999e69d6..823fb49a9ab 100644 --- a/doc/development/database/add_foreign_key_to_existing_column.md +++ b/doc/development/database/add_foreign_key_to_existing_column.md @@ -155,13 +155,13 @@ To limit impact on GitLab.com, a process exists to validate them asynchronously during weekend hours. Due to generally lower traffic and fewer deployments, FK validation can proceed at a lower level of risk. -### Schedule foreign key validation for a low-impact time +#### Schedule foreign key validation for a low-impact time 1. [Schedule the FK to be validated](#schedule-the-fk-to-be-validated). 1. [Verify the MR was deployed and the FK is valid in production](#verify-the-mr-was-deployed-and-the-fk-is-valid-in-production). 1. [Add a migration to validate the FK synchronously](#add-a-migration-to-validate-the-fk-synchronously). -### Schedule the FK to be validated +#### Schedule the FK to be validated 1. Create a merge request containing a post-deployment migration, which prepares the foreign key for asynchronous validation. @@ -198,7 +198,7 @@ def down end ``` -### Verify the MR was deployed and the FK is valid in production +#### Verify the MR was deployed and the FK is valid in production 1. Verify that the post-deploy migration was executed on GitLab.com using ChatOps with `/chatops run auto_deploy status `. If the output returns `db/gprd`, @@ -208,7 +208,7 @@ end 1. Use [Database Lab](database_lab.md) to check if validation was successful. Ensure the output does not indicate the foreign key is `NOT VALID`. -### Add a migration to validate the FK synchronously +#### Add a migration to validate the FK synchronously After the foreign key is valid on the production database, create a second merge request that validates the foreign key synchronously. The schema changes @@ -240,19 +240,20 @@ end ``` -## Test database FK changes locally +### Test database FK changes locally You must test the database foreign key changes locally before creating a merge request. -### Verify the foreign keys validated asynchronously +#### Verify the foreign keys validated asynchronously Use the asynchronous helpers on your local environment to test changes for validating a foreign key: -1. Enable the feature flags by running `Feature.enable(:database_async_foreign_key_validation)` - and `Feature.enable(:database_reindexing)` in the Rails console. +1. Enable the feature flag by running `Feature.enable(:database_async_foreign_key_validation)` + in the Rails console. 1. Run `bundle exec rails db:migrate` so that it creates an entry in the async validation table. -1. Run `bundle exec rails gitlab:db:reindex` so that the FK is validated asynchronously. +1. Run `bundle exec rails gitlab:db:validate_async_constraints:all` so that the FK is validated + asynchronously on all databases. 1. To verify the foreign key, open the PostgreSQL console using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md) command `gdk psql` and run the command `\d+ table_name` to check that your diff --git a/doc/development/database/adding_database_indexes.md b/doc/development/database/adding_database_indexes.md index 1e3a1de9b69..7b29b1b14de 100644 --- a/doc/development/database/adding_database_indexes.md +++ b/doc/development/database/adding_database_indexes.md @@ -38,6 +38,15 @@ when adding a new index: 1. Is the overhead of maintaining the index worth the reduction in query timings? +In some situations, an index might not be required: + +- The table is small (less than `1,000` records) and it's not expected to exponentially grow in size. +- Any existing indexes filter out enough rows. +- The reduction in query timings after the index is added is not significant. + +Additionally, wide indexes are not required to match all filter criteria of queries. We just need +to cover enough columns so that the index lookup has a small enough selectivity. + ## Re-using Queries The first step is to make sure your query re-uses as many existing indexes as @@ -183,6 +192,29 @@ for `index_exists?`, causing a required index to not be created properly. By always requiring a name for certain types of indexes, the chance of error is greatly reduced. +## Testing for existence of indexes + +The easiest way to test for existence of an index by name is to use the `index_name_exists?` method, but the `index_exists?` method can also be used with a name option. For example: + +```ruby +class MyMigration < Gitlab::Database::Migration[2.1] + INDEX_NAME = 'index_name' + + def up + # an index must be conditionally created due to schema inconsistency + unless index_exists?(:table_name, :column_name, name: INDEX_NAME) + add_index :table_name, :column_name, name: INDEX_NAME + end + end + + def down + # no op + end +end +``` + +Keep in mind that concurrent index helpers like `add_concurrent_index`, `remove_concurrent_index`, and `remove_concurrent_index_by_name` already perform existence checks internally. + ## Temporary indexes There may be times when an index is only needed temporarily. @@ -448,7 +480,7 @@ You must test the database index changes locally before creating a merge request the post-deploy migration has been executed in the production database. For more information, see [How to determine if a post-deploy migration has been executed on GitLab.com](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/post_deploy_migration/readme.md#how-to-determine-if-a-post-deploy-migration-has-been-executed-on-gitlabcom). 1. In the case of an [index removed asynchronously](#schedule-the-index-to-be-removed), wait - until the next week so that the index can be created over a weekend. + until the next week so that the index can be removed over a weekend. 1. Use Database Lab [to check if removal was successful](database_lab.md#checking-indexes). [Database Lab](database_lab.md) should report an error when trying to find the removed index. If not, the index may still exist. diff --git a/doc/development/database/avoiding_downtime_in_migrations.md b/doc/development/database/avoiding_downtime_in_migrations.md index 8e1eeee7a42..25310554c24 100644 --- a/doc/development/database/avoiding_downtime_in_migrations.md +++ b/doc/development/database/avoiding_downtime_in_migrations.md @@ -605,7 +605,7 @@ See example [merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_request ### Remove the trigger and old `integer` columns (release N + 2) Using post-deployment migration and the provided `cleanup_conversion_of_integer_to_bigint` helper, -drop the database trigger and the old `integer` columns ([see an example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69714)). +drop the database trigger and the old `integer` columns ([see an example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70351)). ### Remove ignore rules (release N + 3) diff --git a/doc/development/database/batched_background_migrations.md b/doc/development/database/batched_background_migrations.md index c6fe6d16faf..6a6b43e52a0 100644 --- a/doc/development/database/batched_background_migrations.md +++ b/doc/development/database/batched_background_migrations.md @@ -34,10 +34,13 @@ Background migrations can help when: - Populating one column based on JSON stored in another column. - Migrating data that depends on the output of external services. (For example, an API.) -NOTE: -If the batched background migration is part of an important upgrade, it must be announced -in the release post. Discuss with your Project Manager if you're unsure if the migration falls -into this category. +### Notes + +- If the batched background migration is part of an important upgrade, it must be announced + in the release post. Discuss with your Project Manager if you're unsure if the migration falls + into this category. +- You should use the [generator](#generator) to create batched background migrations, + so that required files are created by default. ## Isolation @@ -145,6 +148,49 @@ Make sure the newly-created data is either migrated, or saved in both the old and new version upon creation. Removals in turn can be handled by defining foreign keys with cascading deletes. +### Job retry mechanism + +The batched background migrations retry mechanism ensures that a job is executed again in case of failure. +The following diagram shows the different stages of our retry mechanism: + +```plantuml +@startuml +hide empty description +note as N1 + can_split?: + the failure is due to a query timeout +end note +[*] --> Running +Running --> Failed +note on link + if number of retries <= MAX_ATTEMPTS +end note +Running --> Succeeded +Failed --> Running +note on link + if number of retries > MAX_ATTEMPTS + and can_split? == true + then two jobs with smaller + batch size will be created +end note +Failed --> [*] +Succeeded --> [*] +@enduml +``` + +- `MAX_ATTEMPTS` is defined in the [`Gitlab::Database::BackgroundMigration`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/database/background_migration/batched_job.rb) +class. +- `can_split?` is defined in the [`Gitlab::Database::BatchedJob`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/database/background_migration/batched_job.rb) class. + +### Failed batched background migrations + +The whole batched background migration is marked as `failed` +(`/chatops run batched_background_migrations status MIGRATION_ID` will show +the migration as `failed`) if any of the following are true: + +- There are no more jobs to consume, and there are failed jobs. +- More than [half of the jobs failed since the background migration was started](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/database/background_migration/batched_migration.rb). + ### Requeuing batched background migrations If one of the batched background migrations contains a bug that is fixed in a patch @@ -311,6 +357,22 @@ NOTE: When applying additional filters, it is important to ensure they are properly covered by an index to optimize `EachBatch` performance. In the example above we need an index on `(type, id)` to support the filters. See [the `EachBatch` documentation for more information](iterating_tables_in_batches.md). +## Generator + +The custom generator `batched_background_migration` scaffolds necessary files and +accepts `table_name`, `column_name`, and `feature_category` as arguments. Usage: + +```shell +bundle exec rails g batched_background_migration my_batched_migration --table_name= --column_name= --feature_category= +``` + +This command creates these files: + +- `db/post_migrate/20230214231008_queue_my_batched_migration.rb` +- `spec/migrations/20230214231008_queue_my_batched_migration_spec.rb` +- `lib/gitlab/background_migration/my_batched_migration.rb` +- `spec/lib/gitlab/background_migration/my_batched_migration_spec.rb` + ## Example The `routes` table has a `source_type` field that's used for a polymorphic relationship. @@ -319,8 +381,13 @@ the work is migrating data from the `source_id` column into a new singular forei Because we intend to delete old rows later, there's no need to update them as part of the background migration. -1. Start by defining our migration class, which should inherit - from `Gitlab::BackgroundMigration::BatchedMigrationJob`: +1. Start by using the generator to create batched background migration files: + + ```shell + bundle exec rails g batched_background_migration BackfillRouteNamespaceId --table_name=routes --column_name=id --feature_category=source_code_management + ``` + +1. Update the migration job (subclass of `BatchedMigrationJob`) to copy `source_id` values to `namespace_id`: ```ruby class Gitlab::BackgroundMigration::BackfillRouteNamespaceId < BatchedMigrationJob @@ -344,10 +411,10 @@ background migration. ``` NOTE: - Job classes must be subclasses of `BatchedMigrationJob` to be + Job classes inherit from `BatchedMigrationJob` to ensure they are correctly handled by the batched migration framework. Any subclass of - `BatchedMigrationJob` is initialized with necessary arguments to - execute the batch, as well as a connection to the tracking database. + `BatchedMigrationJob` is initialized with the necessary arguments to + execute the batch, and a connection to the tracking database. 1. Create a database migration that adds a new trigger to the database. Example: @@ -380,12 +447,14 @@ background migration. end ``` -1. Create a post-deployment migration that queues the migration for existing data: +1. Update the created post-deployment migration with required delay and batch sizes: ```ruby class QueueBackfillRoutesNamespaceId < Gitlab::Database::Migration[2.1] MIGRATION = 'BackfillRouteNamespaceId' DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 restrict_gitlab_migration gitlab_schema: :gitlab_main @@ -394,7 +463,9 @@ background migration. MIGRATION, :routes, :id, - job_interval: DELAY_INTERVAL + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE ) end @@ -416,24 +487,6 @@ background migration. - Continues using the data as before. - Ensures that both existing and new data are migrated. -1. In the next release, add a database migration to remove the trigger. - - ```ruby - class RemoveNamepaceIdTriggerFromRoutes < Gitlab::Database::Migration[2.1] - FUNCTION_NAME = 'example_function' - TRIGGER_NAME = 'example_trigger' - - def up - drop_trigger(TRIGGER_NAME, :routes) - drop_function(FUNCTION_NAME) - end - - def down - # Should reverse the trigger and the function in the up method of the migration that added it - end - end - ``` - 1. Add a new post-deployment migration that checks that the batched background migration is completed. For example: @@ -469,6 +522,24 @@ background migration. instance, the data is advisory, and not mission-critical), then you can skip this final step. This step confirms that the migration is completed, and all of the rows were migrated. +1. Add a database migration to remove the trigger. + + ```ruby + class RemoveNamepaceIdTriggerFromRoutes < Gitlab::Database::Migration[2.1] + FUNCTION_NAME = 'example_function' + TRIGGER_NAME = 'example_trigger' + + def up + drop_trigger(TRIGGER_NAME, :routes) + drop_function(FUNCTION_NAME) + end + + def down + # Should reverse the trigger and the function in the up method of the migration that added it + end + end + ``` + After the batched migration is completed, you can safely depend on the data in `routes.namespace_id` being populated. @@ -569,6 +640,37 @@ for more details. more pressure on DB than you expect. Measure on staging, or ask someone to measure on production. 1. Know how much time is required to run the batched background migration. +1. Be careful when silently rescuing exceptions inside job classes. This may lead to + jobs being marked as successful, even in a failure scenario. + + ```ruby + # good + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all(name: 'My Name') + end + end + + # acceptable + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all(name: 'My Name') + rescue Exception => error + logger.error(message: error.message, class: error.class) + + raise + end + end + + # bad + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all(name: 'My Name') + rescue Exception => error + logger.error(message: error.message, class: self.class.name) + end + end + ``` ## Additional tips and strategies @@ -719,6 +821,99 @@ You can view failures in two ways: WHERE transition_logs.next_status = '2' AND migration.job_class_name = "CLASS_NAME"; ``` +### Executing a particular batch on the database testing pipeline + +NOTE: +Only [database maintainers](https://gitlab.com/groups/gitlab-org/maintainers/database/-/group_members?with_inherited_permissions=exclude) can view the database testing pipeline artifacts. Ask one for help if you need to use this method. + +Let's assume that a batched background migration failed on a particular batch on GitLab.com and you want to figure out which query failed and why. At the moment, we don't have a good way to retrieve query information (especially the query parameters) and rerunning the entire migration with more logging would be a long process. + +Fortunately you can leverage our [database migration pipeline](database_migration_pipeline.md) to rerun a particular batch with additional logging and/or fix to see if it solves the problem. + + +For an example see [Draft: Test PG::CardinalityViolation fix](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/110910) but make sure to read the entire section. + +To do that, you need to: + +1. Find the batch `start_id` and `end_id` +1. Create a regular migration +1. Apply a workaround for our migration helpers (optional) +1. Start the database migration pipeline + +#### 1. Find the batch `start_id` and `end_id` + +You should be able to find those in [Kibana](#viewing-failure-error-logs). + +#### 2. Create a regular migration + +Schedule the batch in the `up` block of a regular migration: + +```ruby +def up + instance = Gitlab::BackgroundMigration::YourBackgroundMigrationClass.new( + start_id: , + end_id: , + batch_table: , + batch_column: , + sub_batch_size: , + pause_ms: , + job_arguments: , + connection: connection + ) + + instance.perform +end + + +def down + # no-op +end +``` + +#### 3. Apply a workaround for our migration helpers (optional) + +If your batched background migration touches tables from a schema other than the one you specified by using `restrict_gitlab_migration` helper (example: the scheduling migration has `restrict_gitlab_migration gitlab_schema: :gitlab_main` but the background job uses tables from the `:gitlab_ci` schema) then the migration will fail. To prevent that from happening you'll have to monkey patch database helpers so they don't fail the testing pipeline job: + +1. Add the schema names to [`RestrictGitlabSchema`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb#L57) + +```diff +diff --git a/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb +index b8d1d21a0d2d2a23d9e8c8a0a17db98ed1ed40b7..912e20659a6919f771045178c66828563cb5a4a1 100644 +--- a/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb ++++ b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb +@@ -55,7 +55,7 @@ def unmatched_schemas + end + + def allowed_schemas_for_connection +- Gitlab::Database.gitlab_schemas_for_connection(connection) ++ Gitlab::Database.gitlab_schemas_for_connection(connection) << :gitlab_ci + end + end + end +``` + +1. Add the schema names to [`RestrictAllowedSchemas`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb#L82) + +```diff +diff --git a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb +index 4ae3622479f0800c0553959e132143ec9051898e..d556ec7f55adae9d46a56665ce02de782cb09f2d 100644 +--- a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb ++++ b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb +@@ -79,7 +79,7 @@ def restrict_to_dml_only(parsed) + tables = self.dml_tables(parsed) + schemas = self.dml_schemas(tables) + +- if (schemas - self.allowed_gitlab_schemas).any? ++ if (schemas - (self.allowed_gitlab_schemas << :gitlab_ci)).any? + raise DMLAccessDeniedError, \ + "Select/DML queries (SELECT/UPDATE/DELETE) do access '#{tables}' (#{schemas.to_a}) " \ + "which is outside of list of allowed schemas: '#{self.allowed_gitlab_schemas}'. " \ +``` + +#### 4. Start the database migration pipeline + +Create a Draft merge request with your changes and trigger the manual `db:gitlabcom-database-testing` job. + ### Adding indexes to support batched background migrations Sometimes it is necessary to add a new or temporary index to support a batched background migration. diff --git a/doc/development/database/ci_mirrored_tables.md b/doc/development/database/ci_mirrored_tables.md index bf3a744b936..1e37739bdc4 100644 --- a/doc/development/database/ci_mirrored_tables.md +++ b/doc/development/database/ci_mirrored_tables.md @@ -76,9 +76,8 @@ the source and the target tables in sync: 1. Deleting namespaces/projects. ```mermaid -graph TD - - subgraph "CI database (tables)" +graph LR + subgraph CI["CI Tables"] E[other CI tables] F{queries with joins allowed} G[ci_project_mirrors] @@ -89,17 +88,18 @@ graph TD F---H end - A---B - B---C - B---D + Main["Main Tables"]---L["⛔ ← Joins are not allowed → ⛔"] + L---CI -L["⛔ ← Joins are not allowed → ⛔"] - - subgraph "Main database (tables)" + subgraph Main["Main Tables"] A[other main tables] B{queries with joins allowed} C[projects] D[namespaces] + + A---B + B---C + B---D end ``` diff --git a/doc/development/database/clickhouse/index.md b/doc/development/database/clickhouse/index.md index a26bac261fd..032e4f5f6ee 100644 --- a/doc/development/database/clickhouse/index.md +++ b/doc/development/database/clickhouse/index.md @@ -83,3 +83,65 @@ Quoting the [documentation](https://clickhouse.com/docs/en/sql-reference/stateme > If there's some aggregation in the view query, it's applied only to the batch > of freshly inserted data. Any changes to existing data of the source table > (like update, delete, drop a partition, etc.) do not change the materialized view. + +## Secure and sensible defaults + +ClickHouse instances should follow these security recommendations: + +### Users + +Files: `users.xml` and `config.xml`. + +| Topic | Security Requirement | Reason | +| ----- | -------------------- | ------ | +| [`user_name/password`](https://clickhouse.com/docs/en/operations/settings/settings-users/#user-namepassword) | Usernames **must not** be blank. Passwords **must** use `password_sha256_hex` and **must not** be blank. | `plaintext` and `password_double_sha1_hex` are insecure. If username isn't specified, [`default` is used with no password](https://clickhouse.com/docs/en/operations/settings/settings-users/). | +| [`access_management`](https://clickhouse.com/docs/en/operations/settings/settings-users/#access_management-user-setting) | Use Server [configuration files](https://clickhouse.com/docs/en/operations/configuration-files) `users.xml` and `config.xml`. Avoid SQL-driven workflow. | SQL-driven workflow implies that at least one user has `access_management` which can be avoided via configuration files. These files are easier to audit and monitor too, considering that ["You can't manage the same access entity by both configuration methods simultaneously."](https://clickhouse.com/docs/en/operations/access-rights/#access-control). | +| [`user_name/networks`](https://clickhouse.com/docs/en/operations/settings/settings-users/#user-namenetworks) | At least one of ``, ``, `` **must** be set. Do not use `::/0` to open access for any network. | Network controls. ([Trust cautiously](https://about.gitlab.com/handbook/security/architecture/#trust-cautiously) principle) | +| [`user_name/profile`](https://clickhouse.com/docs/en/operations/settings/settings-users/#user-nameprofile) | Use profiles to set similar properties across multiple users and set limits (from the user interface). | [Least privilege](https://about.gitlab.com/handbook/security/architecture/#assign-the-least-privilege-possible) principle and limits. | +| [`user_name/quota`](https://clickhouse.com/docs/en/operations/settings/settings-users/#user-namequota) | Set quotas for users whenever possible. | Limit resource usage over a period of time or track the use of resources. | +| [`user_name/databases`](https://clickhouse.com/docs/en/operations/settings/settings-users/#user-namedatabases) | Restrict access to data, and avoid users with full access. | [Least privilege](https://about.gitlab.com/handbook/security/architecture/#assign-the-least-privilege-possible) principle. | + +### Network + +Files: `config.xml` + +| Topic | Security Requirement | Reason | +| ----- | -------------------- | ------ | +| [`mysql_port`](https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings/#server_configuration_parameters-mysql_port) | Disable MySQL access unless strictly necessary:
``. | Close unnecessary ports and features exposure. ([Defense in depth](https://about.gitlab.com/handbook/security/architecture/#implement-defense-in-depth) principle) | +| [`postgresql_port`](https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings/#server_configuration_parameters-postgresql_port) | Disable PostgreSQL access unless strictly necessary:
`` | Close unnecessary ports and features exposure. ([Defense in depth](https://about.gitlab.com/handbook/security/architecture/#implement-defense-in-depth) principle) | +| [`http_port/https_port`](https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings/#http-porthttps-port) & [`tcp_port/tcp_port_secure`](https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings/#http-porthttps-port) | Configure [SSL-TLS](https://clickhouse.com/docs/en/guides/sre/configuring-ssl), and disable non SSL ports:
``
``
and enable secure ports:
`8443`
`9440` | Encrypt data in transit. ([Defense in depth](https://about.gitlab.com/handbook/security/architecture/#implement-defense-in-depth) principle) | +| [`interserver_http_host`](https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings/#interserver-http-host) | Disable `interserver_http_host` in favor of `interserver_https_host` (`9010`) if ClickHouse is configured as a cluster. | Encrypt data in transit. ([Defense in depth](https://about.gitlab.com/handbook/security/architecture/#implement-defense-in-depth) principle) | + +### Storage + +| Topic | Security Requirement | Reason | +| ----- | -------------------- | ------ | +| Permissions | ClickHouse runs by default with the `clickhouse` user. Running as `root` is never needed. Use the principle of least privileges for the folders: `/etc/clickhouse-server`, `/var/lib/clickhouse`, `/var/log/clickhouse-server`. These folders must belong to the `clickhouse` user and group, and no other system user must have access to them. | Default passwords, ports and rules are "open doors". ([Fail securely & use secure defaults](https://about.gitlab.com/handbook/security/architecture/#fail-securely--use-secure-defaults) principle) | +| Encryption | Use an encrypted storage for logs and data if RED data is processed. On Kubernetes, the [StorageClass](https://kubernetes.io/docs/concepts/storage/storage-classes/) used must be encrypted. | Encrypt data at rest. ([Defense in depth](https://about.gitlab.com/handbook/security/architecture/#implement-defense-in-depth)) | + +### Logging + +| Topic | Security Requirement | Reason | +| ----- | -------------------- | ------ | +| `logger` | `Log` and `errorlog` **must** be defined and writable by `clickhouse`. | Make sure logs are stored. | +| SIEM | If hosted on GitLab.com, the ClickHouse instance or cluster **must** report [logs to our SIEM](https://internal-handbook.gitlab.io/handbook/security/infrastructure_security_logging/tooling/devo/) (internal link). | [GitLab logs critical information system activity](https://about.gitlab.com/handbook/security/audit-logging-policy.html). | +| Log sensitive data | Query masking rules **must** be used if sensitive data can be logged. See [example masking rules](#example-masking-rules). | [Column level encryption](https://clickhouse.com/docs/en/sql-reference/functions/encryption-functions/) can be used and leak sensitive data (keys) in logs. | + +#### Example masking rules + +```xml + + + hide SSN + (^|\D)\d{3}-\d{2}-\d{4}($|\D) + 000-00-0000 + + + hide encrypt/decrypt arguments + + ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) + + \1(???) + + +``` diff --git a/doc/development/database/clickhouse/merge_request_analytics.md b/doc/development/database/clickhouse/merge_request_analytics.md new file mode 100644 index 00000000000..34da71d6c4c --- /dev/null +++ b/doc/development/database/clickhouse/merge_request_analytics.md @@ -0,0 +1,355 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Merge request analytics with ClickHouse + +The [merge request analytics feature](../../../user/analytics/merge_request_analytics.md) +shows statistics about the merged merge requests in the project and also exposes record-level metadata. +Aggregations include: + +- **Average time to merge**: The duration between the creation time and the merge time. +- **Monthly aggregations**: A chart of 12 months of the merged merge requests. + +Under the chart, the user can see the paginated list of merge requests, 12 months per page. + +You can filter by: + +- Author +- Assignee +- Labels +- Milestone +- Source branch +- Target branch + +## Current performance problems + +- The aggregation queries require specialized indexes, which cost additional + disk space (index-only scans). +- Querying the whole 12 months is slow (statement timeout). Instead, the frontend + requests data per month (12 database queries). +- Even with specialized indexes, making the feature available on the group level + would not be feasible due to the large volume of merge requests. + +## Example queries + +Get the number of merge requests merged in a given month: + +```sql +SELECT COUNT(*) +FROM "merge_requests" +INNER JOIN "merge_request_metrics" ON "merge_request_metrics"."merge_request_id" = "merge_requests"."id" +WHERE (NOT EXISTS + (SELECT 1 + FROM "banned_users" + WHERE (merge_requests.author_id = banned_users.user_id))) + AND "merge_request_metrics"."target_project_id" = 278964 + AND "merge_request_metrics"."merged_at" >= '2022-12-01 00:00:00' + AND "merge_request_metrics"."merged_at" <= '2023-01-01 00:00:00' +``` + +The `merge_request_metrics` table was de-normalized (by adding `target_project_id`) +to improve the first-page load time. The query itself works well for smaller date ranges, +however, it can time out as the date range increases. + +After an extra filter is added, the query becomes more complex because it must also +filter the `merge_requests` table: + +```sql +SELECT COUNT(*) +FROM "merge_requests" +INNER JOIN "merge_request_metrics" ON "merge_request_metrics"."merge_request_id" = "merge_requests"."id" +WHERE (NOT EXISTS + (SELECT 1 + FROM "banned_users" + WHERE (merge_requests.author_id = banned_users.user_id))) + AND "merge_requests"."author_id" IN + (SELECT "users"."id" + FROM "users" + WHERE (LOWER("users"."username") IN (LOWER('ahegyi')))) + AND "merge_request_metrics"."target_project_id" = 278964 + AND "merge_request_metrics"."merged_at" >= '2022-12-01 00:00:00' + AND "merge_request_metrics"."merged_at" <= '2023-01-01 00:00:00' +``` + +To calculate mean time to merge, we also query the total time between the +merge request creation time and merge time. + +```sql +SELECT EXTRACT(epoch + FROM SUM(AGE(merge_request_metrics.merged_at, merge_request_metrics.created_at))) +FROM "merge_requests" +INNER JOIN "merge_request_metrics" ON "merge_request_metrics"."merge_request_id" = "merge_requests"."id" +WHERE (NOT EXISTS + (SELECT 1 + FROM "banned_users" + WHERE (merge_requests.author_id = banned_users.user_id))) + AND "merge_requests"."author_id" IN + (SELECT "users"."id" + FROM "users" + WHERE (LOWER("users"."username") IN (LOWER('ahegyi')))) + AND "merge_request_metrics"."target_project_id" = 278964 + AND "merge_request_metrics"."merged_at" >= '2022-08-01 00:00:00' + AND "merge_request_metrics"."merged_at" <= '2022-09-01 00:00:00' + AND "merge_request_metrics"."merged_at" > "merge_request_metrics"."created_at" +LIMIT 1 +``` + +## Store merge request data in ClickHouse + +Several other use cases exist for storing and querying merge request data in +ClickHouse. In this document, we focus on this particular feature. + +The core data exists in the `merge_request_metrics` and in the `merge_requests` +database tables. Some filters require extra tables to be joined: + +- `banned_users`: Filter out merge requests created by banned users. +- `labels`: A merge request can have one or more assigned labels. +- `assignees`: A merge request can have one or more assignees. +- `merged_at`: The `merged_at` column is located in the `merge_request_metrics` table. + +The `merge_requests` table contains data that can be filtered directly: + +- **Author**: via the `author_id` column. +- **Milestone**: via the `milestone_id` column. +- **Source branch**. +- **Target branch**. +- **Project**: via the `project_id` column. + +### Keep ClickHouse data up to date + +Replicating or syncing the `merge_requests` table is unfortunately not enough. +Separate queries to associated tables are required to insert one de-normalized +`merge_requests` row into the ClickHouse database. + +Change detection is non-trivial to implement. A few corners we could cut: + +- The feature is available for GitLab Premium and GitLab Ultimate customers. + We don't have to sync all the data, but instead sync only the `merge_requests` records + which are part of licensed groups. +- Data changes (often) happen via the `MergeRequest` services, where bumping the + `updated_at` timestamp column is mostly consistent. Some sort of incremental + synchronization process could be implemented. +- We only need to query the merged merge requests. After the merge, the record rarely changes. + +### Database table structure + +The database table structure uses de-normalization to make all required columns +available in one database table. This eliminates the need for `JOINs`. + +```sql +CREATE TABLE merge_requests +( + `id` UInt64, + `project_id` UInt64 DEFAULT 0 NOT NULL, + `author_id` UInt64 DEFAULT 0 NOT NULL, + `milestone_id` UInt64 DEFAULT 0 NOT NULL, + `label_ids` Array(UInt64) DEFAULT [] NOT NULL, + `assignee_ids` Array(UInt64) DEFAULT [] NOT NULL, + `source_branch` String DEFAULT '' NOT NULL, + `target_branch` String DEFAULT '' NOT NULL, + `merged_at` DateTime64(6, 'UTC') NOT NULL, + `created_at` DateTime64(6, 'UTC') DEFAULT now() NOT NULL, + `updated_at` DateTime64(6, 'UTC') DEFAULT now() NOT NULL +) +ENGINE = ReplacingMergeTree(updated_at) +ORDER BY (project_id, merged_at, id); +``` + +Similarly to the [activity data example](gitlab_activity_data.md), we use the +`ReplacingMergeTree` engine. Several columns of the merge request record may change, +so keeping the table up-to-date is important. + +The database table is ordered by the `project_id, merged_at, id` columns. This ordering +optimizes the table data for our use case: querying the `merged_at` column in a project. + +## Rewrite the count query + +First, let's generate some data for the table. + +```sql +INSERT INTO merge_requests (id, project_id, author_id, milestone_id, label_ids, merged_at, created_at) +SELECT id, project_id, author_id, milestone_id, label_ids, merged_at, created_at +FROM generateRandom('id UInt64, project_id UInt8, author_id UInt8, milestone_id UInt8, label_ids Array(UInt8), merged_at DateTime64(6, \'UTC\'), created_at DateTime64(6, \'UTC\')') +LIMIT 1000000; +``` + +NOTE: +Some integer data types were cast as `UInt8` so it is highly probable that they +have same values across different rows. + +The original count query only aggregated data for one month. With ClickHouse, we can +attempt aggregating the data for the whole year. + +PostgreSQL-based count query: + +```sql +SELECT COUNT(*) +FROM "merge_requests" +INNER JOIN "merge_request_metrics" ON "merge_request_metrics"."merge_request_id" = "merge_requests"."id" +WHERE (NOT EXISTS + (SELECT 1 + FROM "banned_users" + WHERE (merge_requests.author_id = banned_users.user_id))) + AND "merge_request_metrics"."target_project_id" = 278964 + AND "merge_request_metrics"."merged_at" >= '2022-12-01 00:00:00' + AND "merge_request_metrics"."merged_at" <= '2023-01-01 00:00:00' +``` + +ClickHouse query: + +```sql +SELECT + toYear(merged_at) AS year, + toMonth(merged_at) AS month, + COUNT(*) +FROM merge_requests +WHERE + project_id = 200 + AND merged_at BETWEEN '2022-01-01 00:00:00' + AND '2023-01-01 00:00:00' +GROUP BY year, month +``` + +The query processed a significantly lower number of rows compared to the generated data. +The `ORDER BY` clause (primary key) is helping the query execution: + +```plaintext +11 rows in set. Elapsed: 0.010 sec. +Processed 8.19 thousand rows, 131.07 KB (783.45 thousand rows/s., 12.54 MB/s.) +``` + +## Rewrite the Mean time to merge query + +The query calculates the mean time to merge as: +`duration(created_at, merged_at) / merge_request_count`. The calculation is done in +two separate steps: + +1. Request the monthly counts and the monthly duration values. +1. Sum the counts to get the yearly count. +1. Sum the durations to get the yearly duration. +1. Divide the durations by the count. + +In ClickHouse, we can calculate the mean time to merge with one query: + +```sql +SELECT + SUM( + dateDiff('second', merged_at, created_at) / 3600 / 24 + ) / COUNT(*) AS mean_time_to_merge -- mean_time_to_merge is in days +FROM merge_requests +WHERE + project_id = 200 + AND merged_at BETWEEN '2022-01-01 00:00:00' + AND '2023-01-01 00:00:00' +``` + +## Filtering + +The database queries above can be used as base queries. You can add more filters. +For example, filtering for a label and a milestone: + +```sql +SELECT + toYear(merged_at) AS year, + toMonth(merged_at) AS month, + COUNT(*) +FROM merge_requests +WHERE + project_id = 200 + AND milestone_id = 15 + AND has(label_ids, 118) + AND -- array includes 118 + merged_at BETWEEN '2022-01-01 00:00:00' + AND '2023-01-01 00:00:00' +GROUP BY year, month +``` + +Optimizing a particular filter is usually done with a database index. This particular +query reads 8000 rows: + +```plaintext +1 row in set. Elapsed: 0.016 sec. +Processed 8.19 thousand rows, 589.99 KB (505.38 thousand rows/s., 36.40 MB/s.) +``` + +Adding an index on `milestone_id`: + +```sql +ALTER TABLE merge_requests +ADD + INDEX milestone_id_index milestone_id TYPE minmax GRANULARITY 10; +ALTER TABLE + merge_requests MATERIALIZE INDEX milestone_id_index; +``` + +On the generated data, adding the index didn't improve the performance. + +### Banned users filter + +A recently added feature in GitLab filters out merge requests where the author is +banned by the admins. The banned users are tracked on the instance level in the +`banned_users` database table. + +#### Idea 1: Enumerate the banned user IDs + +This would require no structural changes to the ClickHouse database schema. +We could query the banned users in the project and filter the values out in query time. + +Get the banned users (in PostgreSQL): + +```sql +SELECT user_id FROM banned_users +``` + +In ClickHouse + +```sql +SELECT + toYear(merged_at) AS year, + toMonth(merged_at) AS month, + COUNT(*) +FROM merge_requests +WHERE + author_id NOT IN (1, 2, 3, 4) AND -- banned users + project_id = 200 + AND milestone_id = 15 + AND has(label_ids, 118) AND -- array includes 118 + merged_at BETWEEN '2022-01-01 00:00:00' + AND '2023-01-01 00:00:00' +GROUP BY year, month +``` + +The problem with this approach is that the number of banned users could increase significantly which would make the query bigger and slower. + +#### Idea 2: replicate the `banned_users` table + +Assuming that the `banned_users table` doesn't grow to millions of rows, we could +attempt to periodically sync the whole table to ClickHouse. With this approach, +a mostly consistent `banned_users` table could be used in the ClickHouse database query: + +```sql +SELECT + toYear(merged_at) AS year, + toMonth(merged_at) AS month, + COUNT(*) +FROM merge_requests +WHERE + author_id NOT IN (SELECT user_id FROM banned_users) AND + project_id = 200 AND + milestone_id = 15 AND + has(label_ids, 118) AND -- array includes 118 + merged_at BETWEEN '2022-01-01 00:00:00' AND '2023-01-01 00:00:00' +GROUP BY year, month +``` + +Alternatively, the `banned_users` table could be stored as a +[dictionary](https://clickhouse.com/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts) +to further improve the query performance. + +#### Idea 3: Alter the feature + +For analytical calculations, it might be acceptable to drop this particular filter. +This approach assumes that including the merge requests of banned users doesn't skew the statistics significantly. diff --git a/doc/development/database/clickhouse/tiered_storage.md b/doc/development/database/clickhouse/tiered_storage.md new file mode 100644 index 00000000000..d9026f47e28 --- /dev/null +++ b/doc/development/database/clickhouse/tiered_storage.md @@ -0,0 +1,138 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Tiered Storages in ClickHouse + +NOTE: +The MergeTree table engine in ClickHouse supports tiered storage. +See the documentation for [Using Multiple Block Devices for Data Storage](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#table_engine-mergetree-multiple-volumes) +for details on setup and further explanation. + +Quoting from the [MergeTree documentation](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#table_engine-mergetree-multiple-volumes): + + + +> MergeTree family table engines can store data on multiple block devices. For example, +> it can be useful when the data of a certain table are implicitly split into "hot" and "cold". +> The most recent data is regularly requested but requires only a small amount of space. +> On the contrary, the fat-tailed historical data is requested rarely. + + + +When used with remote storage backends such as +[Amazon S3](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#table_engine-mergetree-s3), +this makes a very efficient storage scheme. It allows for storage policies, which +allows data to be on local disks for a period of time and then move it to object storage. + +An [example configuration](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#table_engine-mergetree-multiple-volumes_configure) can look like this: + +```xml + + + + /mnt/fast_ssd/clickhouse/ + + + false + s3 + https://storage.googleapis.com/${BUCKET_NAME}/${ROOT_FOLDER}/ + ${SERVICE_ACCOUNT_HMAC_KEY} + ${SERVICE_ACCOUNT_HMAC_SECRET} + /var/lib/clickhouse/disks/gcs/ + + ... + + ... + + + + + + fast_ssd + + + gcs + + + 0.2 + + + .... + +``` + +In this storage policy, two volumes are defined `hot` and `cold`. After the `hot` volume is filled with occupancy of `disk_size * move_factor`, the data is being moved to Google Cloud Storage (GCS). + +If this storage policy is not the default, create tables by attaching the storage policies. For example: + +```sql +CREATE TABLE key_value_table ( + event_date Date, + key String, + value String, +) ENGINE = MergeTree +ORDER BY (key) +PARTITION BY toYYYYMM(event_date) +SETTINGS storage_policy = 'move_from_local_disks_to_gcs' +``` + +NOTE: +In this storage policy, the move happens implicitly. It is also possible to keep +_hot_ data on local disks for a fixed period of time and then move them as _cold_. + +This approach is possible with +[Table TTLs](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree/#mergetree-table-ttl), +which are also available with MergeTree table engine. + +The ClickHouse documentation shows this feature in detail, in the example of +[implementing a hot - warm - cold architecture](https://clickhouse.com/docs/en/guides/developer/ttl/#implementing-a-hotwarmcold-architecture). + +You can take a similar approach for the example shown above. First, adjust the storage policy: + +```xml + + ... + + + + + fast_ssd + + + gcs + + + + .... + +``` + +Then create the table as: + +```sql +CREATE TABLE another_key_value_table ( + event_date Date, + key String, + value String, +) ENGINE = MergeTree +ORDER BY (key) +PARTITION BY toYYYYMM(event_date) +TTL + event_date TO VOLUME 'hot', + event_date + INTERVAL 1 YEAR TO VOLUME 'cold' +SETTINGS storage_policy = 'local_disk_and_gcs'; +``` + +This creates the table so that data older than 1 year (evaluated against the +`event_date` column) is moved to GCS. Such a storage policy can be helpful for append-only +tables (like audit events) where only the most recent data is accessed frequently. +You can drop the data altogether, which can be a regulatory requirement. + +We don't mention modifying TTLs in this guide, but that is possible as well. +See ClickHouse documentation for +[modifying TTL](https://clickhouse.com/docs/en/sql-reference/statements/alter/ttl/#modify-ttl) +for details. diff --git a/doc/development/database/creating_enums.md b/doc/development/database/creating_enums.md index e2ae36f7481..908656dae84 100644 --- a/doc/development/database/creating_enums.md +++ b/doc/development/database/creating_enums.md @@ -63,7 +63,7 @@ module EE module Pipeline override :failure_reasons def failure_reasons - super.merge(activity_limit_exceeded: 2) + super.merge(job_activity_limit_exceeded: 2) end end end @@ -73,9 +73,9 @@ end This works as-is, however, it has a couple of downside that: - Someone could define a key/value pair in EE that is **conflicted** with a value defined in FOSS. - For example, define `activity_limit_exceeded: 1` in `EE::Enums::Pipeline`. + For example, define `job_activity_limit_exceeded: 1` in `EE::Enums::Pipeline`. - When it happens, the feature works totally different. - For example, we cannot figure out `failure_reason` is either `config_error` or `activity_limit_exceeded`. + For example, we cannot figure out `failure_reason` is either `config_error` or `job_activity_limit_exceeded`. - When it happens, we have to ship a database migration to fix the data integrity, which might be impossible if you cannot recover the original value. @@ -88,7 +88,7 @@ module EE module Pipeline override :failure_reasons def failure_reasons - super.merge(activity_limit_exceeded: 1_000, size_limit_exceeded: 1_001) + super.merge(job_activity_limit_exceeded: 1_000, size_limit_exceeded: 1_001) end end end @@ -98,7 +98,7 @@ end This looks working as a workaround, however, this approach has some downsides that: - Features could move from EE to FOSS or vice versa. Therefore, the offset might be mixed between FOSS and EE in the future. - For example, when you move `activity_limit_exceeded` to FOSS, you see `{ unknown_failure: 0, config_error: 1, activity_limit_exceeded: 1_000 }`. + For example, when you move `job_activity_limit_exceeded` to FOSS, you see `{ unknown_failure: 0, config_error: 1, job_activity_limit_exceeded: 1_000 }`. - The integer column for the `enum` is likely created [as `SMALLINT`](#creating-enums). Therefore, you need to be careful of that the offset doesn't exceed the maximum value of 2 bytes integer. @@ -110,7 +110,7 @@ class Pipeline < ApplicationRecord enum failure_reason: { unknown_failure: 0, config_error: 1, - activity_limit_exceeded: 2 + job_activity_limit_exceeded: 2 } end ``` diff --git a/doc/development/database/database_dictionary.md b/doc/development/database/database_dictionary.md index b7e6fa4b5b3..84b76ddc34c 100644 --- a/doc/development/database/database_dictionary.md +++ b/doc/development/database/database_dictionary.md @@ -12,7 +12,8 @@ locate the feature categories responsible for specific database tables. ## Location Database dictionary metadata files are stored in the `gitlab` project under `db/docs/` for the `main` and `ci` databases. -For the `geo` database, the dictionary files are stored under `ee/db/docs/`. +For the `embedding` database, the dictionary files are stored under `ee/db/embedding/docs/`. +For the `geo` database, the dictionary files are stored under `ee/db/geo/docs/`. ## Example dictionary file @@ -26,6 +27,7 @@ feature_categories: description: Represents a Terraform state backend introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26619 milestone: '13.0' +gitlab_schema: gitlab_main ``` ## Adding tables @@ -50,7 +52,8 @@ When adding a table, you should: - `gitlab_main` table: `db/docs/` - `gitlab_ci` table: `db/docs/` - `gitlab_shared` table: `db/docs/` - - `gitlab_geo` table: `ee/db/docs/` + - `gitlab_embedding` table: `ee/db/embedding/docs/` + - `gitlab_geo` table: `ee/db/geo/docs/` 1. Name the file `.yml`, and include as much information as you know about the table. 1. Include this file in the commit with the migration that creates the table. @@ -78,7 +81,8 @@ When dropping a table, you should: - `gitlab_main` table: `db/docs/deleted_tables/` - `gitlab_ci` table: `db/docs/deleted_tables/` - `gitlab_shared` table: `db/docs/deleted_tables/` - - `gitlab_geo` table: `ee/db/docs/deleted_tables/` + - `gitlab_embedding` table: `ee/db/embedding/docs/deleted_tables/` + - `gitlab_geo` table: `ee/db/geo/docs/deleted_tables/` 1. Add the fields `removed_by_url` and `removed_in_milestone` to the dictionary file. 1. Include this change in the commit with the migration that drops the table. @@ -104,7 +108,8 @@ When adding a new view, you should: - `gitlab_main` view: `db/docs/views/` - `gitlab_ci` view: `db/docs/views/` - `gitlab_shared` view: `db/docs/views/` - - `gitlab_geo` view: `ee/db/docs/views/` + - `gitlab_embedding` view: `ee/db/embedding/docs/views/` + - `gitlab_geo` view: `ee/db/geo/docs/views/` 1. Name the file `.yml`, and include as much information as you know about the view. 1. Include this file in the commit with the migration that creates the view. @@ -132,6 +137,7 @@ When dropping a view, you should: - `gitlab_main` view: `db/docs/deleted_views/` - `gitlab_ci` view: `db/docs/deleted_views/` - `gitlab_shared` view: `db/docs/deleted_views/` - - `gitlab_geo` view: `ee/db/docs/deleted_views/` + - `gitlab_embedding` view: `ee/db/embedding/docs/deleted_views/` + - `gitlab_geo` view: `ee/db/geo/docs/deleted_views/` 1. Add the fields `removed_by_url` and `removed_in_milestone` to the dictionary file. 1. Include this change in the commit with the migration that drops the view. diff --git a/doc/development/database/database_lab.md b/doc/development/database/database_lab.md index 162fc597cc4..357133d8bca 100644 --- a/doc/development/database/database_lab.md +++ b/doc/development/database/database_lab.md @@ -12,6 +12,17 @@ on replicated production data. Unlike a typical read-only production replica, in also create, update, and delete rows. You can also test the performance of schema changes, like additional indexes or columns, in an isolated copy of production data. +## Database Lab quick start + +1. [Visit the console](https://console.postgres.ai/). +1. Select **Sign in with Google**. (Not GitLab, as you need Google SSO to connect with our project.) +1. After you sign in, select the GitLab organization and then visit "Ask Joe" in the sidebar. +1. Select the database you're testing against: + - Most queries for the GitLab project run against `gitlab-production-tunnel-pg12`. + - If the query is for a CI table, select `gitlab-production-ci`. + - If the query is for the container registry, select `gitlab-production-registry`. +1. Type `explain ` in the chat box to get a plan. + ## Access Database Lab Engine Access to the DLE is helpful for: @@ -21,27 +32,25 @@ Access to the DLE is helpful for: To access the DLE's services, you can: -- Perform query testing in the `#database_lab` Slack channel, or in the Postgres.ai web console. +- Perform query testing in the Postgres.ai web console. Employees access both services with their GitLab Google account. Query testing provides `EXPLAIN` (analyze, buffers) plans for queries executed there. - Migration testing by triggering a job as a part of a merge request. - Direct `psql` access to DLE instead of a production replica. Available to authorized users only. - To request `psql` access, file an [access request](https://about.gitlab.com/handbook/business-technology/team-member-enablement/onboarding-access-requests/access-requests/#individual-or-bulk-access-request). + To request `psql` access, file an [access request](https://about.gitlab.com/handbook/business-technology/end-user-services/onboarding-access-requests/access-requests/#individual-or-bulk-access-request). For more assistance, use the `#database` Slack channel. NOTE: If you need only temporary access to a production replica, instead of a Database Lab clone, follow the runbook procedure for connecting to the -[database console with Teleport](https://gitlab.com/gitlab-com/runbooks/-/blob/master/docs/Teleport/Connect_to_Database_Console_via_Teleport.md). -This procedure is similar to [Rails console access with Teleport](https://gitlab.com/gitlab-com/runbooks/-/blob/master/docs/Teleport/Connect_to_Rails_Console_via_Teleport.md#how-to-use-teleport-to-connect-to-rails-console). +[database console with Teleport](https://gitlab.com/gitlab-com/runbooks/-/blob/master/docs/teleport/Connect_to_Database_Console_via_Teleport.md). +This procedure is similar to [Rails console access with Teleport](https://gitlab.com/gitlab-com/runbooks/-/blob/master/docs/teleport/Connect_to_Rails_Console_via_Teleport.md#how-to-use-teleport-to-connect-to-rails-console). ### Query testing You can access Database Lab's query analysis features either: -- In the `#database_lab` Slack channel. Shows everyone's commands and results, but - your own commands are still isolated in their own clone. - In [the Postgres.ai web console](https://console.postgres.ai/GitLab/joe-instances). Shows only the commands you run. @@ -86,7 +95,7 @@ Caveats: [`ci_builds`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/db/docs/ci_builds.yml#L14), use `gitlab-production-ci`. - Database Lab typically has a small delay of a few hours. If more up-to-date information - is required, you can instead request access to a replica [via Teleport](https://gitlab.com/gitlab-com/runbooks/-/blob/master/docs/Teleport/Connect_to_Database_Console_via_Teleport.md) + is required, you can instead request access to a replica [via Teleport](https://gitlab.com/gitlab-com/runbooks/-/blob/master/docs/teleport/Connect_to_Database_Console_via_Teleport.md) For example: `\d index_design_management_designs_on_project_id` produces: @@ -121,6 +130,79 @@ For information on testing migrations, review our ### Access the console with `psql` +NOTE: +You must have `AllFeaturesUser` [`psql` access](#access-database-lab-engine) to access the console with `psql`. + +#### Simplified access through `pgai` Ruby gem + +[@mbobin](https://gitlab.com/mbobin) created the [`pgai` Ruby Gem](https://gitlab.com/mbobin/pgai/#pgai) that +greatly simplifies access to a database clone, with support for: + +- Access to all database clones listed in the [Postgres.ai instances page](https://console.postgres.ai/gitlab/instances); +- Multiple `psql` sessions on the same clone. + +If you have `AllFeaturesUser` [`psql` access](#access-database-lab-engine), you can follow the steps below to configure +the `pgai` Gem: + +1. To get started, you need to gather some values from the [Postgres.ai instances page](https://console.postgres.ai/gitlab/instances): + + 1. Navigate to the instance that you want to configure and the on right side of the screen. + 1. Under **Connection**, select **Connect**. The menu might be collapsed. + + A pop-up with everything that's needed for configuration appears, using this format: + + ```shell + dblab init --url http://127.0.0.1:1234 --token TOKEN --environment-id + ``` + + ```shell + ssh -NTML 1234:localhost: @ -i ~/.ssh/id_rsa + ``` + +1. Add the following snippet to your SSH configuration file at `~/.ssh/config`, replacing the variable values: + + ```plaintext + Host pgai-proxy + HostName + User + IdentityFile ~/.ssh/id_ed25519 + ``` + +1. Run the following command so you can accept the server key fingerprint: + + ```shell + ssh pgai-proxy + ``` + +1. Run the following commands: + + ```shell + gem install pgai + + # Grab an access token: https://console.postgres.ai/gitlab/tokens + # GITLAB_USER is your GitLab handle + pgai config --dbname=gitlabhq_dblab --prefix=$GITLAB_USER --proxy=pgai-proxy + + # Grab the respective port values from https://console.postgres.ai/gitlab/instances + # for the instances you'll be using (in this case, for the `main` database instance) + pgai env add --alias main --id --port + ``` + +1. Once this one-time configuration is done, you can use `pgai connect` to connect to a particular database. For + instance, to connect to the `main` database: + + ```shell + pgai connect main + ``` + +1. Once done with the clone, you can destroy it: + + ```shell + pgai destroy main + ``` + +#### Manual access through the Postgres.ai instances page + Team members with [`psql` access](#access-database-lab-engine), can gain direct access to a clone via `psql`. Access to `psql` enables you to see data, not just metadata. diff --git a/doc/development/database/database_migration_pipeline.md b/doc/development/database/database_migration_pipeline.md index 06e16b4c7f1..a9d525e2a41 100644 --- a/doc/development/database/database_migration_pipeline.md +++ b/doc/development/database/database_migration_pipeline.md @@ -9,13 +9,13 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/database-team/team-tasks/-/issues/171) in GitLab 14.2. With the [automated migration testing pipeline](https://gitlab.com/gitlab-org/database-team/gitlab-com-database-testing) -we can automatically test migrations in a production-like environment (similar to `#database-lab`). +we can automatically test migrations in a production-like environment (using [Database Lab](database_lab.md)). It is based on an [architecture blueprint](../../architecture/blueprints/database_testing/index.md). Migration testing is enabled in the [GitLab project](https://gitlab.com/gitlab-org/gitlab) for changes that add a new database migration. Trigger this job manually by running the `db:gitlabcom-database-testing` job within in `test` stage. To avoid wasting resources, -only run this job when your MR is ready for review. +only run this job when your MR is ready for review. Additionally, ensure that the MR has the "database" label for the pipeline to appear in the test stage. The job starts a pipeline on the [ops GitLab instance](https://ops.gitlab.net/). For security reasons, access to the pipeline is restricted to database maintainers. @@ -73,3 +73,41 @@ Some additional information is included at the bottom of the comment: migration (ending in `.log`) are available there, and only accessible by database maintainers or with an access request. Details of the specific batched background migration batches sampled are also available. + +## Test changes to the database testing pipeline + +To test a change to the database testing pipeline itself, you need: + +1. A merge request against GitLab Org. +1. The change to be tested must be present on a branch on GitLab Ops. + +Use this self-documented script to test a merge request on GitLab Org against an arbitrary branch on GitLab Ops: + +```shell +#! /usr/bin/env bash + +# The following must be set on a per-invocation basis: +TESTING_TRIGGER_TOKEN='[REDACTED]' # Testing trigger token created in the CI section of the project +CI_COMMIT_REF_NAME='55-post-notice-on-failure' # The branch on ops that you want to run against +CI_MERGE_REQUEST_IID='117901' # Merge request ID of the MR on gitlab.com that you want to test +SHA="fed6dd8a58d75a0e053a4972765b4fc08c5814a3" # The commit SHA of the HEAD of the branch you want to test on gitlab-org/gitlab + +# The following should not be changed between invocations: +CI_JOB_URL='https://gitlab.com/gitlab-org/database-team/gitlab-com-database-testing/-/jobs/1590162939' +# It doesn't appear that CI_JOB_URL has to be set to anything in particular for the pipeline to run +# successfully, but this would normally be the URL to the upstream job that invokes the DB testing pipeline. +CI_MERGE_REQUEST_PROJECT_ID='278964' # gitlab-org/gitlab numeric ID. Shouldn't change. +CI_PROJECT_ID="gitlab-org/gitlab" # The slug identifying gitlab-org/gitlab. + +curl --verbose --request POST \ + --form "token=$TESTING_TRIGGER_TOKEN" \ + --form "ref=$CI_COMMIT_REF_NAME" \ + --form "variables[TOP_UPSTREAM_MERGE_REQUEST_IID]=$CI_MERGE_REQUEST_IID" \ + --form "variables[TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID]=$CI_MERGE_REQUEST_PROJECT_ID" \ + --form "variables[TOP_UPSTREAM_SOURCE_JOB]=$CI_JOB_URL" \ + --form "variables[TOP_UPSTREAM_SOURCE_PROJECT]=$CI_PROJECT_ID" \ + --form "variables[VALIDATION_PIPELINE]=true" \ + --form "variables[GITLAB_COMMIT_SHA]=$SHA" \ + --form "variables[TRIGGER_SOURCE]=$CI_JOB_URL" \ + "https://ops.gitlab.net/api/v4/projects/429/trigger/pipeline" +``` diff --git a/doc/development/database/database_reviewer_guidelines.md b/doc/development/database/database_reviewer_guidelines.md index aa92134018d..933bbe9c060 100644 --- a/doc/development/database/database_reviewer_guidelines.md +++ b/doc/development/database/database_reviewer_guidelines.md @@ -53,9 +53,8 @@ that require a more in-depth discussion between the database reviewers and maint - [Database Office Hours Agenda](https://docs.google.com/document/d/1wgfmVL30F8SdMg-9yY6Y8djPSxWNvKmhR5XmsvYX1EI/edit). - [YouTube playlist with past recordings](https://www.youtube.com/playlist?list=PL05JrBw4t0Kp-kqXeiF7fF7cFYaKtdqXM). -You should also join the [#database-lab](understanding_explain_plans.md#database-lab-engine) -Slack channel and get familiar with how to use Joe, the Slackbot that provides developers -with their own clone of the production database. +Get familiar with using [Database Lab from postgres.ai](database_lab.md), a bot that +provides developers with their own clone of the production database. Understanding and efficiently using `EXPLAIN` plans is at the core of the database review process. The following guides provide a quick introduction and links to follow on more advanced topics: diff --git a/doc/development/database/efficient_in_operator_queries.md b/doc/development/database/efficient_in_operator_queries.md index 78b310ae708..a770dfe6531 100644 --- a/doc/development/database/efficient_in_operator_queries.md +++ b/doc/development/database/efficient_in_operator_queries.md @@ -433,7 +433,7 @@ construct the following table: For the `issue_types` query we can construct a value list without querying a table: ```ruby -value_list = Arel::Nodes::ValuesList.new([[Issue.issue_types[:incident]],[Issue.issue_types[:test_case]]]) +value_list = Arel::Nodes::ValuesList.new([[WorkItems::Type.base_types[:incident]],[WorkItems::Type.base_types[:test_case]]]) issue_type_values = Arel::Nodes::Grouping.new(value_list).as('issue_type_values (value)').to_sql array_scope = Group diff --git a/doc/development/database/index.md b/doc/development/database/index.md index 8f22eaac496..f532e054849 100644 --- a/doc/development/database/index.md +++ b/doc/development/database/index.md @@ -4,7 +4,7 @@ group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Database guides +# Database development guidelines ## Database Reviews @@ -64,6 +64,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w - [Hash indexes](hash_indexes.md) - [Insert into tables in batches](insert_into_tables_in_batches.md) - [Iterating tables in batches](iterating_tables_in_batches.md) +- [Load balancing](load_balancing.md) - [`NOT NULL` constraints](not_null_constraints.md) - [Ordering table columns](ordering_table_columns.md) - [Pagination guidelines](pagination_guidelines.md) @@ -109,6 +110,8 @@ including the major methods: - [Introduction](clickhouse/index.md) - [Optimizing query execution](clickhouse/optimization.md) - [Rebuild GitLab features using ClickHouse 1: Activity data](clickhouse/gitlab_activity_data.md) +- [Rebuild GitLab features using ClickHouse 2: Merge Request analytics](clickhouse/merge_request_analytics.md) +- [Tiered Storage in ClickHouse](clickhouse/tiered_storage.md) ## Miscellaneous diff --git a/doc/development/database/iterating_tables_in_batches.md b/doc/development/database/iterating_tables_in_batches.md index 6357bed8b00..a927242e8d8 100644 --- a/doc/development/database/iterating_tables_in_batches.md +++ b/doc/development/database/iterating_tables_in_batches.md @@ -44,9 +44,13 @@ all of the arguments that `in_batches` supports. You should always use ## Iterating over non-unique columns -One should proceed with extra caution. When you iterate over an attribute that is not unique, -even with the applied max batch size, there is no guarantee that the resulting batches do not -surpass it. The following snippet demonstrates this situation when one attempt to select +You should not use the `each_batch` method with a non-unique column (in the context of the relation) as it +[may result in an infinite loop](https://gitlab.com/gitlab-org/gitlab/-/issues/285097). +Additionally, the inconsistent batch sizes cause performance issues when you +iterate over non-unique columns. Even when you apply a max batch size +when iterating over an attribute, there's no guarantee that the resulting +batches don't surpass it. The following snippet demonstrates this situation +when you attempt to select `Ci::Build` entries for users with `id` between `1` and `10,000`, the database returns `1 215 178` matching rows. @@ -465,6 +469,58 @@ Issue.each_batch(of: 1000) do |relation| end ``` +### Counting records + +For tables with a large amount of data, counting records through queries can result +in timeouts. The `EachBatch` module provides an alternative way to iteratively count +records. The downside of using `each_batch` is the extra count query which is executed +on the yielded relation object. + +The `each_batch_count` method is a more efficient approach that eliminates the need +for the extra count query. By invoking this method, the iteration process can be +paused and resumed as needed. This feature is particularly useful in situations +where error budget violations are triggered after five minutes, such as when performing +counting operations within Sidekiq workers. + +To illustrate, counting records using `EachBatch` involves invoking an additional +count query as follows: + +```ruby +count = 0 + +Issue.each_batch do |relation| + count += relation.count +end + +puts count +``` + +On the other hand, the `each_batch_count` method enables the counting process to be +performed more efficiently (counting is part of the iteration query) without invoking +an extra count query: + +```ruby +count, _last_value = Issue.each_batch_count # last value can be ignored here +``` + +Furthermore, the `each_batch_count` method allows the counting process to be paused +and resumed at any point. This capability is demonstrated in the following code snippet: + +```ruby +stop_at = Time.current + 3.minutes + +count, last_value = Issue.each_batch_count do + Time.current > stop_at # condition for stopping the counting +end + +# Continue the counting later +stop_at = Time.current + 3.minutes + +count, last_value = Issue.each_batch_count(last_count: count, last_value: last_value) do + Time.current > stop_at +end +``` + ### `EachBatch` vs `BatchCount` When adding new counters for Service Ping, the preferred way to count records is using the diff --git a/doc/development/database/load_balancing.md b/doc/development/database/load_balancing.md new file mode 100644 index 00000000000..f623ad1eab0 --- /dev/null +++ b/doc/development/database/load_balancing.md @@ -0,0 +1,59 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Database load balancing + +With database load balancing, read-only queries can be distributed across multiple +PostgreSQL nodes to increase performance. + +This documentation provides a technical overview on how database load balancing +is implemented in GitLab Rails and Sidekiq. + +## Nomenclature + +1. **Host**: Each database host. It could be a primary or a replica. +1. **Primary**: Primary PostgreSQL host that is used for write-only and read-and-write operations. +1. **Replica**: Secondary PostgreSQL hosts that are used for read-only operations. +1. **Workload**: a Rails request or a Sidekiq job that requires database connections. + +## Components + +F few Ruby classes are involved in the load balancing process. All of them are +in the namespace `Gitlab::Database::LoadBalancing`: + +1. `Host` +1. `LoadBalancer` +1. `ConnectionProxy` +1. `Session` + +Each workload begins with a new instance of `Gitlab::Database::LoadBalancing::Session`. +The `Session` keeps track of the database operations that have been performed. It then +determines if the workload requires a connection to either the primary host or a replica host. + +When the workload requires a database connection through `ActiveRecord`, +`ConnectionProxy` first redirects the connection request to `LoadBalancer`. +`ConnectionProxy` requests either a `read` or `read_write` connection from the `LoadBalancer` +depending on a few criteria: + +1. Whether the query is a read-only or it requires write. +1. Whether the `Session` has recorded a write operation previously. +1. Whether any special blocks have been used to prefer primary or replica, such as: + - `use_primary` + - `ignore_writes` + - `use_replicas_for_read_queries` + - `fallback_to_replicas_for_ambiguous_queries` + +`LoadBalancer` then yields the requested connection from the respective database connection pool. +It yields either: + +- A `read_write` connection from the primary's connection pool. +- A `read` connection from the replicas' connection pools. + +When responding to a request for a `read` connection, `LoadBalancer` would +first attempt to load balance the connection across the replica hosts. +It looks for the next `online` replica host and yields a connection from the host's connection pool. +A replica host is considered `online` if it is up-to-date with the primary, based on +either the replication lag size or time. The thresholds for these requirements are configurable. diff --git a/doc/development/database/loose_foreign_keys.md b/doc/development/database/loose_foreign_keys.md index daa022a3de2..91a22d8c26b 100644 --- a/doc/development/database/loose_foreign_keys.md +++ b/doc/development/database/loose_foreign_keys.md @@ -64,7 +64,7 @@ The tool ensures that all aspects of swapping a foreign key are covered. This in - Creating a migration to remove a foreign key. - Updating `db/structure.sql` with the new migration. -- Updating `lib/gitlab/database/gitlab_loose_foreign_keys.yml` to add the new loose foreign key. +- Updating `config/gitlab_loose_foreign_keys.yml` to add the new loose foreign key. - Creating or updating a model's specs to ensure that the loose foreign key is properly supported. The tool is located at `scripts/decomposition/generate-loose-foreign-key`: diff --git a/doc/development/database/multiple_databases.md b/doc/development/database/multiple_databases.md index d22e3209096..6adfdc90cf2 100644 --- a/doc/development/database/multiple_databases.md +++ b/doc/development/database/multiple_databases.md @@ -1,6 +1,6 @@ --- stage: Data Stores -group: Pods +group: Tenant Scale info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -545,7 +545,7 @@ end ``` Don't hesitate to reach out to the -[pods group](https://about.gitlab.com/handbook/engineering/development/enablement/data_stores/pods/) +[Pods group](https://about.gitlab.com/handbook/engineering/development/enablement/data_stores/tenant-scale/) for advice. ##### Avoid `dependent: :nullify` and `dependent: :destroy` across databases @@ -580,6 +580,24 @@ or records that point to nowhere, which might lead to bugs. As such we created ["loose foreign keys"](loose_foreign_keys.md) which is an asynchronous process of cleaning up orphaned records. +## Testing for multiple databases + +In our testing CI pipelines, we test GitLab by default with multiple databases set up, using +both `main` and `ci` databases. But in merge requests, for example when we modify some database-related code or +add the label `~"pipeline:run-single-db"` to the MR, we additionally run our tests in +[two other database modes](../pipelines/index.md#single-database-testing): +`single-db` and `single-db-ci-connection`. + +To handle situations where our tests need to run in specific database modes, we have some RSpec helpers +to limit the modes where tests can run, and skip them on any other modes. + +| Helper name | Test runs | +|---------------------------------------------| --- | +| `skip_if_shared_database(:ci)` | On **multiple databases** | +| `skip_if_database_exists(:ci)` | On **single-db** and **single-db-ci-connection** | +| `skip_if_multiple_databases_are_setup(:ci)` | Only on **single-db** | +| `skip_if_multiple_databases_not_setup(:ci)` | On **single-db-ci-connection** and **multiple databases** | + ## Locking writes on the tables that don't belong to the database schemas When the CI database is promoted and the two databases are fully split, diff --git a/doc/development/database/query_performance.md b/doc/development/database/query_performance.md index 73a6a40f801..10ab726940a 100644 --- a/doc/development/database/query_performance.md +++ b/doc/development/database/query_performance.md @@ -44,7 +44,7 @@ automatically includes these options. If you are making a warm cache query, you see only the `shared hits`. -For example in #database-lab: +For example, using [Database Lab](database_lab.md): ```plaintext Shared buffers: @@ -60,7 +60,7 @@ Buffers: shared hit=7323 If the cache is cold, you also see `reads`. -In #database-lab: +Using [Database Lab](database_lab.md): ```plaintext Shared buffers: diff --git a/doc/development/database/required_stops.md b/doc/development/database/required_stops.md index 46fabb5c1b4..e4f66f4424f 100644 --- a/doc/development/database/required_stops.md +++ b/doc/development/database/required_stops.md @@ -11,6 +11,62 @@ disruptive effect on customers. Before adding a required stop, consider if any alternative approaches exist to avoid a required stop. Sometimes a required stop is unavoidable. In those cases, follow the instructions below. +## Common scenarios that require stops + +### Long running migrations being finalized + +If a migration takes a long time, it could cause a large number of customers to encounter timeouts +during upgrades. The increased support volume may cause us to introduce a required stop. While any +background migration may cause these issues with particularly large customers, we typically only +introduce stops when the impact is widespread. + +- **Cause:** When an upgrade takes more than an hour, omnibus times out. +- **Mitigation:** Schedule finalization for the first minor version after the next required stop. + +### Improperly finalized background migrations + +You may need to introduce a required stop for mitigation when: + +- A background migration is not finalized, and +- A migration is written that depends on that background migration. + +- **Cause:** The dependent migration may fail if the background migration is incomplete. +- **Mitigation:** Ensure that all background migrations are finalized before authoring dependent migrations. + +### Remove a migration + +If a migration is removed, you may need to introduce a required stop to ensure customers +don't miss the required change. + +- **Cause:** Dependent migrations may fail, or the application may not function, because a required + migration was removed. +- **Mitigation:** Ensure migrations are only removed after they've been a part of a planned + required stop. + +### A migration timestamp is very old + +If a migration timestamp is very old (> 3 weeks, or after a before the last stop), +these scenarios may cause issues: + +- If the migration depends on another migration with a newer timestamp but introduced in a + previous release _after_ a required stop, then the new migration may run sequentially sooner + than the prerequisite migration, and thus fail. +- If the migration timestamp ID is before the last, it may be inadvertently squashed when the + team squashes other migrations from the required stop. + +- **Cause:** The migration may fail if it depends on a migration with a later timestamp introduced + in an earlier version. Or, the migration may be inadvertently squashed after a required stop. +- **Mitigation:** Aim for migration timestamps to fall inside the release dates and be sure that + they are not dated prior to the last required stop. + +### Bugs in migration related tooling + +In a few circumstances, bugs in migration related tooling has required us to introduce stops. While we aim +to prevent these in testing, sometimes they happen. + +- **Cause:** There have been a few different causes where we recognized these too late. +- **Mitigation:** Typically we try to backport fixes for migrations, but in some cases this is not possible. + ## Before the required stop is released Before releasing a known required stop, complete these steps. If the required stop diff --git a/doc/development/database/strings_and_the_text_data_type.md b/doc/development/database/strings_and_the_text_data_type.md index 47e89c1ce0f..5cd325bfa56 100644 --- a/doc/development/database/strings_and_the_text_data_type.md +++ b/doc/development/database/strings_and_the_text_data_type.md @@ -68,9 +68,17 @@ is held for a brief amount of time, the time `add_column` needs to complete its depending on how frequently the table is accessed. For example, acquiring an exclusive lock for a very frequently accessed table may take minutes in GitLab.com and requires the use of `with_lock_retries`. -For these reasons, it is advised to add the text limit on a separate migration than the `add_column` one. +When adding a text limit, transactions must be disabled with `disable_ddl_transaction!`. This means adding the column is not rolled back +in case the migration fails afterwards. An attempt to re-run the migration will raise an error because of the already existing column. -For example, consider a migration that adds a new text column `extended_title` to table `sprints`, +For these reasons, adding a text column to an existing table can be done by either: + +- [Add the column and limit in separate migrations.](#add-the-column-and-limit-in-separate-migrations) +- [Add the column and limit in one migration with checking if the column already exists.](#add-the-column-and-limit-in-one-migration-with-checking-if-the-column-already-exists) + +### Add the column and limit in separate migrations + +Consider a migration that adds a new text column `extended_title` to table `sprints`, `db/migrate/20200501000001_add_extended_title_to_sprints.rb`: ```ruby @@ -103,6 +111,33 @@ class AddTextLimitToSprintsExtendedTitle < Gitlab::Database::Migration[2.1] end ``` +### Add the column and limit in one migration with checking if the column already exists + +Consider a migration that adds a new text column `extended_title` to table `sprints`, +`db/migrate/20200501000001_add_extended_title_to_sprints.rb`: + +```ruby +class AddExtendedTitleToSprints < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + def up + with_lock_retries do + add_column :sprints, :extended_title, :text, if_not_exists: true + end + + add_text_limit :sprints, :extended_title, 512 + end + + def down + remove_text_limit :sprints, :extended_title + + with_lock_retries do + remove_column :sprints, :extended_title, if_exists: true + end + end +end +``` + ## Add a text limit constraint to an existing column Adding text limits to existing database columns requires multiple steps split into at least two different releases: diff --git a/doc/development/database/table_partitioning.md b/doc/development/database/table_partitioning.md index 0d5e3c233f6..88b2ccbc6a2 100644 --- a/doc/development/database/table_partitioning.md +++ b/doc/development/database/table_partitioning.md @@ -6,6 +6,13 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Database table partitioning +WARNING: +If you have questions not answered below, check for and add them +to [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/398650). +Tag `@gitlab-org/database-team/triage` and we'll get back to you with an +answer as soon as possible. If you get an answer in Slack, document +it on the issue as well so we can update this document in the future. + Table partitioning is a powerful database feature that allows a table's data to be split into smaller physical tables that act as a single large table. If the application is designed to work with partitioning in mind, @@ -32,31 +39,38 @@ several releases. Due to the limitations of partitioning and the related migrations, you should understand how partitioning fits your use case before attempting to leverage this feature. -## Determining when to use partitioning +## Determine when to use partitioning While partitioning can be very useful when properly applied, it's imperative to identify if the data and workload of a table naturally fit a -partitioning scheme. There are a few details you have to understand -to decide if partitioning is a good fit for your particular -problem. - -First, a table is partitioned on a partition key, which is a column or -set of columns which determine how the data is split across the -partitions. The partition key is used by the database when reading or -writing data, to decide which partitions must be accessed. The -partition key should be a column that would be included in a `WHERE` -clause on almost all queries accessing that table. - -Second, it's necessary to understand the strategy the database uses -to split the data across the partitions. The scheme supported by the -GitLab migration helpers is date-range partitioning, where each partition -in the table contains data for a single month. In this case, the partitioning -key must be a timestamp or date column. In order for this type of +partitioning scheme. Understand a few details to decide if partitioning +is a good fit for your particular problem: + +- **Table partitioning**. A table is partitioned on a partition key, which is a + column or set of columns which determine how the data is split across the + partitions. The partition key is used by the database when reading or + writing data, to decide which partitions must be accessed. The + partition key should be a column that would be included in a `WHERE` + clause on almost all queries accessing that table. + +- **How the data is split**. What strategy does the database use + to split the data across the partitions? The available choices are `range`, + `hash`, and `list`. + +## Determine the appropriate partitioning strategy + +The available partitioning strategy choices are `range`, `hash`, and `list`. + +### Range partitioning + +The scheme best supported by the GitLab migration helpers is date-range partitioning, +where each partition in the table contains data for a single month. In this case, +the partitioning key must be a timestamp or date column. For this type of partitioning to work well, most queries must access data in a certain date range. -For a more concrete example, the `audit_events` table can be used, which -was the first table to be partitioned in the application database +For a more concrete example, consider using the `audit_events` table. +It was the first table to be partitioned in the application database (scheduled for deployment with the GitLab 13.5 release). This table tracks audit entries of security events that happen in the application. In almost all cases, users want to see audit activity that @@ -142,6 +156,31 @@ substantial. Partitioning should only be leveraged if the access patterns of the data support the partitioning strategy, otherwise performance suffers. +### Hash Partitioning + +Hash partitioning splits a logical table into a series of partitioned +tables. Each partition corresponds to the ID range that matches +a hash and remainder. For example, if partitioning `BY HASH(id)`, rows +with `hash(id) % 64 == 1` would end up in the partition +`WITH (MODULUS 64, REMAINDER 1)`. + +When hash partitioning, you must include a `WHERE hashed_column = ?` condition in +every performance-sensitive query issued by the application. If this is not possible, +hash partitioning may not be the correct fit for your use case. + +Hash partitioning has one main advantage: it is the only type of partitioning that +can enforce uniqueness on a single numeric `id` column. (While also possible with +range partitioning, it's rarely the correct choice). + +Hash partitioning has downsides: + +- The number of partitions must be known up-front. +- It's difficult to move new data to an extra partition if current partitions become too large. +- Range queries, such as `WHERE id BETWEEN ? and ?`, are unsupported. +- Lookups by other keys, such as `WHERE other_id = ?`, are unsupported. + +For this reason, it's often best to choose a large number of hash partitions to accommodate future table growth. + ## Partitioning a table (Range) Unfortunately, tables can only be partitioned at their creation, making @@ -203,6 +242,10 @@ Continuing the above example, the migration would look like: class BackfillPartitionAuditEvents < Gitlab::Database::Migration[2.1] include Gitlab::Database::PartitioningMigrationHelpers + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + def up enqueue_partitioning_data_migration :audit_events end @@ -213,17 +256,12 @@ class BackfillPartitionAuditEvents < Gitlab::Database::Migration[2.1] end ``` -This step uses the same mechanism as any background migration, so you -may want to read the [Background Migration](background_migrations.md) -guide for details on that process. Background jobs are scheduled every -2 minutes and copy `50_000` records at a time, which can be used to -estimate the timing of the background migration portion of the -partitioning migration. +This step [queues a batched background migration](batched_background_migrations.md#queueing) internally with BATCH_SIZE and SUB_BATCH_SIZE as `50,000` and `2,500`. Refer [Batched Background migrations guide](batched_background_migrations.md) for more details. ### Step 3: Post-backfill cleanup (Release N+1) -The third step must occur at least one release after the release that -includes the background migration. This gives time for the background +This step must occur at least one release after the release that +includes step (2). This gives time for the background migration to execute properly in self-managed installations. In this step, add another post-deployment migration that cleans up after the background migration. This includes forcing any remaining jobs to @@ -236,6 +274,10 @@ Once again, continuing the example, this migration would look like: class CleanupPartitionedAuditEventsBackfill < Gitlab::Database::Migration[2.1] include Gitlab::Database::PartitioningMigrationHelpers + disable_ddl_transaction! + + restrict_gitlab_migration gitlab_schema: :gitlab_main + def up finalize_backfilling_partitioned_table :audit_events end @@ -246,16 +288,57 @@ class CleanupPartitionedAuditEventsBackfill < Gitlab::Database::Migration[2.1] end ``` -After this migration has completed, the original table and partitioned +After this migration completes, the original table and partitioned table should contain identical data. The trigger installed on the original table guarantees that the data remains in sync going forward. ### Step 4: Swap the partitioned and non-partitioned tables (Release N+1) -The final step of the migration makes the partitioned table ready -for use by the application. This section will be updated when the -migration helper is ready, for now development can be followed in the -[Tracking Issue](https://gitlab.com/gitlab-org/gitlab/-/issues/241267). +This step replaces the non-partitioned table with its partitioned copy, this should be used only after all other migration steps have completed successfully. + +Some limitations to this method MUST be handled before, or during, the swap migration: + +- Secondary indexes and foreign keys are not automatically recreated on the partitioned table. +- Some types of constraints (UNIQUE and EXCLUDE) which rely on indexes, are not automatically recreated + on the partitioned table, since the underlying index will not be present. +- Foreign keys referencing the original non-partitioned table should be updated to reference the + partitioned table. This is not supported in PostgreSQL 11. +- Views referencing the original table are not automatically updated to reference the partitioned table. + +```ruby +# frozen_string_literal: true + +class SwapPartitionedAuditEvents < ActiveRecord::Migration[6.0] + include Gitlab::Database::PartitioningMigrationHelpers + + def up + replace_with_partitioned_table :audit_events + end + + def down + rollback_replace_with_partitioned_table :audit_events + end +end +``` + +After this migration completes: + +- The partitioned table replaces the non-partitioned (original) table. +- The sync trigger created earlier is dropped. + +The partitioned table is now ready for use by the application. + +## Partitioning a table (Hash) + +Hash partitioning divides data into partitions based on a hash of their ID. +It works well only if most queries against the table include a clause like `WHERE id = ?`, +so that PostgreSQL can decide which partition to look in based on the ID or ids being requested. + +Another key downside is that hash partitioning does not allow adding additional partitions after table creation. +The correct number of partitions must be chosen up-front. + +Hash partitioning is the only type of partitioning (aside from some complex uses of list partitioning) that can guarantee +uniqueness of an ID across multiple partitions at the database level. ## Partitioning a table (List) @@ -504,3 +587,48 @@ class Model < ApplicationRecord self.sequence_name = 'model_id_seq' end ``` + +If the partitioning constraint migration takes [more than 10 minutes](../migration_style_guide.md#how-long-a-migration-should-take) to finish, +it can be made to run asynchronously to avoid running the post-migration during busy hours. + +Prepend the following migration `AsyncPrepareTableConstraintsForListPartitioning` +and use `async: true` option. This change marks the partitioning constraint as `NOT VALID` +and enqueues a scheduled job to validate the existing data in the table during the weekend. + +Then the second post-migration `PrepareTableConstraintsForListPartitioning` only +marks the partitioning constraint as validated, because the existing data is already +tested during the previous weekend. + +For example: + +```ruby +class AsyncPrepareTableConstraintsForListPartitioning < Gitlab::Database::Migration[2.1] + include Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers + + disable_ddl_transaction! + + TABLE_NAME = :table_name + PARENT_TABLE_NAME = :p_table_name + FIRST_PARTITION = 100 + PARTITION_COLUMN = :partition_id + + def up + prepare_constraint_for_list_partitioning( + table_name: TABLE_NAME, + partitioning_column: PARTITION_COLUMN, + parent_table_name: PARENT_TABLE_NAME, + initial_partitioning_value: FIRST_PARTITION, + async: true + ) + end + + def down + revert_preparing_constraint_for_list_partitioning( + table_name: TABLE_NAME, + partitioning_column: PARTITION_COLUMN, + parent_table_name: PARENT_TABLE_NAME, + initial_partitioning_value: FIRST_PARTITION + ) + end +end +``` diff --git a/doc/development/database/transaction_guidelines.md b/doc/development/database/transaction_guidelines.md index 26bb6c2ce8f..1648cf58937 100644 --- a/doc/development/database/transaction_guidelines.md +++ b/doc/development/database/transaction_guidelines.md @@ -12,7 +12,7 @@ For further reference, check PostgreSQL documentation about [transactions](https ## Database decomposition and sharding -The [Pods Group](https://about.gitlab.com/handbook/engineering/development/enablement/data_stores/pods/) plans +The [Pods group](https://about.gitlab.com/handbook/engineering/development/enablement/data_stores/tenant-scale/) plans to split the main GitLab database and move some of the database tables to other database servers. We start decomposing the `ci_*`-related database tables first. To maintain the current application diff --git a/doc/development/database/understanding_explain_plans.md b/doc/development/database/understanding_explain_plans.md index 094bd6b346f..560744430f9 100644 --- a/doc/development/database/understanding_explain_plans.md +++ b/doc/development/database/understanding_explain_plans.md @@ -714,8 +714,7 @@ SQL optimization tool - [Joe Bot](https://gitlab.com/postgres-ai/joe). Database Lab Engine provides developers with their own clone of the production database, while Joe Bot helps with exploring execution plans. -Joe Bot is available in the [`#database-lab`](https://gitlab.slack.com/archives/CLJMDRD8C) channel on Slack, -and through its [web interface](https://console.postgres.ai/gitlab/joe-instances). +Joe Bot is available through its [web interface](https://console.postgres.ai/gitlab/joe-instances). With Joe Bot you can execute DDL statements (like creating indexes, tables, and columns) and get query plans for `SELECT`, `UPDATE`, and `DELETE` statements. @@ -792,34 +791,6 @@ Planning time: 0.411 ms Execution time: 0.113 ms ``` -### ChatOps - -GitLab team members can also use our ChatOps solution, available in Slack -using the [`/chatops` slash command](../chatops_on_gitlabcom.md). - -NOTE: -While ChatOps is still available, the recommended way to generate execution plans is to use [Database Lab Engine](#database-lab-engine). - -You can use ChatOps to get a query plan by running the following: - -```sql -/chatops run explain SELECT COUNT(*) FROM projects WHERE visibility_level IN (0, 20) -``` - -Visualising the plan using is also supported: - -```sql -/chatops run explain --visual SELECT COUNT(*) FROM projects WHERE visibility_level IN (0, 20) -``` - -Quoting the query is not necessary. - -For more information about the available options, run: - -```sql -/chatops run explain --help -``` - ## Further reading A more extensive guide on understanding query plans can be found in diff --git a/doc/development/database_review.md b/doc/development/database_review.md index 048482960f4..0e34e550098 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -68,7 +68,7 @@ Refer to [Preparation when adding or modifying queries](#preparation-when-adding A merge request **author**'s role is to: - Decide whether a database review is needed. -- If database review is needed, add the ~database label. +- If database review is needed, add the `~database` label. - [Prepare the merge request for a database review](#how-to-prepare-the-merge-request-for-a-database-review). - Provide the [required](#required) artifacts prior to submitting the MR. @@ -99,7 +99,7 @@ The MR author should request a review from the suggested database the suggested database **maintainer**. If reviewer roulette didn't suggest a database reviewer & maintainer, -make sure you have applied the ~database label and rerun the +make sure you have applied the `~database` label and rerun the `danger-review` CI job, or pick someone from the [`@gl-database` team](https://gitlab.com/groups/gl-database/-/group_members). @@ -122,14 +122,14 @@ the following preparations into account. - When [high-traffic](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/rubocop-migrations.yml#L3) tables are involved in the migration, use the [`enable_lock_retries`](migration_style_guide.md#retry-mechanism-when-acquiring-database-locks) method to enable lock-retries. Review the relevant [examples in our documentation](migration_style_guide.md#usage-with-transactional-migrations) for use cases and solutions. - Ensure RuboCop checks are not disabled unless there's a valid reason to. - When adding an index to a [large table](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/rubocop-migrations.yml#L3), -test its execution using `CREATE INDEX CONCURRENTLY` in the `#database-lab` Slack channel and add the execution time to the MR description: - - Execution time largely varies between `#database-lab` and GitLab.com, but an elevated execution time from `#database-lab` + test its execution using `CREATE INDEX CONCURRENTLY` in [Database Lab](database/database_lab.md) and add the execution time to the MR description: + - Execution time largely varies between Database Lab and GitLab.com, but an elevated execution time from Database Lab can give a hint that the execution on GitLab.com is also considerably high. - - If the execution from `#database-lab` is longer than `1h`, the index should be moved to a [post-migration](database/post_deployment_migrations.md). + - If the execution from Database Lab is longer than `10 minutes`, the [index](database/adding_database_indexes.md) should be moved to a [post-migration](database/post_deployment_migrations.md). Keep in mind that in this case you may need to split the migration and the application changes in separate releases to ensure the index is in place when the code that needs it is deployed. - Manually trigger the [database testing](database/database_migration_pipeline.md) job (`db:gitlabcom-database-testing`) in the `test` stage. - - This job runs migrations in a production-like environment (similar to `#database_lab`) and posts to the MR its findings (queries, runtime, size change). + - This job runs migrations in a [Database Lab](database/database_lab.md) clone and posts to the MR its findings (queries, runtime, size change). - Review migration runtimes and any warnings. #### Preparation when adding data migrations @@ -173,13 +173,11 @@ Include in the MR description: ##### Query Plans - The query plan for each raw SQL query included in the merge request along with the link to the query plan following each raw SQL snippet. -- Provide a public link to the plan from either: - - [postgres.ai](https://postgres.ai/): Follow the link in `#database-lab` and generate a shareable, public link - by selecting **Share** in the upper right corner. - - [explain.depesz.com](https://explain.depesz.com) or [explain.dalibo.com](https://explain.dalibo.com): Paste both the plan and the query used in the form. +- Provide a link to the plan generated using the `explain` command in the [postgres.ai](database/database_lab.md) chatbot. + - If it's not possible to get an accurate picture in Database Lab, you may need to seed a development environment, and instead provide links + from [explain.depesz.com](https://explain.depesz.com) or [explain.dalibo.com](https://explain.dalibo.com). Be sure to paste both the plan + and the query used in the form. - When providing query plans, make sure it hits enough data: - - You can use a GitLab production replica to test your queries on a large scale, - through the `#database-lab` Slack channel or through [ChatOps](database/understanding_explain_plans.md#chatops). - To produce a query plan with enough data, you can use the IDs of: - The `gitlab-org` namespace (`namespace_id = 9970`), for queries involving a group. - The `gitlab-org/gitlab-foss` (`project_id = 13083`) or the `gitlab-org/gitlab` (`project_id = 278964`) projects, for queries involving a project. @@ -188,7 +186,7 @@ Include in the MR description: - That means that no query plan should return 0 records or less records than the provided limit (if a limit is included). If a query is used in batching, a proper example batch with adequate included results should be identified and provided. - If your queries belong to a new feature in GitLab.com and thus they don't return data in production: - You may analyze the query and to provide the plan from a local environment. - - `#database-lab` and [postgres.ai](https://postgres.ai/) both allow updates to data (`exec UPDATE issues SET ...`) and creation of new tables and columns (`exec ALTER TABLE issues ADD COLUMN ...`). + - [postgres.ai](https://postgres.ai/) allows updates to data (`exec UPDATE issues SET ...`) and creation of new tables and columns (`exec ALTER TABLE issues ADD COLUMN ...`). - More information on how to find the number of actual returned records in [Understanding EXPLAIN plans](database/understanding_explain_plans.md) - For query changes, it is best to provide both the SQL queries along with the plan _before_ and _after_ the change. This helps spot differences quickly. @@ -231,7 +229,7 @@ Include in the MR description: - [Check indexes are present for foreign keys](migration_style_guide.md#adding-foreign-key-constraints) - Ensure that migrations execute in a transaction or only contain concurrent index/foreign key helpers (with transactions disabled) - - If an index to a large table is added and its execution time was elevated (more than 1h) on `#database-lab`: + - If an index to a large table is added and its execution time was elevated (more than 1h) on [Database Lab](database/database_lab.md): - Ensure it was added in a post-migration. - Maintainer: After the merge request is merged, notify Release Managers about it on `#f_upcoming_release` Slack channel. - Check consistency with `db/structure.sql` and that migrations are [reversible](migration_style_guide.md#reversibility) @@ -269,8 +267,7 @@ Include in the MR description: - Check for any overly complex queries and queries the author specifically points out for review (if any) - If not present, ask the author to provide SQL queries and query plans - (for example, by using [ChatOps](database/understanding_explain_plans.md#chatops) or direct - database access) + using [Database Lab](database/database_lab.md) - For given queries, review parameters regarding data distribution - [Check query plans](database/understanding_explain_plans.md) and suggest improvements to queries (changing the query, schema or adding indexes and similar) diff --git a/doc/development/deprecation_guidelines/index.md b/doc/development/deprecation_guidelines/index.md index e532fa82011..bc14b0f127c 100644 --- a/doc/development/deprecation_guidelines/index.md +++ b/doc/development/deprecation_guidelines/index.md @@ -62,6 +62,8 @@ A breaking change can be considered major if it affects many users, or represent Deprecations should be announced on the [Deprecated feature removal schedule](../../update/deprecations.md). +Deprecations should be announced [no later than the third milestone preceding intended removal](https://about.gitlab.com/handbook/product/gitlab-the-product/#process-for-deprecating-and-removing-a-feature). + Do not include the deprecation announcement in the merge request that introduces a code change for the deprecation. Use a separate MR to create a deprecation entry. For steps to create a deprecation entry, see [Deprecations](https://about.gitlab.com/handbook/marketing/blog/release-posts/#deprecations). @@ -77,7 +79,7 @@ However, at GitLab, we [give agency](https://about.gitlab.com/handbook/values/#g Generally, feature or configuration can be removed/changed only on major release. It also should be [deprecated in advance](https://about.gitlab.com/handbook/marketing/blog/release-posts/#deprecations). -For API removals, see the [GraphQL](../../api/graphql/index.md#deprecation-and-removal-process) and [GitLab API](../../api/rest/index.md#compatibility-guidelines) guidelines. +For API removals, see the [GraphQL](../../api/graphql/index.md#deprecation-and-removal-process) and [GitLab API](../documentation/restful_api_styleguide.md#deprecations) guidelines. For configuration removals, see the [Omnibus deprecation policy](../../administration/package_information/deprecation_policy.md). diff --git a/doc/development/diffs.md b/doc/development/diffs.md deleted file mode 100644 index c84bf57e085..00000000000 --- a/doc/development/diffs.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'merge_request_concepts/diffs/index.md' -remove_date: '2023-04-10' ---- - -This document was moved to [another location](merge_request_concepts/diffs/index.md). - - - - - diff --git a/doc/development/directory_structure.md b/doc/development/directory_structure.md deleted file mode 100644 index 34ee86d9ee5..00000000000 --- a/doc/development/directory_structure.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'software_design.md' -remove_date: '2023-01-24' ---- - -This document was moved to [another location](software_design.md) - - - - - diff --git a/doc/development/distributed_tracing.md b/doc/development/distributed_tracing.md index 79d0ff84713..823a71eb130 100644 --- a/doc/development/distributed_tracing.md +++ b/doc/development/distributed_tracing.md @@ -4,7 +4,7 @@ group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Distributed Tracing - development guidelines +# Distributed tracing development guidelines GitLab is instrumented for distributed tracing. Distributed Tracing in GitLab is currently considered **experimental**, as it has not yet been tested at scale on GitLab.com. @@ -22,6 +22,14 @@ Distributed tracing adds minimal overhead when disabled, but imposes only small enabled and is therefore capable in any environment, including production. For this reason, it can be useful in diagnosing production issues, particularly performance problems. +Services have different levels of support for distributed tracing. Custom +instrumentation code must be added to the application layer in addition to +pre-built instrumentation for the most common libraries. + +For service-specific information, see: + +- [Using Jaeger for Gitaly local development](https://gitlab.com/gitlab-org/gitaly/-/blob/master/doc/jaeger_for_local_development.md) + ## Using Correlation IDs to investigate distributed requests The GitLab application passes correlation IDs between the various components in a request. A @@ -77,15 +85,56 @@ In this example, we have the following hypothetical values: they should be URL encoded. Multiple values should be separated by `&` characters like a URL. +GitLab Rails provides pre-implemented instrumentations for common types of +operations that offer a detailed view of the requests. However, the detailed +information comes at a cost. The resulting traces are long and can be difficult +to process, making it hard to identify bigger underlying issues. To address this +concern, some instrumentations are disabled by default. To enable those disabled +instrumentations, set the following environment variables: + +- `GITLAB_TRACING_TRACK_CACHES`: enable tracking cache operations, such as cache +read, write, or delete. +- `GITLAB_TRACING_TRACK_REDIS`: enable tracking Redis operations. Most Redis +operations are for caching, though. + ## Using Jaeger in the GitLab Development Kit The first tracing implementation that GitLab supports is Jaeger, and the -[GitLab Development Kit](https://gitlab.com/gitlab-org/gitlab-development-kit/) supports distributed tracing with -Jaeger out-of-the-box. +[GitLab Development Kit](https://gitlab.com/gitlab-org/gitlab-development-kit/) +supports distributed tracing with Jaeger out-of-the-box. GDK automatically adds +`GITLAB_TRACING` environment variables to add services. + +Configure GDK for Jaeger by editing the `gdk.yml` file and adding the following +settings: + +```yaml +tracer: + build_tags: tracer_static tracer_static_jaeger + jaeger: + enabled: true + listen_address: 127.0.0.1 + version: 1.43.0 +``` -The easiest way to access tracing from a GDK environment is through the -[performance-bar](../administration/monitoring/performance/performance_bar.md). This can be shown -by typing `p` `b` in the browser window. +After modifying the `gdk.yml` file, reconfigure your GDK by running +the `gdk reconfigure` command. This ensures that your GDK is properly configured +and ready to use. + +The above configuration sets the `tracer_static` and `tracer_static_jaeger` +build tags when rebuilding services written in Go for the first time. Any +changes made afterward require rebuilding them with those build tags. You can +either: + +- Add those build tags to the default set of build tags. +- Manually attach them to the build command. For example, Gitaly supports adding + build tag out of the box. You can run + `make all WITH_BUNDLED_GIT=YesPlease BUILD_TAGS="tracer_static tracer_static_jaeger"`. + +After reconfiguration, Jaeger dashboard is available at +`http://localhost:16686`. Another way to access tracing from a GDK environment +is through the +[performance-bar](../administration/monitoring/performance/performance_bar.md). +This can be shown by typing `p` `b` in the browser window. Once the performance bar is enabled, select **Trace** in the performance bar to go to the Jaeger UI. diff --git a/doc/development/distribution/index.md b/doc/development/distribution/index.md new file mode 100644 index 00000000000..168e3854eca --- /dev/null +++ b/doc/development/distribution/index.md @@ -0,0 +1,35 @@ +--- +stage: Systems +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +description: "GitLab's development guidelines for Distribution" +--- + +# Contribute to GitLab Distribution + +Learn how to add new components and services to the GitLab application. + +## Support all package methods + +Additions must support both Omnibus GitLab and Cloud Native GitLab. Changes +to one must be made to the other to retain feature parity. + +## Contributing + +The primary projects handled by Distribution are listed below. For more +information, visit the [Distribution team engineering handbook page](https://about.gitlab.com/handbook/engineering/development/enablement/systems/distribution/) +or select one of the subsections in the navigation bar. + +### GitLab application + +- [Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab) +- [Cloud Native GitLab (CNG)](https://gitlab.com/gitlab-org/build/CNG) +- [GitLab Operator](https://gitlab.com/gitlab-org/cloud-native/gitlab-operator) +- [GitLab Chart](https://gitlab.com/gitlab-org/charts/gitlab) + +### Components and tools + +- [Omnibus GitLab Builder](https://gitlab.com/gitlab-org/gitlab-omnibus-builder) +- [Omnibus Fork](https://gitlab.com/gitlab-org/omnibus) +- [GitLab Logger](https://gitlab.com/gitlab-org/cloud-native/gitlab-logger) +- [Issue Bot](https://gitlab.com/gitlab-org/distribution/issue-bot) diff --git a/doc/development/documentation/alpha_beta.md b/doc/development/documentation/alpha_beta.md new file mode 100644 index 00000000000..d421aae3cb9 --- /dev/null +++ b/doc/development/documentation/alpha_beta.md @@ -0,0 +1,49 @@ +--- +info: For assistance with this Style Guide page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects +stage: none +group: unassigned +--- + +# Documenting Experiment and Beta features + +Some features are not generally available and are instead considered +[Experiment or Beta](../../policy/alpha-beta-support.md). + +When you document a feature in one of these three statuses: + +- Add `(Experiment)` or `(Beta)` in parentheses after the page or topic title. +- Do not include `(Experiment)` or `(Beta)` in the left nav. +- Ensure the version history lists the feature's status. + +These features are usually behind a feature flag, which follow [these documentation guidelines](feature_flags.md). + +If you add details of how users should enroll, or how to contact the team with issues, +the `FLAG:` note should be above these details. + +For example: + +```markdown +## Great new feature (Experiment) + +> [Introduced](link) in GitLab 15.10. This feature is an [Experiment](/policy/alpha-beta-support.md). + +FLAG: +On self-managed GitLab, by default this feature is not available. +To make it available, ask an administrator to enable the feature flag named `example_flag`. +On GitLab.com, this feature is not available. This feature is not ready for production use. + +Use this great new feature when you need to do this new thing. + +This feature is an [Experiment](/policy/alpha-beta-support.md). To join +the list of users testing this feature, do this thing. If you find a bug, +[open an issue](link). +``` + +When the feature is ready for production, remove: + +- The text in parentheses. +- Any language about the feature not being ready for production in the body + description. +- The feature flag information if available. + +Ensure the version history is up-to-date by adding a note about the production release. diff --git a/doc/development/documentation/contribute.md b/doc/development/documentation/contribute.md new file mode 100644 index 00000000000..8b08743c6e9 --- /dev/null +++ b/doc/development/documentation/contribute.md @@ -0,0 +1,83 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Contribute to the GitLab documentation + +Everyone is welcome to update the GitLab documentation! + +## Work without an issue + +You don't need an issue to update the documentation. + +On [https://docs.gitlab.com](https://docs.gitlab.com), at the bottom of any page, +you can select **View page source** or **Edit in Web IDE** and [get started with a merge request](#open-your-merge-request). + +You can alternately: + +- Choose a page [in the `/doc` directory](https://gitlab.com/gitlab-org/gitlab/-/tree/master/doc) + and edit it from there. +- Try installing and running the [Vale linting tool](testing.md#vale) + and fixing the resulting issues. + +When you're developing code, the workflow for updating docs is slightly different. +For details, see the [merge request workflow](../contributing/merge_request_workflow.md). + +## Search available issues + +If you're looking for an open issue, you can +[review the list of documentation issues curated specifically for new contributors](https://gitlab.com/gitlab-org/gitlab/-/issues/?sort=created_date&state=opened&label_name%5B%5D=documentation&label_name%5B%5D=docs-only&label_name%5B%5D=Seeking%20community%20contributions&first_page_size=20). + +When you find an issue you'd like to work on: + +- If the issue is already assigned to someone, pick a different one. +- If the issue is unassigned, add a comment and ask to work on the issue. For a Hackathon, use `@docs-hackathon`. Otherwise, use `@gl-docsteam`. For example: + + ```plaintext + @docs-hackathon I would like to work on this issue + ``` + +- Do not ask for more than three issues at a time. + +## Open your merge request + +When you are ready to update the documentation: + +1. Go to the [GitLab repository](https://gitlab.com/gitlab-org/gitlab). +1. In the upper-right corner, select **Fork**. Forking makes a copy of the repository on GitLab.com. +1. In your fork, find the documentation page in the `\doc` directory. +1. If you know Git, make your changes and open a merge request. + If not, follow these steps: + 1. In the upper-right corner, select **Edit** if it is visible. + If it is not, select the down arrow (**{chevron-lg-down}**) next to + **Open in Web IDE** or **Gitpod**, and select **Edit**. + 1. In the **Commit message** text box, enter a commit message. + Use 3-5 words, start with a capital letter, and do not end with a period. + 1. Select **Commit changes**. + 1. On the left sidebar, select **Merge requests**. + 1. Select **New merge request**. + 1. For the source branch, select your fork and branch. If you did not create a branch, select `master`. + For the target branch, select the [GitLab repository](https://gitlab.com/gitlab-org/gitlab) `master` branch. + 1. Select **Compare branches and continue**. A new merge request opens. + 1. Select the **Documentation** template. In the description, write a brief summary of the changes and link to the related issue, if there is one. + 1. Select **Create merge request**. + +## Ask for help + +Ask for help from the Technical Writing team if you: + +- Need help to choose the correct place for documentation. +- Want to discuss a documentation idea or outline. +- Want to request any other help. + +To identify someone who can help you: + +1. Locate the Technical Writer for the relevant + [DevOps stage group](https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments). +1. Either: + - If urgent help is required, directly assign the Technical Writer in the issue or in the merge request. + - If non-urgent help is required, ping the Technical Writer in the issue or merge request. + +If you are a member of the GitLab Slack workspace, you can request help in the `#docs` channel. diff --git a/doc/development/documentation/feature_flags.md b/doc/development/documentation/feature_flags.md index 37be2178592..854faa839ef 100644 --- a/doc/development/documentation/feature_flags.md +++ b/doc/development/documentation/feature_flags.md @@ -7,33 +7,32 @@ description: "GitLab development - how to document features deployed behind feat # Document features deployed behind feature flags -GitLab uses [feature flags](../feature_flags/index.md) to strategically roll -out the deployment of its own features. The way we document a feature behind a -feature flag depends on its state (enabled or disabled). When the state -changes, the developer who made the change **must update the documentation** -accordingly. +GitLab uses [feature flags](../feature_flags/index.md) to roll +out the deployment of its own features. + +When the state of a feature flag changes, the developer who made the change +**must update the documentation**. ## When to document features behind a feature flag Every feature introduced to the codebase, even if it's behind a disabled feature flag, must be documented. For more information, see -[the discussion that led to this decision](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47917#note_459984428). +[the discussion that led to this decision](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47917#note_459984428). [Experiment or Beta](../../policy/alpha-beta-support.md) features are usually behind a feature flag, and must also be documented. For more information, see [Document Experiment or Beta features](alpha_beta.md). + +When the feature is [implemented in multiple merge requests](../feature_flags/index.md#feature-flags-in-gitlab-development), +discuss the plan with your technical writer. -When the feature is [implemented over multiple merge requests](../feature_flags/index.md#feature-flags-in-gitlab-development), -discuss the exact documentation plan with your technical writer. Consider -creating a dedicated documentation issue if the feature: +You can create a documentation issue and delay the documentation if the feature: - Is far-reaching (makes changes across many areas of GitLab), like navigation changes. - Includes many MRs. - Affects more than a few documentation pages. - Is not fully functional if the feature flag is enabled for testing. -If you and the technical writer agree to delay the product documentation (for example, until after testing), -collaborate with the TW to create a documentation issue detailing the plan for adding the content. -The PM and EM should be included in the discussions to make sure the task of adding the documentation is assigned and scheduled. -Despite any planned delays, every feature flag in the codebase is automatically listed at -, -even when the feature is not fully functional. +The PM, EM, and writer should make sure the documentation work is assigned and scheduled. + +Every feature flag in the codebase is [in the documentation](../../user/feature_flags.md), +even when the feature is not fully functional or otherwise documented. ## How to add feature flag documentation @@ -57,13 +56,6 @@ Possible version history entries are: > - [Generally available](issue-link) in GitLab X.Y. Feature flag `flag_name` removed. ``` -You can combine entries if they happened in the same release: - -```markdown -> - Introduced in GitLab 14.2 [with a flag](../../administration/feature_flags.md) named `ci_include_rules`. Disabled by default. -> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/337507) in GitLab 14.3. -``` - ## Use a note to describe the state of the feature flag Information about feature flags should be in a `FLAG` note at the start of the topic (just below the version history). @@ -144,3 +136,41 @@ And, when the feature is done and fully available to all users: > - [Enabled on GitLab.com](https://gitlab.com/issue/etc) in GitLab 13.9. > - [Generally available](issue-link) in GitLab 14.0. Feature flag `forti_token_cloud` removed. ``` + +## Simplify long version history + +The version history can get long, but you can sometimes simplify or remove entries. + +Combine entries if they happened in the same release: + +- Before: + + ```markdown + > - [Introduced](issue-link) in GitLab 14.2 [with a flag](../../administration/feature_flags.md) named `ci_include_rules`. Disabled by default. + > - [Enabled on GitLab.com](issue-link) in GitLab 14.3. + > - [Enabled on self-managed](issue-link) in GitLab 14.3. + ``` + +- After: + + ```markdown + > - [Introduced](issue-link) in GitLab 14.2 [with a flag](../../administration/feature_flags.md) named `ci_include_rules`. Disabled by default. + > - [Enabled on GitLab.com and self-managed](issue-link) in GitLab 14.3. + ``` + +Remove `Enabled on GitLab.com` entries when the feature is enabled by default for both GitLab.com and self-managed: + +- Before: + + ```markdown + > - [Introduced](issue-link) in GitLab 15.6 [with a flag](../../administration/feature_flags.md) named `ci_hooks_pre_get_sources_script`. Disabled by default. + > - [Enabled on GitLab.com](issue-link) in GitLab 15.9. + > - [Generally available](issue-link) in GitLab 15.10. Feature flag `ci_hooks_pre_get_sources_script` removed. + ``` + +- After: + + ```markdown + > - [Introduced](issue-link) in GitLab 15.6 [with a flag](../../administration/feature_flags.md) named `ci_hooks_pre_get_sources_script`. Disabled by default. + > - [Generally available](issue-link) in GitLab 15.10. Feature flag `ci_hooks_pre_get_sources_script` removed. + ``` diff --git a/doc/development/documentation/index.md b/doc/development/documentation/index.md index fd9e885e097..6811b3d6584 100644 --- a/doc/development/documentation/index.md +++ b/doc/development/documentation/index.md @@ -107,8 +107,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w --- ``` -If you need help determining the correct stage, read [Ask for help](workflow.md#ask-for-help). - ### Redirection metadata The following metadata should be added when a page is moved to another location: @@ -116,21 +114,6 @@ The following metadata should be added when a page is moved to another location: - `redirect_to`: The relative path and filename (with an `.md` extension) of the location to which visitors should be redirected for a moved page. [Learn more](redirects.md). -- `disqus_identifier`: Identifier for Disqus commenting system. Used to keep - comments with a page that has been moved to a new URL. - [Learn more](redirects.md#redirections-for-pages-with-disqus-comments). - -### Comments metadata - -The [docs website](site_architecture/index.md) has comments (provided by Disqus) -enabled by default. In case you want to disable them (for example in index pages), -set it to `false`: - -```yaml ---- -comments: false ---- -``` ### Additional page metadata @@ -157,21 +140,31 @@ You can use a Rake task to update the `CODEOWNERS` file. #### Update the `CODEOWNERS` file -To update the `CODEOWNERS` file: +When groups or [TW assignments](https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments) +change, you must update the `CODEOWNERS` file: -1. Review the [TW team assignments](https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments) - in the [`codeowners.rake`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/tasks/gitlab/tw/codeowners.rake) - file. If any assignments have changed: - 1. Update the `codeowners.rake` file with the changes. - 1. Assign the merge request to a technical writing manager for review and merge. -1. After the changes to `codeowners.rake` are merged, go to the root of the `gitlab` repository. +1. Update the [stage and group metadata](#stage-and-group-metadata) for any affected doc pages, if necessary. If there are many changes, you can do this step in a separate MR. +1. Update the [`codeowners.rake`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/tasks/gitlab/tw/codeowners.rake) file with the changes. +1. Go to the root of the `gitlab` repository. 1. Run the Rake task with this command: `bundle exec rake tw:codeowners` -1. Review the command output for any pages that need attention to - their metadata. Handle any needed changes in a separate merge request. -1. Add the changes to the CODEOWNERS file to Git: `git add .gitlab/CODEOWNERS` -1. Commit your changes to your branch, and push your branch up to `origin`. +1. Review the changes in the `CODEOWNERS` file. +1. Add and commit all your changes and push your branch up to `origin`. 1. Create a merge request and assign it to a technical writing manager for review. +When updating the `codeowners.rake` file: + +- To specify multiple writers for a single group, use a space between writer names: + + ```plaintext + CodeOwnerRule.new('Group Name', '@writer1 @writer2'), + ``` + +- For a group that does not have an assigned writer, include the group name in the file and comment out the line: + + ```plaintext + # CodeOwnerRule.new('Group Name', ''), + ``` + ## Move, rename, or delete a page See [redirects](redirects.md). @@ -194,9 +187,8 @@ Further needs for what would make the doc even better should be immediately addr in a follow-up merge request or issue. If the release version you want to add the documentation to has already been -frozen or released, use the label `~"Pick into X.Y"` to get it merged into -the correct release. Avoid picking into a past release as much as you can, as -it increases the work of the release managers. +released, follow the [patch release runbook](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/patch/engineers.md) +to get it merged into the current version. ## GitLab `/help` @@ -359,28 +351,6 @@ feedback: false The default is to leave it there. If you want to omit it from a document, you must check with a technical writer before doing so. -## Disqus - -We have integrated the docs site with Disqus (introduced by -[!151](https://gitlab.com/gitlab-org/gitlab-docs/-/merge_requests/151)), -allowing our users to post comments. - -To omit only the comments from the feedback section, use the following key in -the front matter: - -```yaml ---- -comments: false ---- -``` - -We're hiding comments only in main index pages, such as [the main documentation index](../../index.md), -since its content is too broad to comment on. Before omitting Disqus, you must -check with a technical writer. - -Note that after adding `feedback: false` to the front matter, it will omit -Disqus, therefore, don't add both keys to the same document. - The click events in the feedback section are tracked with Google Tag Manager. The conversions can be viewed on Google Analytics by navigating to **Behavior > Events > Top events > docs**. diff --git a/doc/development/documentation/redirects.md b/doc/development/documentation/redirects.md index 4cfe8be713a..f080625ea9c 100644 --- a/doc/development/documentation/redirects.md +++ b/doc/development/documentation/redirects.md @@ -81,7 +81,6 @@ To redirect a page to another page in the same repository: - Replace both instances of `../newpath/to/file/index.md` with the new file path. - Replace both instances of `YYYY-MM-DD` with the expiration date, as explained in the template. -1. If the page has Disqus comments, follow [the steps for pages with Disqus comments](#redirections-for-pages-with-disqus-comments). 1. If the page had images that aren't used on any other pages, delete them. After your changes are committed, search for and update all links that point to the old file: @@ -159,25 +158,13 @@ If you create a new page and then rename it before it's added to a release on th Instead of following that procedure, ask a Technical Writer to manually add the redirect to [`redirects.yaml`](https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/content/_data/redirects.yaml). -## Redirections for pages with Disqus comments +## Exceptions to creating a redirect -If the documentation page being relocated already has Disqus comments, -we need to preserve the Disqus thread. +In some cases you can skip adding the redirect and just delete the file. The page +must have already been removed from (or never existed in) the navigation, and one +of the following must be true: -Disqus uses an identifier per page, and for , the page identifier -is configured to be the page URL. Therefore, when we change the document location, -we need to preserve the old URL as the same Disqus identifier. - -To do that, add to the front matter the variable `disqus_identifier`, -using the old URL as value. For example, let's say we moved the document -available under `https://docs.gitlab.com/my-old-location/README.html` to a new location, -`https://docs.gitlab.com/my-new-location/index.html`. - -Into the **new document** front matter, we add the following information. You must -include the filename in the `disqus_identifier` URL, even if it's `index.html` or `README.html`. - -```yaml ---- -disqus_identifier: 'https://docs.gitlab.com/my-old-location/README.html' ---- -``` +- The page was added and removed in the same release, so it was never included in + a self-managed release. +- The page does not contain any content of value, like a placeholder page or a page + with extremely low usage statistics. diff --git a/doc/development/documentation/restful_api_styleguide.md b/doc/development/documentation/restful_api_styleguide.md index a92d58ead96..93afbf7e6dd 100644 --- a/doc/development/documentation/restful_api_styleguide.md +++ b/doc/development/documentation/restful_api_styleguide.md @@ -132,7 +132,7 @@ To deprecate an attribute: ``` To widely announce a deprecation, or if it's a breaking change, -[update the deprecations and removals documentation pages](../deprecation_guidelines/index.md#update-the-deprecations-and-removals-documentation-pages). +[update the REST API deprecations and removals page](../../api/rest/deprecations.md). ## Method description diff --git a/doc/development/documentation/site_architecture/global_nav.md b/doc/development/documentation/site_architecture/global_nav.md index ef6f3c0ae18..3e0534220a8 100644 --- a/doc/development/documentation/site_architecture/global_nav.md +++ b/doc/development/documentation/site_architecture/global_nav.md @@ -69,8 +69,7 @@ Sometimes pages for deprecated features are not in the global nav, depending on All other pages should be in the global nav. The technical writing team runs a report to determine which pages are not in the nav. -For now this report is manual, but [an issue exists](https://gitlab.com/gitlab-org/gitlab-docs/-/issues/1212) -to automate it. +The team reviews this list each month. ### Where to add diff --git a/doc/development/documentation/site_architecture/index.md b/doc/development/documentation/site_architecture/index.md index d4553614fad..849308f3086 100644 --- a/doc/development/documentation/site_architecture/index.md +++ b/doc/development/documentation/site_architecture/index.md @@ -33,7 +33,7 @@ Then you can use one of these approaches: of the documentation in the external repository. The landing page is indexed and searchable on , but the rest of the documentation is not. For example, the [GitLab Workflow extension for VS Code](../../../user/project/repository/vscode.md). - We do not encourage the use of [pages with lists of links](../topic_types/index.md#topics-and-resources), + We do not encourage the use of [pages with lists of links](../topic_types/index.md#pages-and-topics-to-avoid), so only use this option if the recommended options are not feasible. ## Monthly release process (versions) diff --git a/doc/development/documentation/styleguide/index.md b/doc/development/documentation/styleguide/index.md index 74437ea46c9..00307583be4 100644 --- a/doc/development/documentation/styleguide/index.md +++ b/doc/development/documentation/styleguide/index.md @@ -60,7 +60,7 @@ GitLab adds all troubleshooting information to the documentation, no matter how unlikely a user is to encounter a situation. GitLab Support maintains their own -[troubleshooting content](../../../administration/index.md#support-team-documentation) +[troubleshooting content](../../../administration/troubleshooting/index.md) in the GitLab documentation. ### The documentation includes all media types @@ -630,7 +630,7 @@ You can nest lists in other lists. ## Tables Tables should be used to describe complex information in a straightforward -manner. Note that in many cases, an unordered list is sufficient to describe a +manner. In many cases, an unordered list is sufficient to describe a list of items with a single, simple description per item. But, if you have data that's best described by a matrix, tables are the best choice. @@ -718,6 +718,15 @@ This is overridden by the [documentation-specific punctuation rules](#punctuatio Links help the docs adhere to the [single source of truth](#documentation-is-the-single-source-of-truth-ssot) principle. +However, you should avoid putting too many links on any page. Too many links can hinder readability. + +- Do not duplicate links on the same page. For example, on **Page A**, do not link to **Page B** multiple times. +- Avoid multiple links in a single paragraph. +- Avoid multiple links in a single task. +- On any one page, try not to use more than 15 links to other pages. +- Consider using [Related topics](../topic_types/index.md#related-topics) to reduce links that interrupt the flow of a task. +- Try to avoid anchor links to sections on the same page. Let users rely on the right navigation instead. + ### Links within the same repository To link to another page in the same repository, @@ -779,14 +788,18 @@ As much as possible, use text that follows one of these patterns: For example: -- `For more information, see [merge requests](../../../user/project/merge_requests/index.md).` -- `To create a review app, see [review apps](../../../ci/review_apps/index.md).` +- `For more information, see [merge requests](LINK).` +- `To create a review app, see [review apps](LINK).` You can expand on this text by using phrases like `For more information about this feature, see...` -Do not to use alternate phrases, like `Learn more about...` or -`To read more...`. +Do not to use the following constructions: + +- `Learn more about...` +- `To read more...`. +- `For more information, see the [Merge requests](LINK) page.` +- `For more information, see the [Merge requests](LINK) documentation.` #### Descriptive text rather than `here` @@ -1445,7 +1458,7 @@ For tab titles, be brief and consistent. Ensure they are parallel, and start eac For example: - `Linux package (Omnibus)`, `Helm chart (Kubernetes)` (when documenting configuration edits, follow the - [configuration edits guide](#configuration-documentation-for-different-installation-methods)) + [configuration edits guide](#how-to-document-different-installation-methods)) - `15.1 and earlier`, `15.2 and later` Until we implement automated testing for broken links to tabs ([Issue 1355](https://gitlab.com/gitlab-org/gitlab-docs/-/issues/1355)), do not link directly to a single tab, even though they do have unique URL parameters. @@ -1542,24 +1555,28 @@ If the document resides outside of the `doc/` directory, use the full path instead of the relative link: `https://docs.gitlab.com/ee/administration/restart_gitlab.html`. -### Installation guide +### How to document different installation methods -In [step 2 of the installation guide](../../../install/installation.md#2-ruby), -we install Ruby from source. To update the guide for a new Ruby version: +GitLab supports five official installation methods. If you're referring to +words as part of sentences and titles, use the following phrases: -- Change the version throughout the code block. -- Replace the sha256sum. It's available on the - [downloads page](https://www.ruby-lang.org/en/downloads/) of the Ruby website. - -### Configuration documentation for different installation methods +- Linux package +- Helm chart +- GitLab Operator +- Docker +- Self-compiled -GitLab supports four installation methods: +It's OK to add the explanatory parentheses when +[using tabs](#use-tabs-to-describe-a-self-managed-configuration-procedure): - Linux package (Omnibus) - Helm chart (Kubernetes) +- GitLab Operator (Kubernetes) - Docker - Self-compiled (source) +### Use tabs to describe a self-managed configuration procedure + Configuration procedures can require users to edit configuration files, reconfigure GitLab, or restart GitLab. In this case: @@ -1572,8 +1589,8 @@ GitLab, or restart GitLab. In this case: - The final step to reconfigure or restart GitLab can be used verbatim since it's the same every time. -You can copy and paste the following snippet when describing a configuration -edit: +When describing a configuration edit, you can use and edit to your liking the +following snippet: ````markdown diff --git a/doc/development/documentation/styleguide/word_list.md b/doc/development/documentation/styleguide/word_list.md index e64fd4df7ff..094de2bd724 100644 --- a/doc/development/documentation/styleguide/word_list.md +++ b/doc/development/documentation/styleguide/word_list.md @@ -57,6 +57,7 @@ Instead of: - In GitLab 14.4 and above... - In GitLab 14.4 and higher... +- In GitLab 14.4 and newer... ## access level @@ -82,6 +83,11 @@ If you can add the phrase "by zombies" to the phrase, the construction is passive. For example, `The button is selected by zombies` is passive. `Zombies select the button` is active. +## Admin Area + +Use title case for **Admin Area** to refer to the area of the UI that you access when you select **Main menu > Admin**. +This area of the UI says **Admin Area** at the top of the page and on the menu. + ## administrator Use **administrator access** instead of **admin** when talking about a user's access level. @@ -99,10 +105,9 @@ Instead of: - To do this thing, you must have the Admin role. -## Admin Area +## advanced search -Use title case **Admin Area** to refer to the area of the UI that you access when you select **Main menu > Admin**. -This area of the UI says **Admin Area** at the top of the page and on the menu. +Use lowercase for **advanced search** to refer to the faster, more efficient search across the entire GitLab instance. ## agent @@ -113,7 +118,7 @@ For example: - Install the agent in your cluster. - Select an agent from the list. -Do not use title case **GitLab Agent** or **GitLab Agent for Kubernetes**. +Do not use title case for **GitLab Agent** or **GitLab Agent for Kubernetes**. ## agent access token @@ -145,13 +150,6 @@ Instead of: This phrasing is more active and is from the user perspective, rather than the person who implemented the feature. [View details in the Microsoft style guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/a/allow-allows). -## Alpha - -Use uppercase for **Alpha**. For example: **The XYZ feature is in Alpha.** or **This Alpha release is ready to test.** - -You might also want to link to [this section](../../../policy/alpha-beta-support.md#alpha-features) -in the handbook when writing about Alpha features. - ## analytics Use lowercase for **analytics** and its variations, like **contribution analytics** and **issue analytics**. @@ -210,7 +208,7 @@ Try to avoid **below** when referring to an example or table in a documentation Use uppercase for **Beta**. For example: **The XYZ feature is in Beta.** or **This Beta release is ready to test.** -You might also want to link to [this section](../../../policy/alpha-beta-support.md#beta-features) +You might also want to link to [this section](../../../policy/alpha-beta-support.md#beta) in the handbook when writing about Beta features. ## blacklist @@ -282,8 +280,7 @@ CI/CD is always uppercase. No need to spell it out on first use. ## CI/CD minutes -Use **CI/CD minutes** instead of **CI minutes**, **pipeline minutes**, **pipeline minutes quota**, or -**CI pipeline minutes**. This decision was made in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/342813). +Do not use **CI/CD minutes**. This term was renamed to [**units of compute**](#units-of-compute). ## click @@ -293,7 +290,7 @@ Do not use **click**. Instead, use **select** with buttons, links, menu items, a ## cloud native When you're talking about using a Kubernetes cluster to host GitLab, you're talking about a **cloud-native version of GitLab**. -This version is different than the larger, more monolithic **Omnibus package** that is used to deploy GitLab. +This version is different than the larger, more monolithic **Linux package** that is used to deploy GitLab. You can also use **cloud-native GitLab** for short. It should be hyphenated and lowercase. @@ -301,6 +298,12 @@ You can also use **cloud-native GitLab** for short. It should be hyphenated and Use **collapse** instead of **close** when you are talking about expanding or collapsing a section in the UI. +## command line + +Use **From the command line** to introduce commands. + +Hyphenate when using as an adjective. For example, **a command-line tool**. + ## confirmation dialog Use **confirmation dialog** to describe the dialog box that asks you to confirm your action. For example: @@ -408,6 +411,7 @@ Use: Instead of: - In GitLab 14.1 and lower. +- In GitLab 14.1 and older. ## easily @@ -476,6 +480,13 @@ Instead of: Use **expand** instead of **open** when you are talking about expanding or collapsing a section in the UI. +## Experiment + +Use uppercase for **Experiment**. For example: **The XYZ feature is an Experiment.** or **This Experiment is ready to test.** + +You might also want to link to [this section](../../../policy/alpha-beta-support.md#experiment) +in the handbook when writing about Experiment features. + ## FAQ We want users to find information quickly, and they rarely search for the term **FAQ**. @@ -514,6 +525,17 @@ Filtering is different from [searching](#search). Do not use **foo** in product documentation. You can use it in our API and contributor documentation, but try to use a clearer and more meaningful example instead. +## fork + +A **fork** is a project that was created from a **upstream project** by using the +[forking process](../../../topics/git/terminology.md#fork). + +The **upstream project** (also known as the **source project**) and the **fork** have a **fork relationship** and are +**linked**. + +If the [**fork relationship** is removed](../../../user/project/repository/forking_workflow.md#unlink-a-fork), the +**fork** is **unlinked** from the **upstream project**. + ## future tense When possible, use present tense instead of future tense. For example, use **after you execute this command, GitLab displays the result** instead of **after you execute this command, GitLab will display the result**. ([Vale](../testing.md#vale) rule: [`FutureTense.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/FutureTense.yml)) @@ -530,9 +552,12 @@ Use title case for **Geo**. Do not make **GitLab** possessive (GitLab's). This guidance follows [GitLab Trademark Guidelines](https://about.gitlab.com/handbook/marketing/brand-and-product-marketing/brand/brand-activation/trademark-guidelines/). -## GitLab.com +## GitLab Flavored Markdown -**GitLab.com** refers to the GitLab instance managed by GitLab itself. +When possible, spell out [**GitLab Flavored Markdown**](../../../user/markdown.md). +([Vale](../testing.md#vale) rule: [`GLFM.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/SubstitutionSuggestions.yml)) + +If you must abbreviate, do not use **GFM**. Use **GLFM** instead. ## GitLab Helm chart, GitLab chart @@ -545,26 +570,41 @@ Do not use **the `gitlab` chart**, **the GitLab Chart**, or **the cloud-native c You use the **GitLab Helm chart** to deploy **cloud-native GitLab** in a Kubernetes cluster. -## GitLab Flavored Markdown +If you use it in a context of describing the +[different installation methods](index.md#how-to-document-different-installation-methods). +use `Helm chart (Kubernetes)`. -When possible, spell out [**GitLab Flavored Markdown**](../../../user/markdown.md). -([Vale](../testing.md#vale) rule: [`GLFM.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/SubstitutionSuggestions.yml)) +## GitLab Pages -If you must abbreviate, do not use **GFM**. Use **GLFM** instead. +For consistency and branding, use **GitLab Pages** rather than **Pages**. + +However, if you use **GitLab Pages** for the first mention on a page or in the UI, +you can use **Pages** thereafter. + +## GitLab Runner + +Use title case for **GitLab Runner**. This is the product you install. For more information about the decision for this usage, +see [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/233529). + +See also: + +- [runners](#runner-runners) +- [runner managers](#runner-manager-runner-managers) +- [runner workers](#runner-worker-runner-workers) ## GitLab SaaS **GitLab SaaS** refers to the product license that provides access to GitLab.com. It does not refer to the GitLab instance managed by GitLab itself. -## GitLab Runner - -Use title case for **GitLab Runner**. This is the product you install. See also [runners](#runner-runners) and [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/233529). - ## GitLab self-managed Use **GitLab self-managed** to refer to the product license for GitLab instances managed by customers themselves. +## GitLab.com + +**GitLab.com** refers to the GitLab instance managed by GitLab itself. + ## guide We want to speak directly to users. On `docs.gitlab.com`, do not use **guide** as part of a page title. @@ -632,13 +672,29 @@ Do not use Latin abbreviations. Use **that is** instead. ([Vale](../testing.md#v Do not use **in order to**. Use **to** instead. ([Vale](../testing.md#vale) rule: [`Wordy.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/Wordy.yml)) +## Installation from source + +When referring to the installation method using the self-compiled code, refer to it +as **self-compiled**. + +Use: + +- For self-compiled installations... + +Instead of: + +- For installations from source... + +For more information, see the +[different installation methods](index.md#how-to-document-different-installation-methods). + ## -ing words Remove **-ing** words whenever possible. They can be difficult to translate, and more precise terms are usually available. For example: - Instead of **The files using storage are deleted**, use **The files that use storage are deleted**. -- Instead of **Delete files using the Edit button**, use **Delete files by using the Edit button**. +- Instead of **Delete files using the Edit button**, use **Use the Edit button to delete files**. - Instead of **Replicating your server is required**, use **You must replicate your server**. ## issue @@ -699,6 +755,7 @@ Instead of: - In GitLab 14.1 and higher... - In GitLab 14.1 and above... +- In GitLab 14.1 and newer... ## list @@ -730,7 +787,7 @@ Do not use **limitations**. Use **known issues** instead. ## log in, log on -Do not use **log in** or **log on**. Use [sign in](#sign-in) instead. If the user interface has **Log in**, you can use it. +Do not use **log in** or **log on**. Use [sign in](#sign-in-sign-in) instead. If the user interface has **Log in**, you can use it. ## logged-in user, logged in user @@ -747,6 +804,7 @@ Use: Instead of: - In GitLab 14.1 and lower. +- In GitLab 14.1 and older. ## Maintainer @@ -803,6 +861,23 @@ Use lowercase for **merge requests**. If you use **MR** as the acronym, spell it Use lowercase for **milestones**. +## Minimal Access + +When writing about the Minimal Access role: + +- Use a capital **M** and a capital **A**. +- Write it out: + - Use: if you are assigned the Minimal Access role + - Instead of: if you are a Minimal Access user + +- When the Minimal Access role is the minimum required role: + - Use: at least the Minimal Access role + - Instead of: the Minimal Access role or higher + +Do not use bold. + +Do not use **Minimal Access permissions**. A user who is assigned the Minimal Access role has a set of associated permissions. + ## n/a, N/A, not applicable When possible, use **not applicable**. Spelling out the phrase helps non-English speaking users and avoids @@ -835,6 +910,20 @@ When the variable is **optional**: - You can set the variable. +## newer + +Do not use **newer** when talking about version numbers. + +Use: + +- In GitLab 14.4 and later... + +Instead of: + +- In GitLab 14.4 and higher... +- In GitLab 14.4 and above... +- In GitLab 14.4 and newer... + ## normal, normally Don't use **normal** to mean the usual, typical, or standard way of doing something. @@ -865,6 +954,35 @@ Instead of: - Note that you can change the settings. +## older + +Do not use **older** when talking about version numbers. + +Use: + +- In GitLab 14.1 and earlier. + +Instead of: + +- In GitLab 14.1 and lower. +- In GitLab 14.1 and older. + +## Omnibus GitLab + +When referring to the installation method that uses the Linux package, refer to it +as **Linux package**. + +Use: + +- For installations that use the Linux package... + +Instead of: + +- For installations that use Omnibus GitLab... + +For more information, see the +[different installation methods](index.md#how-to-document-different-installation-methods). + ## on When documenting how to select high-level UI elements, use the word **on**. @@ -933,6 +1051,17 @@ An Owner is the highest role a user can have. Use title case for the GitLab Package Registry. +## page + +If you write a phrase like, "On the **Issues** page," ensure steps for how to get to the page are nearby. Otherwise, people might not know what the **Issues** page is. + +The page name should be visible in the UI at the top of the page. +If it is not, you should be able to get the name from the breadcrumb. + +The docs should match the case in the UI, and the page name should be bold. For example: + +- On the **Test cases** page, ... + ## permissions Do not use [**roles**](#roles) and **permissions** interchangeably. Each user is assigned a role. Each role includes a set of permissions. @@ -947,6 +1076,17 @@ Use lowercase for **personal access token**. Do not use **please**. For details, see the [Microsoft style guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/p/please). +## Premium + +Use **Premium**, in uppercase, for the subscription tier. When you refer to **Premium** +in the context of other subscription tiers, follow [the subscription tier](#subscription-tier) guidance. + +## prerequisites + +Use **prerequisites** when documenting the steps before a task. Do not use **requirements**. + +For more information, see [the task topic type](../topic_types/task.md). + ## press Use **press** when talking about keyboard keys. For example: @@ -1009,6 +1149,12 @@ Do not use **Reporter permissions**. A user who is assigned the Reporter role ha Use title case for **Repository Mirroring**. +## requirements + +Use **prerequisites** when documenting the steps before a task. Do not use **requirements**. + +For more information, see [the task topic type](../topic_types/task.md). + ## respectively Avoid **respectively** and be more precise instead. @@ -1045,6 +1191,14 @@ Use lowercase for **runners**. These are the agents that run CI/CD jobs. See als When referring to runners, if you have to specify that the runners are installed on a customer's GitLab instance, use **self-managed** rather than **self-hosted**. +## runner manager, runner managers + +Use lowercase for **runner managers**. These are a type of runner that can create multiple runners for autoscaling. See also [GitLab Runner](#gitlab-runner). + +## runner worker, runner workers + +Use lowercase for **runner workers**. This is the process created by the runner on the host computing platform to run jobs. See also [GitLab Runner](#gitlab-runner). + ## (s) Do not use **(s)** to make a word optionally plural. It can slow down comprehension. For example: @@ -1120,14 +1274,17 @@ Use **setup** as a noun, and **set up** as a verb. For example: - Your remote office setup is amazing. - To set up your remote office correctly, consider the ergonomics of your work area. -## sign in +## sign in, sign-in -Use **sign in** or **sign in to**. +Use **sign in** or **sign in to** as a verb to describe the action of signing in. Do not use **sign on** or **sign into**, or **log on**, **log in**, or **log into**. If the user interface has different words, use those. +You can use **sign-in** as a noun or adjective. For example, **sign-in page** or +**sign-in restrictions**. + You can use **single sign-on**. ## sign up @@ -1202,6 +1359,13 @@ Use lowercase for **terminal**. For example: - Open a terminal. - From a terminal, run the `docker login` command. +## Terraform Module Registry + +Use title case for the GitLab Terraform Module Registry, but use lowercase `m` when +talking about non-specific modules. For example: + +- You can publish a Terraform module to your project's Terraform Module Registry. + ## text box Use **text box** instead of **field** or **box** when referring to the UI element. @@ -1277,6 +1441,24 @@ For example: See also [**enter**](#enter). +## Ultimate + +Use **Ultimate**, in uppercase, for the subscription tier. When you refer to **Ultimate** +in the context of other subscription tiers, follow [the subscription tier](#subscription-tier) guidance. + +## units of compute + +Use **units of compute** instead of these (or similar) terms: + +- **CI/CD minutes** +- **CI minutes** +- **pipeline minutes** +- **CI pipeline minutes** +- **pipeline minutes quota** + +This language is still being standardized in the documentation and UI beginning in March, 2023. +For more information, see [issue 5218](https://gitlab.com/gitlab-com/Product/-/issues/5218). + ## units of measurement Use a space between the number and the unit of measurement. For example, **128 GB**. @@ -1314,7 +1496,10 @@ See also [downgrade](#downgrade) and [roll back](#roll-back). ## upper left, upper right -Use **upper left** and **upper right** instead of **top left** and **top right**. Hyphenate as adjectives (for example, **upper-left corner**). +Use **upper-left corner** and **upper-right corner** to provide direction in the UI. +If the UI element is not in a corner, use **upper left** and **upper right**. + +Do not use **top left** and **top right**. For details, see the [Microsoft style guide](https://learn.microsoft.com/en-us/style-guide/a-z-word-list-term-collections/u/upper-left-upper-right). diff --git a/doc/development/documentation/testing.md b/doc/development/documentation/testing.md index 4f61b2424eb..4a7e206da20 100644 --- a/doc/development/documentation/testing.md +++ b/doc/development/documentation/testing.md @@ -49,17 +49,21 @@ To run tests locally, it's important to: ### Lint checks Lint checks are performed by the [`lint-doc.sh`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/scripts/lint-doc.sh) -script and can be executed as follows: +script and can be executed with the help of a Rake task as follows: -1. Go to the `gitlab` directory. +1. Go to your `gitlab` directory. 1. Run: ```shell - MD_DOC_PATH=path/to/my_doc.md scripts/lint-doc.sh + rake lint:markdown ``` -Where `MD_DOC_PATH` points to the file or directory you would like to run lint checks for. -If you omit it completely, it defaults to the `doc/` directory. +To specify a single file or directory you would like to run lint checks for, run: + +```shell +MD_DOC_PATH=path/to/my_doc.md rake lint:markdown +``` + The output should be similar to: ```plaintext @@ -77,7 +81,7 @@ The output should be similar to: This requires you to either: - Have the [required lint tools installed](#local-linters) on your computer. -- A working Docker installation, in which case an image with these tools pre-installed is used. +- A working Docker or containerd installation, to use an image with these tools pre-installed. ### Documentation link tests diff --git a/doc/development/documentation/topic_types/concept.md b/doc/development/documentation/topic_types/concept.md index 66af8780b9b..c9aedf940a2 100644 --- a/doc/development/documentation/topic_types/concept.md +++ b/doc/development/documentation/topic_types/concept.md @@ -37,9 +37,19 @@ Do not include links to related tasks. The navigation provides links to tasks. ## Concept topic titles -For the title text, use a noun. For example, `Widgets` or `GDK dependency management`. +For the title text, use a noun. For example: -If a noun is ambiguous, you can add a gerund. For example, `Documenting versions` instead of `Versions`. +- `Widgets` +- `GDK dependency management` + +If you need more descriptive words, use the `ion` version of the word, rather than `ing`. For example: + +- `Object migration` instead of `Migrating objects` or `Migrate objects` + +Words that end in `ing` are hard to translate and take up more space, and active verbs are used for task topics. +For details, see [the Google style guide](https://developers.google.com/style/headings#heading-and-title-text). + +### Titles to avoid Avoid these topic titles: diff --git a/doc/development/documentation/topic_types/glossary.md b/doc/development/documentation/topic_types/glossary.md new file mode 100644 index 00000000000..4985101a391 --- /dev/null +++ b/doc/development/documentation/topic_types/glossary.md @@ -0,0 +1,70 @@ +--- +stage: none +group: Style Guide +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Glossary topic type + +A glossary provides a list of unfamiliar terms and their definitions to help users understand a specific +GitLab feature. + +Each glossary item provides a single term and its associated definition. The definition should answer the questions: + +- **What** is this? +- **Why** would you use it? + +For glossary terms: + +- Do not use jargon. +- Do not use internal terminology or acronyms. +- Ensure the correct usage is defined in the [word list](../styleguide/word_list.md). + +## Alternatives to glossaries + +Glossaries should provide short, concise term-definition pairs. + +- If a definition requires more than a brief explanation, use a concept topic instead. +- If you find yourself explaining how to use the feature, use a task topic instead. + +## Glossary example + +Glossary topics should be in this format. Use an unordered list primarily. You can use a table if you need to apply +additional categorization. + +Try to include glossary topics on pages that explain the feature, rather than as a standalone page. + +```markdown +## FeatureName glossary + +This glossary provides definitions for terms related to FeatureName. + +- **Term A**: Term A does this thing. +- **Term B**: Term B does this thing. +- **Term C**: Term C does this thing. +``` + +If you use the table format: + +```markdown +## FeatureName glossary + +This glossary provides definitions for terms related to FeatureName. + +| Term | Definition | Additional category | +|--------|-------------------------|---------------------| +| Term A | Term A does this thing. | | +| Term B | Term B does this thing. | | +| Term C | Term C does this thing. | | +``` + +## Glossary topic titles + +Use `FeatureName glossary`. + +Don't use alternatives to `glossary`. For example: + +- `Terminology` +- `Glossary of terms` +- `Glossary of common terms` +- `Definitions` diff --git a/doc/development/documentation/topic_types/index.md b/doc/development/documentation/topic_types/index.md index cfc231c268a..37ed876fc59 100644 --- a/doc/development/documentation/topic_types/index.md +++ b/doc/development/documentation/topic_types/index.md @@ -16,22 +16,52 @@ Each topic on a page should be one of the following topic types: Even if a page is short, the page usually starts with a concept and then includes a task or reference topic. -The tech writing team sometimes uses the acronym `CTRT` to refer to our topic types. +The tech writing team sometimes uses the acronym `CTRT` to refer to the topic types. The acronym refers to the first letter of each topic type. -In addition to the four primary topic types, we also have a page type for -[Tutorials](tutorial.md) and [Get started](#get-started). +## Other page and topic types + +In addition to the four primary topic types, you can use the following: + +- Page types: [Tutorials](tutorial.md) and [Get started](#get-started) +- Topic type: [Related topics](#related-topics) +- Page or topic type: [Glossaries](glossary.md) + +## Pages and topics to avoid + +You should avoid: + +- Pages that are exclusively links to other pages. The only exception are + top-level pages that aid with navigation. +- Topics that have one or two sentences only. In these cases: + - Incorporate the information in another topic. + - If the sentence links to another page, use a [Related topics](#related-topics) link instead. + +## Topic title guidelines + +In general, for topic titles: + +- Be clear and direct. Make every word count. +- Use articles and prepositions. +- Follow [capitalization](../styleguide/index.md#capitalization) guidelines. +- Do not repeat text from earlier topic titles. For example, if the page is about merge requests, + instead of `Troubleshooting merge requests`, use only `Troubleshooting`. + +See also [guidelines for heading levels in Markdown](../styleguide/index.md#heading-levels-in-markdown). ## Related topics If inline links are not sufficient, you can create a topic called **Related topics** and include an unordered list of related topics. This topic should be above the Troubleshooting section. +Links in this section should be brief and scannable. They are usually not +full sentences, and so should not end in a period. + ```markdown ## Related topics -- [Configure your pipeline](link-to-topic). -- [Trigger a pipeline manually](link-to-topic). +- [CI/CD variables](link-to-topic) +- [Environment variables](link-to-topic) ``` ## Get started @@ -57,22 +87,3 @@ consider using subsections for each distinct task. In the left nav, use `Get started` as the text. On the page itself, spell out the full name. For example, `Get started with application security`. - -## Topics and resources - -Some pages are solely a list of links to other documentation. - -We do not encourage this page type. Lists of links can get out-of-date quickly -and offer little value to users, who prefer to search to find information. - -## Topic title guidelines - -In general, for topic titles: - -- Be clear and direct. Make every word count. -- Use articles and prepositions. -- Follow [capitalization](../styleguide/index.md#capitalization) guidelines. -- Do not repeat text from earlier topic titles. For example, if the page is about merge requests, - instead of `Troubleshooting merge requests`, use only `Troubleshooting`. - -See also [guidelines for heading levels in Markdown](../styleguide/index.md#heading-levels-in-markdown). diff --git a/doc/development/documentation/topic_types/task.md b/doc/development/documentation/topic_types/task.md index 8d23a5f322e..2ddfd841ec3 100644 --- a/doc/development/documentation/topic_types/task.md +++ b/doc/development/documentation/topic_types/task.md @@ -60,6 +60,22 @@ For example, `Create an issue`. If several tasks on a page share prerequisites, you can create a separate topic with the title `Prerequisites`. +## When a task has only one step + +If you need to write a task that has only one step, make that step an unordered list item. +This format helps the step stand out, while keeping it consistent with the rules +for lists. + +For example: + +```markdown +# Create a merge request + +To create a merge request: + +- In the upper-right corner, select **New merge request**. +``` + ### When more than one way exists to perform a task If more than one way exists to perform a task in the UI, you should @@ -139,4 +155,4 @@ how to write the phrase for each role. ## Related topics -- [View the format for writing task steps](../styleguide/index.md#navigation). +- [How to write task steps](../styleguide/index.md#navigation) diff --git a/doc/development/documentation/topic_types/tutorial.md b/doc/development/documentation/topic_types/tutorial.md index 1b1426a0465..2d57029b786 100644 --- a/doc/development/documentation/topic_types/tutorial.md +++ b/doc/development/documentation/topic_types/tutorial.md @@ -13,9 +13,28 @@ In general, you might consider using a tutorial when: of sub-steps. - The steps cover a variety of GitLab features or third-party tools. -Tutorials are learning aids that complement our core documentation. -They do not introduce new features. -Always use the primary [topic types](index.md) to document new features. +## Tutorial guidance + +- Tutorials are not [tasks](task.md). A task gives instructions for one procedure. + A tutorial combines multiple tasks to achieve a specific goal. +- Tutorials provide a working example. Ideally the reader can create the example the + tutorial describes. If they can't replicate it exactly, they should be able + to replicate something similar. +- Tutorials do not introduce new features. +- Tutorials do not need to adhere to the Single Source of Truth tenet. While it's not + ideal to duplicate content that is available elsewhere, it's worse to force the reader to + leave the page to find what they need. + +## Tutorial file name and location + +For tutorial Markdown files, you can either: + +- Save the file in a directory with the product documentation. +- Create a subfolder under `doc/tutorials` and name the file `index.md`. + +In the left nav, add the tutorial near the relevant feature documentation. + +Add a link to the tutorial on one of the [tutorial pages](../../../tutorials/index.md). ## Tutorial format @@ -31,7 +50,9 @@ To create a website: 1. [Do the first task](#do-the-first-task) 1. [Do the second task](#do-the-second-task) -Prerequisites (optional): +## Prerequisites + +This topic is optional. - Thing 1 - Thing 2 @@ -57,7 +78,7 @@ To do step 2: ``` An example of a tutorial that follows this format is -[Tutorial: Make your first Git commit](../../../tutorials/make_your_first_git_commit.md). +[Tutorial: Make your first Git commit](../../../tutorials/make_first_git_commit/index.md). ## Tutorial page title diff --git a/doc/development/documentation/versions.md b/doc/development/documentation/versions.md index 30a0d4d4cf4..859e7265a2a 100644 --- a/doc/development/documentation/versions.md +++ b/doc/development/documentation/versions.md @@ -13,7 +13,7 @@ including when features were introduced and when they were updated or removed. ## View older documentation versions Previous versions of the documentation are available on `docs.gitlab.com`. -To view a previous version, select the **Versions** button in the upper right. +To view a previous version, in the upper-right corner, select **Versions**. To view versions that are not available on `docs.gitlab.com`: @@ -89,8 +89,9 @@ voters to agree. When features are deprecated and removed, update the related documentation. -API documentation follows these guidelines, but the GraphQL docs use -a [separate process](../api_graphql_styleguide.md#deprecating-schema-items). +NOTE: +A separate process exists for [GraphQL docs](../api_graphql_styleguide.md#deprecating-schema-items) +and [REST API docs](restful_api_styleguide.md#deprecations). ### Deprecate a page or topic diff --git a/doc/development/documentation/workflow.md b/doc/development/documentation/workflow.md index fe5453a4a58..1a4194aebd9 100644 --- a/doc/development/documentation/workflow.md +++ b/doc/development/documentation/workflow.md @@ -4,148 +4,39 @@ group: unassigned info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# How to update GitLab documentation +# Documentation workflow -Anyone can contribute to the GitLab documentation! You can create a merge request for documentation when: +Documentation at GitLab follows a workflow. -- You find errors or other room for improvement in existing documentation. -- You have an idea for all-new documentation that would help a GitLab user or administrator to - accomplish their work with GitLab. +## Before merging -If you are working on a feature or enhancement, use the -[feature workflow process described in the GitLab Handbook](https://about.gitlab.com/handbook/product/ux/technical-writing/workflow/#documentation-for-a-product-change). +Ensure your documentation includes: -## Do not use ChatGPT or AI-generated content for the docs - -GitLab documentation is distributed under the [CC BY-SA 4.0 license](https://creativecommons.org/licenses/by-sa/4.0/), which presupposes that GitLab owns the documentation. - -Under current law in the US and the EU, it’s possible that AI-generated works might either: - -- not be owned by anyone because they weren't created by a human, or -- belong to the AI training data’s creator, if the AI verbatim reproduces content that it trained on - -If the documentation contains AI-generated content, GitLab probably wouldn't own this content, which would risk invalidating the CC BY-SA 4.0 license. +- [Product badges](styleguide/index.md#product-tier-badges). +- The GitLab [version](versions.md) that introduced the feature. +- Accurate [links](styleguide/index.md#links). +- Accurate [user permissions](../../user/permissions.md). -Contributions to GitLab documentation are made under either our [DCO or our CLA terms](https://about.gitlab.com/community/contribute/dco-cla/). In both, contributors have to make certain certifications about the authorship of their work that they can't validly make for AI-generated text. - -For these reasons, do not add AI-generated content to the documentation. - -## How to update the docs - -If you are not a GitLab team member, or do not have the Developer role for the GitLab repository, to update GitLab documentation: - -1. Select an [issue](https://about.gitlab.com/handbook/product/ux/technical-writing/#community-contribution-opportunities) you'd like to work on. - - For a Hackathon, mention `@docs-hackathon` in a comment and ask for the issue to be assigned to you. - To be fair to other contributors, if you see someone has already asked to work on the issue, choose another issue. - - If you're not taking part in a Hackathon, you don't need an issue to open a merge request. - If you are looking for issues to work on and don't see any that suit you, you can always fix [Vale](testing.md#vale) issues. -1. Go to the [GitLab repository](https://gitlab.com/gitlab-org/gitlab). -1. In the upper right, select **Fork**. Forking makes a copy of the repository on GitLab.com. -1. In your fork, find the documentation page in the `\doc` directory. -1. If you know Git, make your changes and open a merge request. - If not, follow these steps: - 1. In the upper right, select **Edit** if it is visible. If it is not, select the down arrow (**{chevron-lg-down}**) next to **Open in Web IDE** or **Gitpod**, and select **Edit**. - 1. In the **Commit message** text box, enter a commit message. Use 3-5 words, start with a capital letter, and do not end with a period. - 1. Select **Commit changes**. - 1. On the left sidebar, select **Merge requests**. - 1. Select **New merge request**. - 1. For the source branch, select your fork and branch. If you did not create a branch, select `master`. - For the target branch, select the [GitLab repository](https://gitlab.com/gitlab-org/gitlab) `master` branch. - 1. Select **Compare branches and continue**. A new merge request opens. - 1. Select the **Documentation** template. In the description, write a brief summary of the changes and link to the related issue, if there is one. - 1. Select **Create merge request**. - -If you need help while working on the page, view: - -- The [Style Guide](styleguide/index.md). -- The [Word list](styleguide/word_list.md) -- The [Markdown Guide](https://about.gitlab.com/handbook/markdown-guide/). - -### Ask for help - -Ask for help from the Technical Writing team if you: - -- Need help to choose the correct place for documentation. -- Want to discuss a documentation idea or outline. -- Want to request any other help. - -To identify someone who can help you: - -1. Locate the Technical Writer for the relevant - [DevOps stage group](https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments). -1. Either: - - If urgent help is required, directly assign the Technical Writer in the issue or in the merge request. - - If non-urgent help is required, ping the Technical Writer in the issue or merge request. - -If you are a member of the GitLab Slack workspace, you can request help in `#docs`. +Ensure you've followed the [style guide](styleguide/index.md) and [word list](styleguide/word_list.md). ## Documentation labels -When you author an issue or merge request, you must add these labels: +When you author an issue or merge request, choose the +[Documentation template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/merge_request_templates/Documentation.md). +It includes these labels, which are added to the merge request: -- A [type label](../contributing/issue_workflow.md#type-labels), either `~"type::feature"` or `~"type::maintenance"`. -- A [stage label](../contributing/issue_workflow.md#stage-labels) and [group label](../contributing/issue_workflow.md#group-labels). +- A [type label](../labels/index.md#type-labels), either `~"type::feature"` or `~"type::maintenance"`. +- A [stage label](../labels/index.md#stage-labels) and [group label](../labels/index.md#group-labels). For example, `~devops::create` and `~group::source code`. -- A `~documentation` [specialization label](../contributing/issue_workflow.md#specialization-labels). +- A `~documentation` [specialization label](../labels/index.md#specialization-labels). A member of the Technical Writing team adds these labels: - A [documentation scoped label](../../user/project/labels.md#scoped-labels) with the `docs::` prefix. For example, `~docs::improvement`. -- The [`~Technical Writing` team label](../contributing/issue_workflow.md#team-labels). - -## Reviewing and merging - -Anyone with the Maintainer role to the relevant GitLab project can -merge documentation changes. Maintainers must make a good-faith effort to ensure that the content: - -- Is clear and sufficiently easy for the intended audience to navigate and understand. -- Meets the [Documentation Guidelines](index.md) and [Style Guide](styleguide/index.md). +- The [`~Technical Writing` team label](../labels/index.md#team-labels). -If the author or reviewer has any questions, they can mention the writer who is assigned to the relevant -[DevOps stage group](https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments). - -The process involves the following: - -- Primary Reviewer. Review by a [code reviewer](https://about.gitlab.com/handbook/engineering/projects/) - or other appropriate colleague to confirm accuracy, clarity, and completeness. This can be skipped - for minor fixes without substantive content changes. -- Technical Writer (Optional). If not completed for a merge request before merging, must be scheduled - post-merge. Schedule post-merge reviews only if an urgent merge is required. To request a: - - Pre-merge review, assign the Technical Writer listed for the applicable - [DevOps stage group](https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments). - - Post-merge review, see [Post-merge reviews](#post-merge-reviews). -- Maintainer. For merge requests, Maintainers: - - Can always request any of the above reviews. - - Review before or after a Technical Writer review. - - Ensure the given release milestone is set. - - Ensure the appropriate labels are applied, including any required to pick a merge request into - a release. - - Ensure that, if there has not been a Technical Writer review completed or scheduled, they - [create the required issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Doc%20Review), assign it to the Technical Writer of the given stage group, - and link it from the merge request. - -The process is reflected in the **Documentation** -[merge request template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/merge_request_templates/Documentation.md). - -### Before merging - -Ensure the following if skipping an initial Technical Writer review: - -- [Product badges](styleguide/index.md#product-tier-badges) are applied. -- The GitLab [version](versions.md) that - introduced the feature is included. -- Changes to topic titles don't affect in-app hyperlinks. -- Specific [user permissions](../../user/permissions.md) are documented. -- New documents are linked from higher-level indexes, for discoverability. -- The style guide is followed: - - For [directories and files](site_architecture/folder_structure.md). - - For [images](styleguide/index.md#images). - -Merge requests that change the location of documentation must always be reviewed by a Technical -Writer before merging. - -### Post-merge reviews +## Post-merge reviews If not assigned to a Technical Writer for review prior to merging, a review must be scheduled immediately after merge by the developer or maintainer. For this, @@ -174,8 +65,25 @@ Remember: - The Technical Writer can also help decide that documentation can be merged without Technical writer review, with the review to occur soon after merge. -## Other ways to help +## Do not use ChatGPT or AI-generated content for the docs + +GitLab documentation is distributed under the [CC BY-SA 4.0 license](https://creativecommons.org/licenses/by-sa/4.0/), which presupposes that GitLab owns the documentation. -If you have ideas for further documentation resources please -[create an issue](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Documentation) -using the Documentation template. +Under current law in the US and the EU, it’s possible that AI-generated works might either: + +- not be owned by anyone because they weren't created by a human, or +- belong to the AI training data's creator, if the AI verbatim reproduces content that it trained on + +If the documentation contains AI-generated content, GitLab probably wouldn't own this content, which would risk invalidating the CC BY-SA 4.0 license. + +Contributions to GitLab documentation are made under either our [DCO or our CLA terms](https://about.gitlab.com/community/contribute/dco-cla/). In both, contributors have to make certain certifications about the authorship of their work that they can't validly make for AI-generated text. + +For these reasons, do not add AI-generated content to the documentation. + +## Related topics + +- [Reviews and levels of edit](https://about.gitlab.com/handbook/product/ux/technical-writing/#reviews) +- [Technical writing assignments](https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments) +- The [Style Guide](styleguide/index.md) +- The [Word list](styleguide/word_list.md) +- The [Markdown Guide](https://about.gitlab.com/handbook/markdown-guide/) diff --git a/doc/development/ee_features.md b/doc/development/ee_features.md index 4eb5bedef1c..bc997c37e66 100644 --- a/doc/development/ee_features.md +++ b/doc/development/ee_features.md @@ -18,6 +18,49 @@ info: To determine the technical writer assigned to the Stage/Group associated w [EE features list](https://about.gitlab.com/features/). +## SaaS only feature + +Use the following guidelines when you develop a feature that is only applicable for SaaS (for example, a CustomersDot integration). + +1. It is recommended you use an application setting. This enables + granular settings so that each SaaS instance can switch things according to + their need. +1. If application setting is not possible, helpers such as `Gitlab.com?` can be + used. However, this comes with drawbacks as listed in the epic to + [remove these helpers](https://gitlab.com/groups/gitlab-org/-/epics/7374). + 1. Consider performance and availability impact on other SaaS instances. For example, + [GitLab JH overrides](https://jihulab.com/gitlab-cn/gitlab/-/blob/main-jh/jh/lib/jh/gitlab/saas.rb) + SaaS helpers, so that it returns true for `Gitlab.com?`. + +### Simulate a SaaS instance + +If you're developing locally and need your instance to simulate the SaaS (GitLab.com) +version of the product: + +1. Export this environment variable: + + ```shell + export GITLAB_SIMULATE_SAAS=1 + ``` + + There are many ways to pass an environment variable to your local GitLab instance. + For example, you can create an `env.runit` file in the root of your GDK with the above snippet. + +1. Enable **Allow use of licensed EE features** to make licensed EE features available to projects + only if the project namespace's plan includes the feature. + + 1. Visit **Admin > Settings > General**. + 1. Expand **Account and limit**. + 1. Select the **Allow use of licensed EE features** checkbox. + 1. Select **Save changes**. + +1. Ensure the group you want to test the EE feature for is actually using an EE plan: + 1. On the top bar, select **Main menu > Admin**. + 1. On the left sidebar, select **Overview > Groups**. + 1. Identify the group you want to modify, and select **Edit**. + 1. Scroll to **Permissions and group features**. For **Plan**, select `Ultimate`. + 1. Select **Save changes**. + ## Implement a new EE feature If you're developing a GitLab Premium or GitLab Ultimate licensed feature, use these steps to @@ -74,9 +117,9 @@ To guard your licensed feature: 1. Optional. If your global feature is also available to namespaces with a paid plan, combine two feature identifiers to allow both administrators and group users. For example: - ```ruby - License.feature_available?(:my_feature_name) || group.licensed_feature_available?(:my_feature_name_for_namespace) # Both admins and group members can see this EE feature - ``` + ```ruby + License.feature_available?(:my_feature_name) || group.licensed_feature_available?(:my_feature_name_for_namespace) # Both admins and group members can see this EE feature + ``` ### Simulate a CE instance when unlicensed @@ -100,15 +143,15 @@ To simulate a CE instance without deleting the license in a GDK: 1. Create an `env.runit` file in the root of your GDK with the line: - ```shell - export FOSS_ONLY=1 - ``` + ```shell + export FOSS_ONLY=1 + ``` 1. Then restart the GDK: - ```shell - gdk restart rails && gdk restart webpack - ``` + ```shell + gdk restart rails && gdk restart webpack + ``` Remove the line in `env.runit` if you want to revert back to an EE installation, and repeat step 2. @@ -137,35 +180,6 @@ To do so: bin/rspec spec/features/ ``` -### Simulate a SaaS instance - -If you're developing locally and need your instance to simulate the SaaS (GitLab.com) -version of the product: - -1. Export this environment variable: - - ```shell - export GITLAB_SIMULATE_SAAS=1 - ``` - - There are many ways to pass an environment variable to your local GitLab instance. - For example, you can create an `env.runit` file in the root of your GDK with the above snippet. - -1. Enable **Allow use of licensed EE features** to make licensed EE features available to projects - only if the project namespace's plan includes the feature. - - 1. Visit **Admin > Settings > General**. - 1. Expand **Account and limit**. - 1. Select the **Allow use of licensed EE features** checkbox. - 1. Click **Save changes**. - -1. Ensure that the group for which you want to test the EE feature, is actually using an EE plan: - 1. On the top bar, select **Main menu > Admin**. - 1. On the left sidebar, select **Overview > Groups**. - 1. Identify the group you want to modify, and select **Edit**. - 1. Scroll to **Permissions and group features**. For **Plan**, select `Ultimate`. - 1. Select **Save changes**. - ### Run CI pipelines in a FOSS context By default, merge request pipelines for development run in an EE-context only. If you are @@ -580,11 +594,11 @@ Resolving an EE template path that is relative to the CE view path doesn't work. For `render` and `render_if_exists`, they search for the EE partial first, and then CE partial. They would only render a particular partial, not all partials with the same name. We could take the advantage of this, so that -the same partial path (for example, `shared/issuable/form/default_templates`) could +the same partial path (for example, `projects/settings/archive`) could be referring to the CE partial in CE (that is, -`app/views/shared/issuable/form/_default_templates.html.haml`), while EE +`app/views/projects/settings/_archive.html.haml`), while EE partial in EE (that is, -`ee/app/views/shared/issuable/form/_default_templates.html.haml`). This way, +`ee/app/views/projects/settings/_archive.html.haml`). This way, we could show different things between CE and EE. However sometimes we would also want to reuse the CE partial in EE partial @@ -594,21 +608,19 @@ would be tedious to do so. In this case, we could as well just use `render_ce` which would ignore any EE partials. One example would be -`ee/app/views/shared/issuable/form/_default_templates.html.haml`: +`ee/app/views/projects/settings/_archive.html.haml`: ```haml -- if @project.feature_available?(:issuable_default_templates) - = render_ce 'shared/issuable/form/default_templates' -- elsif show_promotions? - = render 'shared/promotions/promote_issue_templates' +- return if @project.marked_for_deletion? += render_ce 'projects/settings/archive' ``` In the above example, we can't use -`render 'shared/issuable/form/default_templates'` because it would find the +`render 'projects/settings/archive'` because it would find the same EE partial, causing infinite recursion. Instead, we could use `render_ce` so it ignores any partials in `ee/` and then it would render the CE partial -(that is, `app/views/shared/issuable/form/_default_templates.html.haml`) -for the same path (that is, `shared/issuable/form/default_templates`). This way +(that is, `app/views/projects/settings/_archive.html.haml`) +for the same path (that is, `projects/settings/archive`). This way we could easily wrap around the CE partial. ### Code in `lib/gitlab/background_migration/` diff --git a/doc/development/elasticsearch.md b/doc/development/elasticsearch.md deleted file mode 100644 index 935964a9a90..00000000000 --- a/doc/development/elasticsearch.md +++ /dev/null @@ -1,625 +0,0 @@ ---- -stage: Data Stores -group: Global Search -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments ---- - -# Elasticsearch knowledge - -This area is to maintain a compendium of useful information when working with Elasticsearch. - -Information on how to enable Elasticsearch and perform the initial indexing is in -the [Elasticsearch integration documentation](../integration/advanced_search/elasticsearch.md#enable-advanced-search). - -## Deep Dive - -In June 2019, Mario de la Ossa hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/issues/1`) on the GitLab [Elasticsearch integration](../integration/advanced_search/elasticsearch.md) to share his domain specific knowledge with anyone who may work in this part of the codebase in the future. You can find the [recording on YouTube](https://www.youtube.com/watch?v=vrvl-tN2EaA), and the slides on [Google Slides](https://docs.google.com/presentation/d/1H-pCzI_LNrgrL5pJAIQgvLX8Ji0-jIKOg1QeJQzChug/edit) and in [PDF](https://gitlab.com/gitlab-org/create-stage/uploads/c5aa32b6b07476fa8b597004899ec538/Elasticsearch_Deep_Dive.pdf). Everything covered in this deep dive was accurate as of GitLab 12.0, and while specific details may have changed since then, it should still serve as a good introduction. - -In August 2020, a second Deep Dive was hosted, focusing on [GitLab-specific architecture for multi-indices support](#zero-downtime-reindexing-with-multiple-indices). The [recording on YouTube](https://www.youtube.com/watch?v=0WdPR9oB2fg) and the [slides](https://lulalala.gitlab.io/gitlab-elasticsearch-deepdive/) are available. Everything covered in this deep dive was accurate as of GitLab 13.3. - -## Supported Versions - -See [Version Requirements](../integration/advanced_search/elasticsearch.md#version-requirements). - -Developers making significant changes to Elasticsearch queries should test their features against all our supported versions. - -## Setting up development environment - -See the [Elasticsearch GDK setup instructions](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/doc/howto/elasticsearch.md) - -## Helpful Rake tasks - -- `gitlab:elastic:test:index_size`: Tells you how much space the current index is using, as well as how many documents are in the index. -- `gitlab:elastic:test:index_size_change`: Outputs index size, reindexes, and outputs index size again. Useful when testing improvements to indexing size. - -Additionally, if you need large repositories or multiple forks for testing, please consider [following these instructions](rake_tasks.md#extra-project-seed-options) - -## How does it work? - -The Elasticsearch integration depends on an external indexer. We ship an [indexer written in Go](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). The user must trigger the initial indexing via a Rake task but, after this is done, GitLab itself will trigger reindexing when required via `after_` callbacks on create, update, and destroy that are inherited from [`/ee/app/models/concerns/elastic/application_versioned_search.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/models/concerns/elastic/application_versioned_search.rb). - -After initial indexing is complete, create, update, and delete operations for all models except projects (see [#207494](https://gitlab.com/gitlab-org/gitlab/-/issues/207494)) are tracked in a Redis [`ZSET`](https://redis.io/docs/manual/data-types/#sorted-sets). A regular `sidekiq-cron` `ElasticIndexBulkCronWorker` processes this queue, updating many Elasticsearch documents at a time with the [Bulk Request API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html). - -Search queries are generated by the concerns found in [`ee/app/models/concerns/elastic`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/app/models/concerns/elastic). These concerns are also in charge of access control, and have been a historic source of security bugs so please pay close attention to them! - -## Existing Analyzers/Tokenizers/Filters - -These are all defined in [`ee/lib/elastic/latest/config.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/elastic/latest/config.rb) - -### Analyzers - -#### `path_analyzer` - -Used when indexing blobs' paths. Uses the `path_tokenizer` and the `lowercase` and `asciifolding` filters. - -Please see the `path_tokenizer` explanation below for an example. - -#### `sha_analyzer` - -Used in blobs and commits. Uses the `sha_tokenizer` and the `lowercase` and `asciifolding` filters. - -Please see the `sha_tokenizer` explanation later below for an example. - -#### `code_analyzer` - -Used when indexing a blob's filename and content. Uses the `whitespace` tokenizer and the filters: [`code`](#code), `lowercase`, and `asciifolding` - -The `whitespace` tokenizer was selected to have more control over how tokens are split. For example the string `Foo::bar(4)` needs to generate tokens like `Foo` and `bar(4)` to be properly searched. - -Please see the `code` filter for an explanation on how tokens are split. - -NOTE: -The [Elasticsearch `code_analyzer` doesn't account for all code cases](../integration/advanced_search/elasticsearch_troubleshooting.md#elasticsearch-code_analyzer-doesnt-account-for-all-code-cases). - -#### `code_search_analyzer` - -Not directly used for indexing, but rather used to transform a search input. Uses the `whitespace` tokenizer and the `lowercase` and `asciifolding` filters. - -### Tokenizers - -#### `sha_tokenizer` - -This is a custom tokenizer that uses the [`edgeNGram` tokenizer](https://www.elastic.co/guide/en/elasticsearch/reference/5.5/analysis-edgengram-tokenizer.html) to allow SHAs to be searchable by any sub-set of it (minimum of 5 chars). - -Example: - -`240c29dc7e` becomes: - -- `240c2` -- `240c29` -- `240c29d` -- `240c29dc` -- `240c29dc7` -- `240c29dc7e` - -#### `path_tokenizer` - -This is a custom tokenizer that uses the [`path_hierarchy` tokenizer](https://www.elastic.co/guide/en/elasticsearch/reference/5.5/analysis-pathhierarchy-tokenizer.html) with `reverse: true` to allow searches to find paths no matter how much or how little of the path is given as input. - -Example: - -`'/some/path/application.js'` becomes: - -- `'/some/path/application.js'` -- `'some/path/application.js'` -- `'path/application.js'` -- `'application.js'` - -### Filters - -#### `code` - -Uses a [Pattern Capture token filter](https://www.elastic.co/guide/en/elasticsearch/reference/5.5/analysis-pattern-capture-tokenfilter.html) to split tokens into more easily searched versions of themselves. - -Patterns: - -- `"(\\p{Ll}+|\\p{Lu}\\p{Ll}+|\\p{Lu}+)"`: captures CamelCase and lowerCamelCase strings as separate tokens -- `"(\\d+)"`: extracts digits -- `"(?=([\\p{Lu}]+[\\p{L}]+))"`: captures CamelCase strings recursively. For example: `ThisIsATest` => `[ThisIsATest, IsATest, ATest, Test]` -- `'"((?:\\"|[^"]|\\")*)"'`: captures terms inside quotes, removing the quotes -- `"'((?:\\'|[^']|\\')*)'"`: same as above, for single-quotes -- `'\.([^.]+)(?=\.|\s|\Z)'`: separate terms with periods in-between -- `'([\p{L}_.-]+)'`: some common chars in file names to keep the whole filename intact (for example `my_file-ñame.txt`) -- `'([\p{L}\d_]+)'`: letters, numbers and underscores are the most common tokens in programming. Always capture them greedily regardless of context. - -## Gotchas - -- Searches can have their own analyzers. Remember to check when editing analyzers -- `Character` filters (as opposed to token filters) always replace the original character, so they're not a good choice as they can hinder exact searches - -## Zero downtime reindexing with multiple indices - -NOTE: -This is not applicable yet as multiple indices functionality is not fully implemented. - -Currently GitLab can only handle a single version of setting. Any setting/schema changes would require reindexing everything from scratch. Since reindexing can take a long time, this can cause search functionality downtime. - -To avoid downtime, GitLab is working to support multiple indices that -can function at the same time. Whenever the schema changes, the administrator -will be able to create a new index and reindex to it, while searches -continue to go to the older, stable index. Any data updates will be -forwarded to both indices. Once the new index is ready, an administrator can -mark it active, which will direct all searches to it, and remove the old -index. - -This is also helpful for migrating to new servers, for example, moving to/from AWS. - -Currently we are on the process of migrating to this new design. Everything is hardwired to work with one single version for now. - -### Architecture - -The traditional setup, provided by `elasticsearch-rails`, is to communicate through its internal proxy classes. Developers would write model-specific logic in a module for the model to include in (for example, `SnippetsSearch`). The `__elasticsearch__` methods would return a proxy object, for example: - -- `Issue.__elasticsearch__` returns an instance of `Elasticsearch::Model::Proxy::ClassMethodsProxy` -- `Issue.first.__elasticsearch__` returns an instance of `Elasticsearch::Model::Proxy::InstanceMethodsProxy`. - -These proxy objects would talk to Elasticsearch server directly (see top half of the diagram). - -![Elasticsearch Architecture](img/elasticsearch_architecture.svg) - -In the planned new design, each model would have a pair of corresponding sub-classed proxy objects, in which model-specific logic is located. For example, `Snippet` would have `SnippetClassProxy` and `SnippetInstanceProxy` (being subclass of `Elasticsearch::Model::Proxy::ClassMethodsProxy` and `Elasticsearch::Model::Proxy::InstanceMethodsProxy`, respectively). - -`__elasticsearch__` would represent another layer of proxy object, keeping track of multiple actual proxy objects. It would forward method calls to the appropriate index. For example: - -- `model.__elasticsearch__.search` would be forwarded to the one stable index, since it is a read operation. -- `model.__elasticsearch__.update_document` would be forwarded to all indices, to keep all indices up-to-date. - -The global configurations per version are now in the `Elastic::(Version)::Config` class. You can change mappings there. - -### Creating new version of schema - -NOTE: -This is not applicable yet as multiple indices functionality is not fully implemented. - -Folders like `ee/lib/elastic/v12p1` contain snapshots of search logic from different versions. To keep a continuous Git history, the latest version lives under `ee/lib/elastic/latest`, but its classes are aliased under an actual version (for example, `ee/lib/elastic/v12p3`). When referencing these classes, never use the `Latest` namespace directly, but use the actual version (for example, `V12p3`). - -The version name basically follows the GitLab release version. If setting is changed in 12.3, we will create a new namespace called `V12p3` (p stands for "point"). Raise an issue if there is a need to name a version differently. - -If the current version is `v12p1`, and we need to create a new version for `v12p3`, the steps are as follows: - -1. Copy the entire folder of `v12p1` as `v12p3` -1. Change the namespace for files under `v12p3` folder from `V12p1` to `V12p3` (which are still aliased to `Latest`) -1. Delete `v12p1` folder -1. Copy the entire folder of `latest` as `v12p1` -1. Change the namespace for files under `v12p1` folder from `Latest` to `V12p1` -1. Make changes to files under the `latest` folder as needed - -## Creating a new Advanced Search migration - -> This functionality was introduced by [#234046](https://gitlab.com/gitlab-org/gitlab/-/issues/234046). - -NOTE: -This only supported for indices created with GitLab 13.0 or greater. - -In the [`ee/elastic/migrate/`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/elastic/migrate) folder, create a new file with the filename format `YYYYMMDDHHMMSS_migration_name.rb`. This format is the same for Rails database migrations. - -```ruby -# frozen_string_literal: true - -class MigrationName < Elastic::Migration - # Important: Any updates to the Elastic index mappings must be replicated in the respective - # configuration files: - # - `Elastic::Latest::Config`, for the main index. - # - `Elastic::Latest::Config`, for standalone indices. - - def migrate - end - - # Check if the migration has completed - # Return true if completed, otherwise return false - def completed? - end -end -``` - -Applied migrations are stored in `gitlab-#{RAILS_ENV}-migrations` index. All migrations not executed -are applied by the [`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic/migration_worker.rb) -cron worker sequentially. - -To update Elastic index mappings, apply the configuration to the respective files: - -- For the main index: [`Elastic::Latest::Config`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/elastic/latest/config.rb). -- For standalone indices: `Elastic::Latest::Config`. - -Migrations can be built with a retry limit and have the ability to be [failed and marked as halted](https://gitlab.com/gitlab-org/gitlab/-/blob/66e899b6637372a4faf61cfd2f254cbdd2fb9f6d/ee/lib/elastic/migration.rb#L40). -Any data or index cleanup needed to support migration retries should be handled within the migration. - -### Migration helpers - -The following migration helpers are available in `ee/app/workers/concerns/elastic/`: - -#### `Elastic::MigrationBackfillHelper` - -Backfills a specific field in an index. In most cases, the mapping for the field should already be added. - -Requires the `index_name` and `field_name` methods. - -```ruby -class MigrationName < Elastic::Migration - include Elastic::MigrationBackfillHelper - - private - - def index_name - Issue.__elasticsearch__.index_name - end - - def field_name - :schema_version - end -end -``` - -#### `Elastic::MigrationUpdateMappingsHelper` - -Updates a mapping in an index by calling `put_mapping` with the mapping specified. - -Requires the `index_name` and `new_mappings` methods. - -```ruby -class MigrationName < Elastic::Migration - include Elastic::MigrationUpdateMappingsHelper - - private - - def index_name - Issue.__elasticsearch__.index_name - end - - def new_mappings - { - schema_version: { - type: 'short' - } - } - end -end -``` - -#### `Elastic::MigrationRemoveFieldsHelper` - -Removes specified fields from an index. - -Requires the `index_name`, `document_type` methods. If there is one field to remove, add the `field_to_remove` method, otherwise add `fields_to_remove` with an array of fields. - -Checks in batches if any documents that match `document_type` have the fields specified in Elasticsearch. If documents exist, uses a Painless script to perform `update_by_query`. - -```ruby -class MigrationName < Elastic::Migration - include Elastic::MigrationRemoveFieldsHelper - - batched! - throttle_delay 1.minute - - private - - def index_name - User.__elasticsearch__.index_name - end - - def document_type - 'user' - end - - def fields_to_remove - %w[two_factor_enabled has_projects] - end -end -``` - -The default batch size is `10_000`. You can override this value by specifying `BATCH_SIZE`: - -```ruby -class MigrationName < Elastic::Migration - include Elastic::MigrationRemoveFieldsHelper - - batched! - BATCH_SIZE = 100 - - ... -end -``` - -#### `Elastic::MigrationObsolete` - -Marks a migration as obsolete when it's no longer required. - -```ruby -class MigrationName < Elastic::Migration - include Elastic::MigrationObsolete -end -``` - -#### `Elastic::MigrationHelper` - -Contains methods you can use when a migration doesn't fit the previous examples. - -```ruby -class MigrationName < Elastic::Migration - include Elastic::MigrationHelper - - def migrate - ... - end - - def completed? - ... - end -end -``` - -### Migration options supported by the `Elastic::MigrationWorker` - -[`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic/migration_worker.rb) supports the following migration options: - -- `batched!` - Allow the migration to run in batches. If set, the [`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic/migration_worker.rb) -will re-enqueue itself with a delay which is set using the `throttle_delay` option described below. The batching -must be handled within the `migrate` method, this setting controls the re-enqueuing only. - -- `batch_size` - Sets the number of documents modified during a `batched!` migration run. This size should be set to a value which allows the updates -enough time to finish. This can be tuned in combination with the `throttle_delay` option described below. The batching -must be handled within a custom `migrate` method or by using the [`Elastic::MigrationBackfillHelper`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/concerns/elastic/migration_backfill_helper.rb) -`migrate` method which uses this setting. Default value is 1000 documents. - -- `throttle_delay` - Sets the wait time in between batch runs. This time should be set high enough to allow each migration batch -enough time to finish. Additionally, the time should be less than 30 minutes since that is how often the -[`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic/migration_worker.rb) -cron worker runs. Default value is 5 minutes. - -- `pause_indexing!` - Pause indexing while the migration runs. This setting will record the indexing setting before -the migration runs and set it back to that value when the migration is completed. - -- `space_requirements!` - Verify that enough free space is available in the cluster when the migration runs. This setting - will halt the migration if the storage required is not available when the migration runs. The migration must provide - the space required in bytes by defining a `space_required_bytes` method. - -- `retry_on_failure` - Enable the retry on failure feature. By default, it retries - the migration 30 times. After it runs out of retries, the migration is marked as halted. - To customize the number of retries, pass the `max_attempts` argument: - `retry_on_failure max_attempts: 10` - -```ruby -# frozen_string_literal: true - -class BatchedMigrationName < Elastic::Migration - # Declares a migration should be run in batches - batched! - throttle_delay 10.minutes - pause_indexing! - space_requirements! - retry_on_failure - - # ... -end -``` - -### Multi-version compatibility - -These Advanced Search migrations, like any other GitLab changes, need to support the case where -[multiple versions of the application are running at the same time](multi_version_compatibility.md). - -Depending on the order of deployment, it's possible that the migration -has started or finished and there's still a server running the application code from before the -migration. We need to take this into consideration until we can -[ensure all Advanced Search migrations start after the deployment has finished](https://gitlab.com/gitlab-org/gitlab/-/issues/321619). - -### Reverting a migration - -Because Elasticsearch does not support transactions, we always need to design our -migrations to accommodate a situation where the application -code is reverted after the migration has started or after it is finished. - -For this reason we generally defer destructive actions (for example, deletions after -some data is moved) to a later merge request after the migrations have -completed successfully. To be safe, for self-managed customers we should also -defer it to another release if there is risk of important data loss. - -### Best practices for Advanced Search migrations - -Follow these best practices for best results: - -- When working in batches, keep the batch size under 9,000 documents - and `throttle_delay` for at least 3 minutes. The bulk indexer is set to run - every 1 minute and process a batch of 10,000 documents. These limits - allow the bulk indexer time to process records before another migration - batch is attempted. -- To ensure that document counts are up to date, it is recommended to refresh - the index before checking if a migration is completed. -- Add logging statements to each migration when the migration starts, when a - completion check occurs, and when the migration is completed. These logs - are helpful when debugging issues with migrations. -- Pause indexing if you're using any Elasticsearch Reindex API operations. -- Consider adding a retry limit if there is potential for the migration to fail. - This ensures that migrations can be halted if an issue occurs. - -## Deleting Advanced Search migrations in a major version upgrade - -Since our Advanced Search migrations usually require us to support multiple -code paths for a long period of time, it's important to clean those up when we -safely can. - -We choose to use GitLab major version upgrades as a safe time to remove -backwards compatibility for indices that have not been fully migrated. We -[document this in our upgrade documentation](../update/index.md#upgrading-to-a-new-major-version). -We also choose to replace the migration code with the halted migration -and remove tests so that: - -- We don't need to maintain any code that is called from our Advanced Search - migrations. -- We don't waste CI time running tests for migrations that we don't support - anymore. -- Operators who have not run this migration and who upgrade directly to the - target version will see a message prompting them to reindex from scratch. - -To be extra safe, we will not delete migrations that were created in the last -minor version before the major upgrade. So, if we are upgrading to `%14.0`, -we should not delete migrations that were only added in `%13.12`. This is an -extra safety net as we expect there are migrations that get merged that may -take multiple weeks to finish on GitLab.com. It would be bad if we upgraded -GitLab.com to `%14.0` before the migrations in `%13.12` were finished. Since -our deployments to GitLab.com are automated and we currently don't have -automated checks to prevent this, the extra precaution is warranted. -Additionally, even if we did have automated checks to prevent it, we wouldn't -actually want to hold up GitLab.com deployments on Advanced Search migrations, -as they may still have another week to go, and that's too long to block -deployments. - -### Process for removing migrations - -For every migration that was created 2 minor versions before the major version -being upgraded to, we do the following: - -1. Confirm the migration has actually completed successfully for GitLab.com. -1. Replace the content of the migration with: - - ```ruby - include Elastic::MigrationObsolete - ``` - -1. Delete any spec files to support this migration. -1. Remove any logic handling backwards compatibility for this migration. You - can find this by looking for - `Elastic::DataMigrationService.migration_has_finished?(:migration_name_in_lowercase)`. -1. Create a merge request with these changes. Noting that we should not - accidentally merge this before the major release is started. - -## Performance Monitoring - -### Prometheus - -GitLab exports [Prometheus metrics](../administration/monitoring/prometheus/gitlab_metrics.md) -relating to the number of requests and timing for all web/API requests and Sidekiq jobs, -which can help diagnose performance trends and compare how Elasticsearch timing -is impacting overall performance relative to the time spent doing other things. - -#### Indexing queues - -GitLab also exports [Prometheus metrics](../administration/monitoring/prometheus/gitlab_metrics.md) -for indexing queues, which can help diagnose performance bottlenecks and determine -whether or not your GitLab instance or Elasticsearch server can keep up with -the volume of updates. - -### Logs - -All of the indexing happens in Sidekiq, so much of the relevant logs for the -Elasticsearch integration can be found in -[`sidekiq.log`](../administration/logs/index.md#sidekiqlog). In particular, all -Sidekiq workers that make requests to Elasticsearch in any way will log the -number of requests and time taken querying/writing to Elasticsearch. This can -be useful to understand whether or not your cluster is keeping up with -indexing. - -Searching Elasticsearch is done via ordinary web workers handling requests. Any -requests to load a page or make an API request, which then make requests to -Elasticsearch, will log the number of requests and the time taken to -[`production_json.log`](../administration/logs/index.md#production_jsonlog). These -logs will also include the time spent on Database and Gitaly requests, which -may help to diagnose which part of the search is performing poorly. - -There are additional logs specific to Elasticsearch that are sent to -[`elasticsearch.log`](../administration/logs/index.md#elasticsearchlog) -that may contain information to help diagnose performance issues. - -### Performance Bar - -Elasticsearch requests will be displayed in the -[`Performance Bar`](../administration/monitoring/performance/performance_bar.md), which can -be used both locally in development and on any deployed GitLab instance to -diagnose poor search performance. This will show the exact queries being made, -which is useful to diagnose why a search might be slow. - -### Correlation ID and `X-Opaque-Id` - -Our [correlation ID](distributed_tracing.md#developer-guidelines-for-working-with-correlation-ids) -is forwarded by all requests from Rails to Elasticsearch as the -[`X-Opaque-Id`](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html#_identifying_running_tasks) -header which allows us to track any -[tasks](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html) -in the cluster back the request in GitLab. - -## Troubleshooting - -### Getting `flood stage disk watermark [95%] exceeded` - -You might get an error such as - -```plaintext -[2018-10-31T15:54:19,762][WARN ][o.e.c.r.a.DiskThresholdMonitor] [pval5Ct] - flood stage disk watermark [95%] exceeded on - [pval5Ct7SieH90t5MykM5w][pval5Ct][/usr/local/var/lib/elasticsearch/nodes/0] free: 56.2gb[3%], - all indices on this node will be marked read-only -``` - -This is because you've exceeded the disk space threshold - it thinks you don't have enough disk space left, based on the default 95% threshold. - -In addition, the `read_only_allow_delete` setting will be set to `true`. It will block indexing, `forcemerge`, etc - -```shell -curl "http://localhost:9200/gitlab-development/_settings?pretty" -``` - -Add this to your `elasticsearch.yml` file: - -```yaml -# turn off the disk allocator -cluster.routing.allocation.disk.threshold_enabled: false -``` - -_or_ - -```yaml -# set your own limits -cluster.routing.allocation.disk.threshold_enabled: true -cluster.routing.allocation.disk.watermark.flood_stage: 5gb # ES 6.x only -cluster.routing.allocation.disk.watermark.low: 15gb -cluster.routing.allocation.disk.watermark.high: 10gb -``` - -Restart Elasticsearch, and the `read_only_allow_delete` will clear on its own. - -_from "Disk-based Shard Allocation | Elasticsearch Reference" [5.6](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/disk-allocator.html#disk-allocator) and [6.x](https://www.elastic.co/guide/en/elasticsearch/reference/6.7/disk-allocator.html)_ - -### Disaster recovery/data loss/backups - -The use of Elasticsearch in GitLab is only ever as a secondary data store. -This means that all of the data stored in Elasticsearch can always be derived -again from other data sources, specifically PostgreSQL and Gitaly. Therefore if -the Elasticsearch data store is ever corrupted for whatever reason you can reindex -everything from scratch. - -If your Elasticsearch index is incredibly large it may be too time consuming or -cause too much downtime to reindex from scratch. There aren't any built in -mechanisms for automatically finding discrepancies and resyncing an -Elasticsearch index if it gets out of sync but one tool that may be useful is -looking at the logs for all the updates that occurred in a time range you -believe may have been missed. This information is very low level and only -useful for operators that are familiar with the GitLab codebase. It is -documented here in case it is useful for others. The relevant logs that could -theoretically be used to figure out what needs to be replayed are: - -1. All non-repository updates that were synced can be found in - [`elasticsearch.log`](../administration/logs/index.md#elasticsearchlog) by - searching for - [`track_items`](https://gitlab.com/gitlab-org/gitlab/-/blob/1e60ea99bd8110a97d8fc481e2f41cab14e63d31/ee/app/services/elastic/process_bookkeeping_service.rb#L25) - and these can be replayed by sending these items again through - `::Elastic::ProcessBookkeepingService.track!` -1. All repository updates that occurred can be found in - [`elasticsearch.log`](../administration/logs/index.md#elasticsearchlog) by - searching for - [`indexing_commit_range`](https://gitlab.com/gitlab-org/gitlab/-/blob/6f9d75dd3898536b9ec2fb206e0bd677ab59bd6d/ee/lib/gitlab/elastic/indexer.rb#L41). - Replaying these requires resetting the - [`IndexStatus#last_commit/last_wiki_commit`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/models/index_status.rb) - to the oldest `from_sha` in the logs and then triggering another index of - the project using - [`ElasticCommitIndexerWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic_commit_indexer_worker.rb) -1. All project deletes that occurred can be found in - [`sidekiq.log`](../administration/logs/index.md#sidekiqlog) by searching for - [`ElasticDeleteProjectWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic_delete_project_worker.rb). - These updates can be replayed by triggering another - `ElasticDeleteProjectWorker`. - -With the above methods and taking regular -[Elasticsearch snapshots](https://www.elastic.co/guide/en/elasticsearch/reference/current/snapshot-restore.html) -we should be able to recover from different kinds of data loss issues in a -relatively short period of time compared to indexing everything from -scratch. diff --git a/doc/development/emails.md b/doc/development/emails.md index 2d03d8bca2f..fdcdcec814d 100644 --- a/doc/development/emails.md +++ b/doc/development/emails.md @@ -151,37 +151,10 @@ fork of [`MailRoom`](https://github.com/tpitale/mail_room/), to ensure that we can make updates quickly to the gem if necessary. We try to upstream changes as soon as possible and keep the two projects in sync. -Updating the `gitlab-mail_room` dependency in `Gemfile` is deprecated at -the moment in favor of updating the version in -[Omnibus](https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/master/config/software/mail_room.rb) -(see [example merge request](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/5816)) -and Helm Chart configuration (see [example merge request](https://gitlab.com/gitlab-org/build/CNG/-/merge_requests/854)). +To update MailRoom: -#### Rationale - -This was done because to avoid [thread deadlocks](https://github.com/ruby/net-imap/issues/14), `MailRoom` needs -an updated version of the `net-imap` gem. However, this -[version of the net-imap cannot be installed by an unprivileged user](https://github.com/ruby/net-imap/issues/14) due to -[an error installing the digest gem](https://github.com/ruby/digest/issues/14). -[This bug in the Ruby interpreter](https://bugs.ruby-lang.org/issues/17761) was fixed in Ruby -3.0.2. - -Updating the gem directly in the GitLab Rails `Gemfile` caused a [production incident](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/4053) -since Kubernetes pods do not run as privileged users. - -Note that Omnibus GitLab runs `/opt/gitlab/embedded/bin/mail_room` -instead of `bundle exec rake` to avoid loading the older version. With a -Kubernetes install, the MailRoom version has always been explicitly set -in the Helm Chart configuration rather than the `Gemfile`. - -#### Preserving backwards compatibility - -Removing the `Gemfile` would break incoming email processing for source -installs. For now, source installs are advised to upgrade manually to -the version specified in Omnibus and run `bin/mail_room` directly as -done with Omnibus. - -We can reconsider this deprecation once we upgrade to Ruby 3.0. +1. Update `Gemfile` in GitLab Rails (see [example merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/116494)). +1. Update the Helm Chart configuration (see [example merge request](https://gitlab.com/gitlab-org/build/CNG/-/merge_requests/854)). --- diff --git a/doc/development/experiment_guide/index.md b/doc/development/experiment_guide/index.md index fd5b0ea15e6..5e68921510e 100644 --- a/doc/development/experiment_guide/index.md +++ b/doc/development/experiment_guide/index.md @@ -29,7 +29,7 @@ the documentation on that project if you want to dig into more advanced topics o aware that the documentation there reflects what's in the main branch and may not be the same as the version being used in GitLab. -## Glossary of terms +## Glossary To ensure a shared language, you should understand these fundamental terms we use when communicating about experiments: diff --git a/doc/development/export_csv.md b/doc/development/export_csv.md index 5d35c880ffd..971159e83b9 100644 --- a/doc/development/export_csv.md +++ b/doc/development/export_csv.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/development/fe_guide/accessibility.md b/doc/development/fe_guide/accessibility.md index af45603782f..b00131b12f3 100644 --- a/doc/development/fe_guide/accessibility.md +++ b/doc/development/fe_guide/accessibility.md @@ -345,7 +345,7 @@ Keep in mind that: - When you add `:hover` styles, in most cases you should add `:focus` styles too so that the styling is applied for both mouse **and** keyboard users. - If you remove an interactive element's `outline`, make sure you maintain visual focus state in another way such as with `box-shadow`. -See the [Pajamas Keyboard-only page](https://design.gitlab.com/accessibility-audits/keyboard-only/) for more detail. +See the [Pajamas Keyboard-only page](https://design.gitlab.com/accessibility/keyboard-only) for more detail. ## `tabindex` @@ -516,6 +516,55 @@ GitLab-specific examples are assignee and label dropdowns. Building such widgets require ARIA to make them understandable to screen readers. Proper research and testing should be done to ensure compliance with [WCAG](https://www.w3.org/WAI/standards-guidelines/wcag/). +## Automated accessibility testing with axe + +You can use [axe-core](https://github.com/dequelabs/axe-core) [gems](https://github.com/dequelabs/axe-core-gems) +to run automated accessibility tests in feature tests. + +Axe provides the custom matcher `be_axe_clean`, which can be used like the following: + +```ruby +# spec/features/settings_spec.rb +it 'passes axe automated accessibility testing', :js do + visit_settings_page + + wait_for_requests # ensures page is fully loaded + + expect(page).to be_axe_clean +end +``` + +If needed, you can scope testing to a specific area of the page by using `within`. + +Axe also provides specific [clauses](https://github.com/dequelabs/axe-core-gems/blob/develop/packages/axe-core-rspec/README.md#clauses), +for example: + +```ruby +expect(page).to be_axe_clean.within '[data-testid="element"]' + +# run only WCAG 2.0 Level AA rules +expect(page).to be_axe_clean.according_to :wcag2aa + +# specifies which rule to skip +expect(page).to be_axe_clean.skipping :'link-in-text-block' + +# clauses can be chained +expect(page).to be_axe_clean.within('[data-testid="element"]') + .according_to(:wcag2aa) +``` + +Axe does not test hidden regions, such as inactive menus or modal windows. To test +hidden regions for accessibility, write tests that activate or render the regions visible +and run the matcher again. + +### Known accessibility violations + +This section documents violations where a recommendation differs with the [design system](https://design.gitlab.com/): + +- `link-in-text-block`: For now, use the `skipping` clause to skip `:'link-in-text-block'` + rule to fix the violation. After this is fixed as part of [issue 1444](https://gitlab.com/gitlab-org/gitlab-services/design.gitlab.com/-/issues/1444) + and underline is added to the `GlLink` component, this clause can be removed. + ## Resources ### Viewing the browser accessibility tree diff --git a/doc/development/fe_guide/content_editor.md b/doc/development/fe_guide/content_editor.md index 6d13f419430..25140a067ca 100644 --- a/doc/development/fe_guide/content_editor.md +++ b/doc/development/fe_guide/content_editor.md @@ -1,6 +1,6 @@ --- -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -64,7 +64,7 @@ Instead, you should obtain an instance of the `ContentEditor` class by listening ```html ``` +### `v-bind` limitations + +Avoid using `v-bind="$attrs"` unless absolutely necessary. You might need this when +developing a native control wrapper. (This is a good candidate for a `gitlab-ui` component.) +In any other cases, always prefer using `props` and explicit data flow. + +Using `v-bind="$attrs"` leads to: + +1. A loss in component's contract. The `props` were designed specifically + to address this problem. +1. High maintenance cost for each component in the tree. `v-bind="$attrs"` is specifically + hard to debug because you must scan the whole hierarchy of components to understand + the data flow. +1. Problems during migration to Vue 3. `$attrs` in Vue 3 include event listeners which + could cause unexpected side-effects after Vue 3 migration is completed. + ### Aim to have one API style per component When adding `setup()` property to Vue component, consider refactoring it to Composition API entirely. It's not always feasible, especially for large components, but we should aim to have one API style per component for readability and maintainability. @@ -608,8 +727,7 @@ describe('~/todos/app.vue', () => { }); afterEach(() => { - // IMPORTANT: Clean up the component instance and axios mock adapter - wrapper.destroy(); + // IMPORTANT: Clean up the axios mock adapter mock.restore(); }); diff --git a/doc/development/fe_guide/vuex.md b/doc/development/fe_guide/vuex.md index 01ee50fb6ca..52278c94e9f 100644 --- a/doc/development/fe_guide/vuex.md +++ b/doc/development/fe_guide/vuex.md @@ -6,14 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Vuex -When there's a clear benefit to separating state management from components (for example, due to state complexity) we recommend using [Vuex](https://vuex.vuejs.org) over any other Flux pattern. Otherwise, feel free to manage state in the components. - -Vuex should be strongly considered when: - -- You expect multiple parts of the application to react to state changes. -- There's a need to share data between multiple components. -- There are complex interactions with Backend, for example, multiple API calls. -- The app involves interacting with backend via both traditional REST API and GraphQL (especially when moving the REST API over to GraphQL is a pending backend task). +[Vuex](https://vuex.vuejs.org) should no longer be considered a preferred path to store management and is currently in its legacy phase. This means it is acceptable to add upon existing `Vuex` stores, but we strongly recommend reducing store sizes over time and eventually migrating fully to [Apollo](https://www.apollographql.com/) whenever it's possible. Before adding any new `Vuex` store to an application, first ensure that the `Vue` application you plan to add it into **does not use** `Apollo`. `Vuex` and `Apollo` should not be combined unless absolutely necessary. Please consider reading through [our GraphQL documentation](../fe_guide/graphql.md) for more guidelines on how you can build `Apollo` based applications. The information included in this page is explained in more detail in the official [Vuex documentation](https://vuex.vuejs.org). @@ -97,7 +90,7 @@ In this file, we write the actions that call mutations for handling a list of us ```javascript import * as types from './mutation_types'; import axios from '~/lib/utils/axios_utils'; - import { createAlert } from '~/flash'; + import { createAlert } from '~/alert'; export const fetchUsers = ({ state, dispatch }) => { commit(types.REQUEST_USERS); @@ -305,6 +298,7 @@ export default () => { return new Vue({ el, + name: 'AwesomeVueRoot', store: createStore(el.dataset), render: h => h(AwesomeVueApp) }); @@ -462,10 +456,6 @@ describe('component', () => { createComponent(); }); - afterEach(() => { - wrapper.destroy(); - }); - it('should show a user', async () => { const user = { name: 'Foo', diff --git a/doc/development/fe_guide/widgets.md b/doc/development/fe_guide/widgets.md index edb8559da48..6cd8e6c091c 100644 --- a/doc/development/fe_guide/widgets.md +++ b/doc/development/fe_guide/widgets.md @@ -62,11 +62,11 @@ Because we need different GraphQL queries and mutations for different sidebars, ```javascript export const assigneesQueries = { - [IssuableType.Issue]: { + [TYPE_ISSUE]: { query: getIssueParticipants, mutation: updateAssigneesMutation, }, - [IssuableType.MergeRequest]: { + [TYPE_MERGE_REQUEST]: { query: getMergeRequestParticipants, mutation: updateMergeRequestParticipantsMutation, }, diff --git a/doc/development/feature_categorization/index.md b/doc/development/feature_categorization/index.md index dad94a2aae2..76356db2e87 100644 --- a/doc/development/feature_categorization/index.md +++ b/doc/development/feature_categorization/index.md @@ -226,3 +226,8 @@ For example in [`spec/tooling/danger/specs_spec.rb`](https://gitlab.com/gitlab-o For features that support developers and they are not specific to a product group we use `feature_category: :shared` For example [`spec/lib/gitlab/job_waiter_spec.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/lib/gitlab/job_waiter_spec.rb) + +### Admin section + +Adding feature categories is equally important when adding new parts to the Admin section. Historically, Admin sections were often marked as `not_owned` in the code. Now +you must ensure each new addition to the Admin section is properly annotated using `feature_category` notation. diff --git a/doc/development/feature_development.md b/doc/development/feature_development.md index 141b24161b4..15058134092 100644 --- a/doc/development/feature_development.md +++ b/doc/development/feature_development.md @@ -55,6 +55,7 @@ Consult these topics for information on contributing to specific GitLab features - [Pry debugging](pry_debugging.md) - [Sidekiq debugging](../administration/sidekiq/sidekiq_troubleshooting.md) +- [VS Code debugging](vs_code_debugging.md) ### Git specifics @@ -79,8 +80,8 @@ Consult these topics for information on contributing to specific GitLab features - [Adding a new Redis instance](redis/new_redis_instance.md) - [Sidekiq guidelines](sidekiq/index.md) for working with Sidekiq workers - [Working with Gitaly](gitaly.md) -- [Elasticsearch integration docs](elasticsearch.md) -- [Working with merge request diffs](diffs.md) +- [Advanced search integration docs](advanced_search.md) +- [Working with merge request diffs](merge_request_concepts/diffs/index.md) - [Approval Rules](merge_request_concepts/approval_rules.md) - [Repository mirroring](repository_mirroring.md) - [Uploads development guide](uploads/index.md) @@ -94,6 +95,7 @@ Consult these topics for information on contributing to specific GitLab features - [Shell commands](shell_commands.md) in the GitLab codebase - [Value Stream Analytics development guide](value_stream_analytics.md) - [Application limits](application_limits.md) +- [AI features](ai_features.md) ### Import and Export @@ -127,13 +129,11 @@ See [database guidelines](database/index.md). - [Security Scanners](integrations/secure.md) - [Secure Partner Integration](integrations/secure_partner_integration.md) - [How to run Jenkins in development environment](integrations/jenkins.md) -- [How to run local CodeSandbox integration for Web IDE Live Preview](integrations/codesandbox.md) The following integration guides are internal. Some integrations require access to administrative accounts of third-party services and are available only for GitLab team members to contribute to: -- [Jira app development](https://gitlab.com/gitlab-org/manage/integrations/team/-/blob/main/integrations/jira.md) -- [Slack app development](https://gitlab.com/gitlab-org/manage/integrations/team/-/blob/main/integrations/slack.md) -- [ZenTao development](https://gitlab.com/gitlab-org/manage/integrations/team/-/blob/main/integrations/zentao.md) +- [Jira app development](https://gitlab.com/gitlab-org/manage/integrate/team/-/blob/main/integrations/jira.md) +- [GitLab for Slack app development](https://gitlab.com/gitlab-org/manage/integrate/team/-/blob/main/integrations/slack.md) ## Testing guides diff --git a/doc/development/feature_flags/controls.md b/doc/development/feature_flags/controls.md index 3adf5248b8d..d341cb3f1ba 100644 --- a/doc/development/feature_flags/controls.md +++ b/doc/development/feature_flags/controls.md @@ -40,7 +40,7 @@ easier to measure the impact of both separately. The GitLab feature library (using [Flipper](https://github.com/jnunemaker/flipper), and covered in the -[Feature Flags process](https://about.gitlab.com/handbook/product-development-flow/feature-flag-lifecycle/) guide) supports rolling out changes to a percentage of +[Feature flags process](https://about.gitlab.com/handbook/product-development-flow/feature-flag-lifecycle/) guide) supports rolling out changes to a percentage of time to users. This in turn can be controlled using [GitLab ChatOps](../../ci/chatops/index.md). For an up to date list of feature flag commands please see @@ -84,6 +84,8 @@ When a feature has successfully been environment and verified as safe and working, you can roll out the change to GitLab.com (production). +If a feature is [deprecated](../../update/deprecations.md), do not enable the flag. + #### Communicate the change Some feature flag changes on GitLab.com should be communicated with @@ -113,13 +115,13 @@ incidents or in-progress change issues, for example: 2021-06-29 Canary deployment failing QA tests ``` -Before enabling a feature flag, verify that you are not violating any [Production Change Lock periods](https://about.gitlab.com/handbook/engineering/infrastructure/change-management/#production-change-lock-pcl) and are in compliance with the [Feature Flags and the Change Management Process](https://about.gitlab.com/handbook/engineering/infrastructure/change-management/#feature-flags-and-the-change-management-process). +Before enabling a feature flag, verify that you are not violating any [Production Change Lock periods](https://about.gitlab.com/handbook/engineering/infrastructure/change-management/#production-change-lock-pcl) and are in compliance with the [Feature flags and the Change Management Process](https://about.gitlab.com/handbook/engineering/infrastructure/change-management/#feature-flags-and-the-change-management-process). The following `/chatops` commands should be performed in the Slack `#production` channel. When you begin to enable the feature, please link to the relevant -Feature Flag Rollout Issue within a Slack thread of the first `/chatops` +feature flag rollout issue within a Slack thread of the first `/chatops` command you make so people can understand the change if they need to. To enable a feature for 25% of the time, run the following in Slack: @@ -356,7 +358,7 @@ After turning on the feature flag, you need to [monitor the relevant graphs](htt In this illustration, you can see that the Apdex score started to decline after the feature flag was enabled at `09:46`. The feature flag was then deactivated at `10:31`, and the service returned to the original value: -![Feature Flag Metrics](../img/feature-flag-metrics.png) +![Feature flag metrics](../img/feature-flag-metrics.png) ### Feature flag change logging @@ -408,8 +410,8 @@ take one of the following actions: To remove a feature flag, open **one merge request** to make the changes. In the MR: 1. Add the ~"feature flag" label so release managers are aware of the removal. -1. If the merge request has to be picked into a stable branch, add the - appropriate `~"Pick into X.Y"` label, for example `~"Pick into 13.0"`. +1. If the merge request has to be backported into the current version, follow the + [patch release runbook](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/patch/engineers.md) process. See [the feature flag process](https://about.gitlab.com/handbook/product-development-flow/feature-flag-lifecycle/#including-a-feature-behind-feature-flag-in-the-final-release) for further details. 1. Remove all references to the feature flag from the codebase, including tests. diff --git a/doc/development/feature_flags/index.md b/doc/development/feature_flags/index.md index 7370697b082..87d2da016d6 100644 --- a/doc/development/feature_flags/index.md +++ b/doc/development/feature_flags/index.md @@ -35,7 +35,7 @@ should be leveraged: the status of a feature behind the feature flag in the documentation and with other stakeholders. The issue description should be updated with the feature flag name and whether it is defaulted on or off as soon it is evident that a feature flag is needed. -- Merge requests that introduce a feature flag, update its state, or remove them +- Merge requests that introduce a feature flag, update its state, or remove the existing feature flag because a feature is deemed stable must have the ~"feature flag" label assigned. @@ -83,7 +83,7 @@ used for deploying unfinished code to production. Most feature flags used at GitLab are the `development` type. A `development` feature flag must have a rollout issue -created from the [Feature Flag Roll Out template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/issue_templates/Feature%20Flag%20Roll%20Out.md). +created from the [Feature flag Roll Out template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/issue_templates/Feature%20Flag%20Roll%20Out.md). The format for `development` feature flags is `Feature.(:)`. To enable and disable them, run on the GitLab Rails console: @@ -252,7 +252,7 @@ deleting feature flags. ## Develop with a feature flag -There are two main ways of using Feature Flags in the GitLab codebase: +There are two main ways of using feature flags in the GitLab codebase: - [Backend code (Rails)](#backend) - [Frontend code (VueJS)](#frontend) @@ -505,9 +505,12 @@ Feature.remove(:feature_flag_name) ## Changelog +We want to avoid introducing a changelog when features are not accessible by an end-user either directly (example: ability to use the feature) or indirectly (examples: ability to take advantage of background jobs, performance improvements, or database migration updates). + +- Database migrations are always accessible by an end-user indirectly, as self-managed customers need to be aware of database changes before upgrading. For this reason, they **should** have a changelog entry. - Any change behind a feature flag **disabled** by default **should not** have a changelog entry. - - **Exception:** database migrations **should** have a changelog entry. -- Any change related to a feature flag itself (flag removal, default-on setting) **should** have [a changelog entry](../changelog.md). +- Any change behind a feature flag that is **enabled** by default **should** have a changelog entry. +- Changing the feature flag itself (flag removal, default-on setting) **should** have [a changelog entry](../changelog.md). Use the flowchart to determine the changelog entry type. ```mermaid @@ -519,7 +522,6 @@ Feature.remove(:feature_flag_name) A -->|no changelog| D ``` -- Any change behind a feature flag that is **enabled** by default **should** have a changelog entry. - The changelog for a feature flag should describe the feature and not the flag, unless a default on feature flag is removed keeping the new code (`other` in the flowchart above). - A feature flag can also be used for rolling out a bug fix or a maintenance work. In this scenario, the changelog diff --git a/doc/development/features_inside_dot_gitlab.md b/doc/development/features_inside_dot_gitlab.md index dda2c05288f..3c988ec6b21 100644 --- a/doc/development/features_inside_dot_gitlab.md +++ b/doc/development/features_inside_dot_gitlab.md @@ -9,13 +9,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w We have implemented standard features that depend on configuration files in the `.gitlab/` directory. You can find `.gitlab/` in various GitLab repositories. When implementing new features, please refer to these existing features to avoid conflicts: -- [Custom Dashboards](../operations/metrics/dashboards/index.md#add-a-new-dashboard-to-your-project): `.gitlab/dashboards/`. - [Issue Templates](../user/project/description_templates.md#create-an-issue-template): `.gitlab/issue_templates/`. - [Merge request Templates](../user/project/description_templates.md#create-a-merge-request-template): `.gitlab/merge_request_templates/`. - [GitLab agent](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/configuration_repository.md#layout): `.gitlab/agents/`. -- [CODEOWNERS](../user/project/code_owners.md#set-up-code-owners): `.gitlab/CODEOWNERS`. +- [CODEOWNERS](../user/project/codeowners/index.md#set-up-code-owners): `.gitlab/CODEOWNERS`. - [Route Maps](../ci/review_apps/index.md#route-maps): `.gitlab/route-map.yml`. - [Customize Auto DevOps Helm Values](../topics/autodevops/customize.md#customize-helm-chart-values): `.gitlab/auto-deploy-values.yaml`. - [Insights](../user/project/insights/index.md#configure-project-insights): `.gitlab/insights.yml`. -- [Service Desk Templates](../user/project/service_desk.md#using-customized-email-templates): `.gitlab/service_desk_templates/`. -- [Web IDE](../user/project/web_ide/index.md#web-ide-configuration-file): `.gitlab/.gitlab-webide.yml`. +- [Service Desk Templates](../user/project/service_desk.md#create-customized-email-templates): `.gitlab/service_desk_templates/`. diff --git a/doc/development/fips_compliance.md b/doc/development/fips_compliance.md index 147ff5fa6e9..830a8e3cd2a 100644 --- a/doc/development/fips_compliance.md +++ b/doc/development/fips_compliance.md @@ -59,17 +59,15 @@ listed here that also do not work properly in FIPS mode: - [Container Scanning](../user/application_security/container_scanning/index.md) support for scanning images in repositories that require authentication. - [Code Quality](../ci/testing/code_quality.md) does not support operating in FIPS-compliant mode. - [Dependency scanning](../user/application_security/dependency_scanning/index.md) support for Gradle. -- [Dynamic Application Security Testing (DAST)](../user/application_security/dast/index.md) - does not support operating in FIPS-compliant mode. +- [Dynamic Application Security Testing (DAST)](../user/application_security/dast/index.md) supports a reduced set of analyzers. Browser-based and proxy-based analyzers are not available in FIPS mode today, however DAST API and DAST API Fuzzing images are available. - [License compliance](../user/compliance/license_compliance/index.md). - [Solutions for vulnerabilities](../user/application_security/vulnerabilities/index.md#resolve-a-vulnerability) for yarn projects. - [Static Application Security Testing (SAST)](../user/application_security/sast/index.md) supports a reduced set of [analyzers](../user/application_security/sast/index.md#fips-enabled-images) when operating in FIPS-compliant mode. -- Advanced Search is currently not included in FIPS mode. It must not be enabled to be FIPS-compliant. +- Advanced search is currently not included in FIPS mode. It must not be enabled to be FIPS-compliant. - [Gravatar or Libravatar-based profile images](../administration/libravatar.md) are not FIPS-compliant. -- [Personal Access Tokens](../user/profile/personal_access_tokens.md) are not available for use or creation. Additionally, these package repositories are disabled in FIPS mode: @@ -275,104 +273,55 @@ all: gitlab_charts_custom_config_file: '/path/to/gitlab-environment-toolkit/ansible/environments/gitlab-10k/inventory/charts.yml' ``` -Now create `charts.yml` in the location specified above and specify tags with a `-fips` suffix. For example: +Now create `charts.yml` in the location specified above and specify tags with a `-fips` suffix. -```yaml -global: - image: - pullPolicy: Always - certificates: - image: - tag: master-fips - kubectl: - image: - tag: master-fips - -gitlab: - gitaly: - image: - tag: master-fips - gitlab-exporter: - image: - tag: master-fips - gitlab-shell: - image: - tag: main-fips # The default branch is main, not master - gitlab-mailroom: - image: - tag: master-fips - gitlab-pages: - image: - tag: master-fips - migrations: - image: - tag: master-fips - sidekiq: - image: - tag: master-fips - toolbox: - image: - tag: master-fips - webservice: - image: - tag: master-fips - workhorse: - tag: master-fips - -nginx-ingress: - controller: - image: - repository: registry.gitlab.com/gitlab-org/cloud-native/charts/gitlab-ingress-nginx/controller - tag: v1.2.1-fips - pullPolicy: Always - digest: sha256:c4222b7ab3836b9be2a7649cff4b2e6ead34286dfdf3a7b04eb34fdd3abb0334 -``` - -The above example shows a FIPS-enabled [`nginx-ingress`](https://github.com/kubernetes/ingress-nginx) image. -See our [Charts documentation on FIPS](https://docs.gitlab.com/charts/advanced/fips/index.html) for more details. +See our [Charts documentation on FIPS](https://docs.gitlab.com/charts/advanced/fips/index.html) for more details, including +an [example values file](https://gitlab.com/gitlab-org/charts/gitlab/blob/master/examples/fips/values.yaml) as a reference. You can also use release tags, but the versioning is tricky because each component may use its own versioning scheme. For example, for GitLab v15.2: ```yaml global: + image: + tagSuffix: -fips certificates: image: - tag: 20211220-r0-fips + tag: 20211220-r0 kubectl: image: - tag: 1.18.20-fips + tag: 1.18.20 gitlab: gitaly: image: - tag: v15.2.0-fips + tag: v15.2.0 gitlab-exporter: image: - tag: 11.17.1-fips + tag: 11.17.1 gitlab-shell: image: - tag: v14.9.0-fips + tag: v14.9.0 gitlab-mailroom: image: - tag: v15.2.0-fips + tag: v15.2.0 gitlab-pages: image: - tag: v1.61.0-fips + tag: v1.61.0 migrations: image: - tag: v15.2.0-fips + tag: v15.2.0 sidekiq: image: - tag: v15.2.0-fips + tag: v15.2.0 toolbox: image: - tag: v15.2.0-fips + tag: v15.2.0 webservice: image: - tag: v15.2.0-fips + tag: v15.2.0 workhorse: - tag: v15.2.0-fips + tag: v15.2.0 ``` ## FIPS Performance Benchmarking @@ -496,7 +445,7 @@ irb(main):001:0> require 'openssl'; OpenSSL.fips_mode ### Go -Google maintains a [`dev.boringcrypto` branch](https://github.com/golang/go/tree/dev.boringcrypto) in the Golang compiler +Google maintains a [`dev.boringcrypto` branch](https://github.com/golang/go/tree/dev.boringcrypto) in the Go compiler that makes it possible to statically link BoringSSL, a FIPS-validated module forked from OpenSSL. However, BoringSSL is not intended for public use. diff --git a/doc/development/gemfile.md b/doc/development/gemfile.md index 36ef1bcd834..add93e37024 100644 --- a/doc/development/gemfile.md +++ b/doc/development/gemfile.md @@ -4,7 +4,7 @@ group: unassigned info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Gemfile guidelines +# Gemfile development guidelines When adding a new entry to `Gemfile`, or upgrading an existing dependency pay attention to the following rules. @@ -31,6 +31,13 @@ export BUNDLER_CHECKSUM_VERIFICATION_OPT_IN=1 bundle install ``` +Setting it to `false` can also disable it: + +```shell +export BUNDLER_CHECKSUM_VERIFICATION_OPT_IN=false +bundle install +``` + ### Updating the checksum file This needs to be done for any new, or updated gems. @@ -60,9 +67,22 @@ This means that new dependencies should, at a minimum, meet the following criter - There are no issues open that we know may impact the availability or performance of GitLab. - The project is tested using some form of test automation. The test suite must be passing using the Ruby version currently used by GitLab. +- CI builds for all supported platforms must succeed using the new dependency. For more information, see + how to [build a package for testing](build_test_package.md#building-a-package-for-testing). - If the project uses a C extension, consider requesting an additional review from a C or MRI domain expert. C extensions can greatly impact GitLab stability and performance. +## Gems that require a domain expert approval + +Changes to the following gems require a domain expert review and approval by a backend team member of the group. + +For gems not listed in this table, it's still recommended but not required that you find a domain expert to review changes. + +| Gem | Requires approval by | +| ------ | ------ | +| `doorkeeper` | [Manage:Authentication and Authorization](https://about.gitlab.com/handbook/product/categories/#authentication-and-authorization-group) | +| `doorkeeper-openid_connect` | [Manage:Authentication and Authorization](https://about.gitlab.com/handbook/product/categories/#authentication-and-authorization-group) | + ## Request an Appsec review When adding a new gem to our `Gemfile` or even changing versions in @@ -95,6 +115,8 @@ does not contain any hidden dependencies on our application code. In general, we want to think carefully before doing this as there are also disadvantages: +### Potential disadvantages + 1. Gems - even those maintained by GitLab - do not necessarily go through the same [code review process](code_review.md) as the main Rails application. @@ -106,9 +128,23 @@ also disadvantages: community's needs. In general, if we are not using the latest version of our own gem, that might be a warning sign. +### Create and publish a Ruby gem + In the case where we do want to extract some library code we've written to a gem, go through these steps: +1. Determine a suitable name for the gem. If it's a GitLab-owned gem, prefix + the gem name with `gitlab-`. For example, `gitlab-sidekiq-fetcher`. +1. Create the gem or fork as necessary. +1. Ensure the `gitlab_rubygems` group is an owner of the new gem by running: + + ```shell + gem owner --add gitlab_rubygems + ``` + +1. [Publish the gem to rubygems.org](https://guides.rubygems.org/publishing/#publishing-to-rubygemsorg) +1. Visit `https://rubygems.org/gems/` and verify that the gem published + successfully and `gitlab_rubygems` is also an owner. 1. Start with the code in the Rails application. Here it's fine to have the code in `lib/` and loaded automatically. We can skip this step if the step below makes more sense initially. diff --git a/doc/development/geo.md b/doc/development/geo.md index 710d0eec3b0..5d09532afcb 100644 --- a/doc/development/geo.md +++ b/doc/development/geo.md @@ -48,8 +48,8 @@ for new events and creates background jobs for each specific event type. For example when a repository is updated, the Geo **primary** site creates a Geo event with an associated repository updated event. The Geo Log Cursor daemon picks the event up and schedules a `Geo::ProjectSyncWorker` job which -uses the `Geo::RepositorySyncService` and `Geo::WikiSyncService` classes -to update the repository and the wiki respectively. +uses the `Geo::RepositorySyncService` to update the repository +and `Geo::WikiSyncService` classes to update the wiki. The Geo Log Cursor daemon can operate in High Availability mode automatically. The daemon tries to acquire a lock from time to time and once acquired, it diff --git a/doc/development/git_object_deduplication.md b/doc/development/git_object_deduplication.md index 6014ccbfb39..e98ebe5efe1 100644 --- a/doc/development/git_object_deduplication.md +++ b/doc/development/git_object_deduplication.md @@ -23,15 +23,17 @@ At the Git level, we achieve deduplication by using Git alternates is a mechanism that lets a repository borrow objects from another repository on the same machine. -If we want repository A to borrow from repository B, we first write a -path that resolves to `B.git/objects` in the special file -`A.git/objects/info/alternates`. This establishes the alternates link. -Next, we must perform a Git repack in A. After the repack, any objects -that are duplicated between A and B are deleted from A. Repository -A is now no longer self-contained, but it still has its own refs and -configuration. Objects in A that are not in B remain in A. For this -to work, it is of course critical that **no objects ever get deleted from -B** because A might need them. +To make repository A borrow from repository B: + +1. Establish the alternates link in the special file `A.git/objects/info/alternates` + by writing a path that resolves to `B.git/objects`. +1. In repository A, run `git repack` to remove all objects in repository A that + also exist in repository B. + +After the repack, repository A is no longer self-contained, but still contains its +own refs and configuration. Objects in A that are not in B remain in A. For this +configuration to work, **objects must not be deleted from repository B** because +repository A might need them. WARNING: Do not run `git prune` or `git gc` in object pool repositories, which are diff --git a/doc/development/gitaly.md b/doc/development/gitaly.md index b4f5501ccac..d2232d750b2 100644 --- a/doc/development/gitaly.md +++ b/doc/development/gitaly.md @@ -4,7 +4,7 @@ group: Gitaly info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Gitaly developers guide +# Gitaly development guidelines [Gitaly](https://gitlab.com/gitlab-org/gitaly) is a high-level Git RPC service used by GitLab Rails, Workhorse and GitLab Shell. @@ -14,7 +14,7 @@ Workhorse and GitLab Shell. In May 2019, Bob Van Landuyt -hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/issues/1`) +hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/-/issues/1`) on the [Gitaly project](https://gitlab.com/gitlab-org/gitaly). It included how to contribute to it as a Ruby developer, and shared domain-specific knowledge with anyone who may work in this part of the codebase in the future. @@ -54,16 +54,6 @@ they are merged. - See [below](#running-tests-with-a-locally-modified-version-of-gitaly) for instructions on running GitLab tests with a modified version of Gitaly. - In GDK run `gdk install` and restart GDK using `gdk restart` to use a locally modified Gitaly version for development -### `gitaly-ruby` - -It is possible to implement and test RPCs in Gitaly using Ruby code, -in -[`gitaly-ruby`](https://gitlab.com/gitlab-org/gitaly/tree/master/ruby). -This should make it easier to contribute for developers who are less -comfortable writing Go code. - -For more information, see the [Beginner's guide to Gitaly contributions](https://gitlab.com/gitlab-org/gitaly/-/blob/master/doc/beginners_guide.md). - ## Gitaly-Related Test Failures If your test-suite is failing with Gitaly issues, as a first step, try running: @@ -178,7 +168,8 @@ can replace `tmp/tests/gitaly` with a symlink. This is much faster because it avoids a Gitaly re-install each time you run `rspec`. Make sure this directory contains the files `config.toml` and `praefect.config.toml`. -You can copy them from `config.toml.example` and `config.praefect.toml.example` respectively. +You can copy `config.toml` from `config.toml.example`, and `praefect.config.toml` +from `config.praefect.toml.example`. After copying, make sure to edit them so everything points to the correct paths. ```shell @@ -249,7 +240,7 @@ Re-run steps 2-5 each time you want to try out new changes. [Return to Development documentation](index.md) -## Wrapping RPCs in Feature Flags +## Wrapping RPCs in feature flags Here are the steps to gate a new feature in Gitaly behind a feature flag. @@ -257,13 +248,13 @@ Here are the steps to gate a new feature in Gitaly behind a feature flag. 1. Create a package scoped flag name: - ```golang + ```go var findAllTagsFeatureFlag = "go-find-all-tags" ``` 1. Create a switch in the code using the `featureflag` package: - ```golang + ```go if featureflag.IsEnabled(ctx, findAllTagsFeatureFlag) { // go implementation } else { @@ -273,7 +264,7 @@ Here are the steps to gate a new feature in Gitaly behind a feature flag. 1. Create Prometheus metrics: - ```golang + ```go var findAllTagsRequests = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "gitaly_find_all_tags_requests_total", @@ -297,7 +288,7 @@ Here are the steps to gate a new feature in Gitaly behind a feature flag. 1. Set headers in tests: - ```golang + ```go import ( "google.golang.org/grpc/metadata" diff --git a/doc/development/github_importer.md b/doc/development/github_importer.md index 17b4a28f57d..73497c22b65 100644 --- a/doc/development/github_importer.md +++ b/doc/development/github_importer.md @@ -12,12 +12,13 @@ necessary to import GitHub projects into a GitLab instance. The GitHub importer offers two different types of importers: a sequential importer and a parallel importer. The Rake task `import:github` uses the -sequential importer, while everything else uses the parallel importer. The -difference between these two importers is quite simple: the sequential importer -does all work in a single thread, making it more useful for debugging purposes -or Rake tasks. The parallel importer, on the other hand, uses Sidekiq. +sequential importer, and everything else uses the parallel importer. The +difference between these two importers is: -## Requirements +- The sequential importer does all the work in a single thread, so it's more suited for debugging purposes or Rake tasks. +- The parallel importer uses Sidekiq. + +## Prerequisites - GitLab CE 10.2.0 or newer. - Sidekiq workers that process the `github_importer` and @@ -69,30 +70,38 @@ don't need to perform this work in parallel. This worker imports all pull requests. For every pull request a job for the `Gitlab::GithubImport::ImportPullRequestWorker` worker is scheduled. -### 5. Stage::ImportPullRequestsMergedByWorker +### 5. Stage::ImportCollaboratorsWorker + +This worker imports only direct repository collaborators who are not outside collaborators. +For every collaborator, we schedule a job for the `Gitlab::GithubImport::ImportCollaboratorWorker` worker. + +NOTE: +This stage is optional (controlled by `Gitlab::GithubImport::Settings`) and is selected by default. + +### 6. Stage::ImportPullRequestsMergedByWorker This worker imports the pull requests' _merged-by_ user information. The [_List pull requests_](https://docs.github.com/en/rest/pulls#list-pull-requests) API doesn't provide this information. Therefore, this stage must fetch each merged pull request individually to import this information. A -`Gitlab::GithubImport::ImportPullRequestMergedByWorker` job is scheduled for each fetched pull +`Gitlab::GithubImport::PullRequests::ImportMergedByWorker` job is scheduled for each fetched pull request. -### 6. Stage::ImportPullRequestsReviewRequestsWorker +### 7. Stage::ImportPullRequestsReviewRequestsWorker This worker imports assigned reviewers of pull requests. For each pull request, this worker: - Fetches all assigned review requests. - Schedules a `Gitlab::GithubImport::PullRequests::ImportReviewRequestWorker` job for each fetched review request. -### 7. Stage::ImportPullRequestsReviewsWorker +### 8. Stage::ImportPullRequestsReviewsWorker This worker imports reviews of pull requests. For each pull request, this worker: - Fetches all the pages of reviews. -- Schedules a `Gitlab::GithubImport::ImportPullRequestReviewWorker` job for each fetched review. +- Schedules a `Gitlab::GithubImport::PullRequests::ImportReviewWorker` job for each fetched review. -### 8. Stage::ImportIssuesAndDiffNotesWorker +### 9. Stage::ImportIssuesAndDiffNotesWorker This worker imports all issues and pull request comments. For every issue, we schedule a job for the `Gitlab::GithubImport::ImportIssueWorker` worker. For @@ -108,7 +117,7 @@ label links in the same worker removes the need for performing a separate crawl through the API data, reducing the number of API calls necessary to import a project. -### 9. Stage::ImportIssueEventsWorker +### 10. Stage::ImportIssueEventsWorker This worker imports all issues and pull request events. For every event, we schedule a job for the `Gitlab::GithubImport::ImportIssueEventWorker` worker. @@ -124,7 +133,7 @@ Therefore, both issues and pull requests have a common API for most related thin NOTE: This stage is optional and can consume significant extra import time (controlled by `Gitlab::GithubImport::Settings`). -### 10. Stage::ImportNotesWorker +### 11. Stage::ImportNotesWorker This worker imports regular comments for both issues and pull requests. For every comment, we schedule a job for the @@ -135,7 +144,7 @@ returns comments for both issues and pull requests. This means we have to wait for all issues and pull requests to be imported before we can import regular comments. -### 11. Stage::ImportAttachmentsWorker +### 12. Stage::ImportAttachmentsWorker This worker imports note attachments that are linked inside Markdown. For each entity with Markdown text in the project, we schedule a job of: @@ -154,7 +163,7 @@ Each job: NOTE: It's an optional stage that could consume significant extra import time (controlled by `Gitlab::GithubImport::Settings`). -### 12. Stage::ImportProtectedBranchesWorker +### 13. Stage::ImportProtectedBranchesWorker This worker imports protected branch rules. For every rule that exists on GitHub, we schedule a job of @@ -163,7 +172,7 @@ For every rule that exists on GitHub, we schedule a job of Each job compares the branch protection rules from GitHub and GitLab and applies the strictest of the rules to the branches in GitLab. -### 13. Stage::FinishImportWorker +### 14. Stage::FinishImportWorker This worker completes the import process by performing some housekeeping (such as flushing any caches) and by marking the import as completed. @@ -179,10 +188,10 @@ Advancing stages is done in one of two ways: The first approach should only be used by workers that perform all their work in a single thread, while `AdvanceStageWorker` should be used for everything else. -The way `AdvanceStageWorker` works is fairly simple. When scheduling a job it +When you schedule a job, `AdvanceStageWorker` is given a project ID, a list of Redis keys, and the name of the next stage. The Redis keys (produced by `Gitlab::JobWaiter`) are used to check if the -currently running stage has been completed or not. If the stage has not yet been +running stage has been completed or not. If the stage has not yet been completed `AdvanceStageWorker` reschedules itself. After a stage finishes `AdvanceStageworker` refreshes the import JID (more on this below) and schedule the worker of the next stage. @@ -324,14 +333,6 @@ The last log entry reports the number of objects fetched and imported: } ``` -## Errors when importing large projects - -The GitHub importer may encounter errors when importing large projects. For help with this, see the -documentation for the following use cases: - -- [Alternative way to import notes and diff notes](../user/project/import/github.md#alternative-way-to-import-notes-and-diff-notes) -- [Reduce GitHub API request objects per page](../user/project/import/github.md#reduce-github-api-request-objects-per-page) - ## Metrics dashboards To assess the GitHub importer health, the [GitHub importer dashboard](https://dashboards.gitlab.net/d/importers-github-importer/importers-github-importer) diff --git a/doc/development/gitlab_flavored_markdown/index.md b/doc/development/gitlab_flavored_markdown/index.md index f115ae9a11c..cde83bff32e 100644 --- a/doc/development/gitlab_flavored_markdown/index.md +++ b/doc/development/gitlab_flavored_markdown/index.md @@ -1,12 +1,12 @@ --- stage: Create -group: Editor +group: IDE info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# GitLab Flavored Markdown (GLFM) developer documentation +# GitLab Flavored Markdown (GLFM) development guidelines This page contains the MVC for the developer documentation for GitLab Flavored Markdown (GLFM). For the user documentation about Markdown in GitLab, refer to diff --git a/doc/development/gitlab_flavored_markdown/specification_guide/index.md b/doc/development/gitlab_flavored_markdown/specification_guide/index.md index 806ac3837bf..ae78daa3687 100644 --- a/doc/development/gitlab_flavored_markdown/specification_guide/index.md +++ b/doc/development/gitlab_flavored_markdown/specification_guide/index.md @@ -1,6 +1,6 @@ --- stage: Create -group: Editor +group: IDE info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -178,7 +178,7 @@ strings in the standard In this context, it should not be confused with other similar or related meanings of _example_, such as -[RSpec examples](https://relishapp.com/rspec/rspec-core/docs/example-groups/basic-structure-describe-it). +[RSpec examples](https://rspec.info/features/3-12/rspec-core/example-groups/basic-structure/). See the section on the [`glfm_official_specification.md`](#glfm_official_specificationmd) file for more details on the backtick-delimited Markdown+HTML example syntax. @@ -297,7 +297,7 @@ The Markdown dialect used in the GitLab application has a dual requirement for r 1. Rendering to static read-only HTML format, to be displayed in various places throughout the application. 1. Rendering editable content in the - [Content Editor](https://about.gitlab.com/direction/create/editor/content_editor/), + [Content Editor](https://about.gitlab.com/direction/plan/knowledge/content_editor/), a ["What You See Is What You Get" (WYSIWYG)](https://en.wikipedia.org/wiki/WYSIWYG) editor. The Content Editor supports real-time instant switching between an editable Markdown source and an editable WYSIWYG document. diff --git a/doc/development/gitlab_shell/index.md b/doc/development/gitlab_shell/index.md index 7097fd48cea..0663341f806 100644 --- a/doc/development/gitlab_shell/index.md +++ b/doc/development/gitlab_shell/index.md @@ -4,7 +4,7 @@ group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# GitLab Shell +# GitLab Shell development guidelines [![pipeline status](https://gitlab.com/gitlab-org/gitlab-shell/badges/main/pipeline.svg)](https://gitlab.com/gitlab-org/gitlab-shell/-/pipelines?ref=main) [![coverage report](https://gitlab.com/gitlab-org/gitlab-shell/badges/main/coverage.svg)](https://gitlab.com/gitlab-org/gitlab-shell/-/pipelines?ref=main) [![Code Climate](https://codeclimate.com/github/gitlabhq/gitlab-shell.svg)](https://codeclimate.com/github/gitlabhq/gitlab-shell) @@ -21,8 +21,8 @@ Ruby to build and test, but not to run. GitLab Shell runs on `port 22` on an Omnibus installation. To use a regular SSH service, configure it on an alternative port. -Download and install the current version of Go from [golang.org](https://go.dev/dl/). -We follow the [Golang Release Policy](https://golang.org/doc/devel/release.html#policy) +Download and install the [current version of Go](https://go.dev/dl/). +We follow the [Go Release Policy](https://go.dev/doc/devel/release#policy) and support: - The current stable version. @@ -217,6 +217,6 @@ sequenceDiagram ## Related topics -- [LICENSE](https://gitlab.com/gitlab-org/gitlab-shell/-/blob/main/LICENSE). +- [LICENSE](https://gitlab.com/gitlab-org/gitlab-shell/-/blob/main/LICENSE) - [PROCESS.md](https://gitlab.com/gitlab-org/gitlab-shell/-/blob/main/PROCESS.md) - [Using the GitLab Shell chart](https://docs.gitlab.com/charts/charts/gitlab/gitlab-shell/) diff --git a/doc/development/gitpod_internals.md b/doc/development/gitpod_internals.md index a4674df758d..a4b340916dd 100644 --- a/doc/development/gitpod_internals.md +++ b/doc/development/gitpod_internals.md @@ -25,6 +25,6 @@ You can find this webhook in [Webhook Settings in `gitlab-org/gitlab`](https://g If a webhook failed to connect for a long time, then it may have been disabled in the project. -To re-enable a failing or failed webhook, send a test request in [Webhook Settings](https://gitlab.com/gitlab-org/gitlab/-/hooks). See [Re-enable disabled webhooks page](https://docs.gitlab.com/15.4/ee/user/project/integrations/webhooks.html#re-enable-disabled-webhooks) for more details. +To re-enable a failing or failed webhook, send a test request in [Webhook Settings](https://gitlab.com/gitlab-org/gitlab/-/hooks). See [Re-enable disabled webhooks page](../user/project/integrations/webhooks.md#re-enable-disabled-webhooks) for more details. After re-enabling, check the prebuilds' health in a [project's prebuilds](https://gitpod.io/t/gitlab-org/gitlab/prebuilds) and confirm that prebuilds start without any errors. diff --git a/doc/development/go_guide/go_upgrade.md b/doc/development/go_guide/go_upgrade.md index b3ec0a15c37..7fc18604a3d 100644 --- a/doc/development/go_guide/go_upgrade.md +++ b/doc/development/go_guide/go_upgrade.md @@ -26,7 +26,7 @@ by Distribution: ## Supporting multiple Go versions -Individual Golang projects need to support multiple Go versions because: +Individual Go projects need to support multiple Go versions because: - When a new version of Go is released, we should start integrating it into the CI pipelines to verify compatibility with the new compiler. - We must support the [official Omnibus GitLab Go version](#updating-go-version), which may be behind the latest minor release. @@ -150,6 +150,7 @@ if you need help finding the correct person or labels: | [Alertmanager](https://github.com/prometheus/alertmanager) | [Issue Tracker](https://gitlab.com/gitlab-org/gitlab/-/issues) | | Docker Distribution Pruner | [Issue Tracker](https://gitlab.com/gitlab-org/docker-distribution-pruner) | | Gitaly | [Issue Tracker](https://gitlab.com/gitlab-org/gitaly/-/issues) | +| GitLab CLI (`glab`). | [Issue Tracker](https://gitlab.com/gitlab-org/cli/-/issues) | GitLab Compose Kit | [Issuer Tracker](https://gitlab.com/gitlab-org/gitlab-compose-kit/-/issues) | | GitLab Container Registry | [Issue Tracker](https://gitlab.com/gitlab-org/container-registry) | | GitLab Elasticsearch Indexer | [Issue Tracker](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer/-/issues) | diff --git a/doc/development/go_guide/index.md b/doc/development/go_guide/index.md index 508219cee43..e51542649bb 100644 --- a/doc/development/go_guide/index.md +++ b/doc/development/go_guide/index.md @@ -124,7 +124,7 @@ lint: # Write the code coverage report to gl-code-quality-report.json # and print linting issues to stdout in the format: path/to/file:line description # remove `--issues-exit-code 0` or set to non-zero to fail the job if linting issues are detected - - golangci-lint run --issues-exit-code 0 --out-format code-climate | tee gl-code-quality-report.json | jq -r '.[] | "\(.location.path):\(.location.lines.begin) \(.description)"' + - golangci-lint run --issues-exit-code 0 --print-issued-lines=false --out-format code-climate:gl-code-quality-report.json,line-number artifacts: reports: codequality: gl-code-quality-report.json @@ -216,7 +216,7 @@ When comparing expected and actual values in tests, use and others to improve readability when comparing structs, errors, large portions of text, or JSON documents: -```golang +```go type TestData struct { // ... } @@ -291,7 +291,7 @@ easier to debug. For example: -```golang +```go // Wrap the error return nil, fmt.Errorf("get cache %s: %w", f.Name, err) @@ -438,7 +438,7 @@ up to run `goimports -local gitlab.com/gitlab-org` so that it's applied to every ### Naming branches -Only use the characters `a-z`, `0-9` or `-` in branch names. This restriction is due to the fact that `go get` doesn't work as expected when a branch name contains certain characters, such as a slash `/`: +In addition to the GitLab [branch name rules](../../user/project/repository/branches/index.md#name-your-branch), use only the characters `a-z`, `0-9` or `-` in branch names. This restriction is because `go get` doesn't work as expected when a branch name contains certain characters, such as a slash `/`: ```shell $ go get -u gitlab.com/gitlab-org/security-products/analyzers/report/v3@some-user/some-feature @@ -462,7 +462,7 @@ allocations. **Don't:** -```golang +```go var s2 []string for _, val := range s1 { s2 = append(s2, val) @@ -471,8 +471,8 @@ for _, val := range s1 { **Do:** -```golang -s2 := make([]string, 0, size) +```go +s2 := make([]string, 0, len(s1)) for _, val := range s1 { s2 = append(s2, val) } @@ -494,7 +494,7 @@ If the scanner report is small, less than 35 lines, then feel free to [inline th The [go-cmp](https://github.com/google/go-cmp) package should be used when comparing large structs in tests. It makes it possible to output a specific diff where the two structs differ, rather than seeing the whole of both structs printed out in the test logs. Here is a small example: -```golang +```go package main import ( diff --git a/doc/development/graphql_guide/graphql_pro.md b/doc/development/graphql_guide/graphql_pro.md index ec28ceb4f20..7c5770a4410 100644 --- a/doc/development/graphql_guide/graphql_pro.md +++ b/doc/development/graphql_guide/graphql_pro.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/development/graphql_guide/index.md b/doc/development/graphql_guide/index.md index 9c6a2559e5c..ef30d489832 100644 --- a/doc/development/graphql_guide/index.md +++ b/doc/development/graphql_guide/index.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/development/graphql_guide/monitoring.md b/doc/development/graphql_guide/monitoring.md index 1e4c083653e..328cea2648e 100644 --- a/doc/development/graphql_guide/monitoring.md +++ b/doc/development/graphql_guide/monitoring.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/development/graphql_guide/pagination.md b/doc/development/graphql_guide/pagination.md index daa39875119..a858d9be681 100644 --- a/doc/development/graphql_guide/pagination.md +++ b/doc/development/graphql_guide/pagination.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/development/graphql_guide/reviewing.md b/doc/development/graphql_guide/reviewing.md new file mode 100644 index 00000000000..9c32179a89d --- /dev/null +++ b/doc/development/graphql_guide/reviewing.md @@ -0,0 +1,96 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# GraphQL API merge request checklist + +The GitLab GraphQL API has a fair degree of complexity so it's important that merge requests containing GraphQL changes be reviewed by someone familiar with GraphQL. +You can ping one via the `@gitlab-org/graphql-experts` group in a MR or in the [`#f_graphql` channel](https://gitlab.slack.com/archives/C6MLS3XEU) in Slack (available to GitLab team members only). + +GraphQL queries need to be reviewed for: + +- breaking changes +- authorization +- performance + +## Review criteria + +This is not an exhaustive list. + +### Description with sample query + +Ensure that the description includes a sample query with setup instructions. +Try running the query in [GraphiQL](../api_graphql_styleguide.md#graphiql) on your local GDK instance. + +### No breaking changes (unless after full deprecation cycle) + +Check the MR for any [breaking changes](../api_graphql_styleguide.md#breaking-changes). + +If a feature is marked as an [Experiment](../api_graphql_styleguide.md#mark-schema-items-as-alpha), you can make breaking changes immediately, with no deprecation period. + +For more information, see [deprecation and removal process](../../api/graphql/index.md#deprecation-and-removal-process). + +### Multiversion compatibility + +Ensure that multi-version compatibility is guaranteed. +This generally means frontend and backend code for the same GraphQL feature can't be shipped in the same release. + +For details, see [multiple version compatibility](../multi_version_compatibility.md). + +### Technical writing review + +Changes to the generated API docs require a technical writer review. + +### Changelog + +Public-facing changes that are not marked as an [Experiment](../api_graphql_styleguide.md#mark-schema-items-as-alpha) require a [changelog entry](../changelog.md). + +### Use the framework + +GraphQL is a framework with many moving parts. It's important that the framework is followed. + +- Do not manually invoke framework bits. For example, do not instantiate resolvers during execution and instead let the framework do that. +- You can subclass resolvers, as in `MyResolver.single` (see [deriving resolvers](../api_graphql_styleguide.md#deriving-resolvers)). +- Use the `ready?` method for more complex argument logic (see [correct use of resolver#ready](../api_graphql_styleguide.md#correct-use-of-resolverready)). +- Use the `prepare` method for more complex argument validation (see [validating arguments](../api_graphql_styleguide.md#validating-arguments)). + +For details, see [resolver guide](../api_graphql_styleguide.md#writing-resolvers). + +### Authorization + +Ensure proper authorization is followed and that `authorize :some_ability` is tested in the specs. + +For details, see [authorization guide](authorization.md). + +### Performance + +Ensure: + +- You avoid N+1s with [BatchLoader](batchloader.md) or [Lookahead](../api_graphql_styleguide.md#look-ahead) when appropriate. +- You use [laziness](../api_graphql_styleguide.md#laziness) appropriately. + +### Use appropriate types + +For example: + +- [`TimeType`](../api_graphql_styleguide.md#typestimetype) for Ruby `Time` and `DateTime` objects. +- Global IDs for `id` fields + +For details, see [types](../api_graphql_styleguide.md#types). + +### Appropriate complexity + +Query complexity is a way of quantifying how expensive a query is likely to be. Query complexity limits are defined as constants in the schema. +When a resolver or type is expensive to call we need to ensure that the query complexity reflects that. + +For details, see [max complexity](../api_graphql_styleguide.md#max-complexity), [field complexity](../api_graphql_styleguide.md#field-complexity) and [query limits](../api_graphql_styleguide.md#query-limits). + +### Testing + +- Resolver (unit) specs are deprecated in favour of request (integration) specs. +- Many aspects of our framework are outside the `resolve` method and a request spec is the only way to ensure they behave properly. +- Every GraphQL change MR should ideally have changes to API specs. + +For details, see [testing guide](../api_graphql_styleguide.md#testing). diff --git a/doc/development/i18n/externalization.md b/doc/development/i18n/externalization.md index 2269a28e496..ac14b1b5ea2 100644 --- a/doc/development/i18n/externalization.md +++ b/doc/development/i18n/externalization.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -39,7 +39,7 @@ The following tools are used: - [`gettext_i18n_rails_js`](https://github.com/webhippie/gettext_i18n_rails_js): this gem makes the translations available in JavaScript. It provides the following Rake task: - - `rake gettext:po_to_json`: reads the contents of the PO files and generates JSON files that + - `rake gettext:compile`: reads the contents of the PO files and generates JS files which contain all the available translations. - PO editor: there are multiple applications that can help us work with PO files. A good option is @@ -561,9 +561,8 @@ To include formatting in the translated string, you can do the following: - In Ruby/HAML: ```ruby - html_escape(_('Some %{strongOpen}bold%{strongClose} text.')) % { strongOpen: ''.html_safe, strongClose: ''.html_safe } - - # => 'Some bold text.' + safe_format(_('Some %{strongOpen}bold%{strongClose} text.'), strongOpen: ''.html_safe, strongClose: ''.html_safe) + # => 'Some bold text.' ``` - In JavaScript: @@ -801,8 +800,8 @@ translatable in certain languages. ```haml - zones_link_url = 'https://cloud.google.com/compute/docs/regions-zones/regions-zones' - - zones_link_start = ''.html_safe % { url: zones_link_url } - = html_escape(s_('ClusterIntegration|Learn more about %{zones_link_start}zones%{zones_link_end}')) % { zones_link_start: zones_link_start, zones_link_end: ''.html_safe } + - zones_link_start = safe_format('', url: zones_link_url) + = safe_format(s_('ClusterIntegration|Learn more about %{zones_link_start}zones%{zones_link_end}'), zones_link_start: zones_link_start, zones_link_end: ''.html_safe) ``` - In Vue, instead of: @@ -1007,4 +1006,4 @@ Suppose you want to add translations for a new language, for example, French: To manually test Vue translations: 1. Change the GitLab localization to another language than English. -1. Generate JSON files using `bin/rake gettext:po_to_json` or `bin/rake gettext:compile`. +1. Generate JSON files using `bin/rake gettext:compile`. diff --git a/doc/development/i18n/index.md b/doc/development/i18n/index.md index 757bcc0ebf1..0f9688ec26d 100644 --- a/doc/development/i18n/index.md +++ b/doc/development/i18n/index.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/development/i18n/merging_translations.md b/doc/development/i18n/merging_translations.md index f98e5119916..4f36cbe125a 100644 --- a/doc/development/i18n/merging_translations.md +++ b/doc/development/i18n/merging_translations.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/development/i18n/proofreader.md b/doc/development/i18n/proofreader.md index cc7232cd793..74dba53183c 100644 --- a/doc/development/i18n/proofreader.md +++ b/doc/development/i18n/proofreader.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -19,6 +19,8 @@ are very appreciative of the work done by translators and proofreaders! - Tsegaselassie Tadesse - [GitLab](https://gitlab.com/tsega), [Crowdin](https://crowdin.com/profile/tsegaselassi) - Arabic - Proofreaders needed. +- Belarusian + - Anton Katsuba - [GitLab](https://gitlab.com/coinvariant), [Crowdin](https://crowdin.com/profile/aerialfiddle) - Bosnian - Haris Delalić - [GitLab](https://gitlab.com/haris.delalic), [Crowdin](https://crowdin.com/profile/haris.delalic) - Bulgarian @@ -103,7 +105,7 @@ are very appreciative of the work done by translators and proofreaders! - Portuguese - Diogo Trindade - [GitLab](https://gitlab.com/luisdiogo2071317), [Crowdin](https://crowdin.com/profile/ldiogotrindade) - Portuguese, Brazilian - - Paulo George Gomes Bezerra - [GitLab](https://gitlab.com/paulobezerra), [Crowdin](https://crowdin.com/profile/paulogomes.rep) + - Paulo George Gomes Bezerra - [GitLab](https://gitlab.com/paulobezerra) - André Gama - [GitLab](https://gitlab.com/andregamma), [Crowdin](https://crowdin.com/profile/ToeOficial) - Eduardo Addad de Oliveira - [GitLab](https://gitlab.com/eduardoaddad), [Crowdin](https://crowdin.com/profile/eduardoaddad) - Horberlan Brito - [GitLab](https://gitlab.com/horberlan), [Crowdin](https://crowdin.com/profile/horberlan) @@ -126,6 +128,7 @@ are very appreciative of the work done by translators and proofreaders! - Proofreaders needed. - Spanish - Pedro Garcia - [GitLab](https://gitlab.com/pedgarrod), [Crowdin](https://crowdin.com/profile/breaking_pitt) + - David Elizondo - [GitLab](https://gitlab.com/daelmo), [Crowdin](https://crowdin.com/profile/daelmo) - Swedish - Johannes Nilsson - [GitLab](https://gitlab.com/nlssn), [Crowdin](https://crowdin.com/profile/nlssn) - Turkish diff --git a/doc/development/i18n/translation.md b/doc/development/i18n/translation.md index bfc4d817c73..cf6ee16f157 100644 --- a/doc/development/i18n/translation.md +++ b/doc/development/i18n/translation.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/development/image_scaling.md b/doc/development/image_scaling.md index d182bd8333e..48b780b50bf 100644 --- a/doc/development/image_scaling.md +++ b/doc/development/image_scaling.md @@ -1,6 +1,6 @@ --- -stage: Manage -group: Organization +stage: Data Stores +group: Tenant Scale info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/development/img/feature-flag-metrics.png b/doc/development/img/feature-flag-metrics.png index ce8702d47eb..f94c0fc3e46 100644 Binary files a/doc/development/img/feature-flag-metrics.png and b/doc/development/img/feature-flag-metrics.png differ diff --git a/doc/development/import_export.md b/doc/development/import_export.md index 17e733e8904..b8493ef7a6e 100644 --- a/doc/development/import_export.md +++ b/doc/development/import_export.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/development/import_project.md b/doc/development/import_project.md index 7f5a0faf8fb..ed5854f8833 100644 --- a/doc/development/import_project.md +++ b/doc/development/import_project.md @@ -4,28 +4,29 @@ group: unassigned info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Test Import Project +# Test import project For testing, we can import our own [GitLab CE](https://gitlab.com/gitlab-org/gitlab-foss/) project (named `gitlabhq` in this case) under a group named `qa-perf-testing`. Project tarballs that can be used for testing can be found over on the [performance-data](https://gitlab.com/gitlab-org/quality/performance-data) project. A different project could be used if required. -There are several options for importing the project into your GitLab environment. They are detailed as follows with the assumption that the recommended group `qa-perf-testing` and project `gitlabhq` are being set up. +You can import the project into your GitLab environment in a number of ways. They are detailed as follows with the +assumption that the recommended group `qa-perf-testing` and project `gitlabhq` are being set up. ## Importing the project -There are several ways to import a project. +Use one of these methods to import the test project. -### Importing via UI +### Import by using the UI -The first option is to [import the Project tarball file via the GitLab UI](../user/project/settings/import_export.md#import-a-project-and-its-data): +The first option is to [import the project tarball file by using the GitLab UI](../user/project/settings/import_export.md#import-a-project-and-its-data): -1. Create the group `qa-perf-testing` -1. Import the [GitLab FOSS project tarball](https://gitlab.com/gitlab-org/quality/performance-data/-/blob/master/projects_export/gitlabhq_export.tar.gz) into the Group. +1. Create the group `qa-perf-testing`. +1. Import the [GitLab FOSS project tarball](https://gitlab.com/gitlab-org/quality/performance-data/-/blob/master/projects_export/gitlabhq_export.tar.gz) into the group. It should take up to 15 minutes for the project to fully import. You can head to the project's main page for the current status. This method ignores all the errors silently (including the ones related to `GITALY_DISABLE_REQUEST_LIMITS`) and is used by GitLab users. For development and testing, check the other methods below. -### Importing via the `import-project` script +### Import by using the `import-project` script A convenient script, [`bin/import-project`](https://gitlab.com/gitlab-org/quality/performance/blob/master/bin/import-project), is provided with [performance](https://gitlab.com/gitlab-org/quality/performance) project to import the Project tarball into a GitLab environment via API from the terminal. @@ -42,7 +43,7 @@ bin/import-project --help The process should take up to 15 minutes for the project to import fully. The script checks the status periodically and exits after the import has completed. -### Importing via GitHub +### Import by using GitHub There is also an option to [import the project via GitHub](../user/project/import/github.md): @@ -51,126 +52,12 @@ There is also an option to [import the project via GitHub](../user/project/impor This method takes longer to import than the other methods and depends on several factors. It's recommended to use the other methods. -### Importing via a Rake task - -> The [Rake task](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/tasks/gitlab/import_export/import.rake) was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20724) in GitLab 12.6, replacing a GitLab.com Ruby script. - -This script was introduced in GitLab 12.6 for importing large GitLab project exports. - -As part of this script we also disable direct upload to avoid situations where a huge archive is being uploaded to GCS (while being inside a transaction, which can cause idle transaction timeouts). - -We can run this script from the terminal: - -Parameters: - -| Attribute | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `username` | string | yes | User name | -| `namespace_path` | string | yes | Namespace path | -| `project_path` | string | yes | Project name | -| `archive_path` | string | yes | Path to the exported project tarball you want to import | - -```shell -bundle exec rake "gitlab:import_export:import[root, group/subgroup, testingprojectimport, /path/to/file.tar.gz]" -``` - -If you're running Omnibus, run the following Rake task: - -```shell -gitlab-rake "gitlab:import_export:import[root, group/subgroup, testingprojectimport, /path/to/file.tar.gz]" -``` - -#### Enable verbose output - -To make the import Rake task more verbose, use the `IMPORT_DEBUG` environment variable: - -```shell -IMPORT_DEBUG=true gitlab-rake "gitlab:import_export:import[root, group/subgroup, testingprojectimport, /path/to/file.tar.gz]" -``` - -#### Troubleshooting - -Check the common errors listed below, what they mean, and how to fix them. - -##### `Exception: undefined method 'name' for nil:NilClass` - -The `username` is not valid. - -##### `Exception: undefined method 'full_path' for nil:NilClass` - -The `namespace_path` does not exist. -For example, one of the groups or subgroups is mistyped or missing -or you've specified the project name in the path. - -The task only creates the project. -If you want to import it to a new group or subgroup then create it first. - -##### `Exception: No such file or directory @ rb_sysopen - (filename)` - -The specified project export file in `archive_path` is missing. - -##### `Exception: Permission denied @ rb_sysopen - (filename)` - -The specified project export file cannot be accessed by the `git` user. - -Setting the file owner to `git:git`, changing the file permissions to `0400`, and moving it to a -public folder (for example `/tmp/`) fixes the issue. - -##### `Name can contain only letters, digits, emojis ...` - -```plaintext -Name can contain only letters, digits, emojis, '_', '.', '+', dashes, or spaces. It must start with a letter, -digit, emoji, or '_', and Path can contain only letters, digits, '_', '-', or '.'. It cannot start -with '-', end in '.git', or end in '.atom'. -``` - -The project name specified in `project_path` is not valid for one of the specified reasons. - -Only put the project name in `project_path`. For example, if you provide a path of subgroups -it fails with this error as `/` is not a valid character in a project name. - -##### `Name has already been taken and Path has already been taken` - -A project with that name already exists. - -##### `Exception: Error importing repository into (namespace) - No space left on device` - -The disk has insufficient space to complete the import. - -During import, the tarball is cached in your configured `shared_path` directory. Verify the -disk has enough free space to accommodate both the cached tarball and the unpacked -project files on disk. - -##### Import is successful, but with a `Total number of not imported relations: XX` message, and issues are not created during the import - -If you receive a `Total number of not imported relations: XX` message, and issues -aren't created during the import, check [exceptions_json.log](../administration/logs/index.md#exceptions_jsonlog). -You might see an error like `N is out of range for ActiveModel::Type::Integer with limit 4 bytes`, -where `N` is the integer exceeding the 4-byte integer limit. If that's the case, you -are likely hitting the issue with rebalancing of `relative_position` field of the issues. - -```ruby -# Check the current maximum value of relative_position -Issue.where(project_id: Project.find(ID).root_namespace.all_projects).maximum(:relative_position) - -# Run the rebalancing process and check if the maximum value of relative_position has changed -Issues::RelativePositionRebalancingService.new(Project.find(ID).root_namespace.all_projects).execute -Issue.where(project_id: Project.find(ID).root_namespace.all_projects).maximum(:relative_position) -``` - -Repeat the import attempt after that and check if the issues are imported successfully. - -##### Gitaly calls error when importing - -If you're attempting to import a large project into a development environment, you may see Gitaly throw an error about too many calls or invocations, for example: - -```plaintext -Error importing repository into qa-perf-testing/gitlabhq - GitalyClient#call called 31 times from single request. Potential n+1? -``` +### Import by using a Rake task -This is due to a [n+1 calls limit being set for development setups](gitaly.md#toomanyinvocationserror-errors). You can work around this by setting `GITALY_DISABLE_REQUEST_LIMITS=1` as an environment variable, restarting your development environment and importing again. +To import the test project by using a Rake task, see +[Import large projects](../administration/raketasks/project_import_export.md#import-large-projects). -### Importing via the Rails console +### Import by using the Rails console The last option is to import a project using a Rails console: @@ -245,8 +132,9 @@ bundle exec rails r /path_to_script/script.rb project_name /path_to_extracted_p ## Access token setup -Many of the tests also require a GitLab Personal Access Token. This is due to numerous endpoints themselves requiring authentication. +Many of the tests also require a GitLab personal access token because numerous endpoints require authentication themselves. -[The official GitLab docs detail how to create this token](../user/profile/personal_access_tokens.md#create-a-personal-access-token). The tests require that the token is generated by an administrator and that it has the `API` and `read_repository` permissions. +[The GitLab documentation details how to create this token](../user/profile/personal_access_tokens.md#create-a-personal-access-token). +The tests require that the token is generated by an administrator and that it has the `API` and `read_repository` permissions. Details on how to use the Access Token with each type of test are found in their respective documentation. diff --git a/doc/development/index.md b/doc/development/index.md index a39a83b257a..55e594c537a 100644 --- a/doc/development/index.md +++ b/doc/development/index.md @@ -1,5 +1,4 @@ --- -comments: false type: index, dev stage: none group: Development @@ -7,15 +6,15 @@ info: "See the Technical Writers assigned to Development Guidelines: https://abo description: "Development Guidelines: learn how to contribute to GitLab." --- -# Contribute to the development of GitLab +# Contribute to development Learn how to contribute to the development of the GitLab product. This content is intended for GitLab team members as well as members of the wider community. - [Contribute to GitLab development](contributing/index.md) -- [Contribute to Omnibus development](https://docs.gitlab.com/omnibus/development/) -- [Contribute to GitLab Pages development](pages/index.md) - [Contribute to GitLab Runner development](https://docs.gitlab.com/runner/development/) -- [Contribute to Helm chart development](https://docs.gitlab.com/charts/development/) +- [Contribute to GitLab Pages development](pages/index.md) +- [Contribute to GitLab distribution development](distribution/index.md) +- [Contribute to the GitLab Design System](https://design.gitlab.com/get-started/contributing) - [Contribute to the GitLab documentation](documentation/index.md) diff --git a/doc/development/integrations/codesandbox.md b/doc/development/integrations/codesandbox.md deleted file mode 100644 index 4553ed2966f..00000000000 --- a/doc/development/integrations/codesandbox.md +++ /dev/null @@ -1,155 +0,0 @@ ---- -stage: none -group: Development -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -remove_date: '2023-02-01' ---- - -# Set up local CodeSandbox development environment (removed) - -WARNING: -This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/108627) in GitLab 15.8 -and is planned for removal in 15.9. This change is a breaking change. - -This guide walks through setting up a local [CodeSandbox repository](https://github.com/codesandbox/codesandbox-client) and integrating it with a local GitLab instance. CodeSandbox -is used to power the Web IDE [Live Preview feature](../../user/project/web_ide/index.md#live-preview-removed). Having a local CodeSandbox setup is useful for debugging upstream issues or -creating upstream contributions like [this one](https://github.com/codesandbox/codesandbox-client/pull/5137). - -## Initial setup - -Before using CodeSandbox with your local GitLab instance, you must: - -1. Enable HTTPS on your GDK. CodeSandbox uses Service Workers that require `https`. - Follow the GDK [NGINX configuration instructions](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/nginx.md) to enable HTTPS for GDK. -1. Clone the [`codesandbox-client` project](https://github.com/codesandbox/codesandbox-client) - locally. If you plan on contributing upstream, you might want to fork and clone first. -1. Optional. Use correct `python` and `nodejs` versions. Otherwise, `yarn` may fail to - install or build some packages. If you're using `asdf` you can run the following commands: - - ```shell - asdf local nodejs 10.14.2 - asdf local python 2.7.18 - ``` - -1. Run the following commands in the `codesandbox-client` project checkout: - - ```shell - # This might be necessary for the `prepublishOnly` job that is run later - yarn global add lerna - - # Install packages - yarn - ``` - - You can run `yarn build:clean` to clean up the build assets. - -## Use local GitLab instance with local CodeSandbox - -GitLab integrates with two parts of CodeSandbox: - -- An npm package called `smooshpack` (called `sandpack` in the `codesandbox-client` project). - This exposes an entrypoint for us to kick off CodeSandbox's bundler. -- A server that houses CodeSandbox assets for bundling and previewing. This is hosted - on a separate server for security. - -Each time you want to run GitLab and CodeSandbox together, you need to perform the -steps in the following sections. - -### Use local `smooshpack` for GitLab - -GitLab usually satisfies its `smooshpack` dependency with a remote module, but we want -to use a locally-built module. To build and use a local `smooshpack` module: - -1. In the `codesandbox-client` project directory, run: - - ```shell - cd standalone-packages/sandpack - yarn link - - # (Optional) you might want to start a development build - yarn run start - ``` - - Now, in the GitLab project, you can run `yarn link "smooshpack"`. `yarn` looks - for `smooshpack` **on disk** as opposed to the one hosted remotely. - -1. In the `gitlab` project directory, run: - - ```shell - # Remove and reinstall node_modules just to be safe - rm -rf node_modules - yarn install - - # Use the "smooshpack" package on disk - yarn link "smooshpack" - ``` - -### Fix possible GDK webpack problem - -`webpack` in GDK can fail to find packages inside a linked package. This step can help -you avoid `webpack` breaking with messages saying that it can't resolve packages from -`smooshpack/dist/sandpack.es5.js`. - -In the `codesandbox-client` project directory, run: - -```shell -cd standalone-packages - -mkdir node_modules -ln -s $PATH_TO_LOCAL_GITLAB/node_modules/core-js ./node_modules/core-js -``` - -### Start building CodeSandbox app assets - -In the `codesandbox-client` project directory: - -```shell -cd packages/app - -yarn start:sandpack-sandbox -``` - -### Create HTTPS proxy for CodeSandbox `sandpack` assets - -Because we need `https`, we need to create a proxy to the webpack server. We can use -[`http-server`](https://www.npmjs.com/package/http-server), which can do this proxying -out of the box: - -```shell -npx http-server --proxy http://localhost:3000 -S -C $PATH_TO_CERT_PEM -K $PATH_TO_KEY_PEM -p 8044 -d false -``` - -### Update `bundler_url` setting in GitLab (removed) - -WARNING: -This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/108627) in GitLab 15.8 -and is planned for removal in 15.9. This change is a breaking change. - -We need to update our `application_setting_implementation.rb` to point to the server that hosts the -CodeSandbox `sandpack` assets. For instance, if these assets are hosted by a server at `https://sandpack.local:8044`: - -```patch -diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb -index 6eed627b502..1824669e881 100644 ---- a/app/models/application_setting_implementation.rb -+++ b/app/models/application_setting_implementation.rb -@@ -391,7 +391,7 @@ def static_objects_external_storage_enabled? - # This will eventually be configurable - # https://gitlab.com/gitlab-org/gitlab/-/issues/208161 - def web_ide_clientside_preview_bundler_url -- 'https://sandbox-prod.gitlab-static.net' -+ 'https://sandpack.local:8044' - end - - private - -``` - -NOTE: -You can apply this patch by copying it to your clipboard and running `pbpaste | git apply`. - -You may want to restart the GitLab Rails server after making this change: - -```shell -gdk restart rails-web -``` diff --git a/doc/development/integrations/index.md b/doc/development/integrations/index.md index ceb64ba2bb7..4c66dbfa1a4 100644 --- a/doc/development/integrations/index.md +++ b/doc/development/integrations/index.md @@ -1,11 +1,11 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments description: "GitLab's development guidelines for Integrations" --- -# Integrations development guide +# Integrations development guidelines This page provides development guidelines for implementing [GitLab integrations](../../user/project/integrations/index.md), which are part of our [main Rails project](https://gitlab.com/gitlab-org/gitlab). @@ -23,9 +23,11 @@ if you need clarification or spot any outdated information. - For example, `Integrations::FooBar` in `app/models/integrations/foo_bar.rb`. - For certain types of integrations, you can also build on these base classes: - `Integrations::BaseChatNotification` + - `Integrations::BaseCi` - `Integrations::BaseIssueTracker` - `Integrations::BaseMonitoring` - `Integrations::BaseSlashCommands` + - `Integrations::BaseThirdPartyWiki` - For integrations that primarily trigger HTTP calls to external services, you can also use the `Integrations::HasWebHook` concern. This reuses the [webhook functionality](../../user/project/integrations/webhooks.md) in GitLab through an associated `ServiceHook` model, and automatically records request logs @@ -37,9 +39,6 @@ if you need clarification or spot any outdated information. has_one :foo_bar_integration, class_name: 'Integrations::FooBar' ``` -1. TEMPORARY: Accommodate the current migration to [rename "services" to "integrations"](#rename-services-to-integrations): - - Add the integration's camel-cased name (`'FooBar'`) to `Gitlab::Integrations::StiType::NAMESPACED_INTEGRATIONS`. - ### Define properties Integrations can define arbitrary properties to store their configuration with the class method `Integration.prop_accessor`. @@ -237,9 +236,7 @@ module Integrations end ``` -### Expose the integration in the API - -#### REST API +### Expose the integration in the REST API To expose the integration in the [REST API](../../api/integrations.md): @@ -258,46 +255,6 @@ Sensitive fields are not exposed over the API. Sensitive fields are those fields - `token` - `webhook` -#### GraphQL API - -Integrations use the `Types::Projects::ServiceType` type by default, -which only exposes the `type` and `active` properties. - -To expose additional properties, you can write a class implementing `ServiceType`: - -```ruby -# in app/graphql/types/project/services/foo_bar_service_type.rb -module Types - module Projects - module Services - class FooBarServiceType < BaseObject - graphql_name 'FooBarService' - implements(Types::Projects::ServiceType) - authorize :read_project - - field :frobinity, - GraphQL::Types::Float, - null: true, - description: 'The level of frobinity.' - - field :foo_label, - GraphQL::Types::String, - null: true, - description: 'The foo label to apply.' - end - end - end -end -``` - -Each property you want to expose should have a field defined for it. You can also expose any public instance method of the integration. - -Contact a member of the Integrations team to discuss the best authorization. - -Reference documentation for GraphQL is automatically generated. - -You can also refer to our [GraphQL API style guide](../api_graphql_styleguide.md). - ## Availability of integrations By default, integrations are available on the project, group, and instance level. @@ -315,7 +272,7 @@ When developing a new integration, we also recommend you gate the availability b You can provide help text in the integration form, including links to off-site documentation, as described above in [Customize the frontend form](#customize-the-frontend-form). Refer to -our [usability guidelines](https://design.gitlab.com/usability/helping-users/) for help text. +our [usability guidelines](https://design.gitlab.com/usability/contextual-help) for help text. For more detailed documentation, provide a page in `doc/user/project/integrations`, and link it from the [Integrations overview](../../user/project/integrations/index.md). @@ -345,26 +302,8 @@ The strings should use the integration name as [namespace](../i18n/externalizati ## Ongoing migrations and refactorings -The Integrations team is in the process of some larger migrations that developers should be aware of. - -### [Rename "services" to "integrations"](https://gitlab.com/groups/gitlab-org/-/epics/2504) - -The "integrations" in GitLab were historically called "services", which frequently caused -confusion with our "service" classes in `app/services`. We sometimes also called -them "project services" because they were initially only available on projects, which is -not the case anymore. - -We decided to change the naming from "services" and "project services" to "integrations". -This refactoring is an ongoing effort, and there are still references to the old names in some places. - -Developers should be especially aware that we still use the old class names for the STI column -`integrations.type`. For example, a class `Integrations::FooBar` still stores -the old name `FooBarService` in the database. This mapping is handled via `Gitlab::Integrations::StiType` -and should be mostly transparent to the rest of the app. - -### [Consolidate integration settings](https://gitlab.com/groups/gitlab-org/-/epics/3955) - -We want to unify the way integration properties are defined. +Developers should be aware that the Integrations team is in the process of +[unifying the way integration properties are defined](https://gitlab.com/groups/gitlab-org/-/epics/3955). ## Integration examples diff --git a/doc/development/integrations/jenkins.md b/doc/development/integrations/jenkins.md index 6baccdca327..43487486f97 100644 --- a/doc/development/integrations/jenkins.md +++ b/doc/development/integrations/jenkins.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -26,9 +26,9 @@ GitLab does not allow requests to localhost or the local network by default. Whe 1. Log into your GitLab instance as an administrator. 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Settings > Network**. -1. Expand **Outbound requests** and check the following checkboxes: +1. Expand **Outbound requests**, and select the following checkboxes: - - **Allow requests to the local network from web hooks and services** + - **Allow requests to the local network from webhooks and integrations** - **Allow requests to the local network from system hooks** For more details about GitLab webhooks, see [Webhooks and insecure internal web services](../../security/webhooks.md). diff --git a/doc/development/integrations/jira_connect.md b/doc/development/integrations/jira_connect.md index eca4d9775c5..b8815131852 100644 --- a/doc/development/integrations/jira_connect.md +++ b/doc/development/integrations/jira_connect.md @@ -1,10 +1,10 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Set up a development environment +# Set up a Jira development environment The following are required to install and test the app: @@ -72,25 +72,10 @@ To avoid external dependencies like Gitpod and a Jira Cloud instance, use the [J 1. Go to `http://localhost:9292`. 1. Paste the token and select **Install GitLab.com Jira Cloud app**. -### Troubleshooting - -If the app install failed, you might need to delete `jira_connect_installations` from your database. - -1. Open the [database console](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md#access-postgresql). -1. Run `TRUNCATE TABLE jira_connect_installations CASCADE;`. - -#### Not authorized to access the file - -If you use Gitpod and you get an error about Jira not being able to access the descriptor file, you might need to make the GDK port public by following these steps: - -1. Open your GitLab workspace in Gitpod. -1. When the GDK is running, select **Ports** in the bottom-right corner. -1. On the left sidebar, select the port the GDK is listening to (typically `3000`). -1. If the port is marked as private, select the lock icon to make it public. - ## Test the GitLab OAuth authentication flow -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81126) in GitLab 14.9 [with a flag](../../administration/feature_flags.md) named `jira_connect_oauth`. Disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81126) in GitLab 14.9 [with a flag](../../administration/feature_flags.md) named `jira_connect_oauth`. Disabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/117648) in GitLab 16.0. Feature flag `jira_connect_oauth` removed. GitLab for Jira users can authenticate with GitLab using GitLab OAuth. @@ -99,27 +84,34 @@ This feature is not ready for production use. The feature flag should only be en The following steps describe setting up an environment to test the GitLab OAuth flow: -1. Start a Gitpod session and open the rails console. - - ```shell - bundle exec rails console - ``` - -1. Enable the feature flag. - - ```shell - Feature.enable(:jira_connect_oauth) - ``` - +1. Start a Gitpod session. 1. On your GitLab instance, go to **Admin > Applications**. 1. Create a new application with the following settings: - - Name: `Jira Connect` - - Redirect URI: `YOUR_GITPOD_INSTANCE/-/jira_connect/oauth_callbacks` - - Scopes: `api` - - Trusted: **No** - - Confidential: **No** + - Name: `Jira Connect` + - Redirect URI: `YOUR_GITPOD_INSTANCE/-/jira_connect/oauth_callbacks` + - Scopes: `api` + - Trusted: **No** + - Confidential: **No** 1. Copy the Application ID. 1. Go to **Admin > Settings > General**. -1. Scroll down and expand the GitLab for Jira App section. +1. Expand **GitLab for Jira App**. 1. Go to [gitpod.io/variables](https://gitpod.io/variables). 1. Paste the Application ID into the **Jira Connect Application ID** field and select **Save changes**. + +## Troubleshooting + +### App installation fails + +If the app installation fails, you might need to delete `jira_connect_installations` from your database. + +1. Open the [database console](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md#access-postgresql). +1. Run `TRUNCATE TABLE jira_connect_installations CASCADE;`. + +### Not authorized to access the file + +If you use Gitpod and you get an error about Jira not being able to access the descriptor file, you might need to make the GDK port public by following these steps: + +1. Open your GitLab workspace in Gitpod. +1. When the GDK is running, select **Ports** in the bottom-right corner. +1. On the left sidebar, select the port the GDK is listening to (typically `3000`). +1. If the port is marked as private, select the lock icon to make it public. diff --git a/doc/development/integrations/secure.md b/doc/development/integrations/secure.md index 002579d9b83..ee94e57a247 100644 --- a/doc/development/integrations/secure.md +++ b/doc/development/integrations/secure.md @@ -124,7 +124,7 @@ the project repository contains Java source code and the `dependency_scanning` f ```yaml mysec_dependency_scanning: rules: - - if: $DEPENDENCY_SCANNING_DISABLED + - if: $DEPENDENCY_SCANNING_DISABLED == 'true' when: never - if: $GITLAB_FEATURES =~ /\bdependency_scanning\b/ exists: @@ -198,7 +198,7 @@ SAST and Dependency Scanning scanners must scan the files in the project directo To be consistent with the official Container Scanning for GitLab, scanners must scan the Docker image whose name and tag are given by -`CI_APPLICATION_REPOSITORY` and `CI_APPLICATION_TAG`, respectively. If the `DOCKER_IMAGE` +`CI_APPLICATION_REPOSITORY` and `CI_APPLICATION_TAG`. If the `DOCKER_IMAGE` CI/CD variable is provided, then the `CI_APPLICATION_REPOSITORY` and `CI_APPLICATION_TAG` variables are ignored, and the image specified in the `DOCKER_IMAGE` variable is scanned instead. @@ -234,22 +234,13 @@ then `artifacts:reports:dependency_scanning` must be set to `depscan.json`. ### Exit code -Following the POSIX exit code standard, the scanner exits with 0 for success and any number from 1 to 255 for anything else. +Following the POSIX exit code standard, the scanner exits with either `0` for success or `1` for failure. Success also includes the case when vulnerabilities are found. When a CI job fails, security report results are not ingested by GitLab, even if the job -[allows failure](../../ci/yaml/index.md#allow_failure). The report artifacts are still uploaded to GitLab and available +[allows failure](../../ci/yaml/index.md#allow_failure). However, the report artifacts are still uploaded to GitLab and available for [download in the pipeline security tab](../../user/application_security/vulnerability_report/pipeline.md#download-security-scan-outputs). -When executing a scanning job using the [Docker-in-Docker privileged mode](../../user/application_security/sast/index.md#requirements), -we reserve the following standard exit codes. - -| Orchestrator Exit Code | Description | -|------------------------|----------------------------------| -| 3 | No match, no compatible analyzer | -| 4 | Project directory empty | -| 5 | No compatible Docker image | - ### Logging The scanner should log error messages and warnings so that users can easily investigate @@ -412,8 +403,6 @@ The `id` should not collide with any other analyzers or scanners another integra ##### Scan Primary Identifiers -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/368284) in GitLab 15.4 [with a flag](../../administration/feature_flags.md) named `sec_mark_dropped_findings_as_resolved`. Disabled by default. - The `scan.primary_identifiers` field is an optional field containing an array of [primary identifiers](../../user/application_security/terminology/index.md#primary-identifier)). This is an exhaustive list of all rulesets for which the analyzer performed the scan. @@ -422,7 +411,7 @@ Even when the [`Vulnerabilities`](#vulnerabilities) array for a given scan may b should contain the complete list of potential identifiers to inform the Rails application of which rules were executed. -When populated, the Rails application automatically resolves previously detected vulnerabilities as no +When populated, the Rails application [may automatically resolve previously detected vulnerabilities](../../user/application_security/iac_scanning/index.md#automatic-vulnerability-resolution) as no longer relevant when their primary identifier is not included. ##### Name, message, and description diff --git a/doc/development/integrations/secure_partner_integration.md b/doc/development/integrations/secure_partner_integration.md index 853541144fb..ab94cad3bdc 100644 --- a/doc/development/integrations/secure_partner_integration.md +++ b/doc/development/integrations/secure_partner_integration.md @@ -42,7 +42,7 @@ best place to integrate your own product and its results into GitLab. - Pipeline jobs serve a variety of purposes. Jobs can do scanning for and have implications for app security, corporate policy, or compliance. When complete, the job reports back on its status and creates a - [job artifact](../../ci/pipelines/job_artifacts.md) as a result. + [job artifact](../../ci/jobs/job_artifacts.md) as a result. - The [Merge Request Security Widget](../../ci/testing/index.md#security-reports) displays the results of the pipeline's security checks and the developer can review them. The developer can review both a summary and a detailed version @@ -84,7 +84,7 @@ and complete an integration with the Secure stage. to successfully display your own product's results with the rest of GitLab. - See detailed [technical directions](secure.md) for this step. - Read more about [job report artifacts](../../ci/yaml/index.md#artifactsreports). - - Read about [job artifacts](../../ci/pipelines/job_artifacts.md). + - Read about [job artifacts](../../ci/jobs/job_artifacts.md). - Your report artifact must be in one of our currently supported formats. For more information, see the [documentation on reports](secure.md#report). - Documentation for [SAST reports](../../user/application_security/sast/index.md#reports-json-format). @@ -95,7 +95,7 @@ and complete an integration with the Secure stage. and add the label `devops::secure`. - Once the job is completed, the data can be seen: - In the [Merge Request Security Report](../../ci/testing/index.md#security-reports) ([MR Security Report data flow](https://gitlab.com/snippets/1910005#merge-request-view)). - - While [browsing a Job Artifact](../../ci/pipelines/job_artifacts.md). + - While [browsing a Job Artifact](../../ci/jobs/job_artifacts.md). - In the [Security Dashboard](../../user/application_security/security_dashboard/index.md) ([Dashboard data flow](https://gitlab.com/snippets/1910005#project-and-group-dashboards)). 1. Optional: Provide a way to interact with results as Vulnerabilities: - Users can interact with the findings from your artifact within their workflow. They can dismiss the findings or accept them and create a backlog issue. diff --git a/doc/development/interacting_components.md b/doc/development/interacting_components.md index f4becd06245..7fb711795c1 100644 --- a/doc/development/interacting_components.md +++ b/doc/development/interacting_components.md @@ -29,5 +29,5 @@ See also [File Storage in GitLab](file_storage.md). ### Forks GitLab supports a great amount of features for [merge requests](../user/project/merge_requests/index.md). One -of them is the ability to create merge requests from and to [forks](../user/project/repository/forking_workflow.md#creating-a-fork), +of them is the ability to create merge requests from and to [forks](../user/project/repository/forking_workflow.md#create-a-fork), which should also be highly considered and tested upon development phase. diff --git a/doc/development/internal_api/index.md b/doc/development/internal_api/index.md index b19e431ebc6..c1c0177609b 100644 --- a/doc/development/internal_api/index.md +++ b/doc/development/internal_api/index.md @@ -37,13 +37,11 @@ is stored in a file at the path configured in `config/gitlab.yml` by default this is in the root of the rails app named `.gitlab_shell_secret` -To authenticate using that token, clients read the contents of that -file, and include the token Base64 encoded in a `secret_token` parameter -or in the `Gitlab-Shared-Secret` header. +To authenticate using that token, clients: -NOTE: -The internal API used by GitLab Pages, and GitLab agent server (`kas`) uses JSON Web Token (JWT) -authentication, which is different from GitLab Shell. +1. Read the contents of that file. +1. Use the file contents to generate a JSON Web Token (`JWT`). +1. Pass the JWT in the `Gitlab-Shell-Api-Request` header. ## Git Authentication @@ -78,7 +76,7 @@ POST /internal/allowed Example request: ```shell -curl --request POST --header "Gitlab-Shared-Secret: " \ +curl --request POST --header "Gitlab-Shell-Api-Request: " \ --data "key_id=11&project=gnuwget/wget2&action=git-upload-pack&protocol=ssh" \ "http://localhost:3001/api/v4/internal/allowed" ``` @@ -128,7 +126,7 @@ information for LFS clients when the repository is accessed over SSH. Example request: ```shell -curl --request POST --header "Gitlab-Shared-Secret: " \ +curl --request POST --header "Gitlab-Shell-Api-Request: " \ --data "key_id=11&project=gnuwget/wget2" "http://localhost:3001/api/v4/internal/lfs_authenticate" ``` @@ -148,12 +146,12 @@ curl --request POST --header "Gitlab-Shared-Secret: " \ ## Authorized Keys Check This endpoint is called by the GitLab Shell authorized keys -check. Which is called by OpenSSH for +check. Which is called by OpenSSH or GitLab SSHD for [fast SSH key lookup](../../administration/operations/fast_ssh_key_lookup.md). | Attribute | Type | Required | Description | |:----------|:-------|:---------|:------------| -| `key` | string | yes | SSH key as passed by OpenSSH to GitLab Shell | +| `key` | string | yes | An authorized key used for public key authentication. | ```plaintext GET /internal/authorized_keys @@ -162,7 +160,7 @@ GET /internal/authorized_keys Example request: ```shell -curl --request GET --header "Gitlab-Shared-Secret: " "http://localhost:3001/api/v4/internal/authorized_keys?key=" +curl --request GET --header "Gitlab-Shell-Api-Request: " "http://localhost:3001/api/v4/internal/authorized_keys?key=" ``` Example response: @@ -197,7 +195,7 @@ GET /internal/discover Example request: ```shell -curl --request GET --header "Gitlab-Shared-Secret: " "http://localhost:3001/api/v4/internal/discover?key_id=7" +curl --request GET --header "Gitlab-Shell-Api-Request: " "http://localhost:3001/api/v4/internal/discover?key_id=7" ``` Example response: @@ -226,7 +224,7 @@ GET /internal/check Example request: ```shell -curl --request GET --header "Gitlab-Shared-Secret: " "http://localhost:3001/api/v4/internal/check" +curl --request GET --header "Gitlab-Shell-Api-Request: " "http://localhost:3001/api/v4/internal/check" ``` Example response: @@ -263,7 +261,7 @@ GET /internal/two_factor_recovery_codes Example request: ```shell -curl --request POST --header "Gitlab-Shared-Secret: " \ +curl --request POST --header "Gitlab-Shell-Api-Request: " \ --data "key_id=7" "http://localhost:3001/api/v4/internal/two_factor_recovery_codes" ``` @@ -311,7 +309,7 @@ POST /internal/personal_access_token Example request: ```shell -curl --request POST --header "Gitlab-Shared-Secret: " \ +curl --request POST --header "Gitlab-Shell-Api-Request: " \ --data "user_id=29&name=mytokenname&scopes[]=read_user&scopes[]=read_repository&expires_at=2020-07-24" \ "http://localhost:3001/api/v4/internal/personal_access_token" ``` @@ -348,7 +346,7 @@ POST /internal/error_tracking/allowed Example request: ```shell -curl --request POST --header "Gitlab-Shared-Secret: " \ +curl --request POST --header "Gitlab-Shell-Api-Request: " \ --data "project_id=111&public_key=generated-error-tracking-key" \ "http://localhost:3001/api/v4/internal/error_tracking/allowed" ``` @@ -379,7 +377,7 @@ POST /internal/pre_receive Example request: ```shell -curl --request POST --header "Gitlab-Shared-Secret: " \ +curl --request POST --header "Gitlab-Shell-Api-Request: " \ --data "gl_repository=project-7" "http://localhost:3001/api/v4/internal/pre_receive" ``` @@ -412,7 +410,7 @@ POST /internal/post_receive Example Request: ```shell -curl --request POST --header "Gitlab-Shared-Secret: " \ +curl --request POST --header "Gitlab-Shell-Api-Request: " \ --data "gl_repository=project-7" --data "identifier=user-1" \ --data "changes=0000000000000000000000000000000000000000 fd9e76b9136bdd9fe217061b497745792fe5a5ee gh-pages\n" \ "http://localhost:3001/api/v4/internal/post_receive" @@ -811,6 +809,107 @@ Example response: - CustomersDot +## Storage limit exclusions + +The namespace storage limit exclusion endpoints manage storage limit exclusions on top-level namespaces on GitLab.com. +These endpoints can only be consumed in the Admin Area of GitLab.com. + +### Retrieve storage limit exclusions + +Use a GET request to retrieve all `Namespaces::Storage::LimitExclusion` records. + +```plaintext +GET /namespaces/storage/limit_exclusions +``` + +Example request: + +```shell +curl --request GET \ + --url "https://gitlab.com/v4/namespaces/storage/limit_exclusions" \ + --header 'PRIVATE-TOKEN: ' +``` + +Example response: + +```json +[ + { + "id": 1, + "namespace_id": 1234, + "namespace_name": "A Namespace Name", + "reason": "a reason to exclude the Namespace" + }, + { + "id": 2, + "namespace_id": 4321, + "namespace_name": "Another Namespace Name", + "reason": "another reason to exclude the Namespace" + }, +] +``` + +### Create a storage limit exclusion + +Use a POST request to create an `Namespaces::Storage::LimitExclusion`. + +```plaintext +POST /namespaces/:id/storage/limit_exclusion +``` + +| Attribute | Type | Required | Description | +|:------------|:--------|:---------|:------------| +| `reason` | string | yes | The reason to exclude the namespace. | + +Example request: + +```shell +curl --request POST \ + --url "https://gitlab.com/v4/namespaces/123/storage/limit_exclusion" \ + --header 'Content-Type: application/json' \ + --header 'PRIVATE-TOKEN: ' \ + --data '{ + "reason": "a reason to exclude the Namespace" + }' +``` + +Example response: + +```json +{ + "id": 1, + "namespace_id": 1234, + "namespace_name": "A Namespace Name", + "reason": "a reason to exclude the Namespace" +} +``` + +### Delete a storage limit exclusion + +Use a DELETE request to delete a `Namespaces::Storage::LimitExclusion` for a namespace. + +```plaintext +DELETE /namespaces/:id/storage/limit_exclusion +``` + +Example request: + +```shell +curl --request DELETE \ + --url "https://gitlab.com/v4/namespaces/123/storage/limit_exclusion" \ + --header 'PRIVATE-TOKEN: ' +``` + +Example response: + +```plaintext +204 +``` + +### Known consumers + +- GitLab.com Admin Area + ## CI/CD minutes provisioning The CI/CD Minutes endpoints are used by [CustomersDot](https://gitlab.com/gitlab-org/customers-gitlab-com) (`customers.gitlab.com`) diff --git a/doc/development/internal_users.md b/doc/development/internal_users.md index 67000d969af..ce13324507d 100644 --- a/doc/development/internal_users.md +++ b/doc/development/internal_users.md @@ -40,9 +40,12 @@ For this bot: Other examples of internal users: -- [Alert Bot](../operations/metrics/alerts.md#trigger-actions-from-alerts) +- [Alert Bot](../operations/incident_management/alerts.md#trigger-actions-from-alerts) - [Ghost User](../user/profile/account/delete_account.md#associated-records) - [Support Bot](../user/project/service_desk.md#support-bot-user) - Visual Review Bot -- Resource access tokens (including [project access tokens](../user/project/settings/project_access_tokens.md)). - These are implemented as `project_bot` users with a `PersonalAccessToken`. +- Resource access tokens, including: + - [Project access tokens](../user/project/settings/project_access_tokens.md). + - [Group access tokens](../user/group/settings/group_access_tokens.md). + + These are implemented as `project_{project_id}_bot_{random_string}` i.e. `group_{group_id}_bot_{random_string}` users, with a `PersonalAccessToken`. diff --git a/doc/development/issue_types.md b/doc/development/issue_types.md index 465f539e026..d0d513af781 100644 --- a/doc/development/issue_types.md +++ b/doc/development/issue_types.md @@ -4,7 +4,7 @@ group: Project Management info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Issue Types (DEPRECATED) +# Issue Types (deprecated) WARNING: We are deprecating Issue Types as of GitLab 14.2 in favor of [Work Items and Work Item Types](work_items.md). diff --git a/doc/development/json.md b/doc/development/json.md index 8a2575401fb..bdb7f73ab62 100644 --- a/doc/development/json.md +++ b/doc/development/json.md @@ -4,7 +4,7 @@ group: unassigned info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# JSON Guidelines +# JSON development guidelines At GitLab we handle a lot of JSON data. To best ensure we remain performant when handling large JSON encodes or decodes, we use our own JSON class diff --git a/doc/development/kubernetes.md b/doc/development/kubernetes.md index e44e2af4371..e537b4c4c76 100644 --- a/doc/development/kubernetes.md +++ b/doc/development/kubernetes.md @@ -1,10 +1,10 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Kubernetes integration - development guidelines +# Kubernetes integration development guidelines This document provides various guidelines when developing for the GitLab [Kubernetes integration](../user/infrastructure/clusters/index.md). diff --git a/doc/development/labels/index.md b/doc/development/labels/index.md new file mode 100644 index 00000000000..50b6ec74575 --- /dev/null +++ b/doc/development/labels/index.md @@ -0,0 +1,347 @@ +--- +stage: none +group: Development +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Labels + +To allow for asynchronous issue handling, we use [milestones](https://gitlab.com/groups/gitlab-org/-/milestones) +and [labels](https://gitlab.com/gitlab-org/gitlab/-/labels). Leads and product managers handle most of the +scheduling into milestones. Labeling is a task for everyone. (For some projects, labels can be set only by GitLab team members and not by community contributors). + +Most issues will have labels for at least one of the following: + +- Type. For example: `~"type::feature"`, `~"type::bug"`, or `~"type::maintenance"`. +- Stage. For example: `~"devops::plan"` or `~"devops::create"`. +- Group. For example: `~"group::source code"`, `~"group::knowledge"`, or `~"group::editor"`. +- Category. For example: `~"Category:Code Analytics"`, `~"Category:DevOps Reports"`, or `~"Category:Templates"`. +- Feature. For example: `~wiki`, `~ldap`, `~api`, `~issues`, or `~"merge requests"`. +- Department: `~UX`, `~Quality` +- Team: `~"Technical Writing"`, `~Delivery` +- Specialization: `~frontend`, `~backend`, `~documentation` +- Release Scoping: `~Deliverable`, `~Stretch`, `~"Next Patch Release"` +- Priority: `~"priority::1"`, `~"priority::2"`, `~"priority::3"`, `~"priority::4"` +- Severity: `~"severity::1"`, `~"severity::2"`, `~"severity::3"`, `~"severity::4"` + +Please add `~"breaking change"` label if the issue can be considered as a [breaking change](../deprecation_guidelines/index.md). + +Please add `~security` label if the issue is related to application security. + +All labels, their meaning and priority are defined on the +[labels page](https://gitlab.com/gitlab-org/gitlab/-/labels). + +If you come across an issue that has none of these, and you're allowed to set +labels, you can _always_ add the type, stage, group, and often the category/feature labels. + +## Type labels + +Type labels are very important. They define what kind of issue this is. Every +issue should have one and only one. + +The SSOT for type and subtype labels is [available in the handbook](https://about.gitlab.com/handbook/engineering/metrics/#work-type-classification). + +A number of type labels have a priority assigned to them, which automatically +makes them float to the top, depending on their importance. + +Type labels are always lowercase, and can have any color, besides blue (which is +already reserved for category labels). + +The descriptions on the [labels page](https://gitlab.com/groups/gitlab-org/-/labels) +explain what falls under each type label. + +The GitLab handbook documents [when something is a bug](https://about.gitlab.com/handbook/product/product-processes/#bug-issues) and [when it is a feature request](https://about.gitlab.com/handbook/product/product-processes/#feature-issues). + +## Stage labels + +Stage labels specify which [stage](https://about.gitlab.com/handbook/product/categories/#hierarchy) the issue belongs to. + +### Naming and color convention + +Stage labels respects the `devops::` naming convention. +`` is the stage key as it is in the single source of truth for stages at + +with `_` replaced with a space. + +For instance, the "Manage" stage is represented by the `~"devops::manage"` label in +the `gitlab-org` group since its key under `stages` is `manage`. + +The current stage labels can be found by [searching the labels list for `devops::`](https://gitlab.com/groups/gitlab-org/-/labels?search=devops::). + +These labels are [scoped labels](../../user/project/labels.md#scoped-labels) +and thus are mutually exclusive. + +The Stage labels are used to generate the [direction pages](https://about.gitlab.com/direction/) automatically. + +## Group labels + +Group labels specify which [groups](https://about.gitlab.com/company/team/structure/#product-groups) the issue belongs to. + +It's highly recommended to add a group label, as it's used by our triage +automation to +[infer the correct stage label](https://about.gitlab.com/handbook/engineering/quality/triage-operations/#auto-labelling-of-issues-and-merge-requests). + +### Naming and color convention + +Group labels respects the `group::` naming convention and +their color is `#A8D695`. +`` is the group key as it is in the single source of truth for groups at +, +with `_` replaced with a space. + +For instance, the "Pipeline Execution" group is represented by the +`~"group::pipeline execution"` label in the `gitlab-org` group since its key +under `stages.manage.groups` is `pipeline_execution`. + +The current group labels can be found by [searching the labels list for `group::`](https://gitlab.com/groups/gitlab-org/-/labels?search=group::). + +These labels are [scoped labels](../../user/project/labels.md#scoped-labels) +and thus are mutually exclusive. + +You can find the groups listed in the [Product Stages, Groups, and Categories](https://about.gitlab.com/handbook/product/categories/) page. + +We use the term group to map down product requirements from our product stages. +As a team needs some way to collect the work their members are planning to be assigned to, we use the `~group::` labels to do so. + +## Category labels + +From the handbook's +[Product stages, groups, and categories](https://about.gitlab.com/handbook/product/categories/#hierarchy) +page: + +> Categories are high-level capabilities that may be a standalone product at +another company, such as Portfolio Management, for example. + +It's highly recommended to add a category label, as it's used by our triage +automation to +[infer the correct group and stage labels](https://about.gitlab.com/handbook/engineering/quality/triage-operations/#auto-labelling-of-issues). + +If you are an expert in a particular area, it makes it easier to find issues to +work on. You can also subscribe to those labels to receive an email each time an +issue is labeled with a category label corresponding to your expertise. + +### Naming and color convention + +Category labels respects the `Category:` naming convention and +their color is `#428BCA`. +`` is the category name as it is in the single source of truth for categories at +. + +For instance, the "DevOps Reports" category is represented by the +`~"Category:DevOps Reports"` label in the `gitlab-org` group since its +`devops_reports.name` value is "DevOps Reports". + +If a category's label doesn't respect this naming convention, it should be specified +with [the `label` attribute](https://about.gitlab.com/handbook/marketing/digital-experience/website/#category-attributes) +in . + +## Feature labels + +From the handbook's +[Product stages, groups, and categories](https://about.gitlab.com/handbook/product/categories/#hierarchy) +page: + +> Features: Small, discrete functionalities, for example Issue weights. Some common +features are listed within parentheses to facilitate finding responsible PMs by keyword. + +It's highly recommended to add a feature label if no category label applies, as +it's used by our triage automation to +[infer the correct group and stage labels](https://about.gitlab.com/handbook/engineering/quality/triage-operations/#auto-labelling-of-issues). + +If you are an expert in a particular area, it makes it easier to find issues to +work on. You can also subscribe to those labels to receive an email each time an +issue is labeled with a feature label corresponding to your expertise. + +Examples of feature labels are `~wiki`, `~ldap`, `~api`, `~issues`, and `~"merge requests"`. + +### Naming and color convention + +Feature labels are all-lowercase. + +## Workflow labels + +Issues use the following workflow labels to specify the current issue status: + +- `~"workflow::awaiting security release"` +- `~"workflow::blocked"` +- `~"workflow::complete"` +- `~"workflow::design"` +- `~"workflow::feature-flagged"` +- `~"workflow::in dev"` +- `~"workflow::in review"` +- `~"workflow::planning breakdown"` +- `~"workflow::problem validation"` +- `~"workflow::production"` +- `~"workflow::ready for design"` +- `~"workflow::ready for development"` +- `~"workflow::refinement"` +- `~"workflow::scheduling"` +- `~"workflow::solution validation"` +- `~"workflow::start"` +- `~"workflow::validation backlog"` +- `~"workflow::verification"` + +## Facet labels + +To track additional information or context about created issues, developers may +add _facet labels_. Facet labels are also sometimes used for issue prioritization +or for measurements (such as time to close). An example of a facet label is the +`~"customer"` label, which indicates customer interest. + +## Department labels + +The current department labels are: + +- `~"UX"` +- `~"Quality"` + +## Team labels + +**Important**: Most of the historical team labels (like Manage or Plan) are +now deprecated in favor of [Group labels](#group-labels) and [Stage labels](#stage-labels). + +Team labels specify what team is responsible for this issue. +Assigning a team label makes sure issues get the attention of the appropriate +people. + +The current team labels are: + +- `~"Delivery"~` +- `~"Technical Writing"` + +### Naming and color convention + +Team labels are always capitalized so that they show up as the first label for +any issue. + +## Specialization labels + +These labels narrow the [specialization](https://about.gitlab.com/company/team/structure/#specialist) on a unit of work. + +- `~"frontend"` +- `~"backend"` +- `~"documentation"` + +## Release scoping labels + +Release Scoping labels help us clearly communicate expectations of the work for the +release. There are three levels of Release Scoping labels: + +- `~"Deliverable"`: Issues that are expected to be delivered in the current + milestone. +- `~"Stretch"`: Issues that are a stretch goal for delivering in the current + milestone. If these issues are not done in the current release, they will + strongly be considered for the next release. +- `~"Next Patch Release"`: Issues to put in the next patch release. Work on these + first, and follow the [patch release runbook](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/patch/engineers.md) to backport the bug fix to the current version. + +Each issue scheduled for the current milestone should be labeled `~"Deliverable"~` +or `~"Stretch"`. Any open issue for a previous milestone should be labeled +`~"Next Patch Release"`, or otherwise rescheduled to a different milestone. + +## Priority labels + +We have the following priority labels: + +- `~"priority::1"` +- `~"priority::2"` +- `~"priority::3"` +- `~"priority::4"` + +Please refer to the issue triage [priority label](https://about.gitlab.com/handbook/engineering/quality/issue-triage/#priority) section in our handbook to see how it's used. + +## Severity labels + +We have the following severity labels: + +- `~"severity::1"` +- `~"severity::2"` +- `~"severity::3"` +- `~"severity::4"` + +Please refer to the issue triage [severity label](https://about.gitlab.com/handbook/engineering/quality/issue-triage/#severity) section in our handbook to see how it's used. + +## Label for community contributors + +There are many issues that have a clear solution with uncontroversial benefit to GitLab users. +However, GitLab might not have the capacity for all these proposals in the current roadmap. +These issues are labeled `~"Seeking community contributions"` because we welcome merge requests to resolve them. + +Community contributors can submit merge requests for any issue they want, but +the `~"Seeking community contributions"` label has a special meaning. It points to +changes that: + +1. We already agreed on, +1. Are well-defined, +1. Are likely to get accepted by a maintainer. + +We want to avoid a situation when a contributor picks an +~"Seeking community contributions" issue and then their merge request gets closed, +because we realize that it does not fit our vision, or we want to solve it in a +different way. + +We manually add the `~"Seeking community contributions"` label to issues +that fit the criteria described above. +We do not automatically add this label, because it requires human evaluation. + +We recommend people that have never contributed to any open source project to +look for issues labeled `~"Seeking community contributions"` with a +[weight of 1](https://gitlab.com/groups/gitlab-org/-/issues?sort=created_date&state=opened&label_name[]=Seeking+community+contributions&assignee_id=None&weight=1) or the `~"quick win"` +[label](https://gitlab.com/gitlab-org/gitlab/-/issues?scope=all&state=opened&label_name[]=quick%20win&assignee_id=None) +attached to it. +More experienced contributors are very welcome to tackle +[any of them](https://gitlab.com/groups/gitlab-org/-/issues?sort=created_date&state=opened&label_name[]=Seeking+community+contributions&assignee_id=None). + +For more complex features that have a weight of 2 or more and clear scope, we recommend looking at issues +with the [label `~"Community Challenge"`](https://gitlab.com/gitlab-org/gitlab/-/issues?sort=created_date&state=opened&label_name[]=Seeking+community+contributions&label_name[]=Community+challenge). +If your MR for the `~"Community Challenge"` issue gets merged, you will also have a chance to win a custom +GitLab merchandise. + +If you've decided that you would like to work on an issue, please @-mention +the [appropriate product manager](https://about.gitlab.com/handbook/product/#who-to-talk-to-for-what) +as soon as possible. The product manager will then pull in appropriate GitLab team +members to further discuss scope, design, and technical considerations. This will +ensure that your contribution is aligned with the GitLab product and minimize +any rework and delay in getting it merged into main. + +GitLab team members who apply the `~"Seeking community contributions"` label to an issue +should update the issue description with a responsible product manager, inviting +any potential community contributor to @-mention per above. + +## Stewardship label + +For issues related to the open source stewardship of GitLab, +there is the `~"stewardship"` label. + +This label is to be used for issues in which the stewardship of GitLab +is a topic of discussion. For instance if GitLab Inc. is planning to add +features from GitLab EE to GitLab CE, related issues would be labeled with +`~"stewardship"`. + +A recent example of this was the issue for +[bringing the time tracking API to GitLab CE](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/25517#note_20019084). + +## Technical and UX debt + +In order to track things that can be improved in the GitLab codebase, +we use the `~"technical debt"` label in the [GitLab issue tracker](https://gitlab.com/gitlab-org/gitlab/-/issues). +We use the `~"UX debt"` label when we choose to deviate from the MVC, in a way that harms the user experience. + +These labels should be added to issues that describe things that can be improved, +shortcuts that have been taken, features that need additional attention, and all +other things that have been left behind due to high velocity of development. +For example, code that needs refactoring should use the `~"technical debt"` label, +something that didn't ship according to our Design System guidelines should +use the `~"UX debt"` label. + +Everyone can create an issue, though you may need to ask for adding a specific +label, if you do not have permissions to do it by yourself. Additional labels +can be combined with these labels, to make it easier to schedule +the improvements for a release. + +Issues tagged with these labels have the same priority like issues +that describe a new feature to be introduced in GitLab, and should be scheduled +for a release by the appropriate person. + +Make sure to mention the merge request that the `~"technical debt"` issue or +`~"UX debt"` issue is associated with in the description of the issue. diff --git a/doc/development/lfs.md b/doc/development/lfs.md index ec91f9f3c8b..bb327bf5d04 100644 --- a/doc/development/lfs.md +++ b/doc/development/lfs.md @@ -3,15 +3,147 @@ stage: Create group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- - -# Git LFS developer information +# Git LFS development guidelines This page contains developer-centric information for GitLab team members. For the user documentation, see [Git Large File Storage](../topics/git/lfs/index.md). +## Controllers and Services + +### Repositories::GitHttpClientController + +The methods for authentication defined here are inherited by all the other LFS controllers. + +### Repositories::LfsApiController + +#### `#batch` + +After authentication the `batch` action is the first action called by the Git LFS +client during downloads and uploads (such as pull, push, and clone). + +### Repositories::LfsStorageController + +#### `#upload_authorize` + +Provides payload to Workhorse including a path for Workhorse to save the file to. Could be remote object storage. + +#### `#upload_finalize` + +Handles requests from Workhorse that contain information on a file that workhorse already uploaded (see [this middleware](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/middleware/multipart.rb)) so that `gitlab` can either: + +- Create an `LfsObject`. +- Connect an existing `LfsObject` to a project with an `LfsObjectsProject`. + +### LfsObject and LfsObjectsProject + +- Only one `LfsObject` is created for a file with a given `oid` (a SHA256 checksum of the file) and file size. +- `LfsObjectsProject` associate `LfsObject`s with `Project`s. They determine if a file can be accessed through a project. +- These objects are also used for calculating the amount of LFS storage a given project is using. + For more information, see + [`ProjectStatistics#update_lfs_objects_size`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/project_statistics.rb#L82-84). + +### Repositories::LfsLocksApiController + +Handles the lock API for LFS. Delegates mostly to corresponding services: + +- `Lfs::LockFileService` +- `Lfs::UnlockFileService` +- `Lfs::LocksFinderService` + +These services create and delete `LfsFileLock`. + +#### `#verify` + +- This endpoint responds with a payload that allows a client to check if there are any files being pushed that have locks that belong to another user. +- A client-side `lfs.locksverify` configuration can be set so that the client aborts the push if locks exist that belong to another user. +- The existence of locks belonging to other users is also [validated on the server side](https://gitlab.com/gitlab-org/gitlab/-/blob/65f0c6e59121b62c9b0f89b810ef5186969bb4d2/lib/gitlab/checks/diff_check.rb#L69). + +## Example authentication + +```mermaid +sequenceDiagram +autonumber + alt Over HTTPS + Git client-->>Git client: user-supplied credentials + else Over SSH + Git client->>gitlab-shell: git-lfs-authenticate + activate gitlab-shell + activate GitLab Rails + gitlab-shell->>GitLab Rails: POST /api/v4/internal/lfs_authenticate + GitLab Rails-->>gitlab-shell: token with expiry + deactivate gitlab-shell + deactivate GitLab Rails + end +``` + +1. Clients can be configured to store credentials in a few different ways. + See the [Git LFS documentation on authentication](https://github.com/git-lfs/git-lfs/blob/bea0287cdd3acbc0aa9cdf67ae09b6843d3ffcf0/docs/api/authentication.md#git-credentials). +1. Running `gitlab-lfs-authenticate` on `gitlab-shell`. See the [Git LFS documentation concerning `gitlab-lfs-authenticate`](https://github.com/git-lfs/git-lfs/blob/bea0287cdd3acbc0aa9cdf67ae09b6843d3ffcf0/docs/api/server-discovery.md#ssh). +1. `gitlab-shell`makes a request to the GitLab API. +1. [Responding to shell with token](https://gitlab.com/gitlab-org/gitlab/-/blob/7a2f7a31a88b6085ea89b8ba188a4d92d5fada91/lib/api/internal/base.rb#L168) which is used in subsequent requests. See [Git LFS documentation concerning authentication](https://github.com/git-lfs/git-lfs/blob/bea0287cdd3acbc0aa9cdf67ae09b6843d3ffcf0/docs/api/authentication.md). + +## Example clone + +```mermaid +sequenceDiagram + Note right of Git client: Typical Git clone things happen first + Note right of Git client: Authentication for LFS comes next + activate GitLab Rails + autonumber + Git client->>GitLab Rails: POST project/namespace/info/lfs/objects/batch + GitLab Rails-->>Git client: payload with objects + deactivate GitLab Rails + loop each object in payload + Git client->>GitLab Rails: GET project/namespace/gitlab-lfs/objects/:oid/ (<- This URL is from the payload) + GitLab Rails->>Workhorse: SendfileUpload + Workhorse-->> Git client: Binary data + end +``` + +1. Git LFS requests the ability to download files with authorization header from authorization. +1. `gitlab` responds with the list of objects and where to find them. See + [LfsApiController#batch](https://gitlab.com/gitlab-org/gitlab/-/blob/7a2f7a31a88b6085ea89b8ba188a4d92d5fada91/app/controllers/repositories/lfs_api_controller.rb#L25). +1. Git LFS makes a request for each file for the `href` in the previous response. See + [how downloads are handled with the basic transfer mode](https://github.com/git-lfs/git-lfs/blob/bea0287cdd3acbc0aa9cdf67ae09b6843d3ffcf0/docs/api/basic-transfers.md#downloads). +1. `gitlab` redirects to the remote URL if remote object storage is enabled. See + [SendFileUpload](https://gitlab.com/gitlab-org/gitlab/-/blob/7a2f7a31a88b6085ea89b8ba188a4d92d5fada91/app/controllers/concerns/send_file_upload.rb#L4). + +## Example push + +```mermaid +sequenceDiagram + Note right of Git client: Typical Git push things happen first. + Note right of Git client: Suthentication for LFS comes next. + autonumber + activate GitLab Rails + Git client ->> GitLab Rails: POST project/namespace/info/lfs/objects/batch + GitLab Rails-->>Git client: payload with objects + deactivate GitLab Rails + loop each object in payload + Git client->>Workhorse: PUT project/namespace/gitlab-lfs/objects/:oid/:size (URL is from payload) + Workhorse->>GitLab Rails: PUT project/namespace/gitlab-lfs/objects/:oid/:size/authorize + GitLab Rails-->>Workhorse: response with where path to upload + Workhorse->>Workhorse: Upload + Workhorse->>GitLab Rails: PUT project/namespace/gitlab-lfs/objects/:oid/:size/finalize + end +``` + +1. Git LFS requests the ability to upload files. +1. `gitlab` responds with the list of objects and uploads to find them. See + [LfsApiController#batch](https://gitlab.com/gitlab-org/gitlab/-/blob/7a2f7a31a88b6085ea89b8ba188a4d92d5fada91/app/controllers/repositories/lfs_api_controller.rb#L27). +1. Git LFS makes a request for each file for the `href` in the previous response. See + [how uploads are handled with the basic transfer mode](https://github.com/git-lfs/git-lfs/blob/bea0287cdd3acbc0aa9cdf67ae09b6843d3ffcf0/docs/api/basic-transfers.md#uploads). +1. `gitlab` responds with a payload including a path for Workhorse to save the file to. + Could be remote object storage. See + [LfsStorageController#upload_authorize](https://gitlab.com/gitlab-org/gitlab/-/blob/96250de93a410e278ef659a3d38b056f12024636/app/controllers/repositories/lfs_storage_controller.rb#L42). +1. Workhorse does the work of saving the file. +1. Workhorse makes a request to `gitlab` with information on the uploaded file so + that `gitlab` can create an `LfsObject`. See + [LfsStorageController#upload_finalize](https://gitlab.com/gitlab-org/gitlab/-/blob/96250de93a410e278ef659a3d38b056f12024636/app/controllers/repositories/lfs_storage_controller.rb#L51). + ## Deep Dive -In April 2019, Francisco Javier López hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/issues/1`) +In April 2019, Francisco Javier López hosted a Deep Dive (GitLab team members only: `https://gitlab.com/gitlab-org/create-stage/-/issues/1`) on the GitLab [Git LFS](../topics/git/lfs/index.md) implementation to share domain-specific knowledge with anyone who may work in this part of the codebase in the future. You can find the [recording on YouTube](https://www.youtube.com/watch?v=Yyxwcksr0Qc), diff --git a/doc/development/logging.md b/doc/development/logging.md index 538fc7ccfe1..908fa16d3f8 100644 --- a/doc/development/logging.md +++ b/doc/development/logging.md @@ -4,7 +4,7 @@ group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# GitLab Developers Guide to Logging +# Logging development guidelines [GitLab Logs](../administration/logs/index.md) play a critical role for both administrators and GitLab team members to diagnose problems in the field. @@ -87,6 +87,28 @@ importer progresses. Here's what to do: end ``` + Note that by default, `Gitlab::JsonLogger` will include application context metadata in the log entry. If your + logger is expected to be called outside of an application request (for example, in a `rake` task) or by low-level + code that may be involved in building the application context (for example, database connection code), you should + call the class method `exclude_context!` for your logger class, like so: + + ```ruby + module Gitlab + module Database + module LoadBalancing + class Logger < ::Gitlab::JsonLogger + exclude_context! + + def self.file_name_noext + 'database_load_balancing' + end + end + end + end + end + + ``` + 1. In your class where you want to log, you might initialize the logger as an instance variable: ```ruby @@ -169,6 +191,28 @@ Resources: - [Elasticsearch mapping - avoiding type gotchas](https://www.elastic.co/guide/en/elasticsearch/guide/current/mapping.html#_avoiding_type_gotchas) - [Elasticsearch mapping types]( https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html) +#### Include a class attribute + +Structured logs should always include a `class` attribute to make all entries logged from a particular place in the code findable. +To automatically add the `class` attribute, you can include the +[`Gitlab::Loggable` module](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/loggable.rb) and use the `build_structured_payload` method. + +```ruby +class MyClass + include ::Gitlab::Loggable + + def my_method + logger.info(build_structured_payload(message: 'log message', project_id: project_id)) + end + + private + + def logger + @logger ||= Gitlab::AppJsonLogger.build + end +end +``` + #### Logging durations Similar to timezones, choosing the right time unit to log can impose avoidable overhead. So, whenever @@ -180,7 +224,7 @@ suffix and `duration` within its name (for example, `view_duration_s`). ## Multi-destination Logging -GitLab is transitioning from unstructured/plaintext logs to structured/JSON logs. During this transition period some logs are recorded in multiple formats through multi-destination logging. +GitLab transitioned from structured to JSON logs. However, through multi-destination logging, the logs can be recorded in multiple formats. ### How to use multi-destination logging diff --git a/doc/development/merge_request_application_and_rate_limit_guidelines.md b/doc/development/merge_request_application_and_rate_limit_guidelines.md deleted file mode 100644 index 07788400adf..00000000000 --- a/doc/development/merge_request_application_and_rate_limit_guidelines.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'merge_request_concepts/rate_limits.md' -remove_date: '2023-04-23' ---- - -This document was moved to [another location](merge_request_concepts/rate_limits.md). - - - - - diff --git a/doc/development/merge_request_concepts/approval_rules.md b/doc/development/merge_request_concepts/approval_rules.md index d119644cd7c..d6000d48706 100644 --- a/doc/development/merge_request_concepts/approval_rules.md +++ b/doc/development/merge_request_concepts/approval_rules.md @@ -4,7 +4,7 @@ group: Code Review info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Approval Rules development guide +# Approval rules development guidelines This document explains the backend design and flow of all related functionality about [merge request approval rules](../../user/project/merge_requests/approvals/index.md). diff --git a/doc/development/merge_request_concepts/diffs/frontend.md b/doc/development/merge_request_concepts/diffs/frontend.md index 6bd6d80af94..ff163050e1f 100644 --- a/doc/development/merge_request_concepts/diffs/frontend.md +++ b/doc/development/merge_request_concepts/diffs/frontend.md @@ -84,40 +84,9 @@ The most important part of the metadata response is the diff file names. This da app to render the file browser inside of the diffs app, without waiting for all batch diffs requests to complete. -When the metadata response is received, the diff file data gets sent to a web worker. The web worker -exists to allow for this data, which for larger merge requests could be huge, to be processed off -the main thread. Processing this data involves getting the data into the correct structure +When the metadata response is received, the diff file data is processed into the correct structure that the frontend requires to render the file browser in either tree view or list view. -```mermaid -graph TD - A[fetchDiffFilesMeta] - B[Create web worker] - C[Fetch data] - D[SET_DIFF_METADATA] - E[Post worker data] - F[Worker message event listener] - K[SET_TREE_DATA] - - G[TreeWorker] - H[onMessage] - I[generateTreeList] - J[postMessage sortTree] - - A -->B - E -->F - B -->C - C -->D - D -->E - - G -->H - H -->I - I -->J - J -->F - - F -->K -``` - The structure for this file object is: ```javascript @@ -128,6 +97,11 @@ The structure for this file object is: "type": "", "tree": [], "changed": true, + "diffLoaded": false, + "filePaths": { + "old": file.old_path, + "new": file.new_path + }, "tempFile": false, "deleted": false, "fileHash": "", diff --git a/doc/development/merge_request_concepts/diffs/index.md b/doc/development/merge_request_concepts/diffs/index.md index 8ef3f01aba9..c2dec5c4753 100644 --- a/doc/development/merge_request_concepts/diffs/index.md +++ b/doc/development/merge_request_concepts/diffs/index.md @@ -17,7 +17,7 @@ We rely on different sources to present diffs. These include: In January 2019, Oswaldo Ferreira hosted a Deep Dive (GitLab team members only: -`https://gitlab.com/gitlab-org/create-stage/issues/1`) on GitLab Diffs and Commenting on Diffs +`https://gitlab.com/gitlab-org/create-stage/-/issues/1`) on GitLab Diffs and Commenting on Diffs functionality to share domain-specific knowledge with anyone who may work in this part of the codebase in the future: diff --git a/doc/development/merge_request_concepts/index.md b/doc/development/merge_request_concepts/index.md index 14d9582ad84..8bec5e0c0b0 100644 --- a/doc/development/merge_request_concepts/index.md +++ b/doc/development/merge_request_concepts/index.md @@ -34,7 +34,7 @@ This area of the merge request is where all of the options and commit messages a Reports are widgets within the merge request that report information about changes within the merge request. These widgets provide information to better help the author understand the changes and further improvements to the proposed changes. -[Design Documentation](https://design.gitlab.com/regions/merge-request-reports/) +[Design Documentation](https://design.gitlab.com/patterns/merge-request-reports) ![merge request reports](../img/merge_request_reports_v14_7.png) @@ -61,7 +61,7 @@ Additionally, approval settings provide configuration options to define how thos Examples of approval rules and settings include: - [Merge request approval rules](../../user/project/merge_requests/approvals/rules.md) -- [Code owner approvals](../../user/project/code_owners.md) +- [Code owner approvals](../../user/project/codeowners/index.md) - [Security approvals](../../user/application_security/index.md#security-approvals-in-merge-requests) - [Prevent editing approval rules](../../user/project/merge_requests/approvals/settings.md#prevent-editing-approval-rules-in-merge-requests) - [Remove all approvals when commits are added](../../user/project/merge_requests/approvals/settings.md#remove-all-approvals-when-commits-are-added-to-the-source-branch) diff --git a/doc/development/merge_request_concepts/performance.md b/doc/development/merge_request_concepts/performance.md index 740b8f1607b..665b84b2c40 100644 --- a/doc/development/merge_request_concepts/performance.md +++ b/doc/development/merge_request_concepts/performance.md @@ -158,7 +158,7 @@ query. This in turn makes it much harder for this code to overload a database. ## Use read replicas when possible -In a DB cluster we have many read replicas and one primary. A classic use of scaling the DB is to have read-only actions be performed by the replicas. We use [load balancing](../../administration/postgresql/database_load_balancing.md) to distribute this load. This allows for the replicas to grow as the pressure on the DB grows. +In a DB cluster we have many read replicas and one primary. A classic use of scaling the DB is to have read-only actions be performed by the replicas. We use [load balancing](../database/load_balancing.md) to distribute this load. This allows for the replicas to grow as the pressure on the DB grows. By default, queries use read-only replicas, but due to [primary sticking](../../administration/postgresql/database_load_balancing.md#primary-sticking), GitLab uses the @@ -211,7 +211,7 @@ By default, this `Gitlab::SQL::CTE` class forces materialization through adding (this behavior is implemented using a custom Arel node `Gitlab::Database::AsWithMaterialized` under the surface). WARNING: -Upgrading to GitLab 14.0 requires PostgreSQL 12 or higher. +Upgrading to GitLab 14.0 requires PostgreSQL 12 or later. ## Cached Queries @@ -260,7 +260,7 @@ It re-instantiates project object for each build, instead of using the same in-m In this particular case the workaround is fairly easy: ```ruby -ActiveRecord::Associations::Preloader.new.preload(pipeline, [builds: :project]) +ActiveRecord::Associations::Preloader.new(records: pipeline, associations: [builds: :project]).call pipeline.builds.each do |build| build.to_json(only: [:name], include: [project: { only: [:name]}]) diff --git a/doc/development/merge_request_diffs.md b/doc/development/merge_request_diffs.md deleted file mode 100644 index 9ec7e6cae8b..00000000000 --- a/doc/development/merge_request_diffs.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'merge_request_concepts/diffs/development.md' -remove_date: '2023-04-10' ---- - -This document was moved to [another location](merge_request_concepts/diffs/development.md). - - - - - diff --git a/doc/development/merge_request_performance_guidelines.md b/doc/development/merge_request_performance_guidelines.md deleted file mode 100644 index 1af81a8af9f..00000000000 --- a/doc/development/merge_request_performance_guidelines.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'merge_request_concepts/performance.md' -remove_date: '2023-04-10' ---- - -This document was moved to [another location](merge_request_concepts/performance.md). - - - - - diff --git a/doc/development/migration_style_guide.md b/doc/development/migration_style_guide.md index 4625489146e..513659d0f68 100644 --- a/doc/development/migration_style_guide.md +++ b/doc/development/migration_style_guide.md @@ -1,6 +1,6 @@ --- -stage: none -group: unassigned +stage: Data Stores +group: Database info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -156,7 +156,7 @@ regenerate a clean `db/structure.sql` for the migrations you're adding. This script applies all migrations found in `db/migrate` or `db/post_migrate`, so if there are any migrations you don't want to commit to the schema, rename or remove them. If your branch is not -targeting `main` you can set the `TARGET` environment variable. +targeting the default Git branch, you can set the `TARGET` environment variable. ```shell # Regenerate schema against `main` @@ -184,21 +184,7 @@ git checkout origin/master db/structure.sql VERSION= bundle exec rails db:migrate:main ``` -### Adding new tables to the database dictionary - -GitLab connects to two different Postgres databases: `main` and `ci`. New tables should be defined in [`db/docs/`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/db/docs): - -```yaml -table_name: table name exmaple -description: Description example -introduced_by_url: Merge request link -milestone: Milestone example -feature_categories: -- Feature category example -classes: -- Class example -gitlab_schema: gitlab_main -``` +After a table has been created, it should be added to the database dictionary, following the steps mentioned in the [database dictionary guide](database/database_dictionary.md#adding-tables). ## Avoiding downtime @@ -278,6 +264,27 @@ to be longer. Some methods for shortening a name that's too long: `index_vulnerability_findings_remediations_on_remediation_id`. - Instead of columns, specify the purpose of the index, such as `index_users_for_unconfirmation_notification`. +### Migration timestamp age + +The timestamp portion of a migration filename determines the order in which migrations +are run. It's important to maintain a rough correlation between: + +1. When a migration is added to the GitLab codebase. +1. The timestamp of the migration itself. + +A new migration's timestamp should *never* be before the previous hard stop. +Migrations are occasionally squashed, and if a migration is added whose timestamp +falls before the previous hard stop, a problem like what happened in +[issue 408304](https://gitlab.com/gitlab-org/gitlab/-/issues/408304) can occur. + +For example, if we are currently developing against GitLab 16.0, the previous +hard stop is 15.11. 15.11 was released on April 23rd, 2023. Therefore, the +minimum acceptable timestamp would be 20230424000000. + +#### Best practice + +While the above should be considered a hard rule, it is a best practice to try to keep migration timestamps to within three weeks of the date it is anticipated that the migration will be merged upstream, regardless of how much time has elapsed since the last hard stop. + ## Heavy operations in a single transaction When using a single-transaction migration, a transaction holds a database connection @@ -307,7 +314,7 @@ which is a "versioned" class. For new migrations, the latest version should be u can be looked up in `Gitlab::Database::Migration::MIGRATION_CLASSES`) to use the latest version of migration helpers. -In this example, we use version 2.0 of the migration class: +In this example, we use version 2.1 of the migration class: ```ruby class TestMigration < Gitlab::Database::Migration[2.1] @@ -323,7 +330,7 @@ version of migration helpers automatically. Migration helpers and versioning were [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68986) in GitLab 14.3. For merge requests targeting previous stable branches, use the old format and still inherit from -`ActiveRecord::Migration[6.1]` instead of `Gitlab::Database::Migration[2.0]`. +`ActiveRecord::Migration[6.1]` instead of `Gitlab::Database::Migration[2.1]`. ## Retry mechanism when acquiring database locks @@ -345,7 +352,7 @@ on the `users` table once it has been enqueued. More information about PostgreSQL locks: [Explicit Locking](https://www.postgresql.org/docs/current/explicit-locking.html) -For stability reasons, GitLab.com has a specific [`statement_timeout`](../user/gitlab_com/index.md#postgresql) +For stability reasons, GitLab.com has a short `statement_timeout` set. When the migration is invoked, any database query has a fixed time to execute. In a worst-case scenario, the request sits in the lock queue, blocking other queries for the duration of the configured statement timeout, @@ -644,89 +651,14 @@ for more details. ## Adding indexes -Before adding an index, consider if this one is necessary. There are situations in which an index -might not be required, like: - -- The table is small (less than `1,000` records) and it's not expected to exponentially grow in size. -- Any existing indexes filter out enough rows. -- The reduction in query timings after the index is added is not significant. - -Additionally, wide indexes are not required to match all filter criteria of queries, we just need -to cover enough columns so that the index lookup has a small enough selectivity. Please review our -[Adding Database indexes](database/adding_database_indexes.md) guide for more details. - -When adding an index to a non-empty table make sure to use the method -`add_concurrent_index` instead of the regular `add_index` method. -The `add_concurrent_index` method automatically creates concurrent indexes -when using PostgreSQL, removing the need for downtime. - -To use this method, you must disable single-transactions mode -by calling the method `disable_ddl_transaction!` in the body of your migration -class like so: - -```ruby -class MyMigration < Gitlab::Database::Migration[2.1] - disable_ddl_transaction! - - INDEX_NAME = 'index_name' - - def up - add_concurrent_index :table, :column, name: INDEX_NAME - end - - def down - remove_concurrent_index :table, :column, name: INDEX_NAME - end -end -``` - -You must explicitly name indexes that are created with more complex -definitions beyond table name, column names, and uniqueness constraint. -Consult the [Adding Database Indexes](database/adding_database_indexes.md#requirements-for-naming-indexes) -guide for more details. - -If you need to add a unique index, please keep in mind there is the possibility -of existing duplicates being present in the database. This means that should -always _first_ add a migration that removes any duplicates, before adding the -unique index. - -For a small table (such as an empty one or one with less than `1,000` records), -it is recommended to use `add_index` in a single-transaction migration, combining it with other -operations that don't require `disable_ddl_transaction!`. +Before adding an index, consider if one is necessary. The [Adding Database indexes](database/adding_database_indexes.md) guide contains more details to help you decide if an index is necessary and provides best practices for adding indexes. ## Testing for existence of indexes -If a migration requires conditional logic based on the absence or -presence of an index, you must test for existence of that index using -its name. This helps avoids problems with how Rails compares index definitions, -which can lead to unexpected results. For more details, review the -[Adding Database Indexes](database/adding_database_indexes.md#why-explicit-names-are-required) -guide. - -The easiest way to test for existence of an index by name is to use the -`index_name_exists?` method, but the `index_exists?` method can also -be used with a name option. For example: - -```ruby -class MyMigration < Gitlab::Database::Migration[2.1] - INDEX_NAME = 'index_name' - - def up - # an index must be conditionally created due to schema inconsistency - unless index_exists?(:table_name, :column_name, name: INDEX_NAME) - add_index :table_name, :column_name, name: INDEX_NAME - end - end - - def down - # no op - end -end -``` +If a migration requires conditional logic based on the absence or presence of an index, you must test for existence of that index using its name. This helps avoids problems with how Rails compares index definitions, which can lead to unexpected results. -Keep in mind that concurrent index helpers like `add_concurrent_index`, -`remove_concurrent_index`, and `remove_concurrent_index_by_name` already -perform existence checks internally. +For more details, review the [Adding Database Indexes](database/adding_database_indexes.md#testing-for-existence-of-indexes) +guide. ## Adding foreign-key constraints @@ -972,6 +904,8 @@ def down end ``` +After a table has been dropped, it should be added to the database dictionary, following the steps in the [database dictionary guide](database/database_dictionary.md#dropping-tables). + ## Dropping a sequence > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/88387) in GitLab 15.1. @@ -1010,6 +944,20 @@ NOTE: `add_sequence` should be avoided for columns with foreign keys. Adding sequence to these columns is **only allowed** in the down method (restore previous schema state). +## Truncate a table + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/117373) in GitLab 15.11. + +Truncating a table is uncommon, but you can use the `truncate_tables!` method provided by the database team. + +Under the hood, it works like this: + +- Finds the `gitlab_schema` for the tables to be truncated. +- If the `gitlab_schema` for the tables is included in the connection's gitlab_schemas, + it then executes the `TRUNCATE` statement. +- If the `gitlab_schema` for the tables is not included in the connection's + gitlab_schemas, it does nothing. + ## Swapping primary key > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/98645) in GitLab 15.5. diff --git a/doc/development/navigation_sidebar.md b/doc/development/navigation_sidebar.md new file mode 100644 index 00000000000..cd543012ddd --- /dev/null +++ b/doc/development/navigation_sidebar.md @@ -0,0 +1,56 @@ +--- +stage: Manage +group: Foundations +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Navigation sidebar + +Follow these guidelines when contributing additions or changes to the +[redesigned](https://gitlab.com/groups/gitlab-org/-/epics/9044) navigation +sidebar. + +These guidelines reflect the current state of the navigation sidebar. However, +the sidebar is a work in progress, and so is this documentation. + +## Enable the new navigation sidebar + +To enable the new navigation sidebar: + +- Enable the `super_sidebar_nav` feature flag. +- Select your avatar, then turn on the **New navigation** toggle. + +## Adding items to the sidebar + +Before adding an item to the sidebar, ensure you follow [this process](https://about.gitlab.com/handbook/product/ux/navigation/#how-to-propose-a-change-that-impacts-navigation). + +## Adding page-specific Vue content + +Pages can render arbitrary content into the sidebar using the `SidebarPortal` +component. Content passed to its default slot is rendered below that +page's navigation items in the sidebar. + +NOTE: +Only one instance of this component on a given page is supported. This is to +avoid ordering issues and cluttering the sidebar. + +NOTE: +You can use arbitrary content. You should implement nav items by subclassing `::Sidebars::Panel`. +If you must use Vue to render nav items (for example, if you need to use Vue Router) you can make an exception. +However, in the corresponding `panel.rb` file, you must add a comment that explains how the nav items are rendered. + +NOTE: +Do not use the `SidebarPortalTarget` component. It is internal to the sidebar. + +## Snowplow Tracking + +All clicks on the nav items should be automatically tracked in Snowplow, but may require additional input. +We use `data-tracking` attributes on all the elements in the nav to send the data up to Snowplow. +You can test that they're working by [setting up snowplow on your GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/snowplow_micro.md). + +| Field | Data attribute | Notes | Example | +| -- | -- | -- | -- | +| Category | `data-tracking-category` | The page that the user was on when the item was clicked. | `groups:show` | +| Action | `data-tracking-action` | The action taken. In most cases this is `click_link` or `click_menu_item` | `click_link` | +| Label | `data-tracking-label` | A descriptor for what was clicked on. This is inferred by the ID of the item in most cases, but falls back to `item_without_id`. This is one to look out for. | `group_issue_list` | +| Property | `data-tracking-property` | This describes where in the nav the link was clicked. If it's in the main nav panel, then it needs to describe which panel. | `nav_panel_group` | diff --git a/doc/development/organization/index.md b/doc/development/organization/index.md new file mode 100644 index 00000000000..3a23e50caf9 --- /dev/null +++ b/doc/development/organization/index.md @@ -0,0 +1,85 @@ +--- +type: index, dev +stage: Data Stores +group: Tenant Scale +info: "See the Technical Writers assigned to Development Guidelines: https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-development-guidelines" +description: "Development Guidelines: learn about organization when developing GitLab." +--- + +# Organization + +The [Organization initiative](../../user/organization/index.md) focuses on reaching feature parity between +SaaS and self-managed installations. + +## Consolidate groups and projects + +- [Architecture blueprint](../../architecture/blueprints/consolidating_groups_and_projects/index.md) + +One facet of the Organization initiative is to consolidate groups and projects, +addressing the feature disparity between them. Some features, such as epics, are +only available at the group level. Some features, such as issues, are only available +at the project level. Other features, such as milestones, are available to both groups +and projects. + +We receive many requests to add features either to the group or project level. +Moving features around to different levels is problematic on multiple levels: + +- It requires engineering time to move the features. +- It requires UX overhead to maintain mental models of feature availability. +- It creates redundant code. + +When features are copied from one level (project, group, or instance) to another, +the copies often have small, nuanced differences between them. These nuances cause +extra engineering time when fixes are needed, because the fix must be copied to +several locations. These nuances also create different user experiences when the +feature is used in different places. + +A solution for this problem is to consolidate groups and projects into a single +entity, `namespace`. The work on this solution is split into several phases and +is tracked in [epic 6473](https://gitlab.com/groups/gitlab-org/-/epics/6473). + +## How to plan features that interact with Group and ProjectNamespace + +As of now, every Project in the system has a record in the `namespaces` table. This makes it possible to +use common interface to create features that are shared between Groups and Projects. Shared behavior can be added using +a concerns mechanism. Because the `Namespace` model is responsible for `UserNamespace` methods as well, it is discouraged +to use the `Namespace` model for shared behavior for Projects and Groups. + +### Resource-based features + +To migrate resource-based features, existing functionality will need to be supported. This can be achieved in two Phases. + +**Phase 1 - Setup** + +- Link into the namespaces table + - Add a column to the table + - For example, in issues a `project id` points to the projects table. We need to establish a link to the `namespaces` table. + - Modify code so that any new record already has the correct data in it + - Backfill + +**Phase 2 - Prerequisite work** + +- Investigate the permission model as well as any performance concerns related to that. + - Permissions need to be checked and kept in place. +- Investigate what other models need to support namespaces for functionality dependent on features you migrate in Phase 1. +- Adjust CRUD services and APIs (REST and GraphQL) to point to the new column you added in Phase 1. +- Consider performance when fetching resources. + +Introducing new functionality is very much dependent on every single team and feature. + +### Settings-related features + +Right now, cascading settings are available for `NamespaceSettings`. By creating `ProjectNamespace`, +we can use this framework to make sure that some settings are applicable on the project level as well. + +When working on settings, we need to make sure that: + +- They are not used in `join` queries or modify those queries. +- Updating settings is taken into consideration. +- If we want to move from project to project namespace, we follow a similar database process to the one described in Phase 1. + +## Related topics + +- [Consolidating groups and projects](../../architecture/blueprints/consolidating_groups_and_projects/index.md) + architecture documentation +- [Organization user documentation](../../user/organization/index.md) diff --git a/doc/development/packages/debian_repository.md b/doc/development/packages/debian_repository.md index 26a33c548d8..1523d777e7b 100644 --- a/doc/development/packages/debian_repository.md +++ b/doc/development/packages/debian_repository.md @@ -116,7 +116,7 @@ sequenceDiagram ProcessChangesService->>+ExtractChangesMetadataService: Extract changesmetadata ExtractChangesMetadataService->>+ExtractMetadataService: Extract file metadata ExtractMetadataService->>+ParseDebian822Service: run `dpkg --field` to get control file - ExtractMetadataService->>+ExtractDebMetadataService: If .deb or .udeb + ExtractMetadataService->>+ExtractDebMetadataService: If .deb, .udeb or ddeb ExtractDebMetadataService->>+ParseDebian822Service: run `dpkg --field` to get control file ParseDebian822Service-->>-ExtractDebMetadataService: Parse String as Debian RFC822 control data format ExtractDebMetadataService-->>-ExtractMetadataService: Return the parsed control file @@ -127,7 +127,7 @@ sequenceDiagram loop process files listed in .changes ProcessChangesService->>+ExtractMetadataService: Process file ExtractMetadataService->>+ParseDebian822Service: run `dpkg --field` to get control file - ExtractMetadataService->>+ExtractDebMetadataService: If .deb or .udeb + ExtractMetadataService->>+ExtractDebMetadataService: If .deb, .udeb or ddeb ExtractDebMetadataService->>+ParseDebian822Service: run `dpkg --field` to get control file ParseDebian822Service-->>-ExtractDebMetadataService: Parse String as Debian RFC822 control data format ExtractDebMetadataService-->>-ExtractMetadataService: Return the parsed control file diff --git a/doc/development/packages/index.md b/doc/development/packages/index.md index fa0e9f5d926..4f027825422 100644 --- a/doc/development/packages/index.md +++ b/doc/development/packages/index.md @@ -4,7 +4,7 @@ group: Package Registry info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Package and container registry documentation +# Package and container registry development guidelines The documentation for package and container registry development is split into two groups. diff --git a/doc/development/pages/index.md b/doc/development/pages/index.md index e71d7df642c..4769dbf427d 100644 --- a/doc/development/pages/index.md +++ b/doc/development/pages/index.md @@ -1,7 +1,7 @@ --- type: reference, dev -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments description: "GitLab's development guidelines for GitLab Pages" --- @@ -255,7 +255,7 @@ incidents and downtime. To add a new feature flag to GitLab Pages: 1. Create the feature flag in [`internal/feature/feature.go`](https://gitlab.com/gitlab-org/gitlab-pages/-/blob/master/internal/feature/feature.go), which must be **off** by default. -1. Create an issue to track the feature flag using the `Feature Flag` template. +1. Create an issue to track the feature flag using the `Feature flag` template. 1. Add the `~"feature flag"` label to any merge requests that handle feature flags. For GitLab Pages, the feature flags are controlled by environment variables at a global level. diff --git a/doc/development/performance.md b/doc/development/performance.md index 346f70e04b0..291165d8125 100644 --- a/doc/development/performance.md +++ b/doc/development/performance.md @@ -815,6 +815,25 @@ Make sure that you have relevant test data for your filter in the [`spec/fixtures/markdown.md.erb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/fixtures/markdown.md.erb) file. +### Benchmarking specific filters + +A specific filter can be benchmarked by specifying the filter name as an environment variable. +For example, to benchmark the `MarkdownFilter` use + +```plaintext +FILTER=MarkdownFilter bin/rake benchmark:banzai +``` + +which generates the output + +```plaintext +--> Benchmarking MarkdownFilter for FullPipeline +Warming up -------------------------------------- + Markdown 271.000 i/100ms +Calculating ------------------------------------- + Markdown 2.584k (±16.5%) i/s - 23.848k in 10.042503s +``` + ## Reading from files and other data sources Ruby offers several convenience functions that deal with file contents specifically diff --git a/doc/development/permissions.md b/doc/development/permissions.md index bf7f99de1ab..3abadc98501 100644 --- a/doc/development/permissions.md +++ b/doc/development/permissions.md @@ -4,7 +4,7 @@ group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Implementing permissions +# Permission development guidelines There are multiple types of permissions across GitLab, and when implementing anything that deals with permissions, all of them should be considered. @@ -60,7 +60,7 @@ Additionally, the following project features can have different visibility level - Pipelines - Analytics - Requirements -- Security & Compliance +- Security and Compliance - Wiki - Snippets - Pages @@ -198,3 +198,124 @@ Say, for example, we extract the whole endpoint into a service. The `can?` check - If the finder doesn't accept `current_user`, and therefore doesn't check permissions, then probably no. - If the finder accepts `current_user`, and doesn't check permissions, then it would be a good idea to double check other usages of the finder, and we might consider adding authorization. - If the finder accepts `current_user`, and already checks permissions, then either we need to add our case, or the existing checks are appropriate. + +### Refactoring permissions + +#### Finding existing permissions checks + +As mentioned [above](#where-should-permissions-be-checked), permissions are +often checked in multiple locations for a single endpoint or web request. As a +result, finding the list of authorization checks that are run for a given endpoint +can be challenging. + +To assist with this, you can locally set `GITLAB_DEBUG_POLICIES=true`. + +This outputs information about which abilities are checked in the requests +made in any specs that you run. The output also includes the line of code where the +authorization check was made. Caller information is especially helpful in cases +where there is metaprogramming used because those cases are difficult to find by +grepping for ability name strings. + +Example: + +```shell +# example spec run + +GITLAB_DEBUG_POLICIES=true bundle exec rspec spec/controllers/groups_controller_spec.rb:162 + +# permissions debug output when spec is run; if multiple policy checks are run they will all be in the debug output. + +POLICY CHECK DEBUG -> policy: GlobalPolicy, ability: create_group, called_from: ["/gitlab/app/controllers/application_controller.rb:245:in `can?'", "/gitlab/app/controllers/groups_controller.rb:255:in `authorize_create_group!'"] +``` + +This flag is meant to help learn more about authorization checks while +refactoring and should not remain enabled for any specs on the default branch. + +#### Understanding logic for individual abilities + +References to an ability may appear in a `DeclarativePolicy` class many times +and depend on conditions and rules which reference other abilities. As a result, +it can be challenging to know exactly which conditions apply to a particular +ability. + +`DeclarativePolicy` provides a `ability_map` for each Policy class, which +pulls all Rules for an ability into an array. + +Example: + +```ruby +> GroupPolicy.ability_map.map.select { |k,v| k == :read_group_member } +=> {:read_group_member=>[[:enable, #], [:prevent, #]]} + +> GroupPolicy.ability_map.map.select { |k,v| k == :read_group } +=> {:read_group=> + [[:enable, #], + [:enable, #], + [:enable, #], + [:enable, #], + [:enable, #], + [:enable, #], + [:enable, #], + [:prevent, #], + [:enable, #], + [:prevent, #], + [:prevent, #]]} +``` + +`DeclarativePolicy` also provides a `debug` method that can be used to +understand the logic tree for a specific object and actor. The output is similar +to the list of rules from `ability_map`. But, `DeclarativePolicy` stops +evaluating rules once one `prevent`s an ability, so it is possible that +not all conditions are called. + +Example: + +```ruby +policy = GroupPolicy.new(User.last, Group.last) +policy.debug(:read_group) + +- [0] enable when public_group ((@custom_guest_user1 : Group/139)) +- [0] enable when logged_in_viewable ((@custom_guest_user1 : Group/139)) +- [0] enable when admin ((@custom_guest_user1 : Group/139)) +- [0] enable when auditor ((@custom_guest_user1 : Group/139)) +- [14] prevent when all?(~public_group, ~admin, user_banned_from_group) ((@custom_guest_user1 : Group/139)) +- [14] prevent when needs_new_sso_session ((@custom_guest_user1 : Group/139)) +- [16] enable when guest ((@custom_guest_user1 : Group/139)) +- [16] enable when has_projects ((@custom_guest_user1 : Group/139)) +- [16] enable when read_package_registry_deploy_token ((@custom_guest_user1 : Group/139)) +- [16] enable when write_package_registry_deploy_token ((@custom_guest_user1 : Group/139)) + [21] prevent when all?(ip_enforcement_prevents_access, ~owner, ~auditor) ((@custom_guest_user1 : Group/139)) + +=> #, + @enabled=false, + @prevented=true> +``` + +#### Testing that individual policies are equivalent + +You can use the `'equivalent project policy abilities'` shared example to ensure +that 2 project policy abilities are equivalent for all project visibility levels +and access levels. + +Example: + +```ruby + context 'when refactoring read_pipeline_schedule and read_pipeline' do + let(:old_policy) { :read_pipeline_schedule } + let(:new_policy) { :read_pipeline } + + it_behaves_like 'equivalent policies' + end +``` diff --git a/doc/development/pipelines/index.md b/doc/development/pipelines/index.md index 240d98a855f..651fa2d2ce9 100644 --- a/doc/development/pipelines/index.md +++ b/doc/development/pipelines/index.md @@ -47,22 +47,51 @@ In summary: - RSpec tests are dependent on the backend code. - Jest tests are dependent on both frontend and backend code, the latter through the frontend fixtures. +### Predictive Tests Dashboards + +- +- + +### The `detect-tests` CI job + +Most CI/CD pipelines for `gitlab-org/gitlab` will run a [`detect-tests` CI job](https://gitlab.com/gitlab-org/gitlab/-/blob/0c6058def8f182b4a2410db5d08a9550b951b2d8/.gitlab/ci/setup.gitlab-ci.yml#L101-146) in the `prepare` stage to detect which backend/frontend tests should be run based on the files that changed in the given MR. + +The `detect-tests` job will create many files that will contain the backend/frontend tests that should be run. Those files will be read in subsequent jobs in the pipeline, and only those tests will be executed. + ### RSpec predictive jobs #### Determining predictive RSpec test files in a merge request -To identify the RSpec tests that are likely to fail in a merge request, we use the [`test_file_finder` gem](https://gitlab.com/gitlab-org/ci-cd/test_file_finder), with two strategies: +To identify the RSpec tests that are likely to fail in a merge request, we use *static mappings* and *dynamic mappings*. -- dynamic mapping from test coverage tracing (generated via the [`Crystalball` gem](https://github.com/toptal/crystalball)) - ([see where it's used](https://gitlab.com/gitlab-org/gitlab/-/blob/47d507c93779675d73a05002e2ec9c3c467cd698/tooling/bin/find_tests#L15)) -- static mapping maintained in the [`tests.yml` file](https://gitlab.com/gitlab-org/gitlab/-/blob/master/tests.yml) for special cases that cannot - be mapped via coverage tracing ([see where it's used](https://gitlab.com/gitlab-org/gitlab/-/blob/47d507c93779675d73a05002e2ec9c3c467cd698/tooling/bin/find_tests#L12)) +##### Static mappings + +We use the [`test_file_finder` gem](https://gitlab.com/gitlab-org/ci-cd/test_file_finder), with a static mapping maintained in the [`tests.yml` file](https://gitlab.com/gitlab-org/gitlab/-/blob/master/tests.yml) for special cases that cannot + be mapped via coverage tracing ([see where it's used](https://gitlab.com/gitlab-org/gitlab/-/blob/5ab06422826c0d69c615655982a6f969a7f3c6ea/tooling/lib/tooling/find_tests.rb#L17)). The test mappings contain a map of each source files to a list of test files which is dependent of the source file. -In the `detect-tests` job, we use this mapping to identify the predictive tests needed for the current merge request. +##### Dynamic mappings + +First, we use the [`test_file_finder` gem](https://gitlab.com/gitlab-org/ci-cd/test_file_finder), with a dynamic mapping strategy from test coverage tracing (generated via the [`Crystalball` gem](https://github.com/toptal/crystalball)) + ([see where it's used](https://gitlab.com/gitlab-org/gitlab/-/blob/master/tooling/lib/tooling/find_tests.rb#L20)). + +In addition to `test_file_finder`, we have added several advanced mappings to detect even more tests to run: -Later on in [the `rspec fail-fast` job](#fail-fast-job-in-merge-request-pipelines), we run the predictive tests for the current merge request. +- [`FindChanges`](https://gitlab.com/gitlab-org/gitlab/-/blob/28943cbd8b6d7e9a350d00e5ea5bb52123ee14a4/tooling/lib/tooling/find_changes.rb) ([!74003](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74003)) + - Automatically detect Jest tests to run upon backend changes (via frontend fixtures) +- [`PartialToViewsMappings`](https://gitlab.com/gitlab-org/gitlab/-/blob/28943cbd8b6d7e9a350d00e5ea5bb52123ee14a4/tooling/lib/tooling/mappings/partial_to_views_mappings.rb) ([#395016](https://gitlab.com/gitlab-org/gitlab/-/issues/395016)) + - Run view specs when Rails partials included in those views are changed in an MR +- [`JsToSystemSpecsMappings`](https://gitlab.com/gitlab-org/gitlab/-/blob/28943cbd8b6d7e9a350d00e5ea5bb52123ee14a4/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb) ([#386754](https://gitlab.com/gitlab-org/gitlab/-/issues/386754)) + - Run certain system specs if a JavaScript file was changed in an MR +- [`GraphqlBaseTypeMappings`](https://gitlab.com/gitlab-org/gitlab/-/blob/28943cbd8b6d7e9a350d00e5ea5bb52123ee14a4/tooling/lib/tooling/mappings/graphql_base_type_mappings.rb) ([#386756](https://gitlab.com/gitlab-org/gitlab/-/issues/386756)) + - If a GraphQL type class changed, we should try to identify the other GraphQL types that potentially include this type, and run their specs. +- [`ViewToSystemSpecsMappings`](https://gitlab.com/gitlab-org/gitlab/-/blob/28943cbd8b6d7e9a350d00e5ea5bb52123ee14a4/tooling/lib/tooling/mappings/view_to_system_specs_mappings.rb) ([#395017](https://gitlab.com/gitlab-org/gitlab/-/issues/395017)) + - When a view gets changed, we try to find feature specs that would test that area of the code. +- [`ViewToJsMappings`](https://gitlab.com/gitlab-org/gitlab/-/blob/8d7dfb7c043adf931128088b9ffab3b4a39af6f5/tooling/lib/tooling/mappings/view_to_js_mappings.rb) ([#386719](https://gitlab.com/gitlab-org/gitlab/-/issues/386719)) + - If a JS file is changed, we should try to identify the system specs that are covering this JS component. +- [`FindFilesUsingFeatureFlags`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/tooling/lib/tooling/find_files_using_feature_flags.rb) ([#407366](https://gitlab.com/gitlab-org/gitlab/-/issues/407366)) + - If a feature flag was changed, we check which Ruby file is including that feature flag, and we add it to the list of changed files in the detect-tests CI job. The remainder of the job will then detect which frontend/backend tests should be run based on those changed files. #### Exceptional cases @@ -74,6 +103,10 @@ In addition, there are a few circumstances where we would always run the full RS - when the merge request is created in a security mirror - when any CI configuration file is changed (for example, `.gitlab-ci.yml` or `.gitlab/ci/**/*`) +#### Have you encountered a problem with backend predictive tests? + +If so, please have a look at [the Engineering Productivity RUNBOOK on predictive tests](https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/blob/main/runbooks/predictive-tests.md) for instructions on how to act upon predictive tests issues. Additionally, if you identified any test selection gaps, please let `@gl-quality/eng-prod` know so that we can take the necessary steps to optimize test selections. + ### Jest predictive jobs #### Determining predictive Jest test files in a merge request @@ -95,17 +128,20 @@ In addition, there are a few circumstances where we would always run the full Je The `rules` definitions for full Jest tests are defined at `.frontend:rules:jest` in [`rules.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/42321b18b946c64d2f6f788c38844499a5ae9141/.gitlab/ci/rules.gitlab-ci.yml#L938-955). +#### Have you encountered a problem with frontend predictive tests? + +If so, please have a look at [the Engineering Productivity RUNBOOK on predictive tests](https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/blob/main/runbooks/predictive-tests.md) for instructions on how to act upon predictive tests issues. + ### Fork pipelines We run only the predictive RSpec & Jest jobs for fork pipelines, unless the `pipeline:run-all-rspec` label is set on the MR. The goal is to reduce the CI/CD minutes consumed by fork pipelines. -See the [experiment issue](https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/1170). +See the [experiment issue](https://gitlab.com/gitlab-org/quality/quality-engineering/team-tasks/-/issues/1170). ## Fail-fast job in merge request pipelines -To provide faster feedback when a merge request breaks existing tests, we are experimenting with a -fail-fast mechanism. +To provide faster feedback when a merge request breaks existing tests, we implemented a fail-fast mechanism. An `rspec fail-fast` job is added in parallel to all other `rspec` jobs in a merge request pipeline. This job runs the tests that are directly related to the changes @@ -150,8 +186,8 @@ This number can be overridden by setting a CI/CD variable named `RSPEC_FAIL_FAST ## Re-run previously failed tests in merge request pipelines -In order to reduce the feedback time after resolving failed tests for a merge request, the `rspec rspec-pg12-rerun-previous-failed-tests` -and `rspec rspec-ee-pg12-rerun-previous-failed-tests` jobs run the failed tests from the previous MR pipeline. +In order to reduce the feedback time after resolving failed tests for a merge request, the `rspec rspec-pg13-rerun-previous-failed-tests` +and `rspec rspec-ee-pg13-rerun-previous-failed-tests` jobs run the failed tests from the previous MR pipeline. This was introduced on August 25th 2021, with . @@ -159,7 +195,7 @@ This was introduced on August 25th 2021, with B & C ``` +## Merge Trains + +### Why do we need to have a “stable” master branch to enable merge trains? + +If the master branch is unstable (i.e. CI/CD pipelines for the master branch are failing frequently), all of the merge requests pipelines that were added AFTER a faulty merge request pipeline would have to be **cancelled** and **added back to the train**, which would create a lot of delays if the merge train is long. + +### How stable does the master branch have to be for us to enable merge trains? + +We don't have a specific number, but we need to have better numbers for flaky tests failures and infrastructure failures (see the [Master Broken Incidents RCA Dashboard](https://app.periscopedata.com/app/gitlab/1082465/Master-Broken-Incidents-Root-Cause-Analysis)). + +### Could we gradually move to merge trains in our CI/CD configuration? + +There was a proposal from a contributor, but the approach is not without some downsides: [see the original proposal and discussion](https://gitlab.com/gitlab-org/quality/quality-engineering/team-tasks/-/issues/195#note_1117151994). + ## Faster feedback for some merge requests ### Broken Master Fixes @@ -249,11 +299,20 @@ The intent is to ensure that a change doesn't introduce a failure after `gitlab- ### As-if-JH cross project downstream pipeline +#### What it is + +This pipeline is also called [JiHu validation pipeline](https://about.gitlab.com/handbook/ceo/chief-of-staff-team/jihu-support/jihu-validation-pipelines.html), +and it's currently allowed to fail. When that happens, please follow +[What to do when the validation pipeline fails](https://about.gitlab.com/handbook/ceo/chief-of-staff-team/jihu-support/jihu-validation-pipelines.html#what-to-do-when-the-validation-pipeline-failed). + +#### How we run it + The `start-as-if-jh` job triggers a cross project downstream pipeline which runs the GitLab test suite "as if JiHu", meaning as if the pipeline would run in the context of [GitLab JH](../jh_features_review.md). These jobs are only created in the following cases: +- when changes are made to feature flags - when the `pipeline:run-as-if-jh` label is set on the merge request This pipeline runs under the context of a generated branch in the @@ -298,6 +357,9 @@ from the main mirror, rather than the validation project. The whole process looks like this: +NOTE: +We only run `sync-as-if-jh-branch` when there are dependencies changes. + ```mermaid flowchart TD subgraph "JiHuLab.com" @@ -306,7 +368,6 @@ flowchart TD subgraph "GitLab.com" Mirror["gitlab-org/gitlab-jh-mirrors/gitlab"] - Validation["gitlab-org-sandbox/gitlab-jh-validation"] subgraph MR["gitlab-org/gitlab merge request"] Add["add-jh-files job"] @@ -314,10 +375,17 @@ flowchart TD Add --"download artifacts"--> Prepare end - Mirror --"pull mirror with master and main-jh"--> Validation + subgraph "gitlab-org-sandbox/gitlab-jh-validation" + Sync["(*optional) sync-as-if-jh-branch job on branch as-if-jh-code-sync"] + Start["start-as-if-jh job on as-if-jh/* branch"] + AsIfJH["as-if-jh pipeline"] + end + + Mirror --"pull mirror with master and main-jh"--> gitlab-org-sandbox/gitlab-jh-validation Mirror --"download JiHu files with ADD_JH_FILES_TOKEN"--> Add - Prepare --"push as-if-jh branches with AS_IF_JH_TOKEN"--> Validation - Validation --> Pipeline["as-if-jh pipeline"] + Prepare --"push as-if-jh branches with AS_IF_JH_TOKEN"--> Sync + Sync --"push as-if-jh branches with AS_IF_JH_TOKEN"--> Start + Start --> AsIfJH end JH --"pull mirror with corresponding JH branches"--> Mirror @@ -338,10 +406,20 @@ job will create a new branch from the merge request branch, commit the changes, and finally push the branch to the [validation project](https://gitlab.com/gitlab-org-sandbox/gitlab-jh-validation). +Optionally, if the merge requests have changes to the dependencies, we have an +additional step to run `sync-as-if-jh-branch` job to trigger a downstream +pipeline on [`as-if-jh-code-sync` branch](https://gitlab.com/gitlab-org-sandbox/gitlab-jh-validation/-/blob/as-if-jh-code-sync/jh/.gitlab-ci.yml) +in the validation project. This job will perform the same process as +[JiHu code-sync](https://jihulab.com/gitlab-cn/code-sync/-/blob/main-jh/.gitlab-ci.yml), making sure the dependencies changes can be brought to the +as-if-jh branch prior to run the validation pipeline. + +If there are no dependencies changes, we don't run this process. + ##### How we trigger and run the as-if-JH pipeline -After having the `as-if-jh/*` branch, `start-as-if-jh` job will trigger a pipeline -in the [validation project](https://gitlab.com/gitlab-org-sandbox/gitlab-jh-validation) +After having the `as-if-jh/*` branch prepared and optionally synchronized, +`start-as-if-jh` job will trigger a pipeline in the +[validation project](https://gitlab.com/gitlab-org-sandbox/gitlab-jh-validation) to run the cross-project downstream pipeline. ##### How the GitLab JH mirror project is set up @@ -360,7 +438,8 @@ No password is used from mirroring because GitLab JH is a public project. ##### How the GitLab JH validation project is set up -This [GitLab JH validation](https://gitlab.com/gitlab-org-sandbox/gitlab-jh-validation) project is public and CI is enabled, without any project variables. +This [GitLab JH validation](https://gitlab.com/gitlab-org-sandbox/gitlab-jh-validation) project is public and CI is enabled, with temporary +project variables set. It's a pull mirror pulling from [GitLab JH mirror](https://gitlab.com/gitlab-org/gitlab-jh-mirrors/gitlab), mirroring only protected branches, `master` and `main-jh`, overriding @@ -382,6 +461,26 @@ running every day, updating cache. The default CI/CD configuration file is also set at `jh/.gitlab-ci.yml` so it runs exactly like [GitLab JH](https://jihulab.com/gitlab-cn/gitlab/-/blob/main-jh/jh/.gitlab-ci.yml). +Additionally, a special branch +[`as-if-jh-code-sync`](https://gitlab.com/gitlab-org-sandbox/gitlab-jh-validation/-/blob/as-if-jh-code-sync/jh/.gitlab-ci.yml) +is set and protected. Maintainers can push and developers can merge for this +branch. We need to set it so developers can merge because we need to let +developers to trigger pipelines for this branch. This is a compromise +before we resolve [Developer-level users no longer able to run pipelines on protected branches](https://gitlab.com/gitlab-org/gitlab/-/issues/230939). + +It's used to run `sync-as-if-jh-branch` to synchronize the dependencies +when the merge requests changed the dependencies. See +[How we generate the as-if-JH branch](#how-we-generate-the-as-if-jh-branch) +for how it works. + +###### Temporary GitLab JH validation project variables + +- `BUNDLER_CHECKSUM_VERIFICATION_OPT_IN` is set to `false` + - We can remove this variable after JiHu has + [`jh/Gemfile.checksum`](https://jihulab.com/gitlab-cn/gitlab/-/blob/main-jh/jh/Gemfile.checksum) + committed. More context can be found at: + [Setting it to `false` to skip it](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118938#note_1374688877) + ### `rspec:undercoverage` job > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74859) in GitLab 14.6. @@ -392,6 +491,8 @@ to detect, and fail if any changes introduced in the merge request has zero cove The `rspec:undercoverage` job obtains coverage data from the `rspec:coverage` job. +If the `rspec:undercoverage` job detects missing coverage due to a CE method being overridden in EE, add the `pipeline:run-as-if-foss` label to the merge request and start a new pipeline. + In the event of an emergency, or false positive from this job, add the `pipeline:skip-undercoverage` label to the merge request to allow this job to fail. @@ -437,12 +538,12 @@ After that, the next pipeline uses the up-to-date `knapsack/report-master.json` We used to skip tests that are [known to be flaky](../testing_guide/flaky_tests.md#automatic-retries-and-flaky-tests-detection), but we stopped doing so since that could actually lead to actual broken `master`. -Instead, we proactively quarantine any flaky test reported in `#master-broken` incidents -so that they're ultimately fixed by their respective group. +Instead, we introduced +[a fast-quarantining process](../testing_guide/flaky_tests.md#fast-quarantine) +to proactively quarantine any flaky test reported in `#master-broken` incidents. -The automatic skipping of flaky tests can still be enabled by setting the `$SKIP_FLAKY_TESTS_AUTOMATICALLY` variable to `true`. - -See the [experiment issue](https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/1069). +This fast-quarantining process can be disabled by setting the `$FAST_QUARANTINE` +variable to `false`. ### Automatic retry of failing tests in a separate process @@ -451,7 +552,7 @@ RSpec process. The goal is to get rid of most side-effects from previous tests t We keep track of retried tests in the `$RETRIED_TESTS_REPORT_FILE` file saved as artifact by the `rspec:flaky-tests-report` job. -See the [experiment issue](https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/1148). +See the [experiment issue](https://gitlab.com/gitlab-org/quality/quality-engineering/team-tasks/-/issues/1148). ## Compatibility testing @@ -463,10 +564,9 @@ Exceptions to this general guideline should be motivated and documented. ### Ruby versions testing -We're running Ruby 3.0 for the merge requests and the default branch. However, -we're still running Ruby 2.7 for GitLab.com and there are older versions that -we need to maintain, so we also run our test suite against Ruby 2.7 on a -dedicated 2-hourly scheduled pipelines. +We're running Ruby 3.0 on GitLab.com, as well as for merge requests and the default branch. +However, there are older versions for which we need to support Ruby 2.7, so we also run our +test suite against Ruby 2.7 on a dedicated 2-hourly scheduled pipelines. For merge requests, you can add the `pipeline:run-in-ruby2` label to switch the Ruby version used for running the whole test suite to 2.7. When you do @@ -481,22 +581,22 @@ This should let us: ### PostgreSQL versions testing -Our test suite runs against PG12 as GitLab.com runs on PG12 and -[Omnibus defaults to PG12 for new installs and upgrades](../../administration/package_information/postgresql_versions.md). +Our test suite runs against PG13 as GitLab.com runs on PG13 and +[Omnibus defaults to PG13 for new installs and upgrades](../../administration/package_information/postgresql_versions.md). -We do run our test suite against PG11 and PG13 on nightly scheduled pipelines. +We do run our test suite against PG13 on nightly scheduled pipelines. -We also run our test suite against PG11 upon specific database library changes in MRs and `main` pipelines (with the `rspec db-library-code pg11` job). +We also run our test suite against PG13 upon specific database library changes in MRs and `main` pipelines (with the `rspec db-library-code pg13` job). #### Current versions testing -| Where? | PostgreSQL version | Ruby version | -|------------------------------------------------------------------------------------------------|-------------------------------------------------|--------------| -| Merge requests | 12 (default version), 11 for DB library changes | 3.0 (default version) | -| `master` branch commits | 12 (default version), 11 for DB library changes | 3.0 (default version) | -| `maintenance` scheduled pipelines for the `master` branch (every even-numbered hour) | 12 (default version), 11 for DB library changes | 3.0 (default version) | -| `maintenance` scheduled pipelines for the `ruby2` branch (every odd-numbered hour), see below. | 12 (default version), 11 for DB library changes | 2.7 | -| `nightly` scheduled pipelines for the `master` branch | 12 (default version), 11, 13 | 3.0 (default version) | +| Where? | PostgreSQL version | Ruby version | +|------------------------------------------------------------------------------------------------|-------------------------------------------------|-----------------------| +| Merge requests | 13 (default version), 12 for DB library changes | 3.0 (default version) | +| `master` branch commits | 13 (default version), 12 for DB library changes | 3.0 (default version) | +| `maintenance` scheduled pipelines for the `master` branch (every even-numbered hour) | 13 (default version), 12 for DB library changes | 3.0 (default version) | +| `maintenance` scheduled pipelines for the `ruby2` branch (every odd-numbered hour), see below. | 13 (default version), 12 for DB library changes | 2.7 | +| `nightly` scheduled pipelines for the `master` branch | 13 (default version), 12, 14 | 3.0 (default version) | There are 2 pipeline schedules used for testing Ruby 2.7. One is triggering a pipeline in `ruby2-sync` branch, which updates the `ruby2` branch with latest @@ -511,16 +611,6 @@ The `gitlab` job in the `ruby2-sync` branch uses a `gitlab-org/gitlab` project token with `write_repository` scope and `Maintainer` role with no expiration. The token is stored in the `RUBY2_SYNC_TOKEN` variable in `gitlab-org/gitlab`. -#### Long-term plan - -We follow the [PostgreSQL versions shipped with Omnibus GitLab](../../administration/package_information/postgresql_versions.md): - -| PostgreSQL version | 14.1 (July 2021) | 14.2 (August 2021) | 14.3 (September 2021) | 14.4 (October 2021) | 14.5 (November 2021) | 14.6 (December 2021) | -| -------------------| ---------------------- | ---------------------- | ---------------------- | ---------------------- | ---------------------- | ---------------------- | -| PG12 | MRs/`2-hour`/`nightly` | MRs/`2-hour`/`nightly` | MRs/`2-hour`/`nightly` | MRs/`2-hour`/`nightly` | MRs/`2-hour`/`nightly` | MRs/`2-hour`/`nightly` | -| PG11 | `nightly` | `nightly` | `nightly` | `nightly` | `nightly` | `nightly` | -| PG13 | `nightly` | `nightly` | `nightly` | `nightly` | `nightly` | `nightly` | - ### Redis versions testing Our test suite runs against Redis 6 as GitLab.com runs on Redis 6 and @@ -542,6 +632,13 @@ By default, all tests run with [multiple databases](../database/multiple_databas We also run tests with a single database in nightly scheduled pipelines, and in merge requests that touch database-related files. +Single database tests run in two modes: + +1. **Single database with one connection**. Where GitLab connects to all the tables using one connection pool. +This runs through all the jobs that end with `-single-db` +1. **Single database with two connections**. Where GitLab connects to `gitlab_main`, `gitlab_ci` database tables +using different database connections. This runs through all the jobs that end with `-single-db-ci-connection`. + If you want to force tests to run with a single database, you can add the `pipeline:run-single-db` label to the merge request. ## Monitoring @@ -562,7 +659,7 @@ In general, pipelines for an MR fall into one of the following types (from short - [Documentation pipeline](#documentation-pipeline): For MRs that touch documentation. - [Backend pipeline](#backend-pipeline): For MRs that touch backend code. -- [Frontend pipeline](#frontend-pipeline): For MRs that touch frontend code. +- [Review app pipeline](#review-app-pipeline): For MRs that touch frontend code. - [End-to-end pipeline](#end-to-end-pipeline): For MRs that touch code in the `qa/` folder. A "pipeline type" is an abstract term that mostly describes the "critical path" (for example, the chain of jobs for which the sum @@ -599,10 +696,10 @@ graph LR graph RL; classDef criticalPath fill:#f66; - 1-3["compile-test-assets (6 minutes)"]; + 1-3["compile-test-assets (5.5 minutes)"]; class 1-3 criticalPath; click 1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914317&udv=0" - 1-6["setup-test-env (4 minutes)"]; + 1-6["setup-test-env (3.6 minutes)"]; click 1-6 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914315&udv=0" 1-14["retrieve-tests-metadata"]; click 1-14 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356697&udv=0" @@ -614,19 +711,19 @@ graph RL; click 2_5-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations" 2_5-1 --> 1-3 & 1-6 & 1-14 & 1-15; - 3_2-1["rspec:coverage (5.35 minutes)"]; + 3_2-1["rspec:coverage (5 minutes)"]; class 3_2-1 criticalPath; click 3_2-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7248745&udv=0" 3_2-1 -.->|"(don't use needs
because of limitations)"| 2_5-1; - 4_3-1["rspec:undercoverage (3.5 minutes)"]; + 4_3-1["rspec:undercoverage (1.3 minutes)"]; class 4_3-1 criticalPath; click 4_3-1 "https://app.periscopedata.com/app/gitlab/652085/EP---Jobs-Durations?widget=13446492&udv=1005715" 4_3-1 --> 3_2-1; ``` -### Frontend pipeline +### Review app pipeline [Reference pipeline](https://gitlab.com/gitlab-org/gitlab/-/pipelines/431913287). @@ -636,15 +733,15 @@ graph RL; 1-2["build-qa-image (2 minutes)"]; click 1-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914325&udv=0" - 1-5["compile-production-assets (16 minutes)"]; + 1-5["compile-production-assets (12 minutes)"]; class 1-5 criticalPath; click 1-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914312&udv=0" - 2_3-1["build-assets-image (1.3 minutes)"]; + 2_3-1["build-assets-image (1.1 minutes)"]; class 2_3-1 criticalPath; 2_3-1 --> 1-5 - 2_6-1["start-review-app-pipeline (49 minutes)"]; + 2_6-1["start-review-app-pipeline (52 minutes)"]; class 2_6-1 criticalPath; click 2_6-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations" 2_6-1 --> 2_3-1 & 1-2; @@ -660,17 +757,17 @@ graph RL; 1-2["build-qa-image (2 minutes)"]; click 1-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914325&udv=0" - 1-5["compile-production-assets (16 minutes)"]; + 1-5["compile-production-assets (12 minutes)"]; class 1-5 criticalPath; click 1-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914312&udv=0" 1-15["detect-tests"]; click 1-15 "https://app.periscopedata.com/app/gitlab/652085/EP---Jobs-Durations?widget=10113603&udv=1005715" - 2_3-1["build-assets-image (1.3 minutes)"]; + 2_3-1["build-assets-image (1.1 minutes)"]; class 2_3-1 criticalPath; 2_3-1 --> 1-5 - 2_4-1["e2e:package-and-test (102 minutes)"]; + 2_4-1["e2e:package-and-test-ee (103 minutes)"]; class 2_4-1 criticalPath; click 2_4-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914305&udv=0" 2_4-1 --> 1-2 & 2_3-1 & 1-15; diff --git a/doc/development/pipelines/internals.md b/doc/development/pipelines/internals.md index 9ff4e5a35ec..678297eb3e5 100644 --- a/doc/development/pipelines/internals.md +++ b/doc/development/pipelines/internals.md @@ -136,12 +136,10 @@ that are scoped to a single [configuration keyword](../../ci/yaml/index.md#job-k | `.qa-cache` | Allows a job to use a default `cache` definition suitable for QA tasks. | | `.yarn-cache` | Allows a job to use a default `cache` definition suitable for frontend jobs that do a `yarn install`. | | `.assets-compile-cache` | Allows a job to use a default `cache` definition suitable for frontend jobs that compile assets. | -| `.use-pg11` | Allows a job to run the `postgres` 11 and `redis` services (see [`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/global.gitlab-ci.yml) for the specific versions of the services). | -| `.use-pg11-ee` | Same as `.use-pg11` but also use an `elasticsearch` service (see [`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/global.gitlab-ci.yml) for the specific version of the service). | -| `.use-pg12` | Allows a job to use the `postgres` 12 and `redis` services (see [`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/global.gitlab-ci.yml) for the specific versions of the services). | -| `.use-pg12-ee` | Same as `.use-pg12` but also use an `elasticsearch` service (see [`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/global.gitlab-ci.yml) for the specific version of the service). | | `.use-pg13` | Allows a job to use the `postgres` 13 and `redis` services (see [`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/global.gitlab-ci.yml) for the specific versions of the services). | | `.use-pg13-ee` | Same as `.use-pg13` but also use an `elasticsearch` service (see [`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/global.gitlab-ci.yml) for the specific version of the service). | +| `.use-pg14` | Allows a job to use the `postgres` 14 and `redis` services (see [`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/global.gitlab-ci.yml) for the specific versions of the services). | +| `.use-pg14-ee` | Same as `.use-pg14` but also use an `elasticsearch` service (see [`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/global.gitlab-ci.yml) for the specific version of the service). | | `.use-kaniko` | Allows a job to use the `kaniko` tool to build Docker images. | | `.as-if-foss` | Simulate the FOSS project by setting the `FOSS_ONLY='1'` CI/CD variable. | | `.use-docker-in-docker` | Allows a job to use Docker in Docker. | @@ -189,7 +187,6 @@ and included in `rules` definitions via [YAML anchors](../../ci/yaml/yaml_optimi | `if-dot-com-gitlab-org-and-security-merge-request` | Limit jobs creation to merge requests for the `gitlab-org` and `gitlab-org/security` groups on GitLab.com. | | | `if-dot-com-gitlab-org-and-security-tag` | Limit jobs creation to tags for the `gitlab-org` and `gitlab-org/security` groups on GitLab.com. | | | `if-dot-com-ee-schedule` | Limits jobs to scheduled pipelines for the `gitlab-org/gitlab` project on GitLab.com. | | -| `if-security-pipeline-merge-result` | Matches if the pipeline is for a security merge request triggered by `@gitlab-release-tools-bot`. | | diff --git a/doc/development/pipelines/performance.md b/doc/development/pipelines/performance.md index 5f2df91edf3..d0eef0c86bd 100644 --- a/doc/development/pipelines/performance.md +++ b/doc/development/pipelines/performance.md @@ -147,4 +147,4 @@ We no longer use this optimization for `gitlab-org/gitlab` because the [pack-obj allows Gitaly to serve the full CI/CD fetch traffic now. See [Git fetch caching](#git-fetch-caching). The pre-clone step works by using the `CI_PRE_CLONE_SCRIPT` variable -[defined by GitLab.com shared runners](../../ci/runners/saas/linux_saas_runner.md#pre-clone-script). +[defined by GitLab.com shared runners](../../ci/runners/saas/linux_saas_runner.md#pre-clone-script-deprecated). diff --git a/doc/development/product_qualified_lead_guide/index.md b/doc/development/product_qualified_lead_guide/index.md index c72110bc253..fb8ec478840 100644 --- a/doc/development/product_qualified_lead_guide/index.md +++ b/doc/development/product_qualified_lead_guide/index.md @@ -4,7 +4,7 @@ group: Acquisition info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Product Qualified Lead (PQL) development guide +# Product Qualified Lead (PQL) development guidelines The Product Qualified Lead (PQL) funnel connects our users with our team members. Read more about [PQL product principles](https://about.gitlab.com/handbook/product/product-principles/#product-qualified-leads-pqls). @@ -117,7 +117,7 @@ expect(wrapper.findComponent(HandRaiseLeadButton).exists()).toBe(true); The flow of a PQL lead is as follows: 1. A user triggers a [`HandRaiseLeadButton` component](#embed-a-hand-raise-lead-form) on `gitlab.com`. -1. The `HandRaiseLeadButton` submits any information to the following API endpoint: `/-/trials/create_hand_raise_lead`. +1. The `HandRaiseLeadButton` submits any information to the following API endpoint: `/-/subscriptions/hand_raise_leads`. 1. That endpoint reposts the form to the CustomersDot `trials/create_hand_raise_lead` endpoint. 1. CustomersDot records the form data to the `leads` table and posts the form to [Workato](https://about.gitlab.com/handbook/marketing/marketing-operations/workato/). 1. Workato sends the form to Marketo. @@ -170,7 +170,7 @@ sequenceDiagram ```mermaid sequenceDiagram HandRaiseForm Vue Component->>TrialsController#create_hand_raise_lead: GitLab.com frontend sends [lead] to backend - TrialsController#create_hand_raise_lead->>CreateHandRaiseLeadService: [lead] + Subscriptions::HandRaiseLeadsController#create->>CreateHandRaiseLeadService: [lead] CreateHandRaiseLeadService->>SubscriptionPortalClient: [lead] SubscriptionPortalClient->>CustomersDot|TrialsController#create_hand_raise_lead: GitLab.com sends [lead] to CustomersDot ``` diff --git a/doc/development/project_templates.md b/doc/development/project_templates.md index 3320f3134fb..cc53ef77c62 100644 --- a/doc/development/project_templates.md +++ b/doc/development/project_templates.md @@ -8,110 +8,63 @@ info: "To determine the technical writer assigned to the Stage/Group associated ## Adding a new built-in project template -This page provides instructions about how to contribute a -[built-in project template](../user/project/index.md#create-a-project-from-a-built-in-template). - -To contribute a built-in project template, you must complete the following tasks: - -1. [Create a project template for GitLab review](#create-a-project-template-for-review) -1. [Add the template SVG icon to GitLab SVGs](#add-the-template-svg-icon-to-gitlab-svgs) -1. [Create a merge request with vendor details](#create-a-merge-request-with-vendor-details) - -You can contribute the following types of project templates: - -- Enterprise: For users with GitLab Premium and above. -- Non-enterprise: For users with GitLab Free and above. - -### Prerequisites - -To add or update an existing template, you must have the following tools -installed: - -- `wget` -- `tar` - -### Create a project template for review - -1. In your selected namespace, create a public project. -1. Add the project content you want to use in the template. Do not include unnecessary assets or dependencies. For an example, -[see this project](https://gitlab.com/gitlab-org/project-templates/dotnetcore). -1. When the project is ready for review, [create an issue](https://gitlab.com/gitlab-org/gitlab/issues) with a link to your project. - In your issue, mention the Create:Source Code [Backend Engineering Manager and Product Manager](https://about.gitlab.com/handbook/product/categories/#source-code-group) - for the Templates feature. - -### Add the template SVG icon to GitLab SVGs - -If the project template has an SVG icon, you must add it to the -[GitLab SVGs project](https://gitlab.com/gitlab-org/gitlab-svgs/-/blob/main/README.md#adding-icons-or-illustrations) -before you can create a merge request with vendor details. - -### Create a merge request with vendor details - -Before GitLab can implement the project template, you must [create a merge request](../user/project/merge_requests/creating_merge_requests.md) in [`gitlab-org/gitlab`](https://gitlab.com/gitlab-org/gitlab) that includes vendor details about the project. - -1. [Export the project](../user/project/settings/import_export.md#export-a-project-and-its-data) - and save the file as `.tar.gz`, where `` is the short name of the project. - Move this file to the root directory of `gitlab-org/gitlab`. -1. In `gitlab-org/gitlab`, create and check out a new branch. -1. Edit the following files to include the project template: - - For **non-Enterprise** project templates: - - In `lib/gitlab/project_template.rb`, add details about the template - in the `localized_templates_table` method. In the following example, - the short name of the project is `hugo`: - - ```ruby - ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo'), 'https://gitlab.com/pages/hugo', 'illustrations/logos/hugo.svg'), - ``` - - If the project doesn't have an SVG icon, exclude `, 'illustrations/logos/hugo.svg'`. - - - In `spec/support/helpers/project_template_test_helper.rb`, append the short name - of the template in the `all_templates` method. - - In `app/assets/javascripts/projects/default_project_templates.js`, - add details of the template. For example: - - ```javascript - hugo: { - text: s__('ProjectTemplates|Pages/Hugo'), - icon: '.template-option .icon-hugo', - }, - ``` - - If the project doesn't have an SVG icon, use `.icon-gitlab_logo` - instead. - - For **Enterprise** project templates: - - In `ee/lib/ee/gitlab/project_template.rb`, in the `localized_ee_templates_table` method, add details about the template. For example: - - ```ruby - ::Gitlab::ProjectTemplate.new('hipaa_audit_protocol', 'HIPAA Audit Protocol', _('A project containing issues for each audit inquiry in the HIPAA Audit Protocol published by the U.S. Department of Health & Human Services'), 'https://gitlab.com/gitlab-org/project-templates/hipaa-audit-protocol', 'illustrations/logos/asklepian.svg') - ``` - - - In `ee/spec/lib/gitlab/project_template_spec.rb`, add the short name - of the template in the `.all` test. - - In `ee/app/assets/javascripts/projects/default_project_templates.js`, - add the template details. For example: - - ```javascript - hipaa_audit_protocol: { - text: s__('ProjectTemplates|HIPAA Audit Protocol'), - icon: '.template-option .icon-hipaa_audit_protocol', - }, - ``` - -1. Run the following Rake task, where `/` is the - name you gave the template in `lib/gitlab/project_template.rb`: +If you'd like to contribute a new built-in project template to be distributed with GitLab, please do the following: + +1. Create a new public project with the project content you'd like to contribute in a namespace of your choosing. You can view a working example [here](https://gitlab.com/gitlab-org/project-templates/dotnetcore). + - Projects should be as simple as possible and free of any unnecessary assets or dependencies. +1. When the project is ready for review, please create a new issue in [GitLab](https://gitlab.com/gitlab-org/gitlab/issues) with a link to your project. + - In your issue, `@` mention the relevant Backend Engineering Manager and Product Manager for the [Create:Source Code group](https://about.gitlab.com/handbook/product/categories/#source-code-group). + +To make the project template available when creating a new project, the vendoring process will have to be completed: + +1. Create a working template ([example](https://gitlab.com/gitlab-org/project-templates/dotnetcore)) + - 2 types of built-in templates are available within GitLab: + - **Normal templates**: Available in GitLab Core, Starter and above (this is the most common type of built-in template). + - To contribute a normal template: + - Add details of the template in the `localized_templates_table` method in `gitlab/lib/gitlab/project_template.rb`, + - Add details of the template in `spec/lib/gitlab/project_template_spec.rb`, in the test for the `all` method, and + - Add details of the template in `gitlab/app/assets/javascripts/projects/default_project_templates.js`. + - See MR [!25318](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25318) for an example + - **Enterprise templates**: Introduced in GitLab 12.10, that are available only in GitLab Gold & Ultimate. + - To contribute an Enterprise template: + - Add details of the template in the `localized_ee_templates_table` method in `gitlab/ee/lib/ee/gitlab/project_template.rb`, + - Add details of the template in `gitlab/ee/spec/lib/gitlab/project_template_spec.rb`, in the `enterprise_templates` method, and + - Add details of the template in `gitlab/ee/app/assets/javascripts/projects/default_project_templates.js`. + - See MR [!28187](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28187) for an example. + +1. Run the following in the `gitlab` project, where `$name` is the name you gave the template in `gitlab/project_template.rb`: ```shell - bin/rake gitlab:update_project_templates\[/\] + bin/rake gitlab:update_project_templates[$name] ``` -1. Regenerate `gitlab.pot`: - - ```shell - bin/rake gettext:regenerate - ``` - -1. After you run the scripts, there is one new file in `vendor/project_templates/` and four changed files. Commit all changes and push your branch to update the merge request. For an example, see this [merge request](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25318). +1. Run the `bundle_repo` script. Make sure to pass the correct arguments, or the script may damage the folder structure. +1. Add exported project (`$name.tar.gz`) to `gitlab/vendor/project_templates` and remove the resulting build folders `tar-base` and `project`. +1. Run `bin/rake gettext:regenerate` in the `gitlab` project and commit new `.pot` file. +1. Add a changelog entry in the commit message (for example, `Changelog: added`). + For more information, see [Changelog entries](changelog.md). +1. Add an icon to [`gitlab-svgs`](https://gitlab.com/gitlab-org/gitlab-svgs), as shown in + [this example](https://gitlab.com/gitlab-org/gitlab-svgs/merge_requests/195). If a logo + is not available for the project, use the default 'Tanuki' logo instead. +1. Run `yarn run svgs` on `gitlab-svgs` project and commit result. +1. Forward changes in `gitlab-svgs` project to master. This involves: + - Merging your MR in `gitlab-svgs` + - [The bot](https://gitlab.com/gitlab-org/frontend/renovate-gitlab-bot/) + will pick the new release up and create an MR in `gitlab-org/gitlab`. +1. Once the bot-created MR created above is merged, you can rebase your template MR onto the updated `master` to pick up the new svgs. +1. Test everything is working. + +### Contributing an improvement to an existing template + +Existing templates are available in the [project-templates](https://gitlab.com/gitlab-org/project-templates) +group. + +To contribute a change, please open a merge request in the relevant project +and mention `@gitlab-org/manage/import/backend` when you are ready for a review. + +Then, if your merge request gets accepted, either open an issue on +`gitlab` to ask for it to get updated, or open a merge request updating +the vendored template using [these instructions](rake_tasks.md#update-project-templates). ### Test your built-in project with the GitLab Development Kit @@ -124,17 +77,6 @@ Complete the following steps to test the project template in your own GitLab Dev bin/rake gitlab:update_project_templates\[/\] ``` -## Contribute an improvement to an existing template - -To update an existing built-in project template, changes are usually made to the existing template, found in the [project-templates](https://gitlab.com/gitlab-org/project-templates) group. A merge request is made directly against the template and the Create:Source Code [Backend Engineering Manager and Product Manager](https://about.gitlab.com/handbook/product/categories/#source-code-group) pinged for review. - -Sometimes it is necessary to completely replace the template files. In this case the process would be: - -1. Create a merge request in the relevant project of the `project-templates` and `pages` group and mention `@gitlab-org/manage/import/backend` when you are ready for a review. -1. If your merge request is accepted, either: - - [Create an issue](https://gitlab.com/gitlab-org/gitlab/-/issues) to ask for the template to get updated. - - [Create a merge request with vendor details](#create-a-merge-request-with-vendor-details) to update the template. - ## For GitLab team members Please ensure the merge request has been reviewed by the Security Counterpart before merging. diff --git a/doc/development/projections.md b/doc/development/projections.md index 7c727fc0901..caa54e7b912 100644 --- a/doc/development/projections.md +++ b/doc/development/projections.md @@ -28,8 +28,6 @@ You can find a basic list of projection options in - [Alternate File](https://marketplace.visualstudio.com/items?itemName=will-wow.vscode-alternate-file) - [projectionist](https://github.com/jarsen/projectionist) - [`jumpto`](https://github.com/gmdayley/jumpto) -- Atom - - [projectionist-atom](https://atom.io/packages/projectionist-atom) - Command-line - [projectionist](https://github.com/glittershark/projectionist) @@ -41,4 +39,4 @@ This started as a [plugin for vim by tpope](https://github.com/tpope/vim-projectionist) It has since become editor-agnostic and ported to most modern editors. - \ No newline at end of file + diff --git a/doc/development/prometheus_metrics.md b/doc/development/prometheus_metrics.md index 834a20239fc..f72805e0fe9 100644 --- a/doc/development/prometheus_metrics.md +++ b/doc/development/prometheus_metrics.md @@ -4,7 +4,7 @@ group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Working with Prometheus Metrics +# Prometheus metrics development guidelines ## Adding to the library @@ -78,3 +78,15 @@ For example, a histogram with 10 buckets and a label with 100 values would gener entries in the export endpoint. 1. Trigger the relevant page or code that records the new metric. 1. Check that the new metric appears at `/-/metrics`. + +For metrics that are not bounded to a specific context (`request`, `process`, `machine`, `namespace`, etc), +generate them from a cron-based Sidekiq job: + +- For Geo related metrics, check `Geo::MetricsUpdateService`. +- For other "global" / instance-wide metrics, check: `Metrics::GlobalMetricsUpdateService`. + +When exporting data from Sidekiq in an installation with more than one Sidekiq instance, +you are not guaranteed that the same exporter will always be queried. + +You can read more and understand the caveats in [issue 406583](https://gitlab.com/gitlab-org/gitlab/-/issues/406583), +where we also discuss a possible solution using a push-gateway. diff --git a/doc/development/python_guide/index.md b/doc/development/python_guide/index.md index 08d2f46c1cd..e9b52b81c0e 100644 --- a/doc/development/python_guide/index.md +++ b/doc/development/python_guide/index.md @@ -4,7 +4,7 @@ group: unassigned info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Python Development Guidelines +# Python development guidelines GitLab requires Python as a dependency for [reStructuredText](https://docutils.sourceforge.io/rst.html) markup rendering. diff --git a/doc/development/rails_update.md b/doc/development/rails_update.md index a541de020fb..32295cc0e43 100644 --- a/doc/development/rails_update.md +++ b/doc/development/rails_update.md @@ -61,49 +61,49 @@ To efficiently and quickly find which Rails change caused the spec failure you c 1. Clone the `rails` project in a folder of your choice. For example, it might be the GDK root dir: - ```shell - cd - git clone https://github.com/rails/rails.git - ``` + ```shell + cd + git clone https://github.com/rails/rails.git + ``` 1. Replace the `gem 'rails'` line in GitLab `Gemfile` with: - ```ruby - gem 'rails', ENV['RAILS_VERSION'], path: ENV['RAILS_FOLDER'] - ``` + ```ruby + gem 'rails', ENV['RAILS_VERSION'], path: ENV['RAILS_FOLDER'] + ``` 1. Set the `RAILS_FOLDER` environment variable with the folder you cloned Rails into: - ```shell - export RAILS_FOLDER="/rails" - ``` + ```shell + export RAILS_FOLDER="/rails" + ``` 1. Change the directory to `RAILS_FOLDER` and set the range for the `git bisect` command: - ```shell - cd $RAILS_FOLDER - git bisect start - ``` + ```shell + cd $RAILS_FOLDER + git bisect start + ``` - Where `` is the tag where the spec is red and `` is the one with the green spec. - For example, `git bisect start v6.1.4.1 v6.1.3.2` if we're upgrading from version 6.1.3.2 to 6.1.4.1. - Replace `` with the tag where the spec is red and `` with the one with the green spec. For example, `git bisect start v6.1.4.1 v6.1.3.2` if we're upgrading from version 6.1.3.2 to 6.1.4.1. - In the output, you can see how many steps approximately it takes to find the commit. + Where `` is the tag where the spec is red and `` is the one with the green spec. + For example, `git bisect start v6.1.4.1 v6.1.3.2` if we're upgrading from version 6.1.3.2 to 6.1.4.1. + Replace `` with the tag where the spec is red and `` with the one with the green spec. For example, `git bisect start v6.1.4.1 v6.1.3.2` if we're upgrading from version 6.1.3.2 to 6.1.4.1. + In the output, you can see how many steps approximately it takes to find the commit. 1. Start the `git bisect` process and pass spec's filenames to `scripts/rails-update-bisect` as arguments. It can be faster to pick only one example instead of an entire spec file. - ```shell - git bisect run /gitlab/scripts/rails-update-bisect spec/models/ability_spec.rb - # OR - git bisect run /gitlab/scripts/rails-update-bisect spec/models/ability_spec.rb:7 - ``` + ```shell + git bisect run /gitlab/scripts/rails-update-bisect spec/models/ability_spec.rb + # OR + git bisect run /gitlab/scripts/rails-update-bisect spec/models/ability_spec.rb:7 + ``` 1. When the process is completed, `git bisect` prints the commit hash, which you can use to find the corresponding MR in the [`rails/rails`](https://github.com/rails/rails) repository. 1. Execute `git bisect reset` to exit the `bisect` mode. 1. Revert the changes to `Gemfile`: - ```shell - git checkout -- Gemfile - ``` + ```shell + git checkout -- Gemfile + ``` ### Follow-up reading material diff --git a/doc/development/rake_tasks.md b/doc/development/rake_tasks.md index cd7f8cba39b..8261330223d 100644 --- a/doc/development/rake_tasks.md +++ b/doc/development/rake_tasks.md @@ -145,6 +145,58 @@ bin/rake 'gitlab:seed:vulnerabilities' bin/rake 'gitlab:seed:vulnerabilities[group-path/project-path]' ``` +#### Seed a project with environments + +You can seed a project with [environments](../ci/environments/index.md). + +By default, this creates 10 environments, each with the prefix `ENV_`. +Only `project_path` is required to run this command. + +```shell +bundle exec rake "gitlab:seed:project_environments[project_path, seed_count, prefix]" + +# Examples +bundle exec rake "gitlab:seed:project_environments[flightjs/Flight]" +bundle exec rake "gitlab:seed:project_environments[flightjs/Flight, 25, FLIGHT_ENV_]" +``` + +#### Seed CI variables + +You can seed a project, group, or instance with [CI variables](../ci/variables/index.md). + +By default, each command creates 10 CI variables. Variable names are prepended with its own +default prefix (`VAR_` for project-level variables, `GROUP_VAR_` for group-level variables, +and `INSTANCE_VAR_` for instance-level variables). + +Instance-level variables do not have environment scopes. Project-level and group-level variables +use the default `"*"` environment scope if no `environment_scope` is supplied. If `environment_scope` +is set to `"unique"`, each variable is created with its own unique environment. + +```shell +# Seed a project with project-level CI variables +# Only `project_path` is required to run this command. +bundle exec rake "gitlab:seed:ci_variables_project[project_path, seed_count, environment_scope, prefix]" + +# Seed a group with group-level CI variables +# Only `group_name` is required to run this command. +bundle exec rake "gitlab:seed:ci_variables_group[group_name, seed_count, environment_scope, prefix]" + +# Seed an instance with instance-level CI variables +bundle exec rake "gitlab:seed:ci_variables_instance[seed_count, prefix]" + +# Examples +bundle exec rake "gitlab:seed:ci_variables_project[flightjs/Flight]" +bundle exec rake "gitlab:seed:ci_variables_project[flightjs/Flight, 25, staging]" +bundle exec rake "gitlab:seed:ci_variables_project[flightjs/Flight, 25, unique, CI_VAR_]" + +bundle exec rake "gitlab:seed:ci_variables_group[group_name]" +bundle exec rake "gitlab:seed:ci_variables_group[group_name, 25, staging]" +bundle exec rake "gitlab:seed:ci_variables_group[group_name, 25, unique, CI_VAR_]" + +bundle exec rake "gitlab:seed:ci_variables_instance" +bundle exec rake "gitlab:seed:ci_variables_instance[25, CI_VAR_]" +``` + ### Automation If you're very sure that you want to **wipe the current database** and refill @@ -258,7 +310,7 @@ One way to generate the initial list is to run the Rake task `rubocop:todo:gener bundle exec rake rubocop:todo:generate ``` -To generate TODO list for specific RuboCop rules, pass them comma-separated as +To generate TODO list for specific RuboCop rules, pass them comma-seperated as argument to the Rake task: ```shell @@ -372,7 +424,7 @@ a file for quick reference. ## Show obsolete `ignored_columns` -To see a list of all obsolete `ignored_columns` run: +To see a list of all obsolete `ignored_columns` definitions run: ```shell bundle exec rake db:obsolete_ignored_columns diff --git a/doc/development/real_time.md b/doc/development/real_time.md index 2aa48fed8eb..017e308fc03 100644 --- a/doc/development/real_time.md +++ b/doc/development/real_time.md @@ -1,29 +1,347 @@ --- -stage: Plan -group: Project Management +stage: Data Stores +group: Application Performance info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Real-Time Features +# Build and deploy real-time view components -This guide contains instructions on how to safely roll out new real-time -features. +GitLab provides an interactive user experience through individual view components that accept +user input and reflect state changes back to the user. For example, on the Merge Request +page, users can approve, leave comments, interact with the CI/CD pipeline, and more. -Real-time features are implemented using GraphQL Subscriptions. -[Developer documentation](api_graphql_styleguide.md#subscriptions) is available. +However, GitLab often does not reflect state updates in a timely manner. +This means parts of the page display stale data that only update after users reload the page. + +To address this, GitLab has introduced technology and programming APIs that allow view components +to receive state updates in real-time over a WebSocket. + +The following documentation tells you how to build and deploy view components that +receive updates in real-time from the GitLab Ruby on Rails server. + +NOTE: +Action Cable and GraphQL subscriptions are a work-in-progress and under active development. +Developers must evaluate their use case to check if these are the right tools to use. +If you are not sure, ask for help in the [`#f_real-time` internal Slack channel](https://gitlab.slack.com/archives/CUX9Z2N66). + +## Build real-time view components + +Prerequisites: + +Read the: + +- [GraphQL development guide](fe_guide/graphql.md). +- [Vue development guide](fe_guide/vue.md). + +To build a real-time view component on GitLab, you must: + +- Integrate a Vue component with Apollo subscriptions in the GitLab frontend. +- Add and trigger GraphQL subscriptions from the GitLab Ruby on Rails backend. + +### Integrate a Vue component with Apollo subscriptions + +NOTE: +Our current real-time stack assumes that client code is built using Vue as the rendering layer and +Apollo as the state and networking layer. If you are working with a part of +the GitLab frontend that has not been migrated to Vue + Apollo yet, complete that task first. + +Consider a hypothetical `IssueView` Vue component that observes and renders GitLab `Issue` data. +For simplicity, we assume here that all it does is render an issue's title and description: + +```javascript +import issueQuery from '~/issues/queries/issue_view.query.graqhql'; + +export default { + props: { + issueId: { + type: Number, + required: false, + default: null, + }, + }, + apollo: { + // Name of the Apollo query object. Must match the field name bound by `data`. + issue: { + // Query used for the initial fetch. + query: issueQuery, + // Bind arguments used for the initial fetch query. + variables() { + return { + iid: this.issueId, + }; + }, + // Map response data to view properties. + update(data) { + return data.project?.issue || {}; + }, + }, + }, + // Reactive Vue component data. Apollo updates these when queries return or subscriptions fire. + data() { + return { + issue: {}, // It is good practice to return initial state here while the view is loading. + }; + }, +}; + +// The +``` + +Bad: + +```javascript +it('hides the component', () => { + createComponent({ props: { isVisible: false }}) + + expect(wrapper.element).toMatchSnapshot() +}) + +it('shows the component', () => { + createComponent({ props: { isVisible: true }}) + + expect(wrapper.element).toMatchSnapshot() +}) +``` + +Good: + +```javascript +it('hides the component', () => { + createComponent({ props: { isVisible: false }}) + + expect(findMyComponent().exists()).toBe(false) +}) + +it('shows the component', () => { + createComponent({ props: { isVisible: true }}) + + expect(findMyComponent().exists()).toBe(true) +}) +``` + +Not only that, but imagine having passed the wrong prop to your component and having the wrong visibility: the snapshot test would still pass because you would have captured the HTML **with the issue** and so unless you double-checked the output of the snapshot, you would never know that your test is broken. + +#### Example #2 - Presence of text + +Finding text within a component is very easy by using the `vue-test-utils` method `wrapper.text()`. However, there are some cases where it might be tempting to use snapshots when the value returned has a lot of inconsistent spacing due to formatting or HTML nesting. + +In these instances, it is better to assert each string individually and make multiple assertions than to use a snapshot to ignore the spaces. This is because any change to the DOM layout will fail the snapshot test even if the text is still perfectly formatted. + +```vue + +``` + +Bad: + +```javascript +it('renders the text as I expect', () => { + expect(wrapper.text()).toMatchSnapshot() +}) +``` + +Good: + +```javascript +it('renders the code snippet', () => { + expect(findCodeTag().text()).toContain("myFunction()") +}) + +it('renders the paragraph text', () => { + expect(findOtherText().text()).toBe("My second message") +}) +``` + +#### Example #3 - Complex HTML + +When we have very complex HTML, we should focus on asserting specific sensitive and meaningful points rather than capturing it as a whole. The value in a snapshot test is to **warn developers** that they might have accidentally change an HTML structure that they did not intend to change. If the output of the change is hard to read, which is often the case with complex HTML output, then **is the signal itself that something changed** sufficient? And if it is, can it be accomplished without snapshots? + +A good example of a complex HTML output is `GlTable`. Snapshot testing might feel like a good option since you can capture rows and columns structure, but we should instead try to assert text we expect or count the number of rows and columns manually. + +```vue + +``` + +Bad: + +```javascript +it('renders GlTable as I expect', () => { + expect(findGlTable().element).toMatchSnapshot() +}) +``` + +Good: + +```javascript +it('renders the right number of rows', () => { + expect(findGlTable().findAllRows()).toHaveLength(expectedLength) +}) + +it('renders the special icon that only appears on a full moon', () => { + expect(findGlTable().findMoonIcon().exists()).toBe(true) +}) + +it('renders the correct email format', () => { + expect(findGlTable().text()).toContain('my_strange_email@shaddyprovide.com') +}) +``` + +Although more verbose, this now means that our tests are not going to break if `GlTable` changes its internal implementation, we communicate to other developers (or ourselves in 6 months) what is important to preserve when refactoring or adding to our table. + ### How to take a snapshot ```javascript @@ -1211,43 +1434,6 @@ it('renders the component correctly', () => { The above test will create two snapshots. It's important to decide which of the snapshots provide more value for codebase safety. That is, if one of these snapshots changes, does that highlight a possible break in the codebase? This can help catch unexpected changes if something in an underlying dependency changes without our knowledge. -### Pros and Cons - -**Pros** - -- Speed up the creation of unit tests -- Easy to maintain -- Provides a good safety net to protect against accidental breakage of important HTML structures - -**Cons** - -- Is not a catch-all solution that replaces the work of integration or unit tests -- No meaningful assertions or expectations within snapshots -- When carelessly used with [GitLab UI](https://gitlab.com/gitlab-org/gitlab-ui) it can create fragility in tests when the underlying library changes the HTML of a component we are testing - -A good guideline to follow: the more complex the component you may want to steer away from just snapshot testing. But that's not to say you can't still snapshot test and test your component as normal. - -### When to use - -**Use snapshots when** - -- to capture a components rendered output -- to fully or partially match templates -- to match readable data structures -- to verify correctly composed native HTML elements -- as a safety net for critical structures so others don't break it by accident -- Template heavy component -- Not a lot of logic in the component -- Composed of native HTML elements - -### When not to use - -**Don't use snapshots when** - -- To capture large data structures just to have something -- To just have some kind of test written -- To capture highly volatile UI elements without stubbing them (Think of GitLab UI version updates) - ## Get started with feature tests ### What is a feature test @@ -1567,11 +1753,8 @@ Inside the terminal, where capybara is running, you can also execute `next` whic ### Updating ChromeDriver -On MacOS, if you get a ChromeDriver error, make sure to update it by running - -```shell - brew upgrade chromedriver -``` +Starting from `Selenium` 4.6, ChromeDriver can be automatically managed by `Selenium Manager` which comes with the `selenium-webdriver` gem. +You are no longer required to manually keeping chromedriver in sync. --- diff --git a/doc/development/testing_guide/review_apps.md b/doc/development/testing_guide/review_apps.md index b074a9e34f3..1e5ee9f3003 100644 --- a/doc/development/testing_guide/review_apps.md +++ b/doc/development/testing_guide/review_apps.md @@ -47,8 +47,8 @@ Maintainers can elect to use the [process for merging during broken `master`](ht ## Performance Metrics -On every [pipeline](https://gitlab.com/gitlab-org/gitlab/pipelines/125315730) in the `qa` stage, the -`review-performance` job is automatically started: this job does basic +On every Review App child pipeline in the `qa` stage, the +`browser_performance` job is automatically started: this job does basic browser performance testing using a [Sitespeed.io Container](../../ci/testing/browser_performance_testing.md). @@ -227,7 +227,7 @@ Review apps are automatically stopped 2 days after the last deployment thanks to the [Environment auto-stop](../../ci/environments/index.md#stop-an-environment-after-a-certain-time-period) feature. If you need your review app to stay up for a longer time, you can -[pin its environment](../../ci/environments/index.md#override-a-deployments-scheduled-stop-time) or retry the +[pin its environment](../../ci/environments/index.md#override-a-environments-scheduled-stop-date-and-time) or retry the `review-deploy` job to update the "latest deployed at" time. The `review-cleanup` job that automatically runs in scheduled @@ -276,7 +276,7 @@ find a way to limit it to only us.** ## Other resources - [Review apps integration for CE/EE (presentation)](https://docs.google.com/presentation/d/1QPLr6FO4LduROU8pQIPkX1yfGvD13GEJIBOenqoKxR8/edit?usp=sharing) -- [Stability issues](https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/212) +- [Stability issues](https://gitlab.com/gitlab-org/quality/quality-engineering/team-tasks/-/issues/212) ### Helpful command line tools diff --git a/doc/development/testing_guide/testing_levels.md b/doc/development/testing_guide/testing_levels.md index c93a5fd539b..480a53bbefe 100644 --- a/doc/development/testing_guide/testing_levels.md +++ b/doc/development/testing_guide/testing_levels.md @@ -55,6 +55,7 @@ records should use stubs/doubles as much as possible. | `lib/` | `spec/lib/` | RSpec | | | `lib/tasks/` | `spec/tasks/` | RSpec | | | `rubocop/` | `spec/rubocop/` | RSpec | | +| `spec/support/` | `spec/support_specs/` | RSpec | | ### Frontend unit tests @@ -65,7 +66,7 @@ that is not directly perceivable by a user. graph RL plain[Plain JavaScript]; Vue[Vue Components]; - feature-flags[Feature Flags]; + feature-flags[Feature flags]; license-checks[License Checks]; plain---Vuex; @@ -149,7 +150,7 @@ Component tests cover the state of a single component that is perceivable by a u graph RL plain[Plain JavaScript]; Vue[Vue Components]; - feature-flags[Feature Flags]; + feature-flags[Feature flags]; license-checks[License Checks]; plain---Vuex; @@ -243,7 +244,7 @@ Their abstraction level is comparable to how a user would interact with the UI. graph RL plain[Plain JavaScript]; Vue[Vue Components]; - feature-flags[Feature Flags]; + feature-flags[Feature flags]; license-checks[License Checks]; plain---Vuex; @@ -300,7 +301,7 @@ graph RL - **DOM**: Testing on the real DOM ensures your components work in the intended environment. - Part of DOM testing is delegated to [cross-browser testing](https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/45). + Part of DOM testing is delegated to [cross-browser testing](https://gitlab.com/gitlab-org/quality/quality-engineering/team-tasks/-/issues/45). - **Properties or state of components**: On this level, all tests can only perform actions a user would do. For example: to change the state of a component, a click event would be fired. @@ -366,13 +367,12 @@ See also: - The [RSpec testing guidelines](../testing_guide/best_practices.md#rspec). - System / Feature tests in the [Testing Best Practices](best_practices.md#system--feature-tests). -- [Issue #26159](https://gitlab.com/gitlab-org/gitlab/-/issues/26159) which aims at combining those guidelines with this page. ```mermaid graph RL plain[Plain JavaScript]; Vue[Vue Components]; - feature-flags[Feature Flags]; + feature-flags[Feature flags]; license-checks[License Checks]; plain---Vuex; diff --git a/doc/development/testing_guide/testing_migrations_guide.md b/doc/development/testing_guide/testing_migrations_guide.md index 1b1fdcca003..f69f0d1febb 100644 --- a/doc/development/testing_guide/testing_migrations_guide.md +++ b/doc/development/testing_guide/testing_migrations_guide.md @@ -290,7 +290,7 @@ RSpec.describe MigrateIncidentIssuesToIncidentType do .from(issue_type) .to(incident_type) - expect(other_issue.reload.issue_type).to eql(issue_type) + expect(other_issue.reload.issue_type).to eq(issue_type) end end diff --git a/doc/development/uploads/index.md b/doc/development/uploads/index.md index b5509f5934e..a62e8ea2d58 100644 --- a/doc/development/uploads/index.md +++ b/doc/development/uploads/index.md @@ -4,7 +4,7 @@ group: unassigned info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Uploads development guide +# Uploads development guidelines Uploads are an integral part of many GitLab features. To understand how GitLab handles uploads, this page provides an overview of the key mechanisms for transferring files to a storage destination. diff --git a/doc/development/uploads/working_with_uploads.md b/doc/development/uploads/working_with_uploads.md index 6955f9c31cd..5575297ad6b 100644 --- a/doc/development/uploads/working_with_uploads.md +++ b/doc/development/uploads/working_with_uploads.md @@ -95,7 +95,7 @@ They consist of: Example: -```golang +```go u.route("PUT", apiProjectPattern+`packages/nuget/`, mimeMultipartUploader), ``` diff --git a/doc/development/ux/index.md b/doc/development/ux/index.md new file mode 100644 index 00000000000..784a59a3a4a --- /dev/null +++ b/doc/development/ux/index.md @@ -0,0 +1,26 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Contribute to UX design + +## UX Design + +These instructions are specifically for those wanting to make UX design contributions to GitLab. + +The UX department at GitLab uses [Figma](https://www.figma.com/) for all of its designs, and you can see our [Design Repository documentation](https://gitlab.com/gitlab-org/gitlab-design/blob/master/README.md#getting-started) for details on working with our files. + +You may leverage the [Pajamas UI Kit](https://www.figma.com/community/file/781156790581391771) in Figma to create mockups for your proposals. However, we will also gladly accept handmade drawings and sketches, wireframes, manipulated DOM screenshots, or prototypes. You can find design resources documentation in our [Design System](https://design.gitlab.com/). Use it to understand where and when to use common design solutions. + +## Contributing to Pajamas + +To contribute to [Pajamas design system](https://design.gitlab.com/) and the [UI kit](https://www.figma.com/community/file/781156790581391771), follow the [contribution guidelines](https://design.gitlab.com/get-started/contribute) documented in the handbook. While the instructions are code-focused, they will help you understand the overall process of contributing. + +## Contributing to other issues + +1. Review the list of available issues that are currently [accepting UX contribution](https://gitlab.com/groups/gitlab-org/-/issues/?sort=weight&state=opened&label_name%5B%5D=UX&label_name%5B%5D=workflow%3A%3Aready%20for%20design&label_name%5B%5D=Accepting%20UX%20contributions&first_page_size=20). +1. Find an issue that does not have an Assignee to ensure someone else is not working on a solution. Add the `~"workflow::design"` and `~"Community contribution"` labels and mention `@gitlab-com/gitlab-ux/reviewers` to request they assign the issue to you. +1. Add your design proposal to the issue description/[design management](../../user/project/issues/design_management.md) section. Remember to keep the scope of the proposal/change small following our [MVCs guidelines](https://about.gitlab.com/handbook/values/#minimal-viable-change-mvc). +1. If you have any questions or are ready for a review of your proposal, mention `@gitlab-com/gitlab-ux/reviewers` in a comment to make your request. diff --git a/doc/development/value_stream_analytics.md b/doc/development/value_stream_analytics.md index 77a32b62e32..afbd1a76a1b 100644 --- a/doc/development/value_stream_analytics.md +++ b/doc/development/value_stream_analytics.md @@ -4,7 +4,7 @@ group: Optimize info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Value stream analytics development guide +# Value stream analytics development guidelines For information on how to configure value stream analytics (VSA) in GitLab, see our [analytics documentation](../user/analytics/value_stream_analytics.md). @@ -65,7 +65,7 @@ of the stage. Stages are configurable by the user within the pairing rules defin IDs are identical. - The stage event hash ID is later used to store the aggregated data in partitioned database tables. -Historically, value stream analytics defined [7 stages](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/analytics/cycle_analytics/default_stages.rb) +Historically, value stream analytics defined [six stages](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/analytics/cycle_analytics/default_stages.rb) which are always available to the end-users regardless of the subscription. ### Value streams @@ -160,6 +160,7 @@ graph LR; IssueCreated --> IssueLastEdited; IssueCreated --> IssueLabelAdded; IssueCreated --> IssueLabelRemoved; + IssueCreated --> IssueFirstAssignedAt; MergeRequestCreated --> MergeRequestMerged; MergeRequestCreated --> MergeRequestClosed; MergeRequestCreated --> MergeRequestFirstDeployedToProduction; @@ -168,6 +169,13 @@ graph LR; MergeRequestCreated --> MergeRequestLastEdited; MergeRequestCreated --> MergeRequestLabelAdded; MergeRequestCreated --> MergeRequestLabelRemoved; + MergeRequestCreated --> MergeRequestFirstAssignedAt; + MergeRequestFirstAssignedAt --> MergeRequestClosed; + MergeRequestFirstAssignedAt --> MergeRequestLastBuildStarted; + MergeRequestFirstAssignedAt --> MergeRequestLastEdited; + MergeRequestFirstAssignedAt --> MergeRequestMerged; + MergeRequestFirstAssignedAt --> MergeRequestLabelAdded; + MergeRequestFirstAssignedAt --> MergeRequestLabelRemoved; MergeRequestLastBuildStarted --> MergeRequestLastBuildFinished; MergeRequestLastBuildStarted --> MergeRequestClosed; MergeRequestLastBuildStarted --> MergeRequestFirstDeployedToProduction; @@ -184,19 +192,30 @@ graph LR; IssueLabelAdded --> IssueLabelAdded; IssueLabelAdded --> IssueLabelRemoved; IssueLabelAdded --> IssueClosed; + IssueLabelAdded --> IssueFirstAssignedAt; IssueLabelRemoved --> IssueClosed; + IssueLabelRemoved --> IssueFirstAssignedAt; IssueFirstAddedToBoard --> IssueClosed; IssueFirstAddedToBoard --> IssueFirstAssociatedWithMilestone; IssueFirstAddedToBoard --> IssueFirstMentionedInCommit; IssueFirstAddedToBoard --> IssueLastEdited; IssueFirstAddedToBoard --> IssueLabelAdded; IssueFirstAddedToBoard --> IssueLabelRemoved; + IssueFirstAddedToBoard --> IssueFirstAssignedAt; + IssueFirstAssignedAt --> IssueClosed; + IssueFirstAssignedAt --> IssueFirstAddedToBoard; + IssueFirstAssignedAt --> IssueFirstAssociatedWithMilestone; + IssueFirstAssignedAt --> IssueFirstMentionedInCommit; + IssueFirstAssignedAt --> IssueLastEdited; + IssueFirstAssignedAt --> IssueLabelAdded; + IssueFirstAssignedAt --> IssueLabelRemoved; IssueFirstAssociatedWithMilestone --> IssueClosed; IssueFirstAssociatedWithMilestone --> IssueFirstAddedToBoard; IssueFirstAssociatedWithMilestone --> IssueFirstMentionedInCommit; IssueFirstAssociatedWithMilestone --> IssueLastEdited; IssueFirstAssociatedWithMilestone --> IssueLabelAdded; IssueFirstAssociatedWithMilestone --> IssueLabelRemoved; + IssueFirstAssociatedWithMilestone --> IssueFirstAssignedAt; IssueFirstMentionedInCommit --> IssueClosed; IssueFirstMentionedInCommit --> IssueFirstAssociatedWithMilestone; IssueFirstMentionedInCommit --> IssueFirstAddedToBoard; @@ -221,8 +240,11 @@ graph LR; MergeRequestLastBuildFinished --> MergeRequestLabelRemoved; MergeRequestLabelAdded --> MergeRequestLabelAdded; MergeRequestLabelAdded --> MergeRequestLabelRemoved; + MergeRequestLabelAdded --> MergeRequestMerged; + MergeRequestLabelAdded --> MergeRequestFirstAssignedAt; MergeRequestLabelRemoved --> MergeRequestLabelAdded; MergeRequestLabelRemoved --> MergeRequestLabelRemoved; + MergeRequestLabelRemoved --> MergeRequestFirstAssignedAt; ``` ## Default stages @@ -343,9 +365,25 @@ Seed issues and merge requests for value stream analytics: Seed DORA daily metrics for value stream, insights and CI/CD analytics: -1. [Create an environment from the UI](../ci/environments/index.md#create-a-static-environment) named `production`. +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the project's homepage, in the upper-left corner, copy the **Project ID**. You need it in a later step. +1. [Create an environment for your selected project from the UI](../ci/environments/index.md#create-a-static-environment) named `production`. 1. Open the rails console: ```shell rails c ``` + +1. In the rails console, find the created environment by searching for the project ID: + + ```shell + e = Environment.find_by(project_id: , name: "production") + ``` + +1. To seed data for the past 100 days for the environment, run the following command: + + ```shell + 100.times { |i| Dora::DailyMetrics.create(environment_id: e.id, date: (i + 1).days.ago, deployment_frequency: rand(50), incidents_count: rand(5), lead_time_for_changes_in_seconds: rand(50000), time_to_restore_service_in_seconds: rand(100000)) } + ``` + +DORA metric data should now be available for your selected project and any group or subgroup it belongs to. diff --git a/doc/development/vs_code_debugging.md b/doc/development/vs_code_debugging.md new file mode 100644 index 00000000000..08aa4688bfd --- /dev/null +++ b/doc/development/vs_code_debugging.md @@ -0,0 +1,69 @@ +--- +stage: none +group: unassigned +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# VS Code debugging + +This document describes how to set up Rails debugging in [VS Code](https://code.visualstudio.com/). + +## Setup + +1. Install the `debug` gem by running `gem install debug` inside your `gitlab` folder. +1. Add the following configuration to your `.vscode/tasks.json` file: + + ```json + { + "version": "2.0.0", + "tasks": [ + { + "label": "start rdbg", + "type": "shell", + "command": "gdk stop rails-web && GITLAB_RAILS_RACK_TIMEOUT_ENABLE_LOGGING=false PUMA_SINGLE_MODE=true rdbg --open -c -- bin/rails s", + "isBackground": true, + "problemMatcher": { + "owner": "rails", + "pattern": { + "regexp": "^.*$", + }, + "background": { + "activeOnStart": false, + "beginsPattern": "^(ok: down:).*$", + "endsPattern": "^(DEBUGGER: wait for debugger connection\\.\\.\\.)$" + } + } + } + ] + } + ``` + +1. Add the following configuration to your `.vscode/launch.json` file: + + ```json + { + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, see https://go.microsoft.com/fwlink/?linkid=830387. + "version": "0.2.0", + "configurations": [ + { + "type": "rdbg", + "name": "Attach with rdbg", + "request": "attach", + "preLaunchTask": "start rdbg" + } + ] + } + ``` + +## Debugging + +Prerequisite: + +- You must have a running GDK instance. + +To start debugging, do one of the following: + +- Press F5. +- Run the `Debug: Start Debugging` command. diff --git a/doc/development/wikis.md b/doc/development/wikis.md index acb7ed335d3..a814fa76ec9 100644 --- a/doc/development/wikis.md +++ b/doc/development/wikis.md @@ -1,16 +1,14 @@ --- -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments description: "GitLab's development guidelines for Wikis" --- -# Wikis development guide +# Wikis development guidelines > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227027) in GitLab 13.5. -## Overview - The wiki functionality in GitLab is based on [Gollum 4.x](https://github.com/gollum/gollum/). It's used in the [Gitaly](gitaly.md) Ruby service, and accessed from the Rails app through Gitaly RPC calls. @@ -30,7 +28,7 @@ Some notable gems that are used for wikis are: | Component | Description | Gem name | GitLab project | Upstream project | |:--------------|:-----------------------------------------------|:-------------------------------|:--------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------| | `gitlab` | Markup renderer, depends on various other gems | `gitlab-markup` | [`gitlab-org/gitlab-markup`](https://gitlab.com/gitlab-org/gitlab-markup) | [`github/markup`](https://github.com/github/markup) | -| `gitaly-ruby` | Main Gollum library | `gitlab-gollum-lib` | [`gitlab-org/gollum-lib`](https://gitlab.com/gitlab-org/gollum-lib) | [`gollum/gollum-lib`](https://github.com/gollum/gollum-lib) | +| `gollum-lib` | Main Gollum library | `gitlab-gollum-lib` | [`gitlab-org/gollum-lib`](https://gitlab.com/gitlab-org/gollum-lib) | [`gollum/gollum-lib`](https://github.com/gollum/gollum-lib) | | | Gollum Git adapter for Rugged | `gitlab-gollum-rugged_adapter` | [`gitlab-org/gitlab-gollum-rugged_adapter`](https://gitlab.com/gitlab-org/gitlab-gollum-rugged_adapter) | [`gollum/rugged_adapter`](https://github.com/gollum/rugged_adapter) | | | Rugged (also used in Gitaly itself) | `rugged` | - | [`libgit2/rugged`](https://github.com/libgit2/rugged) | diff --git a/doc/development/windows.md b/doc/development/windows.md index bf56e16344a..99085b4b5f8 100644 --- a/doc/development/windows.md +++ b/doc/development/windows.md @@ -13,7 +13,7 @@ This is a guide for how to get a Windows development virtual machine on Google C ## Why Windows in Google Cloud? -Use of Microsoft Windows operating systems on company laptops is banned under the GitLab [Approved Operating Systems policy](https://about.gitlab.com/handbook/security/approved_os.html#windows). +Use of Microsoft Windows operating systems on company laptops is banned under the GitLab [Approved Operating Systems policy](https://about.gitlab.com/handbook/it/operating-systems/#windows). This can make it difficult to develop features for the Windows platforms. Using GCP allows us to have a temporary Windows machine that can be removed once we're done with it. diff --git a/doc/development/work_items_widgets.md b/doc/development/work_items_widgets.md index 5b9602595bb..bafceccdafe 100644 --- a/doc/development/work_items_widgets.md +++ b/doc/development/work_items_widgets.md @@ -137,3 +137,27 @@ Each record in the table defines mapping of a widget to a work item type. Curren | 4 | 1 | 1 | 1 | 0 | MyWeight | false | | 5 | 2 | 0 | 1 | 1 | Other Weight | false | | 6 | 3 | 0 | 1 | 1 | Weight | true | + +## Backend architecture + +You can update widgets using custom fine-grained mutations (for example, `WorkItemCreateFromTask`) or as part of the +`workItemCreate` or `workItemUpdate` mutations. + +### Widget callbacks + +When updating the widget together with the work item's mutation, backend code should be implemented using +callback classes that inherit from `Issuable::Callbacks::Base`. These classes have callback methods +that are named similar to ActiveRecord callbacks and behave similarly. + +Callback classes with the same name as the widget are automatically used. For example, `Issuable::Callbacks::Milestone` +is called when the work item has the milestone widget. To use a different class, you can override the `callback_class` +class method. + +#### Available callbacks + +- `after_initialize` is called after the work item is initialized by the `BuildService` and before + the work item is saved by the `CreateService` and `UpdateService`. This callback runs outside the + creation or update database transaction. +- `after_update_commit` is called after the DB update transaction is committed by the `UpdateService`. +- `after_save_commit` is called after the creation or DB update transaction is committed by the + `CreateService` or `UpdateService`. diff --git a/doc/development/workhorse/configuration.md b/doc/development/workhorse/configuration.md index 9fc106b8f04..e69f16c41f8 100644 --- a/doc/development/workhorse/configuration.md +++ b/doc/development/workhorse/configuration.md @@ -296,4 +296,4 @@ GITLAB_CONTINUOUS_PROFILING="stackdriver?service=workhorse&service_version=1.0.1 ## Related topics -- [LabKit monitoring documentation](https://gitlab.com/gitlab-org/labkit/-/blob/master/monitoring/doc.go). +- [LabKit monitoring documentation](https://gitlab.com/gitlab-org/labkit/-/blob/master/monitoring/doc.go) diff --git a/doc/development/workhorse/index.md b/doc/development/workhorse/index.md index 91795336f78..680dd0c205b 100644 --- a/doc/development/workhorse/index.md +++ b/doc/development/workhorse/index.md @@ -15,9 +15,14 @@ Workhorse itself is not a feature, but there are The canonical source for Workhorse is [`gitlab-org/gitlab/workhorse`](https://gitlab.com/gitlab-org/gitlab/tree/master/workhorse). -Prior to [epic #4826](https://gitlab.com/groups/gitlab-org/-/epics/4826), it was -[`gitlab-org/gitlab-workhorse`](https://gitlab.com/gitlab-org/gitlab-workhorse/tree/master), -but that repository is no longer used for development. + +## Learning Resources + +- Workhorse documentation (this page) +- [GitLab Workhorse Deep Dive: Dependency Proxy](https://www.youtube.com/watch?v=9cRd-k0TRqI) _video_ +- [How Dependency Proxy via Workhorse works](https://gitlab.com/gitlab-org/gitlab/-/issues/370235) +- [Workhorse overview for the Dependency Proxy](https://www.youtube.com/watch?v=WmBibT9oQms) +- [Workhorse architecture discussion](https://www.youtube.com/watch?v=QlHdh-yudtw) ## Install Workhorse diff --git a/doc/development/workspace/index.md b/doc/development/workspace/index.md index 0e0b6943a0b..ca404702d72 100644 --- a/doc/development/workspace/index.md +++ b/doc/development/workspace/index.md @@ -1,159 +1,11 @@ --- -comments: false -type: index, dev -stage: none -group: Development -info: "See the Technical Writers assigned to Development Guidelines: https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-development-guidelines" -description: "Development Guidelines: learn about organization when developing GitLab." +redirect_to: '../organization/index.md' +remove_date: '2023-06-13' --- -# Organization +This document was moved to [another location](../organization/index.md). -The [Organization initiative](../../user/workspace/index.md) focuses on reaching feature parity between -SaaS and self-managed installations. - -## Consolidate groups and projects - -- [Architecture blueprint](../../architecture/blueprints/consolidating_groups_and_projects/index.md) - -One facet of the Organization initiative is to consolidate groups and projects, -addressing the feature disparity between them. Some features, such as epics, are -only available at the group level. Some features, such as issues, are only available -at the project level. Other features, such as milestones, are available to both groups -and projects. - -We receive many requests to add features either to the group or project level. -Moving features around to different levels is problematic on multiple levels: - -- It requires engineering time to move the features. -- It requires UX overhead to maintain mental models of feature availability. -- It creates redundant code. - -When features are copied from one level (project, group, or instance) to another, -the copies often have small, nuanced differences between them. These nuances cause -extra engineering time when fixes are needed, because the fix must be copied to -several locations. These nuances also create different user experiences when the -feature is used in different places. - -A solution for this problem is to consolidate groups and projects into a single -entity, `namespace`. The work on this solution is split into several phases and -is tracked in [epic 6473](https://gitlab.com/groups/gitlab-org/-/epics/6473). - -### Phase 1 - -- [Phase 1 epic](https://gitlab.com/groups/gitlab-org/-/epics/6697). -- **Goals**: - 1. Ensure every project receives a corresponding record in the `namespaces` - table with `type='Project'`. - 1. For user namespaces, the type changes from `NULL` to `User`. - -We should make sure that projects, and the project namespace, are equivalent: - -- **Create project:** use Rails callbacks to ensure a new project namespace is - created for each project. Project namespace records should contain `created_at` and - `updated_at` attributes equal to the project's `created_at`/`updated_at` attributes. -- **Update project:** use the `after_save` callback in Rails to ensure some - attributes are kept in sync between project and project namespaces. - Read [`project#after_save`](https://gitlab.com/gitlab-org/gitlab/blob/6d26634e864d7b748dda0e283eb2477362263bc3/app/models/project.rb#L101-L101) - for more information. -- **Delete project:** use FKs cascade delete or Rails callbacks to ensure when a `Project` - or its `ProjectNamespace` is removed, its corresponding `ProjectNamespace` or `Project` - is also removed. -- **Transfer project to a different group:** make sure that when a project is transferred, - its corresponding project namespace is transferred to the same group. -- **Transfer group:** make sure when transferring a group that all of its sub-projects, - either direct or through descendant groups, have their corresponding project - namespaces transferred correctly as well. -- **Export or import project** - - **Export project** continues to export only the project, and not its project namespace, - in this phase. The project namespace does not contain any specific information - to export at this point. Eventually we want the project namespace to be exported as well. - - **Import project** creates a new project, so the project namespace is created through - Rails `after_save` callback on the project model. -- **Export or import group:** when importing or exporting a `Group`, projects are not - included in the operation. If that feature is changed to include `Project` when its group is - imported or exported, the logic must include their corresponding project namespaces - in the import or export. - -After ensuring these points, run a database migration to create a `ProjectNamespace` -record for every `Project`. Project namespace records created during the migration -should have `created_at` and `updated_at` attributes set to the migration runtime. -The project namespaces' `created_at` and `updated_at` attributes would not match -their corresponding project's `created_at` and `updated_at` attributes. We want -the different dates to help audit any of the created project namespaces, in case we need it. -After this work completes, we must migrate data as described in -[Backfill `ProjectNamespace` for every Project](https://gitlab.com/gitlab-org/gitlab/-/issues/337100). - -### Phase 2 - -- [Phase 2 epic](https://gitlab.com/groups/gitlab-org/-/epics/6768). -- **Goal**: Link `ProjectNamespace` to other entities on the database level. - -In this phase: - -- Communicate the changes company-wide at the engineering level. We want to make - engineers aware of the upcoming changes, even though teams are not expected to - collaborate actively until phase 3. -- Raise awareness to avoid regressions, and conflicting or duplicate work that - can be dealt with before phase 3. - -### Phase 3 - -- [Phase 3 epic](https://gitlab.com/groups/gitlab-org/-/epics/6585). -- **Goal**: Achieve feature parity between the namespace types. -Problems to solve as part of this phase: - -- Routes handling through `ProjectNamespace` rather than `Project`. -- Memberships handling. -- Policies handling. -- Import and export. -- Other interactions between project namespace and project models. - -Phase 3 is when the active migration of features from `Project` to `ProjectNamespace`, -or directly to `Namespace`, happens. - -### How to plan features that interact with Group and ProjectNamespace - -As of now, every Project in the system has a record in the `namespaces` table. This makes it possible to -use common interface to create features that are shared between Groups and Projects. Shared behavior can be added using -a concerns mechanism. Because the `Namespace` model is responsible for `UserNamespace` methods as well, it is discouraged -to use the `Namespace` model for shared behavior for Projects and Groups. - -#### Resource-based features - -To migrate resource-based features, existing functionality will need to be supported. This can be achieved in two Phases. - -**Phase 1 - Setup** - -- Link into the namespaces table - - Add a column to the table - - For example, in issues a `project id` points to the projects table. We need to establish a link to the `namespaces` table. - - Modify code so that any new record already has the correct data in it - - Backfill - -**Phase 2 - Prerequisite work** - -- Investigate the permission model as well as any performance concerns related to that. - - Permissions need to be checked and kept in place. -- Investigate what other models need to support namespaces for functionality dependent on features you migrate in Phase 1. -- Adjust CRUD services and APIs (REST and GraphQL) to point to the new column you added in Phase 1. -- Consider performance when fetching resources. - -Introducing new functionality is very much dependent on every single team and feature. - -#### Settings-related features - -Right now, cascading settings are available for `NamespaceSettings`. By creating `ProjectNamespace`, -we can use this framework to make sure that some settings are applicable on the project level as well. - -When working on settings, we need to make sure that: - -- They are not used in `join` queries or modify those queries. -- Updating settings is taken into consideration. -- If we want to move from project to project namespace, we follow a similar database process to the one described in [Phase 1](#phase-1). - -## Related topics - -- [Consolidating groups and projects](../../architecture/blueprints/consolidating_groups_and_projects/index.md) - architecture documentation -- [Organization user documentation](../../user/workspace/index.md) + + + + diff --git a/doc/drawers/advanced_search_syntax.md b/doc/drawers/advanced_search_syntax.md index 7556c8bdfaf..6e732bd3175 100644 --- a/doc/drawers/advanced_search_syntax.md +++ b/doc/drawers/advanced_search_syntax.md @@ -13,6 +13,7 @@ source: /doc/user/search/advanced_search.md | Syntax | Description | Example | |--------------|---------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------| | `"` | Exact search | [`"gem sidekiq"`](https://gitlab.com/search?group_id=9970&project_id=278964&scope=blobs&search=%22gem+sidekiq%22) | +| `~` | Fuzzy search | [`J~ Doe`](https://gitlab.com/search?scope=users&search=j%7E+doe) | | | | Or | [display | banner](https://gitlab.com/search?group_id=9970&project_id=278964&scope=blobs&search=display+%7C+banner) | | `+` | And | [`display +banner`](https://gitlab.com/search?group_id=9970&project_id=278964&repository_ref=&scope=blobs&search=display+%2Bbanner&snippets=) | | `-` | Exclude | [`display -banner`](https://gitlab.com/search?group_id=9970&project_id=278964&scope=blobs&search=display+-banner) | diff --git a/doc/gitlab-basics/add-file.md b/doc/gitlab-basics/add-file.md index 05731f93605..425b8927520 100644 --- a/doc/gitlab-basics/add-file.md +++ b/doc/gitlab-basics/add-file.md @@ -7,89 +7,95 @@ type: howto # Add a file to a repository **(FREE)** -Adding files to a repository is a small, but key task. Bringing files in to a repository, -such as code, images, or documents, allows them to be tracked by Git, even though they -may have been created elsewhere. - -You can add a file to a repository in your [terminal](#add-a-file-using-the-command-line), and -then push to GitLab. You can also use the [web interface](../user/project/repository/web_editor.md#upload-a-file), -which may be a simpler solution. - -If you need to create a file first, for example a `README.md` text file, that can -also be done from the [terminal](command-line-commands.md#create-a-text-file-in-the-current-directory) or -[web interface](../user/project/repository/web_editor.md#create-a-file). - -## Add a file using the command line - -Open a [terminal/shell](command-line-commands.md), and change into the folder of your -GitLab project. This usually means running the following command until you get -to the desired destination: - -```shell -cd -``` - -[Create a new branch](../tutorials/make_your_first_git_commit.md#create-a-branch-and-make-changes) to add your file into. Submitting changes directly -to the default branch should be avoided unless your project is very small and you're the -only person working on it. - -You can also [switch to an existing branch](start-using-git.md#switch-to-a-branch) -if you have one already. - -Using your standard tool for copying files (for example, Finder in macOS, or File Explorer -on Windows), put the file into a directory in the GitLab project. - -Check if your file is actually present in the directory (if you're on Windows, -use `dir` instead): - -```shell -ls -``` - -You should see the name of the file in the list shown. - -Check the status: - -```shell -git status -``` - -Your file's name should appear in red, so `git` took notice of it! Now add it -to the repository: - -```shell -git add -``` - -Check the status again, your file's name should have turned green: - -```shell -git status -``` - -Commit (save) your file to the repository: - -```shell -git commit -m "DESCRIBE COMMIT IN A FEW WORDS" -``` - -Now you can push (send) your changes (in the branch ``) to GitLab -(the Git remote named 'origin'): - -```shell -git push origin -``` - -Your image is added to your branch in your repository in GitLab. - - +Adding files to a repository is a small, but key, task. No matter where the code, +images, or documents were created, Git tracks them after you add them to your repository. + +## Add an existing file + +To add an existing file to your repository, either: + +- Upload the file from the GitLab UI. +- Add a file to your repository from the command line, then push the file up to GitLab. + +### From the UI + +If you are unfamiliar with the command line, use the +[Web Editor](../user/project/repository/web_editor.md) to upload a file from the GitLab UI: + + + + +1. On the top bar, select **Main menu > Projects** and find your project. +1. From the project dashboard or repository, next to the branch name, select the plus icon (**{plus}**). +1. From the dropdown list, select **Upload file**. +1. Complete the fields. To create a merge request with the uploaded file, ensure the **Start a new merge request with these changes** toggle is turned on. +1. Select **Upload file**. + +### From the command line + +To add a new file from the command line: + +1. Open a terminal (or shell) window. +1. Use the "change directory" (`cd`) command to go to your GitLab project's folder. + Run the `cd DESTINATION` command, changing `DESTINATION` to the location of your folder. +1. Choose a Git branch to work in. You can either: + - [Create a new branch](../tutorials/make_first_git_commit/index.md#create-a-branch-and-make-changes) + to add your file into. Don't submit changes directly to the default branch of your + repository unless your project is very small and you're the only person working on it. + - [Switch to an existing branch](start-using-git.md#switch-to-a-branch). +1. Copy the file into the appropriate directory in your project. Use your standard tool + for copying files, such as Finder in macOS, or File Explorer in Windows. +1. In your terminal window, confirm that your file is present in the directory: + - Windows: Use the `dir` command. + - All other operating systems: Use the `ls` command. + You should see the name of the file in the list shown. +1. Check the status of your file with the `git status` command. Your file's name + should be red. Files listed in red are in your file system, but Git isn't tracking them yet. +1. Tell Git to track this file with the `git add FILENAME` command, replacing `FILENAME` + with the name of your file. +1. Check the status of your file again with the `git status` command. Your file's name + should be green. Files listed in green are tracked locally by Git, but still + need to be committed and pushed. +1. Commit (save) your file to your local copy of your project's Git repository: + + ```shell + git commit -m "DESCRIBE COMMIT IN A FEW WORDS" + ``` + +1. Push (send) your changes from your copy of the repository, up to GitLab. + In this command, `origin` refers to the copy of the repository stored at GitLab. + Replace `BRANCHNAME` with the name of your branch: + + ```shell + git push origin BRANCHNAME + ``` + +1. Git prepares, compresses, and sends the data. Lines from the remote repository + (here, GitLab) are prefixed with `remote:` like this: + + ```plaintext + Enumerating objects: 9, done. + Counting objects: 100% (9/9), done. + Delta compression using up to 10 threads + Compressing objects: 100% (5/5), done. + Writing objects: 100% (5/5), 1.84 KiB | 1.84 MiB/s, done. + Total 5 (delta 3), reused 0 (delta 0), pack-reused 0 + remote: + remote: To create a merge request for BRANCHNAME, visit: + remote: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/new?merge_request%5Bsource_branch%5D=BRANCHNAME + remote: + To https://gitlab.com/gitlab-org/gitlab.git + * [new branch] BRANCHNAME -> BRANCHNAME + branch 'BRANCHNAME' set up to track 'origin/BRANCHNAME'. + ``` + +Your file is now copied from your local copy of the repository, up to the remote +repository at GitLab. To create a merge request, copy the link sent back from the remote +repository and paste it into a browser window. + +## Add a new file + +To create a new file (like a `README.md` text file) in your repository, either: + +- [Create the file](../user/project/repository/web_editor.md#create-a-file) from the GitLab UI. +- Create the file from the terminal. diff --git a/doc/gitlab-basics/command-line-commands.md b/doc/gitlab-basics/command-line-commands.md index 07ab9365693..2850669ce57 100644 --- a/doc/gitlab-basics/command-line-commands.md +++ b/doc/gitlab-basics/command-line-commands.md @@ -1,123 +1,11 @@ --- -stage: Create -group: Source Code -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../user/index.md' +remove_date: '2023-06-09' --- -# Edit files through the command line **(FREE)** +This document was moved to [another location](../user/index.md). -When [working with Git from the command line](start-using-git.md), you need to -use more than just the Git commands. There are several basic commands that you should -learn to make full use of the command line. - -## Start working on your project - -To work on a Git project locally (from your own computer), with the command line, -first you need to [clone (copy) it](start-using-git.md#clone-a-repository) to -your computer. - -## Working with files on the command line - -This section has examples of some basic shell commands that you might find useful. -For more information, search the web for _bash commands_. - -Alternatively, you can edit files using your choice of editor (IDE), or the GitLab user -interface (not locally). - -### Common commands - -The list below is not exhaustive, but contains many of the most commonly used commands. - -| Command | Description | -|--------------------------------|---------------------------------------------| -| `cd NAME-OF-DIRECTORY` | Go into a directory to work in it | -| `cd ..` | Go back one directory | -| `ls` | List what's in the current directory | -| `ls a*` | List what's in the current directory that starts with `a` | -| `ls *.md` | List what's in the current directory that ends with `.md` | -| `mkdir NAME-OF-YOUR-DIRECTORY` | Create a new directory | -| `cat README.md` | Display the contents of a [text file you created previously](#create-a-text-file-in-the-current-directory) | -| `pwd` | Show the current directory | -| `clear` | Clear the shell window | - -### Create a text file in the current directory - -To create a text file from the command line, for example `README.md`, follow these -steps: - -```shell -touch README.md -nano README.md -#### ADD YOUR INFORMATION -#### Press: control + X -#### Type: Y -#### Press: enter -``` - -### Remove a file or directory - -It's easy to delete (remove) a file or directory, but be careful: - -WARNING: -This will **permanently** delete a file. - -```shell -rm NAME-OF-FILE -``` - -WARNING: -This will **permanently** delete a directory and **all** of its contents. - -```shell -rm -r NAME-OF-DIRECTORY -``` - -### View and Execute commands from history - -You can view the history of all the commands you executed from the command line, -and then execute any of them again, if needed. - -First, list the commands you executed previously: - -```shell -history -``` - -Then, choose a command from the list and check the number next to the command (`123`, -for example) . Execute the same full command with: - -```shell -!123 -``` - -### Carry out commands for which the account you are using lacks authority - -Not all commands can be executed from a basic user account on a computer, you may -need administrator's rights to execute commands that affect the system, or try to access -protected data, for example. You can use `sudo` to execute these commands, but you -might be asked for an administrator password. - -```shell -sudo RESTRICTED-COMMAND -``` - -WARNING: -Be careful of the commands you run with `sudo`. Certain commands may cause -damage to your data or system. - -## Sample Git task flow - -If you're completely new to Git, looking through some [sample task flows](https://rogerdudler.github.io/git-guide/) -may help you understand the best practices for using these commands as you work. - - + + + + diff --git a/doc/gitlab-basics/feature_branch_workflow.md b/doc/gitlab-basics/feature_branch_workflow.md index ce4aa0007c6..65445c226ca 100644 --- a/doc/gitlab-basics/feature_branch_workflow.md +++ b/doc/gitlab-basics/feature_branch_workflow.md @@ -2,7 +2,6 @@ stage: Create group: Source Code info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments" -disqus_identifier: 'https://docs.gitlab.com/ee/workflow/workflow.html' --- # Feature branch workflow **(FREE)** diff --git a/doc/gitlab-basics/start-using-git.md b/doc/gitlab-basics/start-using-git.md index 224d1fabb14..fd322b67abe 100644 --- a/doc/gitlab-basics/start-using-git.md +++ b/doc/gitlab-basics/start-using-git.md @@ -15,7 +15,7 @@ You can do many Git operations directly in GitLab. However, the command line is like fixing complex merge conflicts or rolling back commits. If you're new to Git and want to learn by working in your own project, -[learn how to make your first commit](../tutorials/make_your_first_git_commit.md). +[learn how to make your first commit](../tutorials/make_first_git_commit/index.md). For a quick reference of Git commands, download a [Git Cheat Sheet](https://about.gitlab.com/images/press/git-cheat-sheet.pdf). @@ -98,12 +98,12 @@ access on GitLab.com or any other GitLab instance. To use the repository in the examples on this page: 1. Go to [https://gitlab.com/gitlab-tests/sample-project/](https://gitlab.com/gitlab-tests/sample-project/). -1. In the upper right, select **Fork**. +1. In the upper-right corner, select **Fork**. 1. Choose a namespace for your fork. The project becomes available at `https://gitlab.com//sample-project/`. -You can [fork](../user/project/repository/forking_workflow.md#creating-a-fork) any project you have access to. +You can [fork](../user/project/repository/forking_workflow.md#create-a-fork) any project you have access to. ## Clone a repository @@ -152,7 +152,7 @@ between your computer and GitLab. If you have enabled two-factor authentication (2FA) on your account, you cannot use your account password. Instead, you can do one of the following: - [Clone using a token](#clone-using-a-token) with `read_repository` or `write_repository` permissions. - - Install [Git Credential Manager](../user/profile/account/two_factor_authentication.md#git-credential-manager). + - Install an [OAuth credential helper](../user/profile/account/two_factor_authentication.md#oauth-credential-helpers). If you have not enabled 2FA, use your account password. @@ -275,8 +275,10 @@ To create a feature branch: git checkout -b ``` -Branch names cannot contain empty spaces and special characters. Use only lowercase letters, numbers, -hyphens (`-`), and underscores (`_`). +GitLab enforces [branch naming rules](../user/project/repository/branches/index.md#name-your-branch) +to prevent problems, and provides +[branch naming patterns](../user/project/repository/branches/index.md#prefix-branch-names-with-issue-numbers) +to streamline merge request creation. ### Switch to a branch diff --git a/doc/index.md b/doc/index.md index 492320d93aa..958f7fe6111 100644 --- a/doc/index.md +++ b/doc/index.md @@ -2,7 +2,6 @@ stage: none group: unassigned info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false description: 'Learn how to use and administer GitLab, the most scalable Git-based fully integrated platform for software development.' --- @@ -28,7 +27,7 @@ Welcome to the GitLab documentation! | [**New to Git and GitLab?**](tutorials/index.md)
Start learning about Git and GitLab. | [**Contribute to GitLab development**](#contributing-to-gitlab)
Create new GitLab functionality and documentation. | | [**Coming to GitLab from another platform?**](#coming-to-gitlab-from-another-platform)
Learn how to move to GitLab. | [**Build an integration with GitLab**](#build-an-integration-with-gitlab)
Integrate with Jira and other common applications. | | [**Choose a subscription**](subscriptions/index.md)
Determine which subscription tier makes sense for you. | [**Install GitLab**](https://about.gitlab.com/install/)
Install GitLab on different platforms. | -| [**Reference architectures**](administration/reference_architectures/index.md)
View recommended deployments at scale. | [**Update GitLab**](update/index.md)
Update your GitLab self-managed instance to the latest version. | +| [**Reference architectures**](administration/reference_architectures/index.md)
View recommended deployments at scale. | [**Upgrade GitLab**](update/index.md)
Upgrade your GitLab self-managed instance to the latest version. | | [**GitLab releases**](https://about.gitlab.com/releases/)
See what's new in GitLab. | | ## Popular topics @@ -43,10 +42,10 @@ Have a look at some of our most popular topics: | [Activate GitLab EE with a license](user/admin_area/license.md) | Activate GitLab Enterprise Edition functionality with a license. | | [Back up and restore GitLab](raketasks/backup_restore.md) | Rake tasks for backing up and restoring GitLab self-managed instances. | | [GitLab release and maintenance policy](policy/maintenance.md) | Policies for version naming and cadence, and also upgrade recommendations. | -| [Elasticsearch integration](integration/advanced_search/elasticsearch.md) | Integrate Elasticsearch with GitLab to enable advanced searching. | +| [Elasticsearch integration](integration/advanced_search/elasticsearch.md) | Integrate Elasticsearch with GitLab to enable advanced search. | | [Omnibus GitLab database settings](https://docs.gitlab.com/omnibus/settings/database.html) | Database settings for Omnibus GitLab self-managed instances. | | [Omnibus GitLab NGINX settings](https://docs.gitlab.com/omnibus/settings/nginx.html) | NGINX settings for Omnibus GitLab self-managed instances. | -| [Omnibus GitLab SSL configuration](https://docs.gitlab.com/omnibus/settings/ssl.html) | SSL settings for Omnibus GitLab self-managed instances. | +| [Omnibus GitLab SSL configuration](https://docs.gitlab.com/omnibus/settings/ssl/index.html) | SSL settings for Omnibus GitLab self-managed instances. | | [GitLab.com settings](user/gitlab_com/index.md) | Settings used for GitLab.com. | ## The entire DevOps lifecycle @@ -75,7 +74,7 @@ If you are coming to GitLab from another platform, the following information is | Topic | Description | |:----------------------------------------------------|:------------| | [Importing to GitLab](user/project/import/index.md) | Import your projects from GitHub, Bitbucket, GitLab.com, FogBugz, and SVN into GitLab. | -| [Migrating from SVN](user/project/import/svn.md) | Convert a SVN repository to Git and GitLab. | +| [Migrating from SVN](user/project/import/index.md#import-from-subversion) | Convert a SVN repository to Git and GitLab. | ## Build an integration with GitLab diff --git a/doc/install/aws/eks_clusters_aws.md b/doc/install/aws/eks_clusters_aws.md index 191d0f93382..ccbc0752975 100644 --- a/doc/install/aws/eks_clusters_aws.md +++ b/doc/install/aws/eks_clusters_aws.md @@ -23,7 +23,7 @@ Using `eksctl` enables the following when building an EKS Cluster: - You have various cluster configuration options: - Selection of operating system: Amazon Linux 2, Windows, Bottlerocket - Selection of Hardware Architecture: x86, ARM, GPU - - Selection of Kubernetes version (the GitLab-managed clusters for your project's applications have [specific Kubernetes version requirements](../../user/clusters/agent/index.md#supported-cluster-versions)) + - Selection of Kubernetes version (the GitLab-managed clusters for your project's applications have [specific Kubernetes version requirements](../../user/clusters/agent/index.md#gitlab-agent-for-kubernetes-supported-cluster-versions)) - It can deploy high value-add items to the cluster, including: - A bastion host to keep the cluster endpoint private and possible perform performance testing. - Prometheus and Grafana for monitoring. diff --git a/doc/install/aws/gitlab_hybrid_on_aws.md b/doc/install/aws/gitlab_hybrid_on_aws.md index b8c840782b1..e4eb0117410 100644 --- a/doc/install/aws/gitlab_hybrid_on_aws.md +++ b/doc/install/aws/gitlab_hybrid_on_aws.md @@ -48,7 +48,7 @@ The Beta version deploys Aurora PostgreSQL, but the release version will deploy | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | | Overview and Vision | [AWS Quick Start](https://aws.amazon.com/solutions/implementations/amazon-eks/) | [GitLab Environment Toolkit](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/blob/main/README.md) | | Licensing | [Open Source (Apache 2.0)](https://github.com/aws-quickstart/quickstart-eks-gitlab/blob/main/LICENSE.txt) | [GitLab Enterprise Edition license](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/blob/main/LICENSE) ([GitLab Premium tier](https://gitlab.com/gitlab-org/gitlab-environment-toolkit/-/blob/main/README.md)) | -| GitLab Support | [GitLab Beta Support](../../policy/alpha-beta-support.md#beta-features) | [GitLab GA Support](../../policy/alpha-beta-support.md#generally-available-ga) | +| GitLab Support | [GitLab Beta Support](../../policy/alpha-beta-support.md#beta) | [GitLab GA Support](../../policy/alpha-beta-support.md#generally-available-ga) | | GitLab Reference Architecture Compliant | Yes | Yes | | GitLab Performance Tool (GPT) Tested | Yes | Yes | | Amazon Well Architected Compliant | Yes
(via Quick Start program) | Critical portions
reviewed by AWS | diff --git a/doc/install/aws/gitlab_sre_for_aws.md b/doc/install/aws/gitlab_sre_for_aws.md index e68aea00b36..f957dfa8a65 100644 --- a/doc/install/aws/gitlab_sre_for_aws.md +++ b/doc/install/aws/gitlab_sre_for_aws.md @@ -2,7 +2,6 @@ stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false description: Doing SRE for GitLab instances and runners on AWS. --- diff --git a/doc/install/aws/index.md b/doc/install/aws/index.md index 99ba05925a5..4dcf2ce0927 100644 --- a/doc/install/aws/index.md +++ b/doc/install/aws/index.md @@ -2,7 +2,6 @@ stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false description: Read through the GitLab installation methods. type: index --- @@ -87,7 +86,7 @@ Because any given GitLab upgrade might involve data disk updates or database sch - [GitLab Community Edition](https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#Images:visibility=public-images;ownerAlias=782774275127;search=GitLab%20CE;sort=desc:name): The open source version of GitLab. - [GitLab Premium or Ultimate Marketplace (pre-licensed)](https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#Images:visibility=public-images;source=Marketplace;search=GitLab%20EE;sort=desc:name): 5 user license built into per-minute billing. -1. AMI IDs are unique per region, so after you've loaded one of the above, select the desired target region in the upper right of the console to see the appropriate AMIs. +1. AMI IDs are unique per region. After you've loaded any of these editions, in the upper-right corner, select the desired target region of the console to see the appropriate AMIs. 1. After the console is loaded, you can add additional search criteria to narrow further. For instance, type `13.` to find only 13.x versions. 1. To launch an EC2 Machine with one of the listed AMIs, check the box at the start of the relevant row, and select **Launch** near the top of left of the page. diff --git a/doc/install/aws/manual_install_aws.md b/doc/install/aws/manual_install_aws.md index 51ae16ccd17..bd81e0583b5 100644 --- a/doc/install/aws/manual_install_aws.md +++ b/doc/install/aws/manual_install_aws.md @@ -237,7 +237,7 @@ Next, we must associate the **public** subnets to the route table: We also must create two private route tables so that instances in each private subnet can reach the internet via the NAT gateway in the corresponding public subnet in the same availability zone. -1. Follow the same steps as above to create two private route tables. Name them `gitlab-private-a` and `gitlab-private-b` respectively. +1. Follow the same steps as above to create two private route tables. Name them `gitlab-private-a` and `gitlab-private-b`. 1. Next, add a new route to each of the private route tables where the destination is `0.0.0.0/0` and the target is one of the NAT gateways we created earlier. 1. Add the NAT gateway we created in `gitlab-public-10.0.0.0` as the target for the new route in the `gitlab-private-a` route table. 1. Similarly, add the NAT gateway in `gitlab-public-10.0.2.0` as the target for the new route in the `gitlab-private-b`. @@ -336,7 +336,13 @@ Now, it's time to create the database: 1. Select **Standard Create** for the database creation method. 1. Select **PostgreSQL** as the database engine and select the minimum PostgreSQL version as defined for your GitLab version in our [database requirements](../../install/requirements.md#postgresql-requirements). 1. Because this is a production server, let's choose **Production** from the **Templates** section. -1. Under **Settings**, set a DB instance identifier, a master username, and a master password. We use `gitlab-db-ha`, `gitlab`, and a very secure password respectively. Make a note of these as we need them later. +1. Under **Settings**, use: + - `gitlab-db-ha` for the DB instance identifier. + - `gitlab` for a master username. + - A very secure password for the master password. + + Make a note of these as we need them later. + 1. For the DB instance size, select **Standard classes** and select an instance size that meets your requirements from the dropdown list. We use a `db.m4.large` instance. 1. Under **Storage**, configure the following: 1. Select **Provisioned IOPS (SSD)** from the storage type dropdown list. Provisioned IOPS (SSD) storage is best suited for this use (though you can choose General Purpose (SSD) to reduce the costs). Read more about it at [Storage for Amazon RDS](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Storage.html). @@ -483,7 +489,7 @@ Connect to your GitLab instance via **Bastion Host A** using [SSH Agent Forwardi #### Disable Let's Encrypt -Because we're adding our SSL certificate at the load balancer, we do not need the GitLab built-in support for Let's Encrypt. Let's Encrypt [is enabled by default](https://docs.gitlab.com/omnibus/settings/ssl.html#enable-the-lets-encrypt-integration) when using an `https` domain in GitLab 10.7 and later, so we must explicitly disable it: +Because we're adding our SSL certificate at the load balancer, we do not need the GitLab built-in support for Let's Encrypt. Let's Encrypt [is enabled by default](https://docs.gitlab.com/omnibus/settings/ssl/index.html#enable-the-lets-encrypt-integration) when using an `https` domain in GitLab 10.7 and later, so we must explicitly disable it: 1. Open `/etc/gitlab/gitlab.rb` and disable it: @@ -605,7 +611,7 @@ Now that we have our EC2 instance ready, follow the [documentation to install Gi #### Add Support for Proxied SSL -As we are terminating SSL at our [load balancer](#load-balancer), follow the steps at [Supporting proxied SSL](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) to configure this in `/etc/gitlab/gitlab.rb`. +As we are terminating SSL at our [load balancer](#load-balancer), follow the steps at [Supporting proxied SSL](https://docs.gitlab.com/omnibus/settings/ssl/index.html#configure-a-reverse-proxy-or-load-balancer-ssl-termination) to configure this in `/etc/gitlab/gitlab.rb`. Remember to run `sudo gitlab-ctl reconfigure` after saving the changes to the `gitlab.rb` file. @@ -845,6 +851,6 @@ If you see this page when trying to set a password via the web interface, make s ### Some job logs are not uploaded to object storage -When the GitLab deployment is scaled up to more than one node, some job logs may not be uploaded to [object storage](../../administration/object_storage.md) properly. [Incremental logging is required](../../administration/object_storage.md#other-alternatives-to-file-system-storage) for CI to use object storage. +When the GitLab deployment is scaled up to more than one node, some job logs may not be uploaded to [object storage](../../administration/object_storage.md) properly. [Incremental logging is required](../../administration/object_storage.md#alternatives-to-file-system-storage) for CI to use object storage. Enable [incremental logging](../../administration/job_logs.md#enable-or-disable-incremental-logging) if it has not already been enabled. diff --git a/doc/install/azure/index.md b/doc/install/azure/index.md index d92859d518f..088ef50c005 100644 --- a/doc/install/azure/index.md +++ b/doc/install/azure/index.md @@ -191,7 +191,7 @@ To set up the GitLab external URL: 1. Find `external_url` and replace it with your own domain name. For the sake of this example, use the default domain name Azure sets up. Using `https` in the URL - [automatically enables](https://docs.gitlab.com/omnibus/settings/ssl.html#lets-encrypt-integration), + [automatically enables](https://docs.gitlab.com/omnibus/settings/ssl/index.html#lets-encrypt-integration), Let's Encrypt, and sets HTTPS by default: ```ruby diff --git a/doc/install/cloud_providers.md b/doc/install/cloud_providers.md new file mode 100644 index 00000000000..36cda77143f --- /dev/null +++ b/doc/install/cloud_providers.md @@ -0,0 +1,17 @@ +--- +stage: Systems +group: Distribution +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +description: Install GitLab on a cloud provider. +type: index +--- + +# Installing GitLab on a cloud provider **(FREE SELF)** + +You can install GitLab on several cloud providers. + +| Cloud provider | Description | +|---------------------------------------------------------------|-------------| +| [AWS](aws/index.md) | Install GitLab on AWS using the community AMIs provided by GitLab. | +| [Google Cloud Platform (GCP)](google_cloud_platform/index.md) | Install GitLab on a VM in GCP. | +| [Azure](azure/index.md) | Install GitLab from Azure Marketplace. | diff --git a/doc/install/docker.md b/doc/install/docker.md index 40eb3a9796e..d387a4d0abb 100644 --- a/doc/install/docker.md +++ b/doc/install/docker.md @@ -55,6 +55,12 @@ For macOS users, use the user's `$HOME/gitlab` directory: export GITLAB_HOME=$HOME/gitlab ``` +The `GITLAB_HOME` environment variable should be appended to your shell's profile so it is +applied on all future terminal sessions: + +- Bash: `~/.bash_profile` +- ZSH: `~/.zshrc` + The GitLab container uses host mounted volumes to store persistent data: | Local location | Container location | Usage | @@ -305,7 +311,7 @@ point to a valid URL. To receive emails from GitLab you have to configure the [SMTP settings](https://docs.gitlab.com/omnibus/settings/smtp.html) because the GitLab Docker image doesn't have an SMTP server installed. You may also be interested in -[enabling HTTPS](https://docs.gitlab.com/omnibus/settings/ssl.html). +[enabling HTTPS](https://docs.gitlab.com/omnibus/settings/ssl/index.html). After you make all the changes you want, you will need to restart the container to reconfigure GitLab: @@ -456,6 +462,31 @@ web browser under `:8929` and push using SSH under the port `2289`. A `docker-compose.yml` example that uses different ports can be found in the [Docker compose](#install-gitlab-using-docker-compose) section. +### Configure multiple database connections + +In [GitLab 16.0](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/6850), +GitLab defaults to using two database connections that point to the same PostgreSQL database. + +If, for any reason, you wish to switch back to single database connection: + +1. Edit `/etc/gitlab/gitlab.rb` inside the container: + + ```shell + sudo docker exec -it gitlab editor /etc/gitlab/gitlab.rb + ``` + +1. Add the following line: + + ```ruby + gitlab_rails['databases']['ci']['enable'] = false + ``` + +1. Restart the container: + +```shell +sudo docker restart gitlab +``` + ## Upgrade In most cases, upgrading GitLab is as easy as downloading the newest Docker @@ -486,6 +517,12 @@ To upgrade GitLab that was [installed using Docker Engine](#install-gitlab-using sudo docker pull gitlab/gitlab-ee:latest ``` +1. Ensure that the `GITLAB_HOME` environment variable is [defined](#set-up-the-volumes-location): + + ```shell + echo $GITLAB_HOME + ``` + 1. Create the container once again with the [previously specified](#install-gitlab-using-docker-engine) options: diff --git a/doc/install/google_cloud_platform/index.md b/doc/install/google_cloud_platform/index.md index d16ac3e2174..e492b5d75ce 100644 --- a/doc/install/google_cloud_platform/index.md +++ b/doc/install/google_cloud_platform/index.md @@ -118,7 +118,7 @@ here's how you configure GitLab to be aware of the change: ### Configuring HTTPS with the domain name Although not needed, it's strongly recommended to secure GitLab with a -[TLS certificate](https://docs.gitlab.com/omnibus/settings/ssl.html). +[TLS certificate](https://docs.gitlab.com/omnibus/settings/ssl/index.html). ### Configuring the email SMTP settings diff --git a/doc/install/index.md b/doc/install/index.md index 4aef471ad5c..15556117b51 100644 --- a/doc/install/index.md +++ b/doc/install/index.md @@ -2,7 +2,6 @@ stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false description: Read through the GitLab installation methods. type: index --- @@ -11,10 +10,15 @@ type: index You can install GitLab on most GNU/Linux distributions, on several cloud providers, and in Kubernetes clusters. - To get the best experience, you should balance performance, reliability, ease of administration (backups, upgrades, and troubleshooting) with the cost of hosting. -To get started, [choose your installation method](install_methods.md). - -If you already have a running instance, learn how to [configure it](next_steps.md). +- [Requirements](requirements.md) +- [Installation methods](install_methods.md) +- [Cloud provider guides](cloud_providers.md) +- [Offline GitLab](../topics/offline/index.md) +- [Reference architectures](../administration/reference_architectures/index.md) +- [Steps after installing](next_steps.md) +- [Upgrade GitLab](../update/index.md) +- [Install GitLab Runner](https://docs.gitlab.com/runner/install/) +- [Configure GitLab Runner](https://docs.gitlab.com/runner/configuration/) diff --git a/doc/install/install_methods.md b/doc/install/install_methods.md index af15dc3f085..0463c926286 100644 --- a/doc/install/install_methods.md +++ b/doc/install/install_methods.md @@ -2,14 +2,14 @@ stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false description: Read through the GitLab installation methods. type: index --- -# Installation methods +# Installation methods **(FREE SELF)** -You can install GitLab by using any of the following methods. +You can install GitLab on several [cloud providers](cloud_providers.md), +or use one of the following methods. | Installation method | Description | When to choose | |----------------------------------------------------------------|-------------|----------------| @@ -20,16 +20,6 @@ You can install GitLab by using any of the following methods. | [GitLab Environment Toolkit (GET)](https://gitlab.com/gitlab-org/gitlab-environment-toolkit#documentation) | A set of automation tools. | Use to deploy a [reference architecture](../administration/reference_architectures/index.md) on most major cloud providers. Has some [limitations](https://gitlab.com/gitlab-org/gitlab-environment-toolkit#missing-features-to-be-aware-of) and manual setup for production environments. | | [GitLab Operator](https://docs.gitlab.com/operator/) | An installation and management method that follows the [Kubernetes Operator pattern](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/). | Use to run GitLab in an [OpenShift](openshift_and_gitlab/index.md) environment. | -## Cloud providers - -You can install GitLab on several cloud providers. - -| Cloud provider | Description | -|---------------------------------------------------------------|-------------| -| [AWS](aws/index.md) | Install GitLab on AWS using the community AMIs provided by GitLab. | -| [Google Cloud Platform (GCP)](google_cloud_platform/index.md) | Install GitLab on a VM in GCP. | -| [Azure](azure/index.md) | Install GitLab from Azure Marketplace. | - ## Unsupported Linux distributions and Unix-like operating systems - Arch Linux diff --git a/doc/install/installation.md b/doc/install/installation.md index eda9c503e28..28aa37f0d1b 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -11,8 +11,7 @@ using the source files. To set up a **development installation** or for many other installation options, see the [main installation page](index.md). It was created for and tested on **Debian/Ubuntu** operating systems. Read [requirements.md](requirements.md) for hardware and operating system requirements. -If you want to install on RHEL/CentOS, we recommend using the -[Omnibus packages](https://about.gitlab.com/install/). +If you want to install on RHEL/CentOS, you should use the [Omnibus packages](https://about.gitlab.com/install/). This guide is long because it covers many cases and includes all commands you need, this is [one of the few installation scripts that actually work out of the box](https://twitter.com/robinvdvleuten/status/424163226532986880). @@ -46,17 +45,17 @@ If the highest number stable branch is unclear, check the [GitLab blog](https:// ## Software requirements -| Software | Minimum version | Notes | -| ------------------ | --------------- |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [Ruby](#2-ruby) | `2.7` | From GitLab 13.6, Ruby 2.7 is required. Ruby 3.0 is not supported yet (see [the relevant epic](https://gitlab.com/groups/gitlab-org/-/epics/5149) for the current status). You must use the standard MRI implementation of Ruby. We love [JRuby](https://www.jruby.org/) and [Rubinius](https://github.com/rubinius/rubinius#the-rubinius-language-platform), but GitLab needs several Gems that have native extensions. | -| [Go](#3-go) | `1.18` | From GitLab 15.6, Go 1.18 or later is required. | -| [Git](#git) | `2.38.x` | From GitLab 15.8, Git 2.38.x and later is required. It's highly recommended that you use the [Git version provided by Gitaly](#git). | -| [Node.js](#4-node) | `16.15.0` | From GitLab 15.7, Node.js 16.15.0 or later is required. | +| Software | Minimum version | Notes | +|:------------------------|:----------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [Ruby](#2-ruby) | `3.0.x` | From GitLab 15.10, Ruby 3.0 is required. You must use the standard MRI implementation of Ruby. We love [JRuby](https://www.jruby.org/) and [Rubinius](https://github.com/rubinius/rubinius#the-rubinius-language-platform), but GitLab needs several Gems that have native extensions. | +| [RubyGems](#3-rubygems) | `3.4.x` | A specific RubyGems version is not fully needed, but it's recommended to update so you can enjoy some known performance improvements. | +| [Go](#4-go) | `1.18.x` | From GitLab 15.6, Go 1.18 or later is required. | +| [Git](#git) | `2.38.x` | From GitLab 15.8, Git 2.38.x and later is required. It's highly recommended that you use the [Git version provided by Gitaly](#git). | +| [Node.js](#5-node) | `16.15.0` | From GitLab 15.7, Node.js 16.15.0 or later is required. | ## GitLab directory structure -This is the main directory structure you end up with following the instructions -of this page: +When following the instructions on this page, you create this directory structure: ```plaintext |-- home @@ -88,13 +87,14 @@ The GitLab installation consists of setting up the following components: 1. [Packages and dependencies](#1-packages-and-dependencies). 1. [Ruby](#2-ruby). -1. [Go](#3-go). -1. [Node](#4-node). -1. [System users](#5-system-users). -1. [Database](#6-database). -1. [Redis](#7-redis). -1. [GitLab](#8-gitlab). -1. [NGINX](#9-nginx). +1. [RubyGems](#3-rubygems). +1. [Go](#4-go). +1. [Node](#5-node). +1. [System users](#6-system-users). +1. [Database](#7-database). +1. [Redis](#8-redis). +1. [GitLab](#9-gitlab). +1. [NGINX](#10-nginx). ## 1. Packages and dependencies @@ -220,20 +220,25 @@ below to use a system Ruby. Linux distributions generally have older versions of Ruby available, so these instructions are designed to install Ruby from the official source code. -Download Ruby and compile it: +[Install Ruby](https://www.ruby-lang.org/en/documentation/installation/). + +## 3. RubyGems + +Sometimes, a newer version of RubyGems is required than the one bundled with Ruby. + +To update to a specific version: + +```shell +gem update --system 3.4.12 +``` + +Or the latest version: ```shell -mkdir /tmp/ruby && cd /tmp/ruby -curl --remote-name --location --progress-bar "https://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.6.tar.gz" -echo 'e7203b0cc09442ed2c08936d483f8ac140ec1c72e37bb5c401646b7866cb5d10 ruby-2.7.6.tar.gz' | sha256sum -c - && tar xzf ruby-2.7.6.tar.gz -cd ruby-2.7.6 - -./configure --disable-install-rdoc --enable-shared -make -sudo make install +gem update --system ``` -## 3. Go +## 4. Go GitLab has several daemons written in Go. To install GitLab we need a Go compiler. The instructions below assume you use 64-bit @@ -251,7 +256,7 @@ sudo ln -sf /usr/local/go/bin/{go,gofmt} /usr/local/bin/ rm go1.18.8.linux-amd64.tar.gz ``` -## 4. Node +## 5. Node GitLab requires the use of Node to compile JavaScript assets, and Yarn to manage JavaScript dependencies. The current minimum @@ -275,7 +280,7 @@ npm install --global yarn Visit the official websites for [node](https://nodejs.org/en/download/package-manager/) and [yarn](https://classic.yarnpkg.com/en/docs/install/) if you have any trouble with these steps. -## 5. System users +## 6. System users Create a `git` user for GitLab: @@ -283,10 +288,10 @@ Create a `git` user for GitLab: sudo adduser --disabled-login --gecos 'GitLab' git ``` -## 6. Database +## 7. Database NOTE: -In GitLab 12.1 and later, only PostgreSQL is supported. In GitLab 14.0 and later, we [require PostgreSQL 12+](requirements.md#postgresql-requirements). +In GitLab 12.1 and later, only PostgreSQL is supported. In GitLab 16.0 and later, we [require PostgreSQL 13+](requirements.md#postgresql-requirements). 1. Install the database packages. @@ -416,7 +421,7 @@ In GitLab 12.1 and later, only PostgreSQL is supported. In GitLab 14.0 and later gitlabhq_production> \q ``` -## 7. Redis +## 8. Redis See the [requirements page](requirements.md#redis-versions) for the minimum Redis requirements. @@ -499,7 +504,7 @@ fi sudo service redis-server restart ``` -## 8. GitLab +## 9. GitLab ```shell # We'll install GitLab into the home directory of the user "git" @@ -593,6 +598,10 @@ If you want to use HTTPS, see [Using HTTPS](#using-https) for the additional ste ### Configure GitLab DB Settings +NOTE: +From [GitLab 15.9](https://gitlab.com/gitlab-org/gitlab/-/issues/387898), `database.yml` with only a section: `main:` is deprecated. +In GitLab 17.0 and later, you must have the two `main:` and `ci:` sections in your `database.yml`. + ```shell sudo -u git cp config/database.yml.postgresql config/database.yml @@ -604,6 +613,11 @@ sudo -u git cp config/database.yml.postgresql config/database.yml # adapter: postgresql # encoding: unicode # database: gitlabhq_production +# ci: +# adapter: postgresql +# encoding: unicode +# database: gitlabhq_production +# database_tasks: false # sudo -u git -H editor config/database.yml @@ -622,10 +636,10 @@ sudo -u git -H editor config/database.yml sudo -u git -H chmod o-rwx config/database.yml ``` -NOTE: -From GitLab 15.9, `database.yml` with only a section: `main:` is deprecated. -In GitLab 15.10 and later, you should have two sections in your `database.yml`, `main:` and `ci:`. The `ci`: connection [must be to the same database](../administration/postgresql/multiple_databases.md). -In GitLab 17.0 and later, you must have the two `main:` and `ci:` sections in your `database.yml`. +You should have two sections in your `database.yml`: `main:` and `ci:`. The `ci`: +connection [must be to the same database](../administration/postgresql/multiple_databases.md). +If, for any reason, you wish to remain on a single database connection, remove the `ci:` +section from `config/database.yml`. ### Install Gems @@ -872,10 +886,10 @@ sudo -u git -H bundle exec rake gitlab:setup RAILS_ENV=production force=yes # When done, you see 'Administrator account created:' ``` -You can set the Administrator/root password and email by supplying them in environmental variables, `GITLAB_ROOT_PASSWORD` and `GITLAB_ROOT_EMAIL` respectively, as seen below. If you don't set the password (and it is set to the default one), wait to expose GitLab to the public internet until the installation is done and you've logged into the server the first time. During the first login, you are forced to change the default password. An Enterprise Edition license may also be installed at this time by supplying a full path in the `GITLAB_LICENSE_FILE` environment variable. +You can set the Administrator/root password and email by supplying them in environmental variables, `GITLAB_ROOT_PASSWORD` and `GITLAB_ROOT_EMAIL`, as seen below. If you don't set the password (and it is set to the default one), wait to expose GitLab to the public internet until the installation is done and you've logged into the server the first time. During the first login, you are forced to change the default password. An Enterprise Edition subscription may also be activated at this time by supplying the activation code in the `GITLAB_ACTIVATION_CODE` environment variable. ```shell -sudo -u git -H bundle exec rake gitlab:setup RAILS_ENV=production GITLAB_ROOT_PASSWORD=yourpassword GITLAB_ROOT_EMAIL=youremail GITLAB_LICENSE_FILE="/path/to/license" +sudo -u git -H bundle exec rake gitlab:setup RAILS_ENV=production GITLAB_ROOT_PASSWORD=yourpassword GITLAB_ROOT_EMAIL=youremail GITLAB_ACTIVATION_CODE=yourcode ``` ### Secure `secrets.yml` @@ -915,7 +929,7 @@ sudo systemctl start gitlab.target sudo service gitlab start ``` -## 9. NGINX +## 10. NGINX NGINX is the officially supported web server for GitLab. If you cannot or do not want to use NGINX as your web server, see [GitLab recipes](https://gitlab.com/gitlab-org/gitlab-recipes/). @@ -1160,7 +1174,7 @@ prometheus: If you see this message when attempting to clone a repository hosted by GitLab, this is likely due to an outdated NGINX or Apache configuration, or a missing or misconfigured GitLab Workhorse instance. Double-check that you've -[installed Go](#3-go), [installed GitLab Workhorse](#install-gitlab-workhorse), +[installed Go](#4-go), [installed GitLab Workhorse](#install-gitlab-workhorse), and correctly [configured NGINX](#site-configuration). ### `google-protobuf` "LoadError: /lib/x86_64-linux-gnu/libc.so.6: version 'GLIBC_2.14' not found" diff --git a/doc/install/next_steps.md b/doc/install/next_steps.md index c718d895c8a..70b6101b1eb 100644 --- a/doc/install/next_steps.md +++ b/doc/install/next_steps.md @@ -56,7 +56,7 @@ installation. ## Cross-repository Code Search -- [Advanced Search](../integration/advanced_search/elasticsearch.md): Leverage [Elasticsearch](https://www.elastic.co/) or [OpenSearch](https://opensearch.org/) for +- [Advanced search](../integration/advanced_search/elasticsearch.md): Leverage [Elasticsearch](https://www.elastic.co/) or [OpenSearch](https://opensearch.org/) for faster, more advanced code search across your entire GitLab instance. ## Scaling and replication diff --git a/doc/install/openshift_and_gitlab/index.md b/doc/install/openshift_and_gitlab/index.md index 6086716be1c..4f3df3ecff8 100644 --- a/doc/install/openshift_and_gitlab/index.md +++ b/doc/install/openshift_and_gitlab/index.md @@ -31,7 +31,7 @@ The GitLab Operator does not include the GitLab Runner. To install and manage a ### Secure -- [License Compliance](../../user/compliance/license_compliance/index.md) +- [License Compliance via the `License-Scanning.gitlab-ci.yml` CI/CD template](../../user/compliance/license_compliance/index.md). [License scanning of CycloneDX files](../../user/compliance/license_scanning_of_cyclonedx_files/index.md) is supported on OpenShift. - [Code Quality scanning](../../ci/testing/code_quality.md) - [Operational Container Scanning](../../user/clusters/agent/vulnerabilities.md) (Note: Pipeline [Container Scanning](../../user/application_security/container_scanning/index.md) is supported) diff --git a/doc/install/requirements.md b/doc/install/requirements.md index 8c6f469aca2..7fdbdfc2b24 100644 --- a/doc/install/requirements.md +++ b/doc/install/requirements.md @@ -12,10 +12,7 @@ This page includes information about the minimum requirements you need to instal ### Redis versions -GitLab 13.0 and later requires Redis version 5.0 or higher. - -Redis version 6.0 or higher is recommended, as this is what ships with -[Omnibus GitLab](https://docs.gitlab.com/omnibus/) packages starting with GitLab 13.9. +GitLab 16.0 and later requires Redis 6.0 or later. ## Hardware requirements @@ -92,7 +89,7 @@ the following table) as these were used for development and testing: | 13.0 | 11 | | 14.0 | 12.7 | | 15.0 | 12.10 | -| 16.0 (planned) | 13.6 | +| 16.0 | 13.6 | You must also ensure the following extensions are loaded into every GitLab database. [Read more about this requirement, and troubleshooting](postgresql_extensions.md). diff --git a/doc/integration/advanced_search/elasticsearch.md b/doc/integration/advanced_search/elasticsearch.md index 6a2c562377f..e886f2a4b37 100644 --- a/doc/integration/advanced_search/elasticsearch.md +++ b/doc/integration/advanced_search/elasticsearch.md @@ -7,8 +7,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Elasticsearch **(PREMIUM SELF)** -This page describes how to enable Advanced Search. When enabled, -Advanced Search provides faster search response times and [improved search features](../../user/search/advanced_search.md). +This page describes how to enable advanced search. When enabled, +advanced search provides faster search response times and [improved search features](../../user/search/advanced_search.md). ## Version requirements @@ -16,7 +16,7 @@ Advanced Search provides faster search response times and [improved search featu > Support for Elasticsearch 6.8 was [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/350275) in GitLab 15.0. -Advanced Search works with the following versions of Elasticsearch. +Advanced search works with the following versions of Elasticsearch. | GitLab version | Elasticsearch version | |-----------------------|--------------------------| @@ -25,15 +25,16 @@ Advanced Search works with the following versions of Elasticsearch. | GitLab 13.3 - 13.8 | Elasticsearch 6.4 - 7.x | | GitLab 12.7 - 13.2 | Elasticsearch 6.x - 7.x | -Advanced Search follows the [Elasticsearch end-of-life policy](https://www.elastic.co/support/eol). +Advanced search follows the [Elasticsearch end-of-life policy](https://www.elastic.co/support/eol). When we change Elasticsearch supported versions in GitLab, we announce them in [deprecation notes](https://about.gitlab.com/handbook/marketing/blog/release-posts/#deprecations) in monthly release posts before we remove them. ### OpenSearch version requirements -| GitLab version | Elasticsearch version | -|-----------------------|--------------------------| -| GitLab 15.0 or later | OpenSearch 1.x or later | +| GitLab version | OpenSearch version | +|-------------------------|---------------------------| +| GitLab 15.0 to 15.5.2 | OpenSearch 1.x | +| GitLab 15.5.3 and later | OpenSearch 1.x and later | If your version of Elasticsearch or OpenSearch is incompatible, to prevent data loss, indexing pauses and a message is logged in the @@ -46,7 +47,7 @@ If you are using a compatible version and after connecting to OpenSearch, you ge Elasticsearch requires additional resources to those documented in the [GitLab system requirements](../../install/requirements.md). -Memory, CPU, and storage resource amounts vary depending on the amount of data you index into the Elasticsearch cluster. Heavily used Elasticsearch clusters may require more resources. The [`estimate_cluster_size`](#gitlab-advanced-search-rake-tasks) Rake task ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221177) in GitLab 13.10) uses the total repository size to estimate the Advanced Search storage requirements. +Memory, CPU, and storage resource amounts vary depending on the amount of data you index into the Elasticsearch cluster. Heavily used Elasticsearch clusters may require more resources. The [`estimate_cluster_size`](#gitlab-advanced-search-rake-tasks) Rake task ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221177) in GitLab 13.10) uses the total repository size to estimate the advanced search storage requirements. ## Install Elasticsearch @@ -163,20 +164,22 @@ Errors from the [GitLab Elasticsearch Indexer](https://gitlab.com/gitlab-org/git the [`elasticsearch.log`](../../administration/logs/index.md#elasticsearchlog) file and the [`sidekiq.log`](../../administration/logs/index.md#sidekiqlog) file with a `json.exception.class` of `Gitlab::Elastic::Indexer::Error`. These errors may occur when indexing Git repository data. -## Enable Advanced Search +## Enable advanced search -For GitLab instances with more than 50 GB repository data you can follow the instructions for [how to index large instances efficiently](#how-to-index-large-instances-efficiently) below. +Prerequisite: -To enable Advanced Search, you must have administrator access to GitLab: +- You must have administrator access to the instance. + +To enable advanced search: 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Settings > Advanced Search**. NOTE: - To see the Advanced Search section, you need an active GitLab Premium + To see the **Advanced Search** section, you need an active GitLab Premium [license](../../user/admin_area/license.md). -1. Configure the [Advanced Search settings](#advanced-search-configuration) for +1. Configure the [advanced search settings](#advanced-search-configuration) for your Elasticsearch cluster. Do not enable **Search with Elasticsearch enabled** yet. 1. Enable **Elasticsearch indexing** and select **Save changes**. This creates @@ -202,7 +205,9 @@ you might have problems updating documents such as issues because your instance queues a job to index the change, but cannot find a valid Elasticsearch cluster. -### Advanced Search configuration +For GitLab instances with more than 50 GB of repository data, see [How to index large instances efficiently](#how-to-index-large-instances-efficiently). + +### Advanced search configuration The following Elasticsearch settings are available: @@ -223,8 +228,8 @@ The following Elasticsearch settings are available: | `AWS Secret Access Key` | The AWS secret access key. | | `Maximum file size indexed` | See [the explanation in instance limits.](../../administration/instance_limits.md#maximum-file-size-indexed). | | `Maximum field length` | See [the explanation in instance limits.](../../administration/instance_limits.md#maximum-field-length). | -| `Maximum bulk request size (MiB)` | Used by the GitLab Ruby and Golang-based indexer processes. This setting indicates how much data must be collected (and stored in memory) in a given indexing process before submitting the payload to the Elasticsearch Bulk API. For the GitLab Golang-based indexer, you should use this setting with `Bulk request concurrency`. `Maximum bulk request size (MiB)` must accommodate the resource constraints of both the Elasticsearch hosts and the hosts running the GitLab Golang-based indexer from either the `gitlab-rake` command or the Sidekiq tasks. | -| `Bulk request concurrency` | The Bulk request concurrency indicates how many of the GitLab Golang-based indexer processes (or threads) can run in parallel to collect data to subsequently submit to the Elasticsearch Bulk API. This increases indexing performance, but fills the Elasticsearch bulk requests queue faster. This setting should be used together with the Maximum bulk request size setting (see above) and needs to accommodate the resource constraints of both the Elasticsearch hosts and the hosts running the GitLab Golang-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. | +| `Maximum bulk request size (MiB)` | Used by the GitLab Ruby and Go-based indexer processes. This setting indicates how much data must be collected (and stored in memory) in a given indexing process before submitting the payload to the Elasticsearch Bulk API. For the GitLab Go-based indexer, you should use this setting with `Bulk request concurrency`. `Maximum bulk request size (MiB)` must accommodate the resource constraints of both the Elasticsearch hosts and the hosts running the GitLab Go-based indexer from either the `gitlab-rake` command or the Sidekiq tasks. | +| `Bulk request concurrency` | The Bulk request concurrency indicates how many of the GitLab Go-based indexer processes (or threads) can run in parallel to collect data to subsequently submit to the Elasticsearch Bulk API. This increases indexing performance, but fills the Elasticsearch bulk requests queue faster. This setting should be used together with the Maximum bulk request size setting (see above) and needs to accommodate the resource constraints of both the Elasticsearch hosts and the hosts running the GitLab Go-based indexer either from the `gitlab-rake` command or the Sidekiq tasks. | | `Client request timeout` | Elasticsearch HTTP client request timeout value in seconds. `0` means using the system default timeout value, which depends on the libraries that GitLab application is built upon. | WARNING: @@ -294,16 +299,16 @@ For more information, see [Identity and Access Management in Amazon OpenSearch S When using fine-grained access control with a user in the internal database, you should use HTTP basic authentication to connect to OpenSearch. You can provide the master username and password as part of the -OpenSearch URL or in the **Username** and **Password** text boxes in the Advanced Search settings. See +OpenSearch URL or in the **Username** and **Password** text boxes in the advanced search settings. See [Tutorial: Internal user database and HTTP basic authentication](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/fgac-walkthrough-basic.html) for details. ##### Connecting with an IAM user -When using fine-grained access control with IAM credentials, you can provide the credentials in the **AWS OpenSearch IAM credentials** section in the Advanced Search settings. +When using fine-grained access control with IAM credentials, you can provide the credentials in the **AWS OpenSearch IAM credentials** section in the advanced search settings. ##### Permissions for fine-grained access control -The following permissions are required for Advanced Search. See [Creating roles](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/fgac.html#fgac-roles) for details. +The following permissions are required for advanced search. See [Creating roles](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/fgac.html#fgac-roles) for details. ```json { @@ -338,7 +343,7 @@ The following permissions are required for Advanced Search. See [Creating roles] } ``` -The index pattern `*` requires a few permissions for Advanced Search to work. +The index pattern `*` requires a few permissions for advanced search to work. ### Limit the number of namespaces and projects that can be indexed @@ -350,14 +355,14 @@ under **Elasticsearch indexing restrictions** more options become available. You can select namespaces and projects to index exclusively. If the namespace is a group, it includes any subgroups and projects belonging to those subgroups to be indexed as well. -Advanced Search only provides cross-group code/commit search (global) if all name-spaces are indexed. In this particular scenario where only a subset of namespaces are indexed, a global search does not provide a code or commit scope. This is possible only in the scope of an indexed namespace. There is no way to code/commit search in multiple indexed namespaces (when only a subset of namespaces has been indexed). For example if two groups are indexed, there is no way to run a single code search on both. You can only run a code search on the first group and then on the second. +Advanced search only provides cross-group code/commit search (global) if all name-spaces are indexed. In this particular scenario where only a subset of namespaces are indexed, a global search does not provide a code or commit scope. This is possible only in the scope of an indexed namespace. There is no way to code/commit search in multiple indexed namespaces (when only a subset of namespaces has been indexed). For example if two groups are indexed, there is no way to run a single code search on both. You can only run a code search on the first group and then on the second. You can filter the selection dropdown list by writing part of the namespace or project name you're interested in. ![limit namespace filter](img/limit_namespace_filter.png) NOTE: -If no namespaces or projects are selected, no Advanced Search indexing takes place. +If no namespaces or projects are selected, no advanced search indexing takes place. WARNING: If you have already indexed your instance, you must regenerate the index to delete all existing data @@ -385,19 +390,19 @@ For guidance on what to install, see the following Elasticsearch language plugin | Parameter | Description | |-------------------------------------------------------|-------------| | `Enable Chinese (smartcn) custom analyzer: Indexing` | Enables or disables Chinese language support using [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) custom analyzer for newly created indices.| -| `Enable Chinese (smartcn) custom analyzer: Search` | Enables or disables using [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) fields for Advanced Search. Only enable this after [installing the plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html), enabling custom analyzer indexing and recreating the index.| +| `Enable Chinese (smartcn) custom analyzer: Search` | Enables or disables using [`smartcn`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html) fields for advanced search. Only enable this after [installing the plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-smartcn.html), enabling custom analyzer indexing and recreating the index.| | `Enable Japanese (kuromoji) custom analyzer: Indexing` | Enables or disables Japanese language support using [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) custom analyzer for newly created indices.| -| `Enable Japanese (kuromoji) custom analyzer: Search` | Enables or disables using [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) fields for Advanced Search. Only enable this after [installing the plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html), enabling custom analyzer indexing and recreating the index.| +| `Enable Japanese (kuromoji) custom analyzer: Search` | Enables or disables using [`kuromoji`](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html) fields for advanced search. Only enable this after [installing the plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-kuromoji.html), enabling custom analyzer indexing and recreating the index.| -## Disable Advanced Search +## Disable advanced search To disable the Elasticsearch integration: 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Settings > Advanced Search**. -1. Uncheck **Elasticsearch indexing** and **Search with Elasticsearch enabled**. +1. Clear the **Elasticsearch indexing** and **Search with Elasticsearch enabled** checkboxes. 1. Select **Save changes**. -1. Optional. Delete the existing indices: +1. Optional. For Elasticsearch instances that are still online, delete existing indices: ```shell # Omnibus installations @@ -414,7 +419,7 @@ To disable the Elasticsearch integration: 1. Expand **Advanced Search**. 1. Clear the **Pause Elasticsearch indexing** checkbox. -## Zero downtime reindexing +## Zero-downtime reindexing The idea behind this reindexing method is to leverage the [Elasticsearch reindex API](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html) and Elasticsearch index alias feature to perform the operation. We set up an index alias which connects to a @@ -423,7 +428,11 @@ the writes to the `primary` index. Then, we create another index and invoke the index data onto the new index. After the reindexing job is complete, we switch to the new index by connecting the index alias to it which becomes the new `primary` index. At the end, we resume the writes and typical operation resumes. -### Trigger the reindex via the Advanced Search administration +### Using zero-downtime reindexing + +You can use zero-downtime reindexing to configure index settings or mappings that cannot be changed without creating a new index and copying existing data. You should not use zero-downtime reindexing to fix missing data. Zero-downtime reindexing does not add data to the search cluster if the data is not already indexed. You must complete all [advanced search migrations](#advanced-search-migrations) before you start reindexing. + +### Trigger the reindex via the advanced search administration > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34069) in GitLab 13.2. > - A scheduled index deletion and the ability to cancel it was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38914) in GitLab 13.3. @@ -502,15 +511,15 @@ Sometimes, you might want to abandon the unfinished reindex job and resume the i 1. Expand **Advanced Search**. 1. Clear the **Pause Elasticsearch indexing** checkbox. -## Advanced Search migrations +## Advanced search migrations > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/234046) in GitLab 13.6. With reindex migrations running in the background, there's no need for a manual intervention. This usually happens in situations where new features are added to -Advanced Search, which means adding or changing the way content is indexed. +advanced search, which means adding or changing the way content is indexed. -To confirm that the Advanced Search migrations ran, you can check with: +To confirm that the advanced search migrations ran, you can check with: ```shell curl "$CLUSTER_URL/gitlab-production-migrations/_search?q=*" | jq . @@ -554,7 +563,7 @@ To debug issues with the migrations you can check the [`elasticsearch.log` file] ### Retry a halted migration Some migrations are built with a retry limit. If the migration cannot finish within the retry limit, -it is halted and a notification is displayed in the Advanced Search integration settings. +it is halted and a notification is displayed in the advanced search integration settings. It is recommended to check the [`elasticsearch.log` file](../../administration/logs/index.md#elasticsearchlog) to debug why the migration was halted and make any changes before retrying the migration. Once you believe you've fixed the cause of the failure, select "Retry migration", and the migration is scheduled to be retried @@ -575,7 +584,7 @@ version. If you have halted migrations, these need to be resolved and [retried](#retry-a-halted-migration) before proceeding with a major version upgrade. Read more about [upgrading to a new major version](../../update/index.md#upgrading-to-a-new-major-version). -## GitLab Advanced Search Rake tasks +## GitLab advanced search Rake tasks Rake tasks are available to: @@ -587,7 +596,7 @@ The following are some available Rake tasks: | Task | Description | |:--------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [`sudo gitlab-rake gitlab:elastic:info`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Outputs debugging information for the Advanced Search integration. | +| [`sudo gitlab-rake gitlab:elastic:info`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Outputs debugging information for the advanced search integration. | | [`sudo gitlab-rake gitlab:elastic:index`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enables Elasticsearch indexing and run `gitlab:elastic:create_empty_index`, `gitlab:elastic:clear_index_status`, `gitlab:elastic:index_projects`, `gitlab:elastic:index_snippets`, and `gitlab:elastic:index_users`. | | [`sudo gitlab-rake gitlab:elastic:pause_indexing`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Pauses Elasticsearch indexing. Changes are still tracked. Useful for cluster/index migrations. | | [`sudo gitlab-rake gitlab:elastic:resume_indexing`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Resumes Elasticsearch indexing. | @@ -604,7 +613,7 @@ The following are some available Rake tasks: | [`sudo gitlab-rake gitlab:elastic:mark_reindex_failed`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Mark the most recent re-index job as failed. | | [`sudo gitlab-rake gitlab:elastic:list_pending_migrations`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | List pending migrations. Pending migrations include those that have not yet started, have started but not finished, and those that are halted. | | [`sudo gitlab-rake gitlab:elastic:estimate_cluster_size`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Get an estimate of cluster size based on the total repository size. | -| [`sudo gitlab-rake gitlab:elastic:enable_search_with_elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enable advanced search with Elasticsearch. | +| [`sudo gitlab-rake gitlab:elastic:enable_search_with_elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Enables advanced search with Elasticsearch. | | [`sudo gitlab-rake gitlab:elastic:disable_search_with_elasticsearch`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/tasks/gitlab/elastic.rake) | Disables advanced search with Elasticsearch. | ### Environment variables @@ -633,7 +642,7 @@ Indexing project repositories...I, [2019-03-04T21:27:03.083410 #3384] INFO -- : I, [2019-03-04T21:27:05.215266 #3384] INFO -- : Indexing GitLab User / test (ID=33) is done! ``` -## Advanced Search index scopes +## Advanced search index scopes When performing a search, the GitLab index uses the following scopes: @@ -658,6 +667,7 @@ For basic guidance on choosing a cluster configuration you may refer to [Elastic - Generally, you want to use at least a 2-node cluster configuration with one replica, which allows you to have resilience. If your storage usage is growing quickly, you may want to plan horizontal scaling (adding more nodes) beforehand. - It's not recommended to use HDD storage with the search cluster, because it takes a hit on performance. It's better to use SSD storage (NVMe or SATA SSD drives for example). +- You should not use [coordinating-only nodes](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html#coordinating-only-node) with large instances. Coordinating-only nodes are smaller than [data nodes](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html#data-node), which can impact performance and [advanced search migrations](#advanced-search-migrations). - You can use the [GitLab Performance Tool](https://gitlab.com/gitlab-org/quality/performance) to benchmark search performance with different search cluster sizes and configurations. - `Heap size` should be set to no more than 50% of your physical RAM. Additionally, it shouldn't be set to more than the threshold for zero-based compressed oops. The exact threshold varies, but 26 GB is safe on most systems, but can also be as large as 30 GB on some systems. See [Heap size settings](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#heap-size-settings) and [Setting JVM options](https://www.elastic.co/guide/en/elasticsearch/reference/current/jvm-options.html) for more details. - Number of CPUs (CPU cores) per node usually corresponds to the `Number of Elasticsearch shards` setting described below. @@ -671,7 +681,7 @@ For basic guidance on choosing a cluster configuration you may refer to [Elastic - `refresh_interval` is a per index setting. You may want to adjust that from default `1s` to a bigger value if you don't need data in real-time. This changes how soon you see fresh results. If that's important for you, you should leave it as close as possible to the default value. - You might want to raise [`indices.memory.index_buffer_size`](https://www.elastic.co/guide/en/elasticsearch/reference/current/indexing-buffer.html) to 30% or 40% if you have a lot of heavy indexing operations. -### Advanced Search integration settings guidance +### Advanced search integration settings guidance - The `Number of Elasticsearch shards` setting usually corresponds with the number of CPUs available in your cluster. For example, if you have a 3-node cluster with 4 cores each, this means you benefit from having at least 3*4=12 shards in the cluster. It's only possible to change the shards number by using [Split index API](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-split-index.html) or by reindexing to a different index with a changed number of shards. - The `Number of Elasticsearch replicas` setting should most of the time be equal to `1` (each shard has 1 replica). Using `0` is not recommended, because losing one node corrupts the index. @@ -862,6 +872,10 @@ However, some larger installations may wish to tune the merge policy settings: ## Index large instances with dedicated Sidekiq nodes or processes +WARNING: +Most instances should not need to configure this. The steps below use an advanced setting of Sidekiq called [routing rules](../../administration/sidekiq/processing_specific_job_classes.md#routing-rules). +Be sure to fully understand about the implication of using routing rules to avoid losing jobs entirely. + Indexing a large instance can be a lengthy and resource-intensive process that has the potential of overwhelming Sidekiq nodes and processes. This negatively affects the GitLab performance and availability. @@ -871,18 +885,20 @@ additional process dedicated to indexing a set of queues (or queue group). This ensure that indexing queues always have a dedicated worker, while the rest of the queues have another dedicated worker to avoid contention. -For this purpose, use the [queue selectors](../../administration/sidekiq/processing_specific_job_classes.md#queue-selectors) -option that allows a more general selection of queue groups using a [worker matching query](../../administration/sidekiq/processing_specific_job_classes.md#worker-matching-query). +For this purpose, use the [routing rules](../../administration/sidekiq/processing_specific_job_classes.md#routing-rules) +option that allows Sidekiq to route jobs to a specific queue based on [worker matching query](../../administration/sidekiq/processing_specific_job_classes.md#worker-matching-query). -To handle these two queue groups, we generally recommend one of the following two options. You can either: +To handle this, we generally recommend one of the following two options. You can either: - [Use two queue groups on one single node](#single-node-two-processes). - [Use two queue groups, one on each node](#two-nodes-one-process-for-each). -For the steps below, consider: +For the steps below, consider the entry of `sidekiq['routing_rules']`: + +- `["feature_category=global_search", "global_search"]` as all indexing jobs are routed to the `global_search` queue. +- `["*", "default"]` as all other non-indexing jobs are routed to the `default` queue. -- `feature_category=global_search` as an indexing queue group with its own Sidekiq process. -- `feature_category!=global_search` as a non-indexing queue group that has its own Sidekiq process. +At least one process in `sidekiq['queue_groups']` has to include the `mailers` queue, otherwise mailers jobs are not processed at all. ### Single node, two processes @@ -892,11 +908,20 @@ To create both an indexing and a non-indexing Sidekiq process in one node: ```ruby sidekiq['enable'] = true - sidekiq['queue_selector'] = true + sidekiq['queue_selector'] = false + + sidekiq['routing_rules'] = [ + ["feature_category=global_search", "global_search"], + ["*", "default"], + ] + sidekiq['queue_groups'] = [ - "feature_category=global_search", - "feature_category!=global_search" - ] + "global_search", # process that listens to global_search queue + "default,mailers" # process that listens to default and mailers queue + ] + + sidekiq['min_concurrency'] = 20 + sidekiq['max_concurrency'] = 20 ``` 1. Save the file and [reconfigure GitLab](../../administration/restart_gitlab.md) @@ -914,26 +939,42 @@ To handle these queue groups on two nodes: 1. To set up the indexing Sidekiq process, on your indexing Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to: - ```ruby - sidekiq['enable'] = true - sidekiq['queue_selector'] = true - sidekiq['queue_groups'] = [ - "feature_category=global_search" - ] - ``` + ```ruby + sidekiq['enable'] = true + sidekiq['queue_selector'] = false + + sidekiq['routing_rules'] = [ + ["feature_category=global_search", "global_search"], + ["*", "default"], + ] + + sidekiq['queue_groups'] = [ + "global_search", # process that listens to global_search queue + ] + + sidekiq['min_concurrency'] = 20 + sidekiq['max_concurrency'] = 20 + ``` 1. Save the file and [reconfigure GitLab](../../administration/restart_gitlab.md) for the changes to take effect. 1. To set up the non-indexing Sidekiq process, on your non-indexing Sidekiq node, change the `/etc/gitlab/gitlab.rb` file to: - ```ruby - sidekiq['enable'] = true - sidekiq['queue_selector'] = true - sidekiq['queue_groups'] = [ - "feature_category!=global_search" - ] - ``` + ```ruby + sidekiq['enable'] = true + sidekiq['routing_rules'] = [ + ["feature_category=global_search", "global_search"], + ["*", "default"], + ] + + sidekiq['queue_groups'] = [ + "default,mailers" # process that listens to default and mailers queue + ] + + sidekiq['min_concurrency'] = 20 + sidekiq['max_concurrency'] = 20 + ``` to set up a non-indexing Sidekiq process. @@ -945,7 +986,7 @@ for the changes to take effect. Sometimes there may be issues with your Elasticsearch index data and as such GitLab allows you to revert to "basic search" when there are no search results and assuming that basic search is supported in that scope. This "basic -search" behaves as though you don't have Advanced Search enabled at all for +search" behaves as though you don't have advanced search enabled at all for your instance and search using other data sources (such as PostgreSQL data and Git data). diff --git a/doc/integration/advanced_search/elasticsearch_troubleshooting.md b/doc/integration/advanced_search/elasticsearch_troubleshooting.md index c8d11286a3c..0c4895f34fa 100644 --- a/doc/integration/advanced_search/elasticsearch_troubleshooting.md +++ b/doc/integration/advanced_search/elasticsearch_troubleshooting.md @@ -101,12 +101,12 @@ Here are some common pitfalls and how to overcome them. There are a couple of ways to achieve that: - When you perform a search, in the upper-right corner of the search results page, - **Advanced search functionality is enabled** is displayed. + **Advanced search is enabled** is displayed. This is always correctly identifying whether the current project/namespace being searched is using Elasticsearch. - From the Admin Area under **Settings > Advanced Search** check that the - Advanced Search settings are checked. + advanced search settings are checked. Those same settings there can be obtained from the Rails console if necessary: @@ -212,7 +212,7 @@ More [complex Elasticsearch API calls](https://www.elastic.co/guide/en/elasticse If the results: -- Sync up, check that you are using [supported syntax](../../user/search/advanced_search.md#syntax). Advanced Search does not support [exact substring matching](https://gitlab.com/gitlab-org/gitlab/-/issues/325234). +- Sync up, check that you are using [supported syntax](../../user/search/advanced_search.md#syntax). Advanced search does not support [exact substring matching](https://gitlab.com/gitlab-org/gitlab/-/issues/325234). - Do not match up, this indicates a problem with the documents generated from the project. It is best to [re-index that project](../advanced_search/elasticsearch.md#indexing-a-range-of-projects-or-a-specific-project). NOTE: @@ -247,8 +247,8 @@ sudo gitlab-rake gitlab:elastic:clear_locked_projects If `ElasticCommitIndexerWorker` Sidekiq workers are failing with this error during indexing, it usually means that Elasticsearch is unable to keep up with the concurrency of indexing request. To address change the following settings: -- To decrease the indexing throughput you can decrease `Bulk request concurrency` (see [Advanced Search settings](elasticsearch.md#advanced-search-configuration)). This is set to `10` by default, but you change it to as low as 1 to reduce the number of concurrent indexing operations. -- If changing `Bulk request concurrency` didn't help, you can use the [queue selector](../../administration/sidekiq/processing_specific_job_classes.md#queue-selectors) option to [limit indexing jobs only to specific Sidekiq nodes](elasticsearch.md#index-large-instances-with-dedicated-sidekiq-nodes-or-processes), which should reduce the number of indexing requests. +- To decrease the indexing throughput you can decrease `Bulk request concurrency` (see [Advanced search settings](elasticsearch.md#advanced-search-configuration)). This is set to `10` by default, but you change it to as low as 1 to reduce the number of concurrent indexing operations. +- If changing `Bulk request concurrency` didn't help, you can use the [routing rules](../../administration/sidekiq/processing_specific_job_classes.md#routing-rules) option to [limit indexing jobs only to specific Sidekiq nodes](elasticsearch.md#index-large-instances-with-dedicated-sidekiq-nodes-or-processes), which should reduce the number of indexing requests. ### Indexing is very slow or fails with `rejected execution of coordinating operation` messages @@ -437,11 +437,11 @@ Improvements to the `code_analyzer` pattern and filters are being discussed in [ In GitLab 13.9, a change was made where [binary file names are being indexed](https://gitlab.com/gitlab-org/gitlab/-/issues/301083). However, without indexing all projects' data from scratch, only binary files that are added or updated after the GitLab 13.9 release are searchable. -## How does Advanced Search handle private projects? +## How does advanced search handle private projects? -Advanced Search stores all the projects in the same Elasticsearch indices, +Advanced search stores all the projects in the same Elasticsearch indices, however, searches only surface results that can be viewed by the user. -Advanced Search honors all permission checks in the application by +Advanced search honors all permission checks in the application by filtering out projects that a user does not have access to at search time. ### Role mapping when using fine-grained access control with AWS Elasticsearch or OpenSearch diff --git a/doc/integration/alicloud.md b/doc/integration/alicloud.md index d861d32e96a..4270444f0bb 100644 --- a/doc/integration/alicloud.md +++ b/doc/integration/alicloud.md @@ -59,7 +59,7 @@ Sign in to the AliCloud platform and create an application on it. AliCloud gener sudo -u git -H editor config/gitlab.yml ``` -1. Edit the [common configuration file settings](omniauth.md#configure-common-settings) +1. Configure the [common settings](omniauth.md#configure-common-settings) to add `alicloud` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. diff --git a/doc/integration/arkose.md b/doc/integration/arkose.md index ae90f1f9574..8f6cec0ac0a 100644 --- a/doc/integration/arkose.md +++ b/doc/integration/arkose.md @@ -156,5 +156,5 @@ The [Anti-abuse team](https://about.gitlab.com/handbook/engineering/development/ ArkoseLabs also maintains the following resources: - [ArkoseLabs portal](https://portal.arkoselabs.com/) -- [ArkoseLabs Zendesk](https://support.arkoselabs.com/) +- [ArkoseLabs Zendesk](https://support.arkoselabs.com/hc/en-us) - [ArkoseLabs documentation](https://developer.arkoselabs.com/docs/documentation-guide) diff --git a/doc/integration/auth0.md b/doc/integration/auth0.md index ad20057f452..07a750143d6 100644 --- a/doc/integration/auth0.md +++ b/doc/integration/auth0.md @@ -42,7 +42,7 @@ application. sudo -u git -H editor config/gitlab.yml ``` -1. Edit the [common configuration file settings](omniauth.md#configure-common-settings) +1. Configure the [common settings](omniauth.md#configure-common-settings) to add `auth0` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. diff --git a/doc/integration/azure.md b/doc/integration/azure.md index cc479dbf65d..0d8c830c016 100644 --- a/doc/integration/azure.md +++ b/doc/integration/azure.md @@ -16,6 +16,140 @@ For new projects, Microsoft suggests you use the [OpenID Connect protocol](../administration/auth/oidc.md#configure-microsoft-azure), which uses the Microsoft identity platform (v2.0) endpoint. +## Migrate to the OpenID Connect protocol + +To migrate to the OpenID Connect protocol, see [configure multiple OpenID Connect providers](../administration/auth/oidc.md#configure-multiple-openid-connect-providers). + +You must set the `uid_field`, which differs across the providers: + +| Provider | `uid` | Remarks | +|-----------------------------------------------------------------------------------------------------------------|-------|-----------------------------------------------------------------------| +| [`omniauth-azure-oauth2`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/vendor/gems/omniauth-azure-oauth2) | `sub` | Additional attributes `oid`, `tid` are offered within the info object | +| [`omniauth-azure-activedirectory-v2`](https://github.com/RIPAGlobal/omniauth-azure-activedirectory-v2/) | `oid` | You must configure `oid` as `uid_field` when migrating | +| [`omniauth_openid_connect`](https://github.com/omniauth/omniauth_openid_connect/) | `sub` | Specify `uid_field` to use another field | + +To migrate from `omniauth-azure-oauth2` to `omniauth_openid_connect` you +must change the configuration: + +- **For Omnibus installations** + +```diff +gitlab_rails['omniauth_providers'] = [ + { + name: "azure_oauth2", + # label: "Provider name", # optional label for login button, defaults to "Azure AD" + args: { ++ name: "azure_oauth2", ++ strategy_class: "OmniAuth::Strategies::OpenIDConnect", ++ scope: ["openid", "profile", "email"], ++ response_type: "code", ++ issuer: "https://login.microsoftonline.com//v2.0", ++ client_auth_method: "query", ++ discovery: true, ++ uid_field: "sub", ++ client_options: { ++ identifier: "", ++ secret: "", ++ redirect_uri: "https://gitlab.example.com/users/auth/azure_oauth2/callback" ++ } +- client_id: "", +- client_secret: "", +- tenant_id: "", + } + } +] +``` + +- **For installations from source** + +```diff + - { name: 'azure_oauth2', + # label: 'Provider name', # optional label for login button, defaults to "Azure AD" +- args: { client_id: '', +- client_secret: '', +- tenant_id: '' } } ++ icon: "", ++ args: { ++ name: "azure_oauth2", ++ strategy_class: "OmniAuth::Strategies::OpenIDConnect", ++ scope: ["openid","profile","email"], ++ response_type: "code", ++ issuer: 'https://login.microsoftonline.com//v2.0', ++ discovery: true, ++ client_auth_method: 'query', ++ uid_field: 'sub', ++ send_scope_to_token_endpoint: "false", ++ client_options: { ++ identifier: "", ++ secret: "", ++ redirect_uri: "/users/auth/azure_oauth2/callback" ++ } ++ } + } +``` + +To migrate for example from `omniauth-azure-activedirectory-v2` to `omniauth_openid_connect` you +must change the configuration: + +- **For Omnibus installations** + +```diff +gitlab_rails['omniauth_providers'] = [ + { + - name: "azure_activedirectory_v2", + # label: "Provider name", # optional label for login button, defaults to "Azure AD v2" + args: { ++ name: "azure_activedirectory_v2", ++ strategy_class: "OmniAuth::Strategies::OpenIDConnect", ++ scope: ["openid", "profile", "email"], ++ response_type: "code", ++ issuer: "https://login.microsoftonline.com//v2.0", ++ client_auth_method: "query", ++ discovery: true, ++ uid_field: "oid", ++ client_options: { ++ identifier: "", ++ secret: "", ++ redirect_uri: "https://gitlab.example.com/users/auth/azure_activedirectory_v2/callback" ++ } +- client_id: "", +- client_secret: "", +- tenant_id: "", + } + } +] +``` + +- **For installations from source** + +```diff + - { name: 'azure_activedirectory_v2', + # label: 'Provider name', # optional label for login button, defaults to "Azure AD v2" +- args: { client_id: '', +- client_secret: '', +- tenant_id: '' } } ++ icon: "", ++ args: { ++ name: "azure_activedirectory_v2", ++ strategy_class: "OmniAuth::Strategies::OpenIDConnect", ++ scope: ["openid","profile","email"], ++ response_type: "code", ++ issuer: 'https://login.microsoftonline.com//v2.0', ++ discovery: true, ++ client_auth_method: 'query', ++ uid_field: 'oid', ++ send_scope_to_token_endpoint: "false", ++ client_options: { ++ identifier: "", ++ secret: "", ++ redirect_uri: "/users/auth/azure_activedirectory_v2/callback" ++ } ++ } + } +``` + +For more information on other customizations, see [`gitlab_username_claim`](index.md#authentication-sources). + ## Register an Azure application To enable the Microsoft Azure OAuth 2.0 OmniAuth provider, you must register @@ -68,7 +202,7 @@ Alternatively, add the `User.Read.All` application permission. sudo -u git -H editor config/gitlab.yml ``` -1. Edit the [common configuration file settings](omniauth.md#configure-common-settings) +1. Configure the [common settings](omniauth.md#configure-common-settings) to add `azure_oauth2` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. diff --git a/doc/integration/cas.md b/doc/integration/cas.md index 750c9aeb8a4..d2a29161a53 100644 --- a/doc/integration/cas.md +++ b/doc/integration/cas.md @@ -2,79 +2,12 @@ stage: Manage group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-15' +redirect_to: '../administration/auth/index.md' --- -# CAS OmniAuth provider (deprecated) **(FREE SELF)** +# CAS OmniAuth provider (removed) **(FREE SELF)** -WARNING: -This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/369127) in GitLab 15.3 and is planned for -removal in 16.0. - -To enable the CAS OmniAuth provider you must register your application with your -CAS instance. This requires the service URL GitLab supplies to CAS. It should be -something like: `https://gitlab.example.com:443/users/auth/cas3/callback?url`. -Handling for Single Logout (SLO) is enabled by default, so you only have to -configure CAS for back-channel logout. - -1. On your GitLab server, open the configuration file. - - For Omnibus package: - - ```shell - sudo editor /etc/gitlab/gitlab.rb - ``` - - For installations from source: - - ```shell - cd /home/git/gitlab - - sudo -u git -H editor config/gitlab.yml - ``` - -1. Edit the [common configuration file settings](omniauth.md#configure-common-settings) - to add `cas3` as a single sign-on provider. This enables Just-In-Time - account provisioning for users who do not have an existing GitLab account. - -1. Add the provider configuration: - - For Omnibus package: - - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - name: "cas3", - label: "Provider name", # optional label for login button, defaults to "Cas3" - args: { - url: "CAS_SERVER", - login_url: "/CAS_PATH/login", - service_validate_url: "/CAS_PATH/p3/serviceValidate", - logout_url: "/CAS_PATH/logout" - } - } - ] - ``` - - For installations from source: - - ```yaml - - { name: 'cas3', - label: 'Provider name', # optional label for login button, defaults to "Cas3" - args: { - url: 'CAS_SERVER', - login_url: '/CAS_PATH/login', - service_validate_url: '/CAS_PATH/p3/serviceValidate', - logout_url: '/CAS_PATH/logout' } } - ``` - -1. Change 'CAS_PATH' to the root of your CAS instance (such as `cas`). - -1. If your CAS instance does not use default TGC lifetimes, update the `cas3.session_duration` to at least the current TGC maximum lifetime. To explicitly disable SLO, regardless of CAS settings, set this to 0. - -1. Save the configuration file. - -1. For the changes to take effect: - - If you installed via Omnibus, [reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure). - - If you installed from source, [restart GitLab](../administration/restart_gitlab.md#installations-from-source). - -On the sign in page there should now be a CAS tab in the sign in form. +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/369127) +in GitLab 15.3 and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/369128) +in 16.0. diff --git a/doc/integration/datadog.md b/doc/integration/datadog.md index 8b64e3898f9..edae3d0f9bd 100644 --- a/doc/integration/datadog.md +++ b/doc/integration/datadog.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/270123) in GitLab 14.1 -This integration enables you to send CI/CD pipeline and job information to +The Datadog integration enables you to send CI/CD pipeline and job information to [Datadog](https://www.datadoghq.com/). The [Datadog CI Visibility](https://app.datadoghq.com/ci) product helps you monitor for job failures and performance issues, then troubleshoot them. It's based on [Webhooks](../user/project/integrations/webhooks.md), @@ -53,4 +53,4 @@ section of your Datadog account. ## Related topics -- [Datadog CI Visibility](https://docs.datadoghq.com/continuous_integration/) documentation. +- [Datadog CI Visibility documentation](https://docs.datadoghq.com/continuous_integration/) diff --git a/doc/integration/ding_talk.md b/doc/integration/ding_talk.md index ca939dc9f9a..97ffab146a0 100644 --- a/doc/integration/ding_talk.md +++ b/doc/integration/ding_talk.md @@ -6,7 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # DingTalk OAuth 2.0 OmniAuth provider **(FREE SELF)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/341898) in GitLab 14.5. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/341898) in GitLab 14.5. +> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/390855) in GitLab 15.10. You can sign in to GitLab using your DingTalk account. Sign in to DingTalk Open Platform and create an application on it. DingTalk generates a client ID and secret key for you to use. @@ -51,7 +52,7 @@ Sign in to DingTalk Open Platform and create an application on it. DingTalk gene sudo -u git -H editor config/gitlab.yml ``` -1. Edit the [common configuration file settings](omniauth.md#configure-common-settings) +1. Configure the [common settings](omniauth.md#configure-common-settings) to add `dingtalk` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. diff --git a/doc/integration/external-issue-tracker.md b/doc/integration/external-issue-tracker.md index a3c206176b9..c63c2e3fd24 100644 --- a/doc/integration/external-issue-tracker.md +++ b/doc/integration/external-issue-tracker.md @@ -1,10 +1,10 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# External issue tracker **(FREE)** +# External issue trackers **(FREE)** GitLab has an [issue tracker](../user/project/issues/index.md), but you can configure an external issue tracker per GitLab project. diff --git a/doc/integration/facebook.md b/doc/integration/facebook.md index 8b7bdeaa177..aeea798715f 100644 --- a/doc/integration/facebook.md +++ b/doc/integration/facebook.md @@ -31,7 +31,7 @@ Facebook. Facebook generates an app ID and secret key for you to use. 1. Choose "Next" -1. Choose "Skip Quick Start" in the upper right corner +1. In the upper-right corner, select **Skip Quick Start**. 1. Choose "Settings" in the menu on the left @@ -72,7 +72,7 @@ Facebook. Facebook generates an app ID and secret key for you to use. sudo -u git -H editor config/gitlab.yml ``` -1. Edit the [common configuration file settings](omniauth.md#configure-common-settings) +1. Configure the [common settings](omniauth.md#configure-common-settings) to add `facebook` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. diff --git a/doc/integration/github.md b/doc/integration/github.md index 7b8927054ee..00303754d85 100644 --- a/doc/integration/github.md +++ b/doc/integration/github.md @@ -35,7 +35,7 @@ your website could enable the covert redirect attack. ## Enable GitHub OAuth in GitLab -1. Edit the [common configuration file settings](omniauth.md#configure-common-settings) +1. Configure the [common settings](omniauth.md#configure-common-settings) to add `github` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. diff --git a/doc/integration/gitlab.md b/doc/integration/gitlab.md index 168d55b0fa5..59f122a5110 100644 --- a/doc/integration/gitlab.md +++ b/doc/integration/gitlab.md @@ -51,7 +51,7 @@ GitLab.com generates an application ID and secret key for you to use. sudo -u git -H editor config/gitlab.yml ``` -1. Edit the [common configuration file settings](omniauth.md#configure-common-settings) +1. Configure the [common settings](omniauth.md#configure-common-settings) to add `gitlab` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. 1. Add the provider configuration: diff --git a/doc/integration/gitpod.md b/doc/integration/gitpod.md index ee8f46e73df..0ba227c2a85 100644 --- a/doc/integration/gitpod.md +++ b/doc/integration/gitpod.md @@ -1,7 +1,7 @@ --- type: reference, how-to stage: Create -group: Editor +group: IDE info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments" --- @@ -69,6 +69,6 @@ You can launch Gitpod directly from GitLab in one of these ways: 1. Select **Open in Gitpod**. - *From a merge request:* 1. Go to your merge request. - 1. In the upper right corner, select **Code**, then select **Open in Gitpod**. + 1. In the upper-right corner, select **Code**, then select **Open in Gitpod**. Gitpod builds your development environment for your branch. diff --git a/doc/integration/glab/index.md b/doc/integration/glab/index.md index 621472a2083..aae04c36210 100644 --- a/doc/integration/glab/index.md +++ b/doc/integration/glab/index.md @@ -43,19 +43,24 @@ glab mr merge ## Core commands -- `glab alias` -- `glab api` -- `glab auth` -- `glab ci` -- `glab issue` -- `glab label` -- `glab mr` -- `glab project` -- `glab release` -- `glab snippet` -- `glab ssh-key` -- `glab user` -- `glab variable` +- [`glab alias`](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source/alias) +- [`glab api`](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source/api) +- [`glab auth`](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source/auth) +- [`glab check-update`](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source/check-update) +- [`glab ci`](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source/ci) +- [`glab completion`](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source/completion) +- [`glab config`](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source/config) +- [`glab incident`](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source/incident) +- [`glab issue`](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source/issue) +- [`glab label`](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source/label) +- [`glab mr`](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source/mr) +- [`glab release`](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source/release) +- [`glab repo`](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source/repo) +- [`glab schedule`](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source/schedule) +- [`glab snippet`](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source/snippet) +- [`glab ssh-key`](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source/ssh-key) +- [`glab user`](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source/user) +- [`glab variable`](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source/variable) ## Install the CLI @@ -69,15 +74,14 @@ To authenticate with your GitLab account, run `glab auth login`. ## Report issues -Open an issue in the [`gitlab-org/cli` repository](https://gitlab.com/gitlab-org/cli/issues/new) +Open an issue in the [`gitlab-org/cli` repository](https://gitlab.com/gitlab-org/cli/-/issues/new) to send us feedback. ## Related topics - [Install the CLI](https://gitlab.com/gitlab-org/cli/-/blob/main/README.md#installation) - [Documentation](https://gitlab.com/gitlab-org/cli/-/tree/main/docs/source) -- The extension source code is available in the - [`cli`](https://gitlab.com/gitlab-org/cli/) project. +- Extension source code in the [`cli`](https://gitlab.com/gitlab-org/cli/) project ## Troubleshooting diff --git a/doc/integration/gmail_action_buttons_for_gitlab.md b/doc/integration/gmail_action_buttons_for_gitlab.md index 4a233df3899..389d0aeb3f3 100644 --- a/doc/integration/gmail_action_buttons_for_gitlab.md +++ b/doc/integration/gmail_action_buttons_for_gitlab.md @@ -1,14 +1,13 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Gmail actions **(FREE)** GitLab supports [Google actions in email](https://developers.google.com/gmail/markup/actions/actions-overview). - -If correctly set up, emails that require an action are marked in Gmail. +When you configure this integration, emails that require an action are marked in Gmail. ![Gmail actions button](img/gmail_action_buttons_for_gitlab.png) diff --git a/doc/integration/google.md b/doc/integration/google.md index 5eac639f119..d60c1b43ed6 100644 --- a/doc/integration/google.md +++ b/doc/integration/google.md @@ -22,7 +22,7 @@ In Google's side: the randomly generated ID or choose a new one. 1. Refresh the page and you should see your new project in the list 1. Go to the [Google API Console](https://console.developers.google.com/apis/dashboard) -1. Select the previously created project in the upper left corner +1. In the upper-left corner, select the previously created project 1. Select **Credentials** from the sidebar 1. Select **OAuth consent screen** and fill the form with the required information 1. In the **Credentials** tab, select **Create credentials > OAuth client ID** @@ -71,7 +71,7 @@ On your GitLab server: sudo -u git -H editor config/gitlab.yml ``` -1. Edit the [common configuration file settings](omniauth.md#configure-common-settings) +1. Configure the [common settings](omniauth.md#configure-common-settings) to add `google_oauth2` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. 1. Add the provider configuration: diff --git a/doc/integration/index.md b/doc/integration/index.md index 195890ea4d8..f9865199505 100644 --- a/doc/integration/index.md +++ b/doc/integration/index.md @@ -1,8 +1,7 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false --- # Integrate with GitLab **(FREE)** @@ -73,7 +72,7 @@ You can integrate GitLab with the following feature enhancements: or [Kroki](../administration/integration/kroki.md) to use diagrams in AsciiDoc and Markdown documents. - Attach merge requests to [Trello](trello_power_up.md) cards. - Enable integrated code intelligence powered by [Sourcegraph](sourcegraph.md). -- Add [Elasticsearch](advanced_search/elasticsearch.md) for [Advanced Search](../user/search/advanced_search.md). +- Add [Elasticsearch](advanced_search/elasticsearch.md) for [advanced search](../user/search/advanced_search.md). ## Troubleshooting @@ -88,7 +87,7 @@ As a workaround, you can do one of the following: - [Adding trusted root certificates to the server](https://manuals.gfi.com/en/kerio/connect/content/server-configuration/ssl-certificates/adding-trusted-root-certificates-to-the-server-1605.html) - [How do you add a certificate authority (CA) to Ubuntu?](https://superuser.com/questions/437330/how-do-you-add-a-certificate-authority-ca-to-ubuntu) - In Omnibus GitLab, add the certificate to the Omnibus trusted chain: - 1. [Install the self-signed certificate](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). + 1. [Install the self-signed certificate](https://docs.gitlab.com/omnibus/settings/ssl/index.html#install-custom-public-certificates). 1. Concatenate the self-signed certificate with the GitLab trusted certificate. The self-signed certificate might be overwritten during upgrades. diff --git a/doc/integration/jenkins.md b/doc/integration/jenkins.md index 983ff3c54bc..3db02ed1221 100644 --- a/doc/integration/jenkins.md +++ b/doc/integration/jenkins.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -166,7 +166,7 @@ If you get this error message while configuring GitLab, the following are possib - GitLab is unable to reach your Jenkins instance at the address. If your GitLab instance is self-managed, try pinging the Jenkins instance at the domain provided on the GitLab instance. - The Jenkins instance is at a local address and is not included in the - [GitLab installation's allowlist](../security/webhooks.md#create-an-allowlist-for-local-requests). + [GitLab installation's allowlist](../security/webhooks.md#allow-outbound-requests-to-certain-ip-addresses-and-domains). - The credentials for the Jenkins instance do not have sufficient access or are invalid. - The **Enable authentication for `/project` end-point** checkbox is not selected in your [Jenkins plugin configuration](#configure-the-jenkins-server). @@ -190,26 +190,21 @@ This issue can occur when the request exceeds the [webhook timeout](../user/project/integrations/webhooks.md#webhook-fails-or-multiple-webhook-requests-are-triggered), which is set to 10 seconds by default. -Check the [service hook logs](../user/project/integrations/index.md#troubleshooting-integrations) -for request failures or check the `/var/log/gitlab/gitlab-rails/production.log` -file for messages like: +For this issue, check: -```plaintext -WebHook Error => Net::ReadTimeout -``` +- [Integration webhook logs](../user/project/integrations/index.md#troubleshooting) +for request failures. +- `/var/log/gitlab/gitlab-rails/production.log` for messages like: -or + ```plaintext + WebHook Error => Net::ReadTimeout + ``` -```plaintext -WebHook Error => execution expired -``` + or -Or check for duplicate messages in `/var/log/gitlab/gitlab-rail`, like: - -```plaintext -2019-10-25_04:22:41.25630 2019-10-25T04:22:41.256Z 1584 TID-ovowh4tek WebHookWorker JID-941fb7f40b69dff3d833c99b INFO: start -2019-10-25_04:22:41.25630 2019-10-25T04:22:41.256Z 1584 TID-ovowh4tek WebHookWorker JID-941fb7f40b69dff3d833c99b INFO: start -``` + ```plaintext + WebHook Error => execution expired + ``` On self-managed GitLab instances, you can fix this issue by [increasing the webhook timeout value](../administration/instance_limits.md#webhook-timeout). diff --git a/doc/integration/jira/configure.md b/doc/integration/jira/configure.md index 03d742703a1..3f3511c3838 100644 --- a/doc/integration/jira/configure.md +++ b/doc/integration/jira/configure.md @@ -1,26 +1,31 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Configure the Jira integration **(FREE)** +# Jira issue integration **(FREE)** -You can set up the [Jira integration](index.md#jira-integration) -by configuring your project settings in GitLab. -You can also configure these settings at a [group level](../../user/admin_area/settings/project_integration_management.md#manage-group-level-default-settings-for-a-project-integration), -and for self-managed GitLab, at an [instance level](../../user/admin_area/settings/project_integration_management.md#manage-instance-level-default-settings-for-a-project-integration). +The Jira issue integration connects one or more GitLab projects to a Jira instance. You can host the Jira instance yourself or in [Jira Cloud](https://www.atlassian.com/migration/assess/why-cloud). The supported Jira versions are `6.x`, `7.x`, `8.x`, and `9.x`. + +## Configure the integration Prerequisites: -- Ensure your GitLab installation does not use a [relative URL](development_panel.md#limitations). -- For **Jira Server**, ensure you have a Jira username and password. - See [authentication in Jira](index.md#authentication-in-jira). -- For **Jira on Atlassian cloud**, ensure you have an API token - and the email address you used to create the token. - See [authentication in Jira](index.md#authentication-in-jira). +- Your GitLab installation must not use a [relative URL](https://docs.gitlab.com/omnibus/settings/configuration.html#configure-a-relative-url-for-gitlab). +- **For Jira Cloud**, you must have a [Jira Cloud API token](#create-a-jira-cloud-api-token) and + the email address you used to create the token. +- **For Jira Data Center or Jira Server**, you must have one of the following: + - [Jira username and password](jira_server_configuration.md). + - Jira personal access token. + +You can enable the Jira issue integration by configuring your project settings in GitLab. +You can also configure these settings at the: + +- [Group level](../../user/admin_area/settings/project_integration_management.md#manage-group-level-default-settings-for-a-project-integration) +- [Instance level](../../user/admin_area/settings/project_integration_management.md#manage-instance-level-default-settings-for-a-project-integration) for self-managed GitLab -To configure your project: +To configure your project settings in GitLab: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Settings > Integrations**. @@ -34,14 +39,21 @@ To configure your project: [closing reference](../../user/project/issues/managing_issues.md#closing-issues-automatically) is made in GitLab, select **Enable Jira transitions**. 1. Provide Jira configuration information: - - **Web URL**: The base URL to the Jira instance web interface you're linking to - this GitLab project, such as `https://jira.example.com`. - - **Jira API URL**: The base URL to the Jira instance API, such as `https://jira-api.example.com`. - Defaults to the **Web URL** value if not set. Leave blank if using **Jira on Atlassian cloud**. - - **Username or Email**: - For **Jira Server**, use `username`. For **Jira on Atlassian cloud**, use `email`. - - **Password/API token**: - Use `password` for **Jira Server** or `API token` for **Jira on Atlassian cloud**. + - **Web URL**: Base URL for the Jira instance web interface you're linking to + this GitLab project (for example, `https://jira.example.com`). + - **Jira API URL**: Base URL for the Jira instance API (for example, `https://jira-api.example.com`). + If this URL is not set, the **Web URL** value is used by default. For Jira Cloud, leave **Jira API URL** blank. + - **Authentication type**: From the dropdown list, select: + - **Basic** + - **Jira personal access token (Jira Data Center and Jira Server only)** + - **Email or username** (relevant to **Basic** authentication only): + - For Jira Cloud, enter an email. + - For Jira Data Center or Jira Server, enter a username. + - **New API token, password, or Jira personal access token**: + - For **Basic** authentication: + - For Jira Cloud, enter an API token. + - For Jira Data Center or Jira Server, enter a password. + - For **Jira personal access token** authentication, enter a personal access token. 1. To enable users to [view Jira issues](issues.md#view-jira-issues) inside the GitLab project, select **Enable Jira issues** and enter a Jira project key. @@ -52,13 +64,29 @@ To configure your project: can view all issues from the specified Jira project. 1. To enable [issue creation for vulnerabilities](../../user/application_security/vulnerabilities/index.md#create-a-jira-issue-for-a-vulnerability), select **Enable Jira issue creation from vulnerabilities**. -1. Select the **Jira issue type**. If the dropdown list is empty, select refresh (**{retry}**) and try again. +1. Select the **Jira issue type**. If the dropdown list is empty, select **Refresh** (**{retry}**) and try again. 1. To verify the Jira connection is working, select **Test settings**. 1. Select **Save changes**. -Your GitLab project can now interact with all Jira projects in your instance and the project now +Your GitLab project can now interact with all Jira projects in your instance, and the project displays a Jira link that opens the Jira project. +## Create a Jira Cloud API token + +To configure the Jira issue integration for Jira Cloud, you must have a Jira Cloud API token. +To create a Jira Cloud API token: + +1. Sign in to [Atlassian](https://id.atlassian.com/manage-profile/security/api-tokens) + from an account with **write** access to Jira projects. + + The link opens the **API tokens** page. Alternatively, from your Atlassian + profile, select **Account Settings > Security > Create and manage API tokens**. + +1. Select **Create API token**. +1. In the dialog, enter a label for your token and select **Create**. + +To copy the API token, select **Copy** and paste the token somewhere safe. + ## Migrate from Jira Server to Jira Cloud in GitLab To migrate from Jira Server to Jira Cloud in GitLab and maintain your Jira integration: @@ -68,7 +96,9 @@ To migrate from Jira Server to Jira Cloud in GitLab and maintain your Jira integ 1. Select **Jira**. 1. In **Web URL**, enter the new Jira site URL (for example, `https://myjirasite.atlassian.net`). 1. In **Username or Email**, enter the username or email registered on your Jira profile. -1. [Create an API token](jira_cloud_configuration.md), and copy that value. +1. [Create a Jira Cloud API token](#create-a-jira-cloud-api-token), and copy the token value. 1. In **Password or API token**, paste the API token value. 1. Optional. Select **Test settings** to check if the connection is working. 1. Select **Save changes**. + +To update existing Jira issue references in GitLab to use the new Jira site URL, you must [invalidate the Markdown cache](../../administration/invalidate_markdown_cache.md#invalidate-the-cache). diff --git a/doc/integration/jira/connect-app.md b/doc/integration/jira/connect-app.md index 402efc409cb..e60eeb8fba1 100644 --- a/doc/integration/jira/connect-app.md +++ b/doc/integration/jira/connect-app.md @@ -1,69 +1,52 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # GitLab for Jira Cloud app **(FREE)** -You can integrate GitLab and Jira Cloud using the -[GitLab for Jira Cloud](https://marketplace.atlassian.com/apps/1221011/gitlab-com-for-jira-cloud) -app in the Atlassian Marketplace. +With the [GitLab for Jira Cloud](https://marketplace.atlassian.com/apps/1221011/gitlab-com-for-jira-cloud) app, you can connect GitLab and Jira Cloud to sync development information in real time. You can view this information in the [Jira development panel](development_panel.md). -Only Jira users with administrator access can install or configure -the GitLab for Jira Cloud app. +You can use the GitLab for Jira Cloud app to link top-level groups or subgroups. It's not possible to directly link projects or personal namespaces. -## Install the GitLab for Jira Cloud app **(FREE SAAS)** +- **For GitLab.com**: + - [Install the GitLab for Jira Cloud app](#install-the-gitlab-for-jira-cloud-app). +- **For self-managed GitLab**, do one of the following: + - [Connect the GitLab for Jira Cloud app for self-managed instances](#connect-the-gitlab-for-jira-cloud-app-for-self-managed-instances) (GitLab 15.7 and later). + - [Install the GitLab for Jira Cloud app manually](#install-the-gitlab-for-jira-cloud-app-manually). -If you use GitLab.com and Jira Cloud, you can install the GitLab for Jira Cloud app. -If you do not use both of these environments, use the [Jira DVCS Connector](dvcs/index.md) or -[install the GitLab for Jira Cloud app manually](#install-the-gitlab-for-jira-cloud-app-manually). -We recommend the GitLab for Jira Cloud app, because data is -synchronized in real time. The DVCS connector updates data only once per hour. +If you use Jira Data Center or Jira Server, use the [Jira DVCS connector](dvcs/index.md) instead. -To configure the GitLab for Jira Cloud app, you must have -at least the Maintainer role in the GitLab.com namespace. +## Install the GitLab for Jira Cloud app **(FREE SAAS)** -This integration method supports [Smart Commits](dvcs/index.md#smart-commits). +Prerequisites: - -For a walkthrough of the integration with GitLab for Jira Cloud app, watch -[Configure GitLab.com Jira Could Integration using Marketplace App](https://youtu.be/SwR-g1s1zTo) on YouTube. +- You must have at least the Maintainer role for the GitLab group. +- You must have administrator access to the Jira instance. +- Your network must allow inbound and outbound connections between GitLab and Jira. To install the GitLab for Jira Cloud app: -1. In Jira, go to **Jira Settings > Apps > Find new apps**, then search for GitLab. -1. Select **GitLab for Jira Cloud**, then select **Get it now**, or go to the - [App in the marketplace directly](https://marketplace.atlassian.com/apps/1221011/gitlab-com-for-jira-cloud). - - ![Install GitLab.com app on Jira Cloud](img/jira_dev_panel_setup_com_1.png) -1. After installing, to go to the configurations page, select **Get started**. - This page is always available under **Jira Settings > Apps > Manage apps**. - - ![Start GitLab.com app configuration on Jira Cloud](img/jira_dev_panel_setup_com_2.png) -1. To add namespaces, ensure you're signed in to GitLab.com - as a user with at least the Maintainer role. - - ![Sign in to GitLab.com in GitLab for Jira Cloud app](img/jira_dev_panel_setup_com_3_v13_9.png) -1. To open the list of available namespaces, select **Add namespace**. - -1. Identify the namespace you want to link, and select **Link**. - - You must have at least the Maintainer role for the namespace. - - Only Jira site administrators can add or remove namespaces for an installation. +1. In Jira, select **Jira Settings > Apps > Find new apps**, and search for GitLab. +1. Select **GitLab for Jira Cloud**, and select **Get it now**. - ![Link namespace in GitLab for Jira Cloud app](img/jira_dev_panel_setup_com_4_v13_9.png) + Alternatively, [get the app directly from the Atlassian Marketplace](https://marketplace.atlassian.com/apps/1221011/gitlab-com-for-jira-cloud). -NOTE: -The GitLab.com user only needs access when adding a new namespace. For syncing with -Jira, we do not depend on the user's token. +1. To go to the configurations page, select **Get started**. + You can always access this page in **Jira Settings > Apps > Manage apps**. +1. For a list of groups to link, select **Add namespace**. +1. To link to a group, select **Link**. -After a namespace is added: + +For an overview, see +[Configure the GitLab for Jira Cloud app from the Atlassian Marketplace](https://youtu.be/SwR-g1s1zTo). -- All future commits, branches, and merge requests of all projects under that namespace - are synced to Jira. -- From GitLab 13.8, past merge request data is synced to Jira. +After you add a group, the following data is synced to Jira for all projects in that group: -Support for syncing past branch and commit data is tracked [in this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/263240). +- New merge requests, branches, and commits +- Existing merge requests (GitLab 13.8 and later) +- Existing branches and commits (GitLab 15.11 and later) ## Update the GitLab for Jira Cloud app @@ -73,15 +56,15 @@ for details. If the app requires additional permissions, [the update must first be manually approved in Jira](https://developer.atlassian.com/platform/marketplace/upgrading-and-versioning-cloud-apps/#changes-that-require-manual-customer-approval). -## Set up OAuth authentication +## Set up OAuth authentication for self-managed instances **(FREE SELF)** The GitLab for Jira Cloud app is [switching to OAuth authentication](https://gitlab.com/gitlab-org/gitlab/-/issues/387299). To enable OAuth authentication, you must create an OAuth application on the GitLab instance. -Enabling OAuth authentication is: +You must enable OAuth authentication to: -- Required to [connect the GitLab for Jira Cloud app for self-managed instances](#connect-the-gitlab-for-jira-cloud-app-for-self-managed-instances). -- Recommended to [install the GitLab for Jira Cloud app manually](#install-the-gitlab-for-jira-cloud-app-manually). +- [Connect the GitLab for Jira Cloud app for self-managed instances](#connect-the-gitlab-for-jira-cloud-app-for-self-managed-instances). +- [Install the GitLab for Jira Cloud app manually](#install-the-gitlab-for-jira-cloud-app-manually). To create an OAuth application: @@ -96,42 +79,50 @@ To create an OAuth application: 1. Select **Save application**. 1. Copy the **Application ID** value. 1. On the left sidebar, select **Settings > General** (`/admin/application_settings/general`). -1. Expand the **GitLab for Jira App** section. +1. Expand **GitLab for Jira App**. 1. Paste the **Application ID** value into **Jira Connect Application ID**. 1. Select **Save changes**. -1. Optional. Enable the `jira_connect_oauth` [feature flag](../../administration/feature_flags.md) to avoid [authentication problems in some browsers](#browser-displays-a-sign-in-message-when-already-signed-in). ## Connect the GitLab for Jira Cloud app for self-managed instances **(FREE SELF)** > Introduced in GitLab 15.7. -Prerequisites: - -- GitLab.com must serve as a proxy for the instance. -- The instance must be publicly available. -- The instance must be on version 15.7 or later. - You can link self-managed instances after installing the GitLab for Jira Cloud app from the marketplace. Jira apps can only link to one URL per marketplace listing. The official listing links to GitLab.com. -If your instance doesn't meet the prerequisites or you don't want to use the official marketplace listing, you can +NOTE: +With this method, GitLab.com serves as a proxy for Jira traffic from your instance. + +If your instance doesn't meet the [prerequisites](#prerequisites) or you don't want to use the official marketplace listing, you can [install the app manually](#install-the-gitlab-for-jira-cloud-app-manually). -It's not possible to create branches from Jira for self-managed instances. +With this method, it's not possible to create branches from Jira Cloud for self-managed instances. +For more information, see [issue 391432](https://gitlab.com/gitlab-org/gitlab/-/issues/391432). +To create branches from Jira Cloud, [install the app manually](#install-the-gitlab-for-jira-cloud-app-manually). + +### Prerequisites + +- The instance must be publicly available. +- The instance must be on GitLab version 15.7 or later. +- You must set up [OAuth authentication](#set-up-oauth-authentication-for-self-managed-instances). +- Your network must allow inbound and outbound connections between GitLab and Jira. ### Set up your instance +[Prerequisites](#prerequisites) + To set up your self-managed instance for the GitLab for Jira Cloud app in GitLab 15.7 and later: -1. [Set up OAuth authentication](#set-up-oauth-authentication). 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Settings > General** (`/admin/application_settings/general`). -1. Expand the **GitLab for Jira App** section. +1. Expand **GitLab for Jira App**. 1. In **Jira Connect Proxy URL**, enter `https://gitlab.com`. 1. Select **Save changes**. ### Link your instance +[Prerequisites](#prerequisites) + To link your self-managed instance to the GitLab for Jira Cloud app: 1. Install the [GitLab for Jira Cloud app](https://marketplace.atlassian.com/apps/1221011/gitlab-com-for-jira-cloud?tab=overview&hosting=cloud). @@ -144,13 +135,6 @@ To link your self-managed instance to the GitLab for Jira Cloud app: If your GitLab instance is self-managed and you don't want to use the official marketplace listing, you can install the app manually. -Prerequisites: - -- The instance must be publicly available. -- You must set up [OAuth authentication](#set-up-oauth-authentication). - -### Set up your Jira app - Each Jira Cloud application must be installed from a single location. Jira fetches a [manifest file](https://developer.atlassian.com/cloud/jira/platform/connect-app-descriptor/) from the location you provide. The manifest file describes the application to the system. To support @@ -159,30 +143,30 @@ self-managed GitLab instances with Jira Cloud, you can do one of the following: - [Install the application in development mode](#install-the-application-in-development-mode). - [Create a Marketplace listing](#create-a-marketplace-listing). -#### Install the application in development mode +### Prerequisites + +- The instance must be publicly available. +- You must set up [OAuth authentication](#set-up-oauth-authentication-for-self-managed-instances). -You can configure your Atlassian Cloud instance to allow you to install applications -from outside the Marketplace, which allows you to install the application: +### Install the application in development mode + +[Prerequisites](#prerequisites-1) + +To configure your Jira instance so you can install applications +from outside the Marketplace: 1. Sign in to your Jira instance as an administrator. 1. Place your Jira instance into [development mode](https://developer.atlassian.com/cloud/jira/platform/getting-started-with-connect/#step-2--enable-development-mode). 1. Sign in to your GitLab application as a user with administrator access. -1. Install the GitLab application from your Jira instance, as - described in the [Atlassian developer guides](https://developer.atlassian.com/cloud/jira/platform/getting-started-with-connect/#step-3--install-and-test-your-app): +1. Install the GitLab application from your Jira instance as + described in the [Atlassian developer guide](https://developer.atlassian.com/cloud/jira/platform/getting-started-with-connect/#step-3--install-and-test-your-app): 1. In your Jira instance, go to **Apps > Manage Apps** and select **Upload app**: - - ![Button labeled "upload app"](img/jira-upload-app_v13_11.png) - - 1. For **App descriptor URL**, provide the full URL to your manifest file, based + 1. For **App descriptor URL**, provide the full URL to your manifest file based on your instance configuration. By default, your manifest file is located at `/-/jira_connect/app_descriptor.json`. For example, if your GitLab self-managed instance domain is `app.pet-store.cloud`, your manifest file is located at `https://app.pet-store.cloud/-/jira_connect/app_descriptor.json`. 1. Select **Upload**. Jira fetches the content of your `app_descriptor` file and installs it. - 1. If the upload is successful, Jira displays a modal panel: **Installed and ready to go!** - To configure the integration, select **Get started**. - - ![Success modal](img/jira-upload-app-success_v13_11.png) - + 1. To configure the integration, select **Get started**. 1. Disable [development mode](https://developer.atlassian.com/cloud/jira/platform/getting-started-with-connect/#step-2--enable-development-mode) on your Jira instance. The **GitLab for Jira Cloud** app now displays under **Manage apps**. You can also @@ -192,19 +176,20 @@ NOTE: If a GitLab update makes changes to the application descriptor, you must uninstall, then reinstall the application. -#### Create a Marketplace listing +### Create a Marketplace listing + +[Prerequisites](#prerequisites-1) -If you prefer to not use development mode on your Jira instance, you can create -your own Marketplace listing for your instance. This enables your application -to be installed from the Atlassian Marketplace. +If you don't want to use development mode on your Jira instance, you can create +your own Marketplace listing. This way, your application +can be installed from the Atlassian Marketplace. -For full instructions, review the Atlassian [guide to creating a marketplace listing](https://developer.atlassian.com/platform/marketplace/installing-cloud-apps/#creating-the-marketplace-listing). To create a Marketplace listing: 1. Register as a Marketplace vendor. -1. List your application using the application descriptor URL. +1. List your application with the application descriptor URL. - Your manifest file is located at: `https://your.domain/your-path/-/jira_connect/app_descriptor.json` - - We recommend you list your application as `private`, because public + - You should list your application as `private` because public applications can be viewed and installed by any user. 1. Generate test license tokens for your application. @@ -212,22 +197,51 @@ NOTE: This method uses [automated updates](#update-the-gitlab-for-jira-cloud-app) the same way as our GitLab.com Marketplace listing. -## Configure your GitLab instance to serve as a proxy for the GitLab for Jira Cloud app +For more information about creating a Marketplace listing, see the [Atlassian documentation](https://developer.atlassian.com/platform/marketplace/installing-cloud-apps/#creating-the-marketplace-listing). -A GitLab instance can serve as a proxy for other GitLab instances using the GitLab for Jira Cloud app. -This can be useful if you are managing multiple GitLab instance but only want to [manually install](#install-the-gitlab-for-jira-cloud-app-manually) -the GitLab for Jira app once. +## Configure your GitLab instance to serve as a proxy for the GitLab for Jira Cloud app **(FREE SELF)** + +A GitLab instance can serve as a proxy for other GitLab instances through the GitLab for Jira Cloud app. +You might want to use a proxy if you're managing multiple GitLab instances but only want to +[manually install](#install-the-gitlab-for-jira-cloud-app-manually) the GitLab for Jira Cloud app once. To configure your GitLab instance to serve as a proxy: 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Settings > General** (`/admin/application_settings/general`). -1. Expand the **GitLab for Jira App** section. +1. Expand **GitLab for Jira App**. 1. Select **Enable public key storage**. 1. Select **Save changes**. -1. [Install the GitLab for Jira Cloud app manually](#install-the-gitlab-for-jira-cloud-app-manually) +1. [Install the GitLab for Jira Cloud app manually](#install-the-gitlab-for-jira-cloud-app-manually). + +Other GitLab instances that use the proxy must configure the **Jira Connect Proxy URL** and the [OAuth application](#set-up-oauth-authentication-for-self-managed-instances) **Redirect URI** settings to point to the proxy instance. + +## Security considerations + +The GitLab for Jira Cloud app connects GitLab and Jira. Data must be shared between the two applications, and access must be granted in both directions. + +### Access to GitLab through OAuth **(FREE SELF)** + +GitLab does not share an access token with Jira. However, users must authenticate through OAuth to configure the app. + +An access token is retrieved through a [PKCE](https://www.rfc-editor.org/rfc/rfc7636) OAuth flow and stored only on the client side. +The app frontend that initializes the OAuth flow is a JavaScript application that's loaded from GitLab through an iframe on Jira. + +The OAuth application must have the `api` scope, which grants complete read and write access to the API. +This access includes all groups and projects, the container registry, and the package registry. +However, the GitLab for Jira Cloud app only uses this access to: + +- Display groups to link. +- Link groups. + +Access through OAuth is only needed for the time a user configures the GitLab for Jira Cloud app. For more information, see [Access token expiration](../oauth_provider.md#access-token-expiration). + +### Access to Jira through access token -Other GitLab instances using the proxy must configure the **Jira Connect Proxy URL** setting and the [OAuth application](#set-up-oauth-authentication) **Redirect URI** to point to the proxy instance. +Jira shares an access token with GitLab to authenticate and authorize data pushes to Jira. +As part of the app installation process, Jira sends a handshake request to GitLab containing the access token. +The handshake is signed with an [asymmetric JWT](https://developer.atlassian.com/cloud/jira/platform/understanding-jwt-for-connect-apps/), +and the access token is stored encrypted with `AES256-GCM` on GitLab. ## Troubleshooting @@ -240,14 +254,20 @@ when you're already signed in: You need to sign in or sign up before continuing. ``` -The GitLab for Jira Cloud app uses an iframe to add namespaces on the +The GitLab for Jira Cloud app uses an iframe to add groups on the settings page. Some browsers block cross-site cookies, which can lead to this issue. -To resolve this issue, set up [OAuth authentication](#set-up-oauth-authentication) and enable the `jira_connect_oauth` [feature flag](../../administration/feature_flags.md). +To resolve this issue, set up [OAuth authentication](#set-up-oauth-authentication-for-self-managed-instances). ### Manual installation fails -You might get an error if you have installed the GitLab for Jira Cloud app from the official marketplace listing and replaced it with manual installation. To resolve this issue, disable the **Jira Connect Proxy URL** setting. +You might get an error if you have installed the GitLab for Jira Cloud app from the official marketplace listing and replaced it with manual installation: + +```plaintext +The app "gitlab-jira-connect-gitlab.com" could not be installed as a local app as it has previously been installed from Atlassian Marketplace +``` + +To resolve this issue, disable the **Jira Connect Proxy URL** setting. - In GitLab 15.7: @@ -258,7 +278,7 @@ You might get an error if you have installed the GitLab for Jira Cloud app from 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Settings > General** (`/admin/application_settings/general`). - 1. Expand the **GitLab for Jira App** section. + 1. Expand **GitLab for Jira App**. 1. Clear the **Jira Connect Proxy URL** text box. 1. Select **Save changes**. @@ -282,6 +302,37 @@ To resolve this issue on GitLab self-managed, follow one of the solutions below, - If you [installed the GitLab for Jira Cloud app manually](#install-the-gitlab-for-jira-cloud-app-manually): - In GitLab 14.9 and later: - - Contact the [Jira Software Cloud support](https://support.atlassian.com/jira-software-cloud/) and ask to trigger a new installed lifecycle event for the GitLab for Jira Cloud app in your namespace. + - Contact the [Jira Software Cloud support](https://support.atlassian.com/jira-software-cloud/) and ask to trigger a new installed lifecycle event for the GitLab for Jira Cloud app in your group. - In all GitLab versions: - - Re-install the GitLab for Jira Cloud app. This might remove all already synced development panel data. + - Re-install the GitLab for Jira Cloud app. This method might remove all synced data from the Jira development panel. + +### `Failed to update GitLab version` error when setting up the GitLab for Jira Cloud app for self-managed instances + +When you set up the GitLab for Jira Cloud app, you might get the following message after you enter your +self-managed instance URL: + +```plaintext +Failed to update GitLab version. Please try again. +``` + +To resolve this issue, ensure all prerequisites for your installation method have been met: + +- [Prerequisites for connecting the GitLab for Jira Cloud app](#prerequisites) +- [Prerequisites for installing the GitLab for Jira Cloud app manually](#prerequisites-1) + +If you're using GitLab 15.8 and earlier and have previously enabled both the `jira_connect_oauth_self_managed` +and the `jira_connect_oauth` feature flags, you must disable the `jira_connect_oauth_self_managed` flag +due to a [known issue](https://gitlab.com/gitlab-org/gitlab/-/issues/388943). To check for these flags: + +1. Open a [Rails console](../../administration/operations/rails_console.md#starting-a-rails-console-session). +1. Execute the following code: + + ```ruby + # Check if both feature flags are enabled. + # If the flags are enabled, these commands return `true`. + Feature.enabled?(:jira_connect_oauth) + Feature.enabled?(:jira_connect_oauth_self_managed) + + # If both flags are enabled, disable the `jira_connect_oauth_self_managed` flag. + Feature.disable(:jira_connect_oauth_self_managed) + ``` diff --git a/doc/integration/jira/development_panel.md b/doc/integration/jira/development_panel.md index f36b9f2c4c1..009e620f121 100644 --- a/doc/integration/jira/development_panel.md +++ b/doc/integration/jira/development_panel.md @@ -1,102 +1,99 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# GitLab Jira development panel integration **(FREE)** +# Jira development panel **(FREE)** > [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/233149) from GitLab Premium to GitLab Free in 13.4. -With the Jira development panel integration, you can reference Jira issues in GitLab. -When configured, activity (such as pipeline, deployment, and feature flags) displays in the Jira issue's -[development panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/). -From the development panel, you can open a detailed view and -[take various actions](#use-the-integration), including creating a new merge request from a branch: +You can use the Jira development panel to view GitLab activity for a Jira issue directly in Jira. +To set up the Jira development panel: -![Branch, Commit and Pull Requests links on Jira issue](img/jira_dev_panel_jira_setup_3.png) +- **For Jira Cloud**, use the [GitLab for Jira Cloud app](connect-app.md) developed by GitLab. +- **For Jira Data Center or Jira Server**, use the [Jira DVCS connector](dvcs/index.md) developed by Atlassian. -The information displayed in the Jira development panel depends on where you mention the Jira issue ID: - -| Your mention of Jira issue ID in GitLab context | Automated effect in Jira issue | -|---------------------------------------------------|--------------------------------------------------------------------------------------------------------| -| In a merge request title or description | Link to the MR is displayed in the development panel. | -| In a branch name | Link to the branch is displayed in the development panel. | -| In a commit message | Link to the commit is displayed in the development panel. | -| In a commit message with Jira [Smart Commits](https://confluence.atlassian.com/fisheye/using-smart-commits-960155400.html) | Displays your custom comment or logged time spent and/or performs specified issue transition on merge. | - -This integration connects all GitLab projects to projects in the Jira instance in either: - -- A top-level GitLab group: Connects the projects in a group with no parent group, - including the projects in its subgroups. -- A personal namespace: Connects the projects in that personal namespace to Jira. + +For an overview, see [Jira development panel integration](https://www.youtube.com/watch?v=VjVTOmMl85M). -## Use the integration +## Feature availability -After the integration is [set up on GitLab and Jira](#configure-the-integration), you can: +This table shows the features available with the Jira DVCS connector and the GitLab for Jira Cloud app: -- Refer to any Jira issue by its ID (in uppercase) in GitLab branch names, - commit messages, and merge request titles. -- See the linked branches, commits, and merge requests in Jira issues. -- Create GitLab branches from Jira Cloud issues ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/66032) in GitLab 14.2 for the GitLab for Jira app). +| Feature | Jira DVCS connector | GitLab for Jira Cloud app | +|---------------------|------------------------|---------------------------| +| Smart Commits | **{check-circle}** Yes | **{check-circle}** Yes | +| Sync merge requests | **{check-circle}** Yes | **{check-circle}** Yes | +| Sync branches | **{check-circle}** Yes | **{check-circle}** Yes | +| Sync commits | **{check-circle}** Yes | **{check-circle}** Yes | +| Sync existing data | **{check-circle}** Yes | **{check-circle}** Yes | +| Sync builds | **{dotted-circle}** No | **{check-circle}** Yes | +| Sync deployments | **{dotted-circle}** No | **{check-circle}** Yes | +| Sync feature flags | **{dotted-circle}** No | **{check-circle}** Yes | +| Sync interval | Up to 60 minutes | Real time | +| Create branches | **{dotted-circle}** No | **{check-circle}** Yes (GitLab SaaS only) | +| Create merge request from branch | **{check-circle}** Yes | **{check-circle}** Yes | +| Create branch from Jira issue | **{dotted-circle}** No | **{check-circle}** Yes ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/66032) in GitLab 14.2) | -At this time, merge requests are called "pull requests" in Jira issues. -This name may change in a future Jira release. +## Connected projects in GitLab -Select the links to see your GitLab repository data. +The Jira development panel connects a Jira instance with all its projects to the following: -![GitLab commits details on a Jira issue](img/jira_dev_panel_jira_setup_4.png) +- **For the [GitLab for Jira Cloud app](connect-app.md)**, linked GitLab groups or subgroups and their projects +- **For the [Jira DVCS connector](dvcs/index.md)**, linked GitLab groups, subgroups, or personal namespaces and their projects -![GitLab merge requests details on a Jira issue](img/jira_dev_panel_jira_setup_5.png) +## Information displayed in the development panel -### Use Jira Smart Commits +You can [view GitLab activity for a Jira issue](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/) +in the Jira development panel by referring to the Jira issue by ID in GitLab. The information displayed in the development panel +depends on where you mention the Jira issue ID in GitLab. -With Jira [Smart Commits](https://confluence.atlassian.com/fisheye/using-smart-commits-960155400.html), -you can use GitLab to add Jira comments, log time spent on the issue, or apply any issue transition. +| GitLab: where you mention the Jira issue ID | Jira development panel: what information is displayed | +|------------------------------------------------|-------------------------------------------------------| +| Merge request title or description | Link to the merge request
Link to the branch ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/354373) in GitLab 15.11) | +| Branch name | Link to the branch | +| Commit message | Link to the commit | +| [Jira Smart Commit](#jira-smart-commits) | Custom comment, logged time, or workflow transition | -For more information about using Jira Smart Commits to track time against an issue, specify -an issue transition, or add a custom comment, read the Atlassian page -[Using Smart Commits](https://confluence.atlassian.com/fisheye/using-smart-commits-960155400.html). +## Jira Smart Commits -## Configure the integration +Prerequisites: - -For an overview of how to configure the Jira development panel integration, see -[Agile Management - GitLab Jira development panel integration](https://www.youtube.com/watch?v=VjVTOmMl85M). +- You must have GitLab and Jira user accounts with the same email address or username. +- The commands must be in the first line of the commit message. +- The commit message must not span more than one line. -To simplify administration, we recommend that a GitLab group maintainer or group owner -(or, if possible, instance administrator in the case of self-managed GitLab) set up the integration. +Jira Smart Commits are special commands to process a Jira issue. With these commands, you can use GitLab to: -| Jira usage | GitLab.com customers need | GitLab self-managed customers need | -|------------|---------------------------|------------------------------------| -| [Atlassian cloud](https://www.atlassian.com/migration/assess/why-cloud) | The [GitLab for Jira Cloud app](https://marketplace.atlassian.com/apps/1221011/gitlab-com-for-jira-cloud?hosting=cloud&tab=overview) from the [Atlassian Marketplace](https://marketplace.atlassian.com). This method offers real-time sync between GitLab.com and Jira. The method requires inbound connections for the setup and then pushes data to Jira through outbound connections. For more information, see [GitLab for Jira Cloud app](connect-app.md). | The GitLab for Jira Cloud app [installed manually](connect-app.md#install-the-gitlab-for-jira-cloud-app-manually). By default, you can install the app from the [Atlassian Marketplace](https://marketplace.atlassian.com/). The method requires inbound connections for the setup and then pushes data to Jira through outbound connections. For more information, see [Connect the GitLab for Jira Cloud app for self-managed instances](connect-app.md#connect-the-gitlab-for-jira-cloud-app-for-self-managed-instances). | -| Your own server | The [Jira DVCS connector](dvcs/index.md). This method syncs data every hour and works only with inbound connections. The method tries to set up webhooks in GitLab to implement real-time data sync, which does not work without outbound connections. | The [Jira DVCS connector](dvcs/index.md). This method syncs data every hour and works only with inbound connections. The method tries to set up webhooks in GitLab to implement real-time data sync, which does not work without outbound connections. | +- Add a custom comment to a Jira issue. +- Log time against a Jira issue. +- Transition a Jira issue to any status defined in the project workflow. -Each GitLab project can be configured to connect to an entire Jira instance. That means after -configuration, one GitLab project can interact with all Jira projects in that instance. For: +Smart Commits must follow this syntax: -- The [view Jira issues](issues.md#view-jira-issues) feature, you must associate a GitLab project with a - specific Jira project. -- Other features, you do not have to explicitly associate a GitLab project with any single Jira - project. +```plaintext + # +``` -If you have a single Jira instance, you can pre-fill the settings. For more information, read the -documentation for [central administration of project integrations](../../user/admin_area/settings/project_integration_management.md). +You can execute one or more commands in a single commit. -## Limitations +### Smart Commit syntax -- This integration is not supported on GitLab instances under a -[relative URL](https://docs.gitlab.com/omnibus/settings/configuration.html#configure-a-relative-url-for-gitlab) -(for example, `http://example.com/gitlab`). -- [Creating a branch](https://gitlab.com/gitlab-org/gitlab/-/issues/2647) is only supported by the GitLab for Jira app and is not available within the DVCS integration. See [officially supported DVCS features](https://confluence.atlassian.com/adminjiraserver/integrating-with-development-tools-938846890.html) for more information. +| Commands | Syntax | +|-------------------------------------------------|--------------------------------------------------------------| +| Add a comment | `KEY-123 #comment Bug is fixed` | +| Log time | `KEY-123 #time 2w 4d 10h 52m Tracking work time` | +| Close an issue | `KEY-123 #close Closing issue` | +| Log time and close an issue | `KEY-123 #time 2d 5h #close` | +| Add a comment and transition to **In-progress** | `KEY-123 #comment Started working on the issue #in-progress` | -## Troubleshoot the development panel +For more information about how Smart Commits work and what commands are available to use, see: -If you use Jira on your own server, go to the [Atlassian documentation](https://confluence.atlassian.com/jirakb/troubleshoot-the-development-panel-in-jira-server-574685212.html) -for general troubleshooting information. +- [Process issues with Smart Commits](https://support.atlassian.com/jira-software-cloud/docs/process-issues-with-smart-commits/) +- [Using Smart Commits](https://confluence.atlassian.com/fisheye/using-smart-commits-960155400.html) -### Cookies for Oracle's Access Manager +## Troubleshooting -To support Oracle's Access Manager, GitLab sends additional cookies -to enable Basic Auth. The cookie being added to each request is `OBBasicAuth` with -a value of `fromDialog`. +To troubleshoot the Jira development panel on your own server, see the +[Atlassian documentation](https://confluence.atlassian.com/jirakb/troubleshoot-the-development-panel-in-jira-server-574685212.html). diff --git a/doc/integration/jira/dvcs/index.md b/doc/integration/jira/dvcs/index.md index e07f0e579b2..0406fef3f1e 100644 --- a/doc/integration/jira/dvcs/index.md +++ b/doc/integration/jira/dvcs/index.md @@ -1,179 +1,77 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Jira DVCS connector **(FREE)** -Use the Jira DVCS (distributed version control system) connector if you self-host -your Jira instance, and you want to sync information -between GitLab and Jira. If you use Jira Cloud, you should use the -[GitLab for Jira Cloud app](../connect-app.md) unless you specifically need the -DVCS connector. - -When you configure the Jira DVCS connector, make sure your GitLab and Jira instances -are accessible. - -- **Self-managed GitLab**: Your GitLab instance must be accessible by Jira. -- **Jira Server**: Your network must allow access to your instance. -- **Jira Cloud**: Your instance must be accessible through the internet. - -NOTE: -When using GitLab 15.0 and later (including GitLab.com) with Jira Server, you might experience a [session token bug in Jira](https://jira.atlassian.com/browse/JSWSERVER-21389). As a workaround, ensure Jira Server is version 9.1.0 and later or 8.20.11 and later. - -## Smart Commits - -When connecting GitLab with Jira with DVCS, you can process your Jira issues using -special commands, called -[Smart Commits](https://support.atlassian.com/jira-software-cloud/docs/process-issues-with-smart-commits/), -in your commit messages. With Smart Commits, you can: +WARNING: +The Jira DVCS connector for Jira Cloud was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/362168) in GitLab 15.1 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118126) in 16.0. Use the [GitLab for Jira Cloud app](../connect-app.md) instead. +The Jira DVCS connector was also deprecated and removed for Jira 8.13 and earlier. You can only use the Jira DVCS connector with Jira Data Center or Jira Server in Jira 8.14 and later. Upgrade your Jira instance to Jira 8.14 or later, and reconfigure the Jira integration in your GitLab instance. -- Comment on issues. -- Record time-tracking information against issues. -- Transition issues to any status defined in the Jira project's workflow. - -Commands must be in the first line of the commit message. The -[Jira Software documentation](https://support.atlassian.com/jira-software-cloud/docs/process-issues-with-smart-commits/) -contains more information about how Smart Commits work, and what commands are available -for your use. - -For Smart Commits to work, the committing user on GitLab must have a corresponding -user on Jira with the same email address or username. - -### Smart Commit syntax - -Smart Commits should follow the pattern of: +Use the Jira DVCS (distributed version control system) connector if you self-host +your Jira instance with Jira Data Center or Jira Server and want to use the [Jira development panel](../development_panel.md). -```plaintext - # -``` +If you're on Jira Cloud, migrate to the GitLab for Jira Cloud app. For more information, see [Install the GitLab for Jira Cloud app](../connect-app.md#install-the-gitlab-for-jira-cloud-app). -Some examples: +## Configure the Jira DVCS connector -- Add a comment to a Jira issue: `KEY-123 fixes a bug #comment Bug is fixed.` -- Record time tracking: `KEY-123 #time 2w 4d 10h 52m Tracking work time.` -- Close an issue: `KEY-123 #close Closing issue` +### Prerequisites -A Smart Commit message must not span more than one line (no carriage returns) but -you can still perform multiple actions in a single commit. For example: +- Your GitLab instance must be accessible by Jira. +- You must have at least the Maintainer role for the GitLab group. +- Your network must allow inbound and outbound connections between GitLab and Jira. -- Add time tracking, add a comment, and transition to **Closed**: +### Create a GitLab application for DVCS - ```plaintext - KEY-123 #time 2d 5h #comment Task completed ahead of schedule #close - ``` +- **For projects in a single group**, you should create a [group application](../../oauth_provider.md#create-a-group-owned-application). +- **For projects across multiple groups**, you should create a separate GitLab user account for Jira integration work only. + This account ensures regular maintenance does not affect your integration. +- **If you cannot create a group application or separate user account**, you can create instead: + - [An instance-wide application](../../oauth_provider.md#create-an-instance-wide-application) + - [A user-owned application](../../oauth_provider.md#create-a-user-owned-application) -- Add a comment, transition to **In-progress**, and add time tracking: +To create a GitLab application for DVCS: - ```plaintext - KEY-123 #comment started working on the issue #in-progress #time 12d 5h - ``` +1. Go to the [appropriate **Applications** section](../../oauth_provider.md). +1. In the **Name** text box, enter a descriptive name for the integration (for example, `Jira`). +1. In the **Redirect URI** text box, enter the generated **Redirect URL** from + [linking GitLab accounts](https://confluence.atlassian.com/adminjiraserver/linking-gitlab-accounts-1027142272.html). +1. In **Scopes**, select `api` and clear any other checkboxes. + The Jira DVCS connector requires a **write-enabled** `api` scope to automatically create and manage required webhooks. +1. Select **Submit**. +1. Copy the **Application ID** and **Secret** values. + You need these values to configure Jira. -## Configure a GitLab application for DVCS +### Configure Jira for DVCS -For projects in a single group we recommend you create a [group application](../../oauth_provider.md#create-a-group-owned-application). -For projects across multiple groups we recommend you create and use a `jira` user in GitLab, and use the account -only for integration work. A separate account ensures regular account -maintenance does not affect your integration. If a `jira` user or group application is not feasible, -you can set up this integration as an [instance-wide application](../../oauth_provider.md#create-an-instance-wide-application) -or with a [user owned application](../../oauth_provider.md#create-a-user-owned-application) instead. +To configure Jira for DVCS: -1. Navigate to the [appropriate **Applications** section](../../oauth_provider.md). -1. In the **Name** field, enter a descriptive name for the integration, such as `Jira`. -1. In the **Redirect URI** field, enter the URI appropriate for your version of GitLab, - replacing `` with your GitLab instance domain: - - *For GitLab versions 13.0 and later* **and** *Jira versions 8.14 and later,* use the - generated `Redirect URL` from - [Linking GitLab accounts with Jira](https://confluence.atlassian.com/adminjiraserver/linking-gitlab-accounts-1027142272.html). - - *For GitLab versions 13.0 and later* **and** *Jira Cloud,* use `https:///login/oauth/callback`. - - *For GitLab versions 11.3 and later* **and** *Jira versions 8.13 and earlier,* use `https:///login/oauth/callback`. - If you use GitLab.com, the URL is `https://gitlab.com/login/oauth/callback`. - - *For GitLab versions 11.2 and earlier,* use - `https:///-/jira/login/oauth/callback`. +1. Go to your DVCS account. **For Jira Server**, select **Settings (gear) > Applications > DVCS accounts**. +1. To create a new integration, for **Host**, select **GitLab** or **GitLab Self-Managed**. +1. For **Team or User Account**, enter the relative path of a top-level GitLab group that [the GitLab user](#create-a-gitlab-application-for-dvcs) can access. +1. In the **Host URL** text box, enter the appropriate URL. + Replace `` with your GitLab instance domain. + Use `https://`. +1. For **Client ID**, use the [**Application ID** value](#create-a-gitlab-application-for-dvcs). +1. For **Client Secret**, use the [**Secret** value](#create-a-gitlab-application-for-dvcs). +1. Ensure that all other checkboxes are selected. +1. To create the DVCS account, select **Add**, then **Continue**. -1. For **Scopes**, select `api` and clear any other checkboxes. - - The DVCS connector requires a _write-enabled_ `api` scope to automatically create and manage required webhooks. -1. Select **Submit**. -1. Copy the **Application ID** and **Secret** values. - You need them to configure Jira. - -## Configure Jira for DVCS - -Configure this connection when you want to import all GitLab commits and branches, -for the groups you specify, into Jira. This import takes a few minutes and, after -it completes, refreshes every 60 minutes: - -1. Complete the [GitLab configuration](#configure-a-gitlab-application-for-dvcs). -1. Go to your DVCS accounts: - - *For Jira Server,* select **Settings (gear) > Applications > DVCS accounts**. - - *For Jira Cloud,* select **Settings (gear) > Products > DVCS accounts**. -1. To create a new integration, select the appropriate value for **Host**: - - *For Jira versions 8.14 and later:* Select **GitLab** or - **GitLab Self-Managed**. - - *For Jira Cloud or Jira versions 8.13 and earlier:* Select **GitHub Enterprise**. -1. For **Team or User Account**, enter either: - - *For Jira versions 8.14 and later:* - - The relative path of a top-level GitLab group that - [the GitLab user](#configure-a-gitlab-application-for-dvcs) has access to. - - *For Jira Cloud or Jira versions 8.13 and earlier:* - - The relative path of a top-level GitLab group that - [the GitLab user](#configure-a-gitlab-application-for-dvcs) has access to. - - The relative path of your personal namespace. - -1. In the **Host URL** field, enter the URI appropriate for your version of GitLab, - replacing `` with your GitLab instance domain: - - *For GitLab versions 11.3 and later,* use `https://`. - - *For GitLab versions 11.2 and earlier,* use - `https:///-/jira`. - -1. For **Client ID**, use the **Application ID** value from the previous section. -1. For **Client Secret**, use the **Secret** value from the previous section. -1. Ensure that the rest of the checkboxes are selected. -1. To create the DVCS account, select **Add** and then **Continue**. -1. Jira redirects to GitLab where you have to confirm the authorization. - GitLab then redirects back to Jira where the synced - projects should display in the new account. +Jira redirects to GitLab where you have to confirm the authorization. GitLab then redirects back to Jira +where the synced projects are displayed in the new account. The initial sync takes a few minutes. +After the initial sync, it can take up to 60 minutes to refresh. To connect additional GitLab projects from other GitLab top-level groups or personal namespaces, repeat the previous steps with additional Jira DVCS accounts. -After you configure the integration, read more about [how to test and use it](../development_panel.md). - ## Refresh data imported to Jira -Jira imports the commits and branches every 60 minutes for your projects. You -can refresh the data manually from the Jira interface: +Jira imports commits and branches for GitLab projects every 60 minutes. To refresh the data manually in Jira: 1. Sign in to your Jira instance as the user you configured the integration with. 1. Go to **Settings (gear) > Applications**. 1. Select **DVCS accounts**. -1. In the table, for the repository you want to refresh, in the **Last Activity** - column, select the icon. - -## Migrate to the GitLab for Jira Cloud app - -If you are using DVCS with Jira Cloud, you should consider migrating to the GitLab for Jira app. -[DVCS for Jira Cloud will be deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/362168) in GitLab 16.0. - -To get started using the GitLab for Jira Cloud app, follow the guide [Install the GitLab for Jira Cloud app](../connect-app.md#install-the-gitlab-for-jira-cloud-app) . - -### Feature comparison of DVCS and GitLab for Jira Cloud app - -| Feature | DVCS (Jira Cloud) | GitLab for Jira Cloud app | -|--------------------|--------------------|---------------------------------------| -| Smart Commits | **{check-circle}** Yes | **{check-circle}** Yes | -| Sync MRs | **{check-circle}** Yes | **{check-circle}** Yes | -| Sync branches | **{check-circle}** Yes | **{check-circle}** Yes | -| Sync commits | **{check-circle}** Yes | **{check-circle}** Yes| -| Sync builds | **{dotted-circle}** No | **{check-circle}** Yes | -| Sync deployments | **{dotted-circle}** No | **{check-circle}** Yes | -| Sync feature flags | **{dotted-circle}** No | **{check-circle}** Yes | -| Sync interval | 60 Minutes | Real time | -| Create branches | **{dotted-circle}** No | **{check-circle}** Yes (Only GitLab SaaS) | -| Historic data sync | **{check-circle}** Yes | **{dotted-circle}** No | - -### Risks of migrating - -The GitLab for Jira Cloud app has a limited ability to sync historic data. -Only branches, commits, builds, deployments, and feature flags created after installing the GitLab for Jira Cloud app are synced with Jira. +1. In the **Last activity** column, next to the repository you want to refresh, select **Refresh** (**{retry}**). diff --git a/doc/integration/jira/dvcs/troubleshooting.md b/doc/integration/jira/dvcs/troubleshooting.md index e40c188c1c3..541c743b609 100644 --- a/doc/integration/jira/dvcs/troubleshooting.md +++ b/doc/integration/jira/dvcs/troubleshooting.md @@ -1,12 +1,12 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Troubleshooting Jira DVCS connector **(FREE)** -Refer to the items in this section if you're having problems with your DVCS connector. +Refer to the items in this section if you're having problems with your Jira DVCS connector. ## Jira cannot access GitLab server @@ -18,6 +18,12 @@ appear in any logs: Error obtaining access token. Cannot access https://gitlab.example.com from Jira. ``` +## Session token bug in Jira + +When using GitLab 15.0 and later (including GitLab.com) with Jira Server, you might experience +a [session token bug in Jira](https://jira.atlassian.com/browse/JSWSERVER-21389). As a workaround, +ensure Jira Server is version 9.1.0 and later or 8.20.11 and later. + ## SSL and TLS problems Problems with SSL and TLS can cause this error message: @@ -26,12 +32,12 @@ Problems with SSL and TLS can cause this error message: Error obtaining access token. Cannot access https://gitlab.example.com from Jira. ``` -- The [GitLab Jira integration](../index.md) requires +- The [Jira integration](../index.md) requires GitLab to connect to Jira. Any TLS issues that arise from a private certificate authority or self-signed certificate are resolved - [on the GitLab server](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates), + [on the GitLab server](https://docs.gitlab.com/omnibus/settings/ssl/index.html#install-custom-public-certificates), as GitLab is the TLS client. -- The Jira Development panel integration requires Jira to connect to GitLab, which +- The Jira development panel requires Jira to connect to GitLab, which causes Jira to be the TLS client. If your GitLab server's certificate is not issued by a public certificate authority, add the appropriate certificate (such as your organization's root certificate) to the Java Truststore on Jira Server. @@ -74,7 +80,7 @@ Potential resolutions: [Jira DVCS connector setup](index.md#configure-jira-for-dvcs) includes `scope=api` in the query string. 1. If `scope=api` is missing from the URL, edit the - [GitLab account configuration](index.md#configure-a-gitlab-application-for-dvcs). Review + [GitLab account configuration](index.md#create-a-gitlab-application-for-dvcs). Review the **Scopes** field and ensure the `api` checkbox is selected. ## Jira error adding account and no repositories listed @@ -119,7 +125,7 @@ For more information, see the ## `Sync Failed` error when refreshing repository data -If you get a `Sync Failed` error in Jira when [refreshing repository data](index.md#refresh-data-imported-to-jira) for specific projects, check your DVCS connector logs. Look for errors that occur when executing requests to API resources in GitLab. For example: +If you get a `Sync Failed` error in Jira when [refreshing repository data](index.md#refresh-data-imported-to-jira) for specific projects, check your Jira DVCS connector logs. Look for errors that occur when executing requests to API resources in GitLab. For example: ```plaintext Failed to execute request [https://gitlab.com/api/v4/projects/:id/merge_requests?page=1&per_page=100 GET https://gitlab.com/api/v4/projects/:id/merge_requests?page=1&per_page=100 returned a response status of 403 Forbidden] errors: diff --git a/doc/integration/jira/img/jira-upload-app-success_v13_11.png b/doc/integration/jira/img/jira-upload-app-success_v13_11.png deleted file mode 100644 index c0d4c9744b6..00000000000 Binary files a/doc/integration/jira/img/jira-upload-app-success_v13_11.png and /dev/null differ diff --git a/doc/integration/jira/img/jira-upload-app_v13_11.png b/doc/integration/jira/img/jira-upload-app_v13_11.png deleted file mode 100644 index 88d1573f778..00000000000 Binary files a/doc/integration/jira/img/jira-upload-app_v13_11.png and /dev/null differ diff --git a/doc/integration/jira/img/jira_added_user_to_group.png b/doc/integration/jira/img/jira_added_user_to_group.png deleted file mode 100644 index f5120a8d42e..00000000000 Binary files a/doc/integration/jira/img/jira_added_user_to_group.png and /dev/null differ diff --git a/doc/integration/jira/img/jira_create_new_group.png b/doc/integration/jira/img/jira_create_new_group.png deleted file mode 100644 index 4ab7a5eae4e..00000000000 Binary files a/doc/integration/jira/img/jira_create_new_group.png and /dev/null differ diff --git a/doc/integration/jira/img/jira_dev_panel_jira_setup_3.png b/doc/integration/jira/img/jira_dev_panel_jira_setup_3.png deleted file mode 100644 index c58f1d5cecc..00000000000 Binary files a/doc/integration/jira/img/jira_dev_panel_jira_setup_3.png and /dev/null differ diff --git a/doc/integration/jira/img/jira_dev_panel_jira_setup_4.png b/doc/integration/jira/img/jira_dev_panel_jira_setup_4.png deleted file mode 100644 index 81d84cb173d..00000000000 Binary files a/doc/integration/jira/img/jira_dev_panel_jira_setup_4.png and /dev/null differ diff --git a/doc/integration/jira/img/jira_dev_panel_jira_setup_5.png b/doc/integration/jira/img/jira_dev_panel_jira_setup_5.png deleted file mode 100644 index b143fc24355..00000000000 Binary files a/doc/integration/jira/img/jira_dev_panel_jira_setup_5.png and /dev/null differ diff --git a/doc/integration/jira/img/jira_dev_panel_setup_com_1.png b/doc/integration/jira/img/jira_dev_panel_setup_com_1.png deleted file mode 100644 index 18f0d5da043..00000000000 Binary files a/doc/integration/jira/img/jira_dev_panel_setup_com_1.png and /dev/null differ diff --git a/doc/integration/jira/img/jira_dev_panel_setup_com_2.png b/doc/integration/jira/img/jira_dev_panel_setup_com_2.png deleted file mode 100644 index 31dc13e1271..00000000000 Binary files a/doc/integration/jira/img/jira_dev_panel_setup_com_2.png and /dev/null differ diff --git a/doc/integration/jira/img/jira_dev_panel_setup_com_3_v13_9.png b/doc/integration/jira/img/jira_dev_panel_setup_com_3_v13_9.png deleted file mode 100644 index fe791637d31..00000000000 Binary files a/doc/integration/jira/img/jira_dev_panel_setup_com_3_v13_9.png and /dev/null differ diff --git a/doc/integration/jira/img/jira_dev_panel_setup_com_4_v13_9.png b/doc/integration/jira/img/jira_dev_panel_setup_com_4_v13_9.png deleted file mode 100644 index 08787f12b67..00000000000 Binary files a/doc/integration/jira/img/jira_dev_panel_setup_com_4_v13_9.png and /dev/null differ diff --git a/doc/integration/jira/img/jira_group_access.png b/doc/integration/jira/img/jira_group_access.png deleted file mode 100644 index 42a9b4ae564..00000000000 Binary files a/doc/integration/jira/img/jira_group_access.png and /dev/null differ diff --git a/doc/integration/jira/img/jira_issue_detail_view_v13.10.png b/doc/integration/jira/img/jira_issue_detail_view_v13.10.png deleted file mode 100644 index bf1607a35fe..00000000000 Binary files a/doc/integration/jira/img/jira_issue_detail_view_v13.10.png and /dev/null differ diff --git a/doc/integration/jira/img/jira_issue_reference.png b/doc/integration/jira/img/jira_issue_reference.png deleted file mode 100644 index db8bc4f0bb9..00000000000 Binary files a/doc/integration/jira/img/jira_issue_reference.png and /dev/null differ diff --git a/doc/integration/jira/img/jira_project_settings.png b/doc/integration/jira/img/jira_project_settings.png deleted file mode 100644 index d96002b7db8..00000000000 Binary files a/doc/integration/jira/img/jira_project_settings.png and /dev/null differ diff --git a/doc/integration/jira/img/jira_service_close_issue.png b/doc/integration/jira/img/jira_service_close_issue.png deleted file mode 100644 index 73d6498192c..00000000000 Binary files a/doc/integration/jira/img/jira_service_close_issue.png and /dev/null differ diff --git a/doc/integration/jira/img/open_jira_issues_list_v14_6.png b/doc/integration/jira/img/open_jira_issues_list_v14_6.png deleted file mode 100644 index 6f06b7fec9a..00000000000 Binary files a/doc/integration/jira/img/open_jira_issues_list_v14_6.png and /dev/null differ diff --git a/doc/integration/jira/index.md b/doc/integration/jira/index.md index 2ad71a19cf7..2b6395f437b 100644 --- a/doc/integration/jira/index.md +++ b/doc/integration/jira/index.md @@ -1,70 +1,52 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Jira integrations **(FREE)** +# Jira **(FREE)** -If your organization uses [Jira](https://www.atlassian.com/software/jira) issues, -you can [migrate your issues from Jira](../../user/project/import/jira.md) and work -exclusively in GitLab. However, if you'd like to continue to use Jira, you can -integrate it with GitLab. GitLab offers two types of Jira integrations, and you -can use one or both depending on the capabilities you need. We recommend you enable both. +If your organization uses [Jira](https://www.atlassian.com/software/jira), +you can [migrate your issues from Jira to GitLab](../../user/project/import/jira.md). +If you want to continue to use Jira, you can integrate Jira with GitLab instead. -## Compare integrations +## Jira integrations -After you set up one or both of these integrations, you can cross-reference activity -in your GitLab project with any of your projects in Jira. +GitLab offers two types of Jira integrations. You can use one or both integrations +[depending on the capabilities you need](#jira-integration-capabilities). -### Jira integration +### Jira issue integration -This integration connects one or more GitLab projects to a Jira instance. You can host -the Jira instance yourself or in [Atlassian Cloud](https://www.atlassian.com/migration/assess/why-cloud). -The supported Jira versions are `6.x`, `7.x`, `8.x`, and `9.x`. +You can use the [Jira issue integration](configure.md) developed by GitLab with Jira Cloud, Jira Data Center, or Jira Server. With this integration, you can: - -For an overview, see [Agile Management - GitLab-Jira Basic Integration](https://www.youtube.com/watch?v=fWvwkx5_00E&feature=youtu.be). +- View and search Jira issues directly in GitLab. +- Refer to Jira issues by ID in GitLab commits and merge requests. +- Create Jira issues for vulnerabilities. -To set up the integration, [configure the settings](configure.md) in GitLab. +### Jira development panel -### Jira development panel integration +You can use the [Jira development panel](development_panel.md) to [view GitLab activity for an issue](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/) +including related branches, commits, and merge requests. To configure the Jira development panel: -The [Jira development panel integration](development_panel.md) -connects all GitLab projects under a group or personal namespace. When configured, -relevant GitLab information, including related branches, commits, and merge requests, -displays in the [development panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/). +- **For Jira Cloud**, use the [GitLab for Jira Cloud app](connect-app.md) developed by GitLab. +- **For Jira Data Center or Jira Server**, use the [Jira DVCS connector](dvcs/index.md) developed by Atlassian. -To set up the Jira development panel integration, use the GitLab for Jira Cloud app -or the Jira DVCS (distributed version control system) connector, -[depending on your installation](development_panel.md#configure-the-integration). +## Jira integration capabilities -### Direct feature comparison +This table shows the capabilities available with the Jira issue integration and the Jira development panel: -| Capability | Jira integration | Jira development panel integration | +| Capability | Jira issue integration | Jira development panel | |-|-|-| -| Mention a Jira issue ID in a GitLab commit or merge request, and a link to the Jira issue is created. | Yes. | No. | -| Mention a Jira issue ID in GitLab and the Jira issue shows the GitLab issue or merge request. | Yes. A Jira comment with the GitLab issue or MR title links to GitLab. The first mention is also added to the Jira issue under **Web links**. | Yes, in the issue's [development panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/). | -| Mention a Jira issue ID in a GitLab commit message and the Jira issue shows the commit message. | Yes. The entire commit message is displayed in the Jira issue as a comment and under **Web links**. Each message links back to the commit in GitLab. | Yes, in the issue's [development panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/) and optionally with a custom comment on the Jira issue using Jira [Smart Commits](https://confluence.atlassian.com/fisheye/using-smart-commits-960155400.html). | -| Mention a Jira issue ID in a GitLab branch name and the Jira issue shows the branch name. | No. | Yes, in the issue's [development panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/). | -| Add Jira time tracking to an issue. | No. | Yes. Time can be specified using Jira Smart Commits. | -| Use a Git commit or merge request to transition or close a Jira issue. | Yes. Only a single transition type, typically configured to close the issue by setting it to Done. | Yes. Transition to any state using Jira Smart Commits. | -| Display a list of [Jira issues](issues.md#view-jira-issues). | Yes. | No. | -| Create a Jira issue from a [vulnerability or finding](../../user/application_security/vulnerabilities/index.md#create-a-jira-issue-for-a-vulnerability). | Yes. | No. | -| Create a GitLab branch from a Jira issue. | No. | Yes, in the issue's [development panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/). | -| Mention a Jira issue ID in a GitLab merge request, and deployments are synced. | No. | Yes, in the issue's [development panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/). | - -## Authentication in Jira - -The authentication method in Jira depends on whether you host Jira on your own server or on -[Atlassian cloud](https://www.atlassian.com/migration/assess/why-cloud): - -- **Jira Server** supports basic authentication. When connecting, a **username and password** are - required. Connecting to Jira Server using the Central Authentication Service (CAS) is not possible. For more information, read - how to [set up a user in Jira Server](jira_server_configuration.md). -- **Jira on Atlassian cloud** supports authentication through an API token. When connecting to Jira on - Atlassian cloud, an email and API token are required. For more information, read - [create an API token for Jira in Atlassian cloud](jira_cloud_configuration.md). +| Mention a Jira issue ID in a GitLab commit or merge request, and a link to the Jira issue is created | **{check-circle}** Yes | **{dotted-circle}** No | +| Mention a Jira issue ID in GitLab, and the Jira issue shows the GitLab issue or merge request | **{check-circle}** Yes, a Jira comment with the GitLab issue or merge request title links to GitLab. The first mention is also added to the Jira issue under **Web links** | **{check-circle}** Yes, in the issue's [development panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/) | +| Mention a Jira issue ID in a GitLab commit message, and the Jira issue shows the commit message | **{check-circle}** Yes, the entire commit message is displayed in the Jira issue as a comment and under **Web links**. Each message links back to the commit in GitLab | **{check-circle}** Yes, in the issue's development panel and optionally with a custom comment on the Jira issue by using [Jira Smart Commits](https://confluence.atlassian.com/fisheye/using-smart-commits-960155400.html) | +| Mention a Jira issue ID in a GitLab branch name, and the Jira issue shows the branch name | **{dotted-circle}** No | **{check-circle}** Yes, in the issue's development panel | +| Add time tracking to a Jira issue | **{dotted-circle}** No | **{check-circle}** Yes, time can be specified by using Jira Smart Commits | +| Use a Git commit or merge request to transition or close a Jira issue |**{check-circle}** Yes, only a single transition type. Typically configured to close the issue by setting it to **Done** | **{check-circle}** Yes, transition to any state by using Jira Smart Commits | +| [View a list of Jira issues](issues.md#view-jira-issues) | **{check-circle}** Yes | **{dotted-circle}** No | +| [Create a Jira issue for a vulnerability](../../user/application_security/vulnerabilities/index.md#create-a-jira-issue-for-a-vulnerability) | **{check-circle}** Yes | **{dotted-circle}** No | +| Create a GitLab branch from a Jira issue | **{dotted-circle}** No | **{check-circle}** Yes, in the issue's development panel | +| Mention a Jira issue ID in a GitLab merge request, and deployments are synced | **{dotted-circle}** No | **{check-circle}** Yes, in the issue's development panel | ## Privacy considerations @@ -72,8 +54,8 @@ All Jira integrations share data with Jira to make it visible outside of GitLab. If you integrate a private GitLab project with Jira, the private data is shared with users who have access to your Jira project. -The [**Jira project integration**](#jira-integration) posts GitLab data in the form of comments in Jira issues. -The GitLab for Jira Cloud app and Jira DVCS connector share this data through the [**Jira Development Panel**](development_panel.md). +The [Jira issue integration](configure.md) posts GitLab data in the form of comments in Jira issues. +The GitLab for Jira Cloud app and Jira DVCS connector share this data through the [Jira development panel](development_panel.md). This method provides more fine-grained access control because access can be restricted to certain user groups or roles. ## Third-party Jira integrations diff --git a/doc/integration/jira/issues.md b/doc/integration/jira/issues.md index 3a5d8e66b2d..086d478ac15 100644 --- a/doc/integration/jira/issues.md +++ b/doc/integration/jira/issues.md @@ -1,58 +1,59 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Jira integration issue management **(FREE)** +# Jira issue management **(FREE)** -Integrating issue management with Jira requires you to [configure Jira](index.md#jira-integration) -and [enable the Jira integration](configure.md) in GitLab. -After you configure and enable the integration, you can reference and close Jira -issues by mentioning the Jira ID in GitLab commits and merge requests. +You can [manage Jira issues directly in GitLab](configure.md). +You can then refer to Jira issues by ID in GitLab commits and merge requests. +The Jira issue IDs must be in uppercase. -The Jira integration requires to you format any Jira issue IDs in uppercase. - -## Reference Jira issues +## Cross-reference GitLab activity and Jira issues With this integration, you can cross-reference Jira issues while you work in -GitLab issues and merge requests. Mention a Jira issue in a GitLab issue, -merge request, or comment, and GitLab adds a formatted comment to the Jira issue. -The comment links back to your work in GitLab. +GitLab issues, merge requests, and Git. +When you mention a Jira issue in a GitLab issue, merge request, comment, or commit: + +- GitLab links to the Jira issue from the mention in GitLab. +- GitLab adds a formatted comment to the Jira issue that links back to the issue, merge request, or commit in GitLab. -For example, this commit references the Jira issue `GIT-1`: +For example, when this commit refers to a `GIT-1` Jira issue: ```shell git commit -m "GIT-1 this is a test commit" ``` -GitLab adds a reference to the **Issue Links** section of Jira issue `GIT-1`: +GitLab adds to that Jira issue: -![Example of mentioning or closing the Jira issue](img/jira_issue_reference.png) +- A reference in the **Web links** section +- A comment in the **Activity** section that follows this format: -GitLab also adds a comment to the issue, and uses this format: + ```plaintext + USER mentioned this issue in RESOURCE_NAME of [PROJECT_NAME|COMMENTLINK]: + ENTITY_TITLE + ``` -```plaintext -USER mentioned this issue in RESOURCE_NAME of [PROJECT_NAME|COMMENTLINK]: -ENTITY_TITLE -``` + - `USER`: Name of the user who has mentioned the Jira issue with a link to their GitLab user profile. + - `RESOURCE_NAME`: Type of resource (for example, a GitLab commit, issue, or merge request) that has referenced the Jira issue. + - `PROJECT_NAME`: GitLab project name. + - `COMMENTLINK`: Link to where the Jira issue is mentioned. + - `ENTITY_TITLE`: Title of the GitLab commit (first line), issue, or merge request. -- `USER`: The name of the user who mentioned the issue, linked to their GitLab user profile. -- `COMMENTLINK`: A link to where the Jira issue was mentioned. -- `RESOURCE_NAME`: The type of resource, such as a commit or merge request, which referenced the issue. -- `PROJECT_NAME`: The GitLab project name. -- `ENTITY_TITLE`: The title of the merge request, or the first line of the commit. +Only a single cross-reference comment appears in Jira per GitLab issue, merge request, or commit. +For example, multiple comments on a GitLab merge request that reference a Jira issue +create only a single cross-reference comment back to that merge request in Jira. You can [disable comments](#disable-comments-on-jira-issues) on issues. ### Require associated Jira issue for merge requests to be merged **(ULTIMATE)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/280766) in GitLab 13.12, disabled behind `jira_issue_association_on_merge_request` [feature flag](../../administration/feature_flags.md). -> - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61722) in GitLab 14.1. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/335280) in GitLab 14.2. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/280766) in GitLab 13.12 [with a flag](../../administration/feature_flags.md) named `jira_issue_association_on_merge_request`. Disabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/335280) in GitLab 14.2. Feature flag `jira_issue_association_on_merge_request` removed. -You can prevent merge requests from being merged if they do not refer to a Jira issue. -To enforce this: +With this integration, you can prevent merge requests from being merged if they do not refer to a Jira issue. +To enable this feature: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Settings > Merge requests**. @@ -63,6 +64,41 @@ After you enable this feature, a merge request that doesn't reference an associa Jira issue can't be merged. The merge request displays the message **To merge, a Jira issue key must be mentioned in the title or description.** +## Customize Jira issue matching in GitLab + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/112826) in GitLab 15.10. + +You can configure custom rules for how GitLab matches Jira issue keys by defining: + +- [A regex pattern](#use-regular-expression) +- [A prefix](#use-a-prefix) + +When you don't configure custom rules, the [default behavior](https://gitlab.com/gitlab-org/gitlab/-/blob/710d83af298d8896f2b940faf48a46d2feb4cbaf/lib/gitlab/regex.rb#L552) is used. For more information, see the [RE2 wiki](https://github.com/google/re2/wiki/Syntax). + +### Use regular expression + +To define a regex pattern for Jira issue keys: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Integrations**. +1. Select **Jira**. +1. Go to the **Jira issue matching** section. +1. In the **Jira issue regex** text box, enter a regex pattern. +1. Select **Save changes**. + +For more information, see the [Atlassian documentation](https://confluence.atlassian.com/adminjiraserver073/changing-the-project-key-format-861253229.html). + +### Use a prefix + +To define a prefix for Jira issue keys: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Integrations**. +1. Select **Jira**. +1. Go to the **Jira issue matching** section. +1. In the **Jira issue prefix** text box, enter a prefix. +1. Select **Save changes**. + ## Close Jira issues in GitLab If you have configured GitLab transition IDs, you can close a Jira issue directly @@ -79,7 +115,7 @@ For example, use any of these trigger words to close the Jira issue `PROJECT-1`: - `Fixes PROJECT-1` The commit or merge request must target your project's [default branch](../../user/project/repository/branches/default.md). -You can change your project's default branch under [project settings](img/jira_project_settings.png). +You can change your project's default branch in [project settings](../../user/project/settings/index.md). ### Use case for closing issues @@ -89,8 +125,7 @@ Consider this example: 1. You create a merge request in GitLab to build the requested feature. 1. In the merge request, you add the issue closing trigger `Closes PROJECT-7`. 1. When the merge request is merged: - - GitLab closes the Jira issue for you: - ![The GitLab integration closes Jira issue](img/jira_service_close_issue.png) + - GitLab closes the Jira issue for you. - GitLab adds a formatted comment to Jira, linking back to the commit that resolved the issue. You can [disable comments](#disable-comments-on-jira-issues). @@ -98,19 +133,19 @@ Consider this example: > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3622) in GitLab 13.2. -You can browse, search, and view issues from a selected Jira project directly in GitLab, -if your GitLab administrator [has configured it](configure.md). +You can view and search issues from a selected Jira project directly in GitLab, +provided your GitLab administrator [has configured the integration](configure.md#configure-the-integration). -To do this, in GitLab, go to your project and select **Issues > Jira issues**. The issue list -sorts by **Created date** by default, with the newest issues listed at the top: +To view Jira issues: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Issues > Jira issues**. -![Jira issues integration enabled](img/open_jira_issues_list_v14_6.png) +The issues are sorted by **Created date** by default, with the most recently created issues listed at the top. - To display the most recently updated issues first, select **Updated date**. -- You can [search and filter](#search-and-filter-the-issues-list) the issues list. -- In GitLab [versions 13.10 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/299832), - you can select an issue from the list to view it in GitLab: - ![Jira issue detail view](img/jira_issue_detail_view_v13.10.png) +- You can [search and filter the issue list](#search-and-filter-the-issue-list). +- In GitLab 13.10 and later, you can [select an issue from the list to view the issue in GitLab](https://gitlab.com/gitlab-org/gitlab/-/issues/299832). Issues are grouped into tabs based on their [Jira status](https://confluence.atlassian.com/adminjiraserver070/defining-status-field-values-749382903.html): @@ -119,7 +154,7 @@ Issues are grouped into tabs based on their - **Closed** tab: All issues with a Jira status categorized as Done. - **All** tab: All issues of any status. -### Search and filter the issues list **(PREMIUM)** +### Search and filter the issue list **(PREMIUM)** To refine the list of issues, use the search bar to search for any text contained in an issue summary (title) or description. Use any combination diff --git a/doc/integration/jira/jira_cloud_configuration.md b/doc/integration/jira/jira_cloud_configuration.md index 9d52368f528..83321df0420 100644 --- a/doc/integration/jira/jira_cloud_configuration.md +++ b/doc/integration/jira/jira_cloud_configuration.md @@ -1,23 +1,11 @@ --- -stage: Manage -group: Integrations -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: 'configure.md#create-a-jira-cloud-api-token' +remove_date: '2023-07-07' --- -# Create a Jira Cloud API token **(FREE)** +This document was moved to [another location](configure.md#create-a-jira-cloud-api-token). -You need an API token to [integrate with Jira](index.md) -in Atlassian Cloud. To create the API token: - -1. Sign in to [Atlassian](https://id.atlassian.com/manage-profile/security/api-tokens) - using an account with *write* access to Jira projects. - - The link opens the API tokens page. Alternatively, to go to this page from your Atlassian - profile, select **Account Settings > Security > Create and manage API tokens**. -1. Select **Create API token**. -1. In the dialog, enter a label for your token and select **Create**. -1. To copy the API token, select **Copy**, then paste the token somewhere safe. - -You need the newly created token, and the email -address you used when you created it, when you -[configure GitLab](configure.md). + + + + diff --git a/doc/integration/jira/jira_server_configuration.md b/doc/integration/jira/jira_server_configuration.md index 7dee99b024e..c840e1ebde5 100644 --- a/doc/integration/jira/jira_server_configuration.md +++ b/doc/integration/jira/jira_server_configuration.md @@ -1,87 +1,75 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Jira Server credentials **(FREE)** -To [integrate Jira with GitLab](index.md), you must -create a Jira user account for your Jira projects to access projects in GitLab. -This Jira user account must have write access to your Jira projects. To create the -credentials, you must: +To [integrate Jira with GitLab](configure.md), you should create a separate Jira user account for your Jira projects +to access projects in GitLab. This Jira user account must have write access to your Jira projects. +To create the credentials: 1. [Create a Jira Server user](#create-a-jira-server-user). -1. [Create a Jira Server group](#create-a-jira-server-group) for the user to belong to. -1. [Create a permission scheme for your group](#create-a-permission-scheme-for-your-group). +1. [Create a Jira Server group for the user](#create-a-jira-server-group-for-the-user). +1. [Create a permission scheme for the group](#create-a-permission-scheme-for-the-group). + +Alternatively, you can use an existing Jira user account, provided the user belongs to a Jira group that +has been granted the **Administer Projects** [permission scheme](#create-a-permission-scheme-for-the-group). + +After you select a Jira user account, [configure the integration](configure.md#configure-the-integration) in GitLab to use the account. ## Create a Jira Server user -This process creates a user named `gitlab`: +To create a Jira Server user: 1. Sign in to your Jira instance as a Jira administrator. -1. In the upper right corner of the top menu, go to the gear icon and +1. On the top bar, in the upper-right corner, select the gear icon, then select **User Management**. -1. Create a new user account (`gitlab`) with write access to +1. [Create a new user account manually](https://confluence.atlassian.com/adminjiraserver/create-edit-or-remove-a-user-938847025.html#Create,edit,orremoveauser-CreateusersmanuallyinJira) with write access to projects in Jira. - - **Email address**: Jira requires a valid email address, and sends a verification - email, which is required to set up the password. - - **Username**: Jira creates the username by using the email prefix. You can change - this username later. - - **Password**: You must create a password, because the GitLab integration doesn't - support SSO, such as SAML. To create the password, go to the user profile, look up - the username, and set a password. + - **Email address**: You should use a valid email address. + - **Username**: Set the username to `gitlab`. + - **Password**: You must set a password because the Jira issue integration does not + support SSO such as SAML. 1. Select **Create user**. -After you create the user, create a group for it. +Now that you've created a user named `gitlab`, it's time to create a group for the user. -## Create a Jira Server group +## Create a Jira Server group for the user -After you [create a Jira Server user](#create-a-jira-server-user), create a -group to assign permissions to the user. - -This process adds the `gitlab` user you created to a new group named `gitlab-developers`: +To create a Jira Server group for the user: 1. Sign in to your Jira instance as a Jira administrator. -1. In the upper right corner of the top menu, go to the gear icon and +1. On the top bar, in the upper-right corner, select the gear icon, then select **User Management**. -1. On the sidebar, select **Groups**. - - ![Jira create new user](img/jira_create_new_group.png) - +1. On the left sidebar, select **Groups**. 1. In the **Add group** section, enter a **Name** for the group (for example, - `gitlab-developers`), and then select **Add group**. + `gitlab-developers`), then select **Add group**. 1. To add the `gitlab` user to the `gitlab-developers` group, select **Edit members**. - The `gitlab-developers` group should be listed in the leftmost box as a - selected group. + The `gitlab-developers` group appears as a selected group. 1. In the **Add members to selected group(s)** section, enter `gitlab`. 1. Select **Add selected users**. - The `gitlab` user appears in the **Group member(s)** - section. + The `gitlab` user appears as a group member. - ![Jira added user to group](img/jira_added_user_to_group.png) - -Next, create a permission scheme for your group. +Now that you've added the `gitlab` user to a new group named `gitlab-developers`, +it's time to create a permission scheme for the group. -## Create a permission scheme for your group +## Create a permission scheme for the group -After creating the group in Jira, grant permissions to the group by creating a permission scheme: +To create a permission scheme for the group: 1. Sign in to your Jira instance as a Jira administrator. -1. In the upper right corner of the top menu, go to the gear icon and +1. On the top bar, in the upper-right corner, select the gear icon, then select **Issues**. -1. On the sidebar, select **Permission Schemes**. +1. On the left sidebar, select **Permission Schemes**. 1. Select **Add Permission Scheme**, enter a **Name** and (optionally) a - **Description**, and then select **Add**. + **Description**, then select **Add**. 1. In the permissions scheme list, locate your new permissions scheme, and select **Permissions**. 1. Next to **Administer Projects**, select **Edit**. -1. From the **Group** dropdown list, select `gitlab-developers`, and then select **Grant**. - - ![Jira group access](img/jira_group_access.png) +1. From the **Group** dropdown list, select `gitlab-developers`, then select **Grant**. -Write down the new Jira username and its -password, as you need them when -[configuring GitLab](configure.md). +You need the new Jira username and password when you [configure the integration](configure.md#configure-the-integration) in GitLab. diff --git a/doc/integration/jira/troubleshooting.md b/doc/integration/jira/troubleshooting.md index 0e679693614..586f09be751 100644 --- a/doc/integration/jira/troubleshooting.md +++ b/doc/integration/jira/troubleshooting.md @@ -1,17 +1,17 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Troubleshooting Jira integrations **(FREE)** +# Troubleshooting Jira **(FREE)** This page contains a list of common issues you might encounter when working with Jira integrations. ## GitLab cannot comment on a Jira issue If GitLab cannot comment on Jira issues, make sure the Jira user you -set up for the integration has permission to: +set up for the [Jira integration](configure.md) has permission to: - Post comments on a Jira issue. - Transition the Jira issue. @@ -91,16 +91,16 @@ To reset the Jira user's password for all projects with active Jira integrations run the following in a [Rails console](../../administration/operations/rails_console.md#starting-a-rails-console-session): ```ruby -p = Project.find_by_sql("SELECT p.id FROM projects p LEFT JOIN services s ON p.id = s.project_id WHERE s.type = 'JiraService' AND s.active = true") +p = Project.find_by_sql("SELECT p.id FROM projects p LEFT JOIN integrations i ON p.id = i.project_id WHERE i.type_new = 'Integrations::Jira' AND i.active = true") p.each do |project| project.jira_integration.update_attribute(:password, '') end ``` -## `500 Whoops` when accessing a Jira issue in GitLab +## `500 We're sorry` when accessing a Jira issue in GitLab -When accessing a Jira issue in GitLab, you might get a `500 Whoops, something went wrong on our end` error. +When accessing a Jira issue in GitLab, you might get a `500 We're sorry. Something went wrong on our end` error. Check [`production.log`](../../administration/logs/index.md#productionlog) to see if it contains the following exception: ```plaintext diff --git a/doc/integration/kerberos.md b/doc/integration/kerberos.md index f0c1a75041e..c2be5a5a91c 100644 --- a/doc/integration/kerberos.md +++ b/doc/integration/kerberos.md @@ -8,22 +8,13 @@ info: "To determine the technical writer assigned to the Stage/Group associated GitLab can integrate with [Kerberos](https://web.mit.edu/kerberos/) as an authentication mechanism. +- You can configure GitLab so your users can sign in with their Kerberos credentials. +- You can use Kerberos to [prevent](https://web.mit.edu/sipb/doc/working/guide/guide/node20.html) anyone from intercepting or eavesdropping on the transmitted password. + WARNING: GitLab CI/CD doesn't work with a Kerberos-enabled GitLab instance unless the integration is [set to use a dedicated port](#http-git-access-with-kerberos-token-passwordless-authentication). -## Overview - -[Kerberos](https://web.mit.edu/kerberos/) is a secure method for authenticating a request for a service in a -computer network. Kerberos was developed in the Athena Project at the -[Massachusetts Institute of Technology (MIT)](https://web.mit.edu/). The name is taken from Greek -mythology; Kerberos was a three-headed dog who guarded the gates of Hades. - -## Use-cases - -- GitLab can be configured to allow your users to sign with their Kerberos credentials. -- You can use Kerberos to [prevent](https://web.mit.edu/sipb/doc/working/guide/guide/node20.html) anyone from intercepting or eavesdropping on the transmitted password. - ## Configuration For GitLab to offer Kerberos token-based authentication, perform the @@ -99,7 +90,7 @@ to authenticate with Kerberos tokens. #### Enable single sign-on -Edit the [common configuration file settings](omniauth.md#configure-common-settings) +Configure the [common settings](omniauth.md#configure-common-settings) to add `kerberos` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. @@ -356,8 +347,28 @@ as extensions to the Kerberos protocol may result in HTTP authentication headers larger than the default size of 8 kB. Configure `large_client_header_buffers` to a larger value in [the NGINX configuration](https://nginx.org/en/docs/http/ngx_http_core_module.html#large_client_header_buffers). +### Use Keytabs created using AES-only encryption with Windows AD + +When you create a keytab with Advanced Encryption Standard (AES)-only encryption, you must select the **This account supports Kerberos AES <128/256> bit encryption** checkbox for that account in the AD server. Whether the checkbox is 128 or 256 bit depends on the encryption strength used when you created the keytab. To check this, on the Active Directory server: + +1. Open the **Users and Groups** tool. +1. Locate the account that you used to create the keytab. +1. Right-click the account and select **Properties**. +1. In **Account Options** on the **Account** tab, select the appropriate AES encryption support checkbox. +1. Save and close. + ## Troubleshooting +### Using Google Chrome with Kerberos authentication against Windows AD + +When you use Google Chrome to sign in to GitLab with Kerberos, you must enter your full username. For example, `username@domain.com`. + +If you do not enter your full username, the sign-in fails. Check the logs to see the following event message as evidence of this sign-in failure: + +```plain +"message":"OmniauthKerberosController: failed to process Negotiate/Kerberos authentication: gss_accept_sec_context did not return GSS_S_COMPLETE: An unsupported mechanism was requested\nUnknown error"`. +``` + ### Test connectivity between the GitLab and Kerberos servers You can use utilities like [`kinit`](https://web.mit.edu/kerberos/krb5-1.12/doc/user/user_commands/kinit.html) and [`klist`](https://web.mit.edu/kerberos/krb5-1.12/doc/user/user_commands/klist.html) to test connectivity between the GitLab server diff --git a/doc/integration/mattermost/index.md b/doc/integration/mattermost/index.md index 42592a0dff2..e1183f62225 100644 --- a/doc/integration/mattermost/index.md +++ b/doc/integration/mattermost/index.md @@ -10,7 +10,7 @@ NOTE: This document applies to GitLab 11.0 and later. You can run a [GitLab Mattermost](https://gitlab.com/gitlab-org/gitlab-mattermost) -service on your GitLab server. Mattermost is not part of the single application that GitLab is. There is a good integration between with GitLab, and our Omnibus installer allows you to easily install it. But it is a separate application from a separate company. +service on your GitLab server. Mattermost is not part of the single application that GitLab is. There is a good integration between [Mattermost and GitLab](https://mattermost.com/solutions/mattermost-gitlab/), and our Omnibus installer allows you to easily install it. But it is a separate application from a separate company. ## Prerequisites @@ -245,7 +245,7 @@ For local connections, the `mmctl` binary and Mattermost must be run from the sa "LocalModeSocketLocation": "/var/tmp/mattermost_local.socket", ... } - } + } ``` 1. Restart Mattermost: @@ -280,7 +280,7 @@ $ /opt/gitlab/embedded/bin/mmctl auth login http://mattermost.example.com Connection name: test Username: local-user -Password: +Password: credentials for "test": "local-user@http://mattermost.example.com" stored ``` @@ -378,6 +378,10 @@ If this is not the case, there are two options: For a complete list of upgrade notices and special considerations for older versions, see the [Mattermost documentation](https://docs.mattermost.com/administration/important-upgrade-notes.html). +## Upgrading GitLab Mattermost to 15.10 + +GitLab 15.10 ships with Mattermost 7.8. Before upgrading, [connect to the bundled PostgreSQL database](#connecting-to-the-bundled-postgresql-database) to perform the PostgreSQL maintenance described in the [Important Upgrade Notes](https://docs.mattermost.com/administration/important-upgrade-notes.html) provided by Mattermost. + ## Upgrading GitLab Mattermost to 14.6 GitLab 14.6 ships with Mattermost 6.1 including potentially long running database migrations for Mattermost 6.0. For information about upgrading and for ways to reduce the downtime caused by those migrations, read the [Important Upgrade Notes](https://docs.mattermost.com/administration/important-upgrade-notes.html) for both versions. If you need to perform any manual migrations, [connect to the bundled PostgreSQL database](#connecting-to-the-bundled-postgresql-database). diff --git a/doc/integration/oauth2_generic.md b/doc/integration/oauth2_generic.md index 2c0439a328c..79238c78421 100644 --- a/doc/integration/oauth2_generic.md +++ b/doc/integration/oauth2_generic.md @@ -54,7 +54,7 @@ To configure the provider: :::TabTitle Linux package (Omnibus) - 1. Edit the [common configuration file settings](omniauth.md#configure-common-settings) + 1. Configure the [common settings](omniauth.md#configure-common-settings) to add `oauth2_generic` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. 1. Edit `/etc/gitlab/gitlab.rb` to add the configuration for your provider. For example: @@ -98,7 +98,7 @@ To configure the provider: :::TabTitle Helm chart (Kubernetes) - 1. Edit the [common configuration file settings](omniauth.md#configure-common-settings) + 1. Configure the [common settings](omniauth.md#configure-common-settings) to add `oauth2_generic` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. 1. Export the Helm values: @@ -107,39 +107,45 @@ To configure the provider: helm get values gitlab > gitlab_values.yaml ``` - 1. Edit `gitlab_values.yaml`. + 1. Put the following content in a file named `oauth2_generic.yaml` for use as a + [Kubernetes Secret](https://docs.gitlab.com/charts/charts/globals.html#providers): - NOTE: - The following example exposes the `app_secret` value in the main YAML file. - You're strongly advised to use - [Helm secrets](https://docs.gitlab.com/charts/installation/secrets.html) - instead. + ```yaml + name: "oauth2_generic" + label: "Provider name" # optional label for login button defaults to "Oauth2 Generic" + app_id: "" + app_secret: "" + args: + client_options: + site: "" + user_info_url: "/oauth2/v1/userinfo" + authorize_url: "/oauth2/v1/authorize" + token_url: "/oauth2/v1/token" + user_response_structure: + root_path: [] + id_path: ["sub"] + attributes: + email: "email" + name: "name" + authorize_params: + scope: "openid profile email" + strategy_class: "OmniAuth::Strategies::OAuth2Generic" + ``` + + 1. Create the Kubernetes Secret: + + ```shell + kubectl create secret generic -n gitlab-oauth2-generic --from-file=provider=oauth2_generic.yaml + ``` + + 1. Edit `gitlab_values.yaml` and add the provider configuration: ```yaml global: appConfig: omniauth: - enabled: true providers: - - name: "oauth2_generic" - label: "Provider name" # optional label for login button defaults to "Oauth2 Generic" - app_id: "" - app_secret: "" - args: - client_options: - site: "" - user_info_url: "/oauth2/v1/userinfo" - authorize_url: "/oauth2/v1/authorize" - token_url: "/oauth2/v1/token" - user_response_structure: - root_path: [] - id_path: ["sub"] - attributes: - email: "email" - name: "name" - authorize_params: - scope: "openid profile email" - strategy_class: "OmniAuth::Strategies::OAuth2Generic" + - secret: gitlab-oauth2-generic ``` 1. Save the file and apply the new values: @@ -150,7 +156,7 @@ To configure the provider: :::TabTitle Self-compiled (source) - 1. Edit the [common configuration file settings](omniauth.md#configure-common-settings) + 1. Configure the [common settings](omniauth.md#configure-common-settings) to add `oauth2_generic` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. 1. Edit `/home/git/gitlab/config/gitlab.yml`: diff --git a/doc/integration/omniauth.md b/doc/integration/omniauth.md index 61019915c52..cd287d70ca3 100644 --- a/doc/integration/omniauth.md +++ b/doc/integration/omniauth.md @@ -46,7 +46,7 @@ Linux package, Docker, and self-compiled | Helm chart | Description | Default va ----------------------------|------------|-------------|----------- `allow_single_sign_on` | `allowSingleSignOn` | List of providers that automatically create a GitLab account. The provider names are available in the **OmniAuth provider name** column in the [supported providers table](#supported-providers). | `false`, which means that signing in using your OmniAuth provider account without a pre-existing GitLab account is not allowed. You must create a GitLab account first, and then connect it to your OmniAuth provider account through your profile settings. `auto_link_ldap_user` | `autoLinkLdapUser` | Creates an LDAP identity in GitLab for users that are created through an OmniAuth provider. You can enable this setting if you have [LDAP integration](../administration/auth/ldap/index.md) enabled. Requires the `uid` of the user to be the same in both LDAP and the OmniAuth provider. | `false` -`block_auto_created_users` | `blockAutoCreatedUsers` | Blocks users that are automatically created from signing in until they are approved by an administrator. | `true`. If you set the value to `false`, make sure you define providers that you can control, like SAML or Google. Otherwise, any user on the internet can sign in to GitLab without an administrator's approval. +`block_auto_created_users` | `blockAutoCreatedUsers` | Places automatically-created users in a [Pending approval](../user/admin_area/moderate_users.md#users-pending-approval) state (unable to sign in) until they are approved by an administrator. | `true`. If you set the value to `false`, make sure you define providers that you can control, like SAML or Google. Otherwise, any user on the internet can sign in to GitLab without an administrator's approval. ### Configure initial settings @@ -307,7 +307,7 @@ To enable automatic linking for SAML, see the [SAML setup instructions](saml.md# ## Create an external providers list You can define a list of external OmniAuth providers. -Users who create accounts or sign in to GitLab through the listed providers do not get access to [internal projects](../user/public_access.md#internal-projects-and-groups). +Users who create accounts or sign in to GitLab through the listed providers do not get access to [internal projects](../user/public_access.md#internal-projects-and-groups) and are marked as [external users](../user/admin_area/external_users.md). To define the external providers list, use the full name of the provider, for example, `google_oauth2` for Google. For provider names, see the @@ -379,8 +379,12 @@ but we'd like to at least help those with specific needs. ## Keep OmniAuth user profiles up to date -You can enable profile syncing from selected OmniAuth providers. You can sync -all or specific user information. +You can enable profile syncing from selected OmniAuth providers. +You can sync any combination of the following user attributes: + +- `name` +- `email` +- `location` When authenticating using LDAP, the user's name and email are always synced. diff --git a/doc/integration/openid_connect_provider.md b/doc/integration/openid_connect_provider.md index b0d85fae8c2..ff81ec49a9f 100644 --- a/doc/integration/openid_connect_provider.md +++ b/doc/integration/openid_connect_provider.md @@ -20,7 +20,7 @@ OAuth 2.0 protocol. It allows clients to: OIDC performs many of the same tasks as OpenID 2.0, but is API-friendly and usable by native and mobile applications. -On the client side, you can use [OmniAuth::OpenIDConnect](https://github.com/jjbohn/omniauth-openid-connect/) for Rails +On the client side, you can use [OmniAuth::OpenIDConnect](https://github.com/omniauth/omniauth_openid_connect) for Rails applications, or any of the other available [client implementations](https://openid.net/developers/libraries/#connect). The GitLab implementation uses the [doorkeeper-openid_connect](https://github.com/doorkeeper-gem/doorkeeper-openid_connect "Doorkeeper::OpenidConnect website") gem, refer diff --git a/doc/integration/partner_marketplace.md b/doc/integration/partner_marketplace.md index a15457b5b0c..76988af99b2 100644 --- a/doc/integration/partner_marketplace.md +++ b/doc/integration/partner_marketplace.md @@ -1,13 +1,13 @@ --- stage: Fulfillment -group: Commerce Integrations +group: Provision info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Marketplace partner integration guide +# Marketplace partners GitLab supports automation for selected distribution marketplaces to process sales of GitLab products to authorized -channel partners. Marketplace partners can use the GitLab Marketplace APIs to integrate their systems with GitLab to +channel partners. Marketplace partners can use the GitLab Marketplace APIs to integrate their systems with GitLab to sell GitLab subscriptions on their site. This document's target audience is third-party developers for Marketplace partners. @@ -48,17 +48,113 @@ sequenceDiagram Customers Portal ->> Marketplace partner system: Success response with order status ``` -## Prerequisites +## Marketplace API Specification + +OpenAPI specs for the Marketplace APIs are available at [Marketplace interactive API documentation](https://customers.staging.gitlab.com/openapi_docs/marketplace). + +## Access the Marketplace API + +To access the Marketplace API you need to: + +- Request access from GitLab. +- Retrieve an OAuth access token. + +Marketplace API endpoints are secured with [OAuth 2.0](https://oauth.net/2/). OAuth is an authorization framework +that grants 3rd party or client applications, like a Marketplace partner application, limited access to resources on an +HTTP service, like the Customers Portal. + +OAuth 2.0 uses _grant types_ (or _flows_) that describe how a client application gets authorization in +the form of an _access token_. An access token is a string that the client application uses to make authorized requests to +the resource server. + +The Marketplace API uses the `client_credentials` grant type. The client application uses the access token to access its +own resources, instead of accessing resources on behalf of a user. + +### Step 1: Request access -Before a marketplace partner client can use the Marketplace API, GitLab must set up the following configurations for the client: +Before you can use the Marketplace API, you must contact your Marketplace partner Manager or email [`partnerorderops`](mailto:partnerorderops@gitlab.com) +to request access. After you request access, GitLab configures the following accounts and credentials for you: -1. Client credential. Marketplace APIs are secured with OAuth 2.0. A client credential is required to get the OAuth token. +1. Client credentials. Marketplace APIs are secured with OAuth 2.0. The client credentials include the client ID and client secret + that you need to retrieve the OAuth access token. 1. Invoice owner account in Zuora system. Required for invoice processing. 1. Distributor account in Salesforce system. -1. Trading partner account in Salesforce system. +1. Trading partner account in Salesforce system. GitLab adds the Trading Partner ID to a permitted list to pass the validations. -Interested GitLab Partners should contact their GitLab Partner Manager or email [`partnerorderops`](mailto:partnerorderops@gitlab.com). +### Step 2: Retrieve an access token -## Marketplace API Specification +To retrieve an access token, + +- Make a POST request to the [`/oauth/token`](https://customers.staging.gitlab.com/openapi_docs/marketplace#/marketplace/post_oauth_token) endpoint with the following required parameters: + +| Parameter | Type | Required |Description | +|-----------------|--------|----------|------------------------------------------------------------------------------------------------------------------------------------| +| `client_id` | string | yes |ID of your client application record on the Customers Portal. Received from GitLab. | +| `client_secret` | string | yes |Secret of your client application record on the Customers Portal. Received from GitLab. | +| `grant_type` | string | yes |Specifies the type of credential flow. Use `client_credentials`. | +| `scope` | string | yes |Specifies the level of access. Use `marketplace.order:read` for read-only access. Use `marketplace.order:create` for create access. | + +If the request is successful, the response body includes the access token that you can use in subsequent requests. For an example of a successful +response, see the [Marketplace interactive API documentation](https://customers.staging.gitlab.com/openapi_docs/marketplace) + +If the request is unsuccessful, the response body includes an error and error description. The errors can be: + +| Status | Description | +|--------|----------------------------------------------------------------------------------------------------------------------------------------------| +| 400 | Invalid scope. Ensure the `scope` is `marketplace.order:read` or `marketplace.order:create`. | +| 401 | Invalid client. Ensure that there are no typos or extra spaces on the client specific credentials. Incorrect `client_id` or `client_secret` | + +### Step 3: Use the access token + +To use the access token from a client application: + +1. Set the `Authorization` header of the request to `Bearer `. +1. Set parameters or data needed for the endpoint and send the request. + +Example request: + +```shell +curl \ + --url "https://customers.staging.gitlab.com/api/v1/marketplace/subscriptions/:external_subscription_id" \ + --header "Authorization: Bearer NHb_VhZhPOnBTSNfBSzmCmt28lLDWb2xtwr_c3DL148" +``` + +## Create a new customer subscription + +To create a new customer subscription from a Marketplace partner client application, + +- Make an authorized POST request to the +[`/api/v1/marketplace/subscriptions`](https://customers.staging.gitlab.com/openapi_docs/marketplace#/marketplace/post_api_v1_marketplace_subscriptions) +endpoint in the Customers Portal with the following parameters in JSON format: + +| Parameter | Type | Required | Description | +|--------------------------|--------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------| +| `externalSubscriptionId` | string | yes | ID of the subscription on the Marketplace partner system. | +| `tradingPartnerId` | string | yes | ID of the Trading Partner account in Salesforce. Received from GitLab. | +| `customer` | object | yes | Information about the customer. Must include company name. Contact must include `firstName`, `lastName` and `email`. Address must include `country`. | +| `orderLines` | array | yes | Specifies the product purchased. Must include `quantity` and `productId`. | + +If the request is successful, the response body includes the newly created subscription number. For an example of a full request body, +see the [Marketplace interactive API documentation](https://customers.staging.gitlab.com/openapi_docs/marketplace). + +If the subscription creation is unsuccessful, the response body includes an error message with details about the cause of the error. + +## Check the status of a subscription + +To get the status of a given subscription, + +- Make an authorized GET request to the +[`/api/v1/marketplace/subscriptions/{external_subscription_id}`](https://customers.staging.gitlab.com/openapi_docs/marketplace#/marketplace/get_api_v1_marketplace_subscriptions__external_subscription_id_) +endpoint in the Customers Portal. + +The request must include the Marketplace partner system ID of the subscription to fetch the status for. + +If the request is successful, the response body contains the status of the subscription provision. The status can be: + +- Creating +- Created +- Failed +- Provisioned -OpenAPI specs for the Marketplace APIs are available upon request. The specs will be made public before the end of Q1 2023. +If the subscription cannot be found using the passed `external_subscription_id`, the response has +a 404 Not Found status. diff --git a/doc/integration/salesforce.md b/doc/integration/salesforce.md index 16432d3ca5d..8c43f038f86 100644 --- a/doc/integration/salesforce.md +++ b/doc/integration/salesforce.md @@ -48,7 +48,7 @@ To get the credentials (a pair of Client ID and Client Secret), you must [create sudo -u git -H editor config/gitlab.yml ``` -1. Edit the [common configuration file settings](omniauth.md#configure-common-settings) +1. Configure the [common settings](omniauth.md#configure-common-settings) to add `salesforce` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. diff --git a/doc/integration/saml.md b/doc/integration/saml.md index 231709df7f4..f59824c8db6 100644 --- a/doc/integration/saml.md +++ b/doc/integration/saml.md @@ -19,7 +19,7 @@ To set up SAML on GitLab.com, see [SAML SSO for GitLab.com groups](../user/group For more information on: - OmniAuth provider settings, see the [OmniAuth documentation](omniauth.md). -- Commonly-used terms, see the [glossary of common terms](#glossary-of-common-terms). +- Commonly-used terms, see the [glossary](#glossary). ## Configure SAML support in GitLab @@ -28,7 +28,7 @@ For more information on: :::TabTitle Linux package (Omnibus) 1. Make sure GitLab is [configured with HTTPS](https://docs.gitlab.com/omnibus/settings/ssl/). -1. Edit the [common configuration file settings](omniauth.md#configure-common-settings) +1. Configure the [common settings](omniauth.md#configure-common-settings) to add `saml` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. 1. To allow your users to use SAML to sign up without having to manually create @@ -51,7 +51,7 @@ For more information on: 1. Configure the following attributes so your SAML users cannot change them: - - [`NameID`](../user/group/saml_sso/index.md#nameid). + - [`NameID`](../user/group/saml_sso/index.md#manage-user-saml-identity). - `Email` when used with `omniauth_auto_link_saml_user`. If users can change these attributes, they can sign in as other authorized users. @@ -98,7 +98,7 @@ For more information on: :::TabTitle Helm chart (Kubernetes) 1. Make sure GitLab is [configured with HTTPS](https://docs.gitlab.com/charts/installation/tls.html). -1. Edit the [common configuration file settings](omniauth.md#configure-common-settings) +1. Configure the [common settings](omniauth.md#configure-common-settings) to add `saml` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. 1. Export the Helm values: @@ -134,7 +134,7 @@ For more information on: 1. Configure the following attributes so your SAML users cannot change them: - - [`NameID`](../user/group/saml_sso/index.md#nameid). + - [`NameID`](../user/group/saml_sso/index.md#manage-user-saml-identity). - `Email` when used with `omniauth_auto_link_saml_user`. If users can change these attributes, they can sign in as other authorized users. @@ -193,7 +193,7 @@ For more information on: :::TabTitle Docker 1. Make sure GitLab is [configured with HTTPS](https://docs.gitlab.com/omnibus/settings/ssl/). -1. Edit the [common configuration file settings](omniauth.md#configure-common-settings) +1. Configure the [common settings](omniauth.md#configure-common-settings) to add `saml` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. 1. To allow your users to use SAML to sign up without having to manually create @@ -226,7 +226,7 @@ For more information on: 1. Configure the following attributes so your SAML users cannot change them: - - [`NameID`](../user/group/saml_sso/index.md#nameid). + - [`NameID`](../user/group/saml_sso/index.md#manage-user-saml-identity). - `Email` when used with `omniauth_auto_link_saml_user`. If users can change these attributes, they can sign in as other authorized users. @@ -278,7 +278,7 @@ For more information on: :::TabTitle Self-compiled (source) 1. Make sure GitLab is [configured with HTTPS](../install/installation.md#using-https). -1. Edit the [common configuration file settings](omniauth.md#configure-common-settings) +1. Configure the [common settings](omniauth.md#configure-common-settings) to add `saml` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. 1. To allow your users to use SAML to sign up without having to manually create @@ -306,7 +306,7 @@ For more information on: 1. Configure the following attributes so your SAML users cannot change them: - - [`NameID`](../user/group/saml_sso/index.md#nameid). + - [`NameID`](../user/group/saml_sso/index.md#manage-user-saml-identity). - `Email` when used with `omniauth_auto_link_saml_user`. If users can change these attributes, they can sign in as other authorized users. @@ -382,7 +382,7 @@ To configure a SAML application on your IdP, you need at least the following inf - Assertion consumer service URL. - Issuer. -- [`NameID`](../user/group/saml_sso/index.md#nameid). +- [`NameID`](../user/group/saml_sso/index.md#manage-user-saml-identity). - [Email address claim](#configure-assertions). For an example configuration, see [set up identity providers](#set-up-identity-providers). @@ -408,7 +408,8 @@ You can configure GitLab to use multiple SAML IdPs if: - The `strategy_class` is explicitly set because it cannot be inferred from provider name. -[SAML group memberships](#configure-users-based-on-saml-group-membership) and [Group Sync](../user/group/saml_sso/group_sync.md) do not support multiple IdPs. For more information, see [issue 386605](https://gitlab.com/gitlab-org/gitlab/-/issues/386605). +NOTE: +[SAML group memberships](#configure-users-based-on-saml-group-membership) and [Group Sync](../user/group/saml_sso/group_sync.md) do not support multiple IdPs. For more information, see [issue 386605](https://gitlab.com/gitlab-org/gitlab/-/issues/386605). This also includes `required_groups`, as mentioned in [issue 391926](https://gitlab.com/gitlab-org/gitlab/-/issues/391926). To set up multiple SAML IdPs: @@ -509,9 +510,9 @@ To set up multiple SAML IdPs: omniauth: providers: - secret: gitlab-saml - - key: saml + key: saml - secret: gitlab-saml - - key: saml_2 + key: saml_2 ``` To allow your users to use SAML to sign up without having to manually create an @@ -653,7 +654,7 @@ IdPs, contact your provider's support. 1. Complete the SAML general configuration. Enter: - `"Single sign-on URL"`: Use the assertion consumer service URL. - `"Audience URI"`: Use the issuer. - - [`NameID`](../user/group/saml_sso/index.md#nameid). + - [`NameID`](../user/group/saml_sso/index.md#manage-user-saml-identity). - [Assertions](#configure-assertions). 1. In the feedback section, enter that you're a customer and creating an app for internal use. @@ -699,7 +700,7 @@ When configuring the Google Workspace SAML application, record the following inf | | Value | Description | |-------------|--------------|-----------------------------------------------------------------------------------| | SSO URL | Depends | Google Identity Provider details. Set to the GitLab `idp_sso_target_url` setting. | -| Certificate | Downloadable | Run `openssl x509 -in -noout -fingerprint` to generate the SHA1 fingerprint that can be used in the `idp_cert_fingerprint` setting. | +| Certificate | Downloadable | Run `openssl x509 -in -noout -fingerprint -sha1` to generate the SHA1 fingerprint that can be used in the `idp_cert_fingerprint` setting. | Google Workspace Administrator also provides the IdP metadata, Entity ID, and SHA-256 fingerprint. However, GitLab does not need this information to connect to the @@ -734,6 +735,10 @@ For a full list of supported assertions, see the [OmniAuth SAML gem](https://git ## Configure users based on SAML group membership +NOTE: +SAML Group Sync is only supported for the [SAML provider named `saml`](#configure-gitlab-to-use-multiple-saml-idps). +As a result, SAML Group Sync only supports a single SAML provider. For more information, see [issue 366450](https://gitlab.com/gitlab-org/gitlab/-/issues/366450). + You can: - Require users to be members of a certain group. @@ -749,7 +754,7 @@ Support for these groups depends on: - Whether you've installed [GitLab Enterprise Edition (EE)](https://about.gitlab.com/install/). - The [name of the SAML provider](#configure-saml-support-in-gitlab). Group memberships are only supported by a single SAML provider named - `saml`. For more information, see [issue 386605](https://gitlab.com/gitlab-org/gitlab/-/issues/386605). + `saml`. | Group | Tier | GitLab Enterprise Edition (EE) Only? | |------------------------------|--------------------|--------------------------------------| @@ -2135,7 +2140,7 @@ instead of `email`, let GitLab know by setting it on your configuration: By default, the local part of the email address in the SAML response is used to generate the user's GitLab username. -Configure `nickname` in `attribute_statements` to specify one or more attributes that contain a user's desired username: +Configure [`username` or `nickname`](omniauth.md#per-provider-configuration) in `attribute_statements` to specify one or more attributes that contain a user's desired username: ::Tabs @@ -2438,7 +2443,7 @@ The value given is added to the current time at which the response is validated. Before setting the `uid` to a unique attribute, make sure that you have configured the following attributes so your SAML users cannot change them: -- [`NameID`](../user/group/saml_sso/index.md#nameid). +- [`NameID`](../user/group/saml_sso/index.md#manage-user-saml-identity). - `Email` when used with `omniauth_auto_link_saml_user`. If users can change these attributes, they can sign in as other authorized users. @@ -2970,8 +2975,7 @@ Users authenticated with SSO or SAML must not use a password for Git operations over HTTPS. These users can instead: - Set up a [personal access token](../user/profile/personal_access_tokens.md). -- Use the [Git Credential Manager](../user/profile/account/two_factor_authentication.md#git-credential-manager) - which securely authenticates using OAuth. +- Use an [OAuth credential helper](../user/profile/account/two_factor_authentication.md#oauth-credential-helpers). ## Link SAML identity for an existing user @@ -3116,12 +3120,12 @@ such as the following: | Sign SAML assertion | Optional | Validates the integrity of a SAML assertion. When active, signs the whole response. | | Check SAML request signature | Optional | Checks the signature on the SAML response. | | Default RelayState | Optional | Specifies the sub-paths of the base URL that users should end up on after successfully signing in through SAML at your IdP. | -| NameID format | Persistent | See [NameID format details](../user/group/saml_sso/index.md#nameid-format). | +| NameID format | Persistent | See [NameID format details](../user/group/saml_sso/index.md#manage-user-saml-identity). | | Additional URLs | Optional | May include the issuer, identifier, or assertion consumer service URL in other fields on some providers. | For example configurations, see the [notes on specific providers](#set-up-identity-providers). -## Glossary of common terms +## Glossary | Term | Description | |--------------------------------|-------------| diff --git a/doc/integration/security_partners/index.md b/doc/integration/security_partners/index.md index 1ad8ab03914..5453b3417ab 100644 --- a/doc/integration/security_partners/index.md +++ b/doc/integration/security_partners/index.md @@ -15,7 +15,7 @@ each security partner: - [Anchore](https://docs.anchore.com/current/docs/configuration/integration/ci_cd/gitlab/) - [Bridgecrew](https://docs.bridgecrew.io/docs/integrate-with-gitlab-self-managed) - [Checkmarx](https://checkmarx.atlassian.net/wiki/spaces/SD/pages/1929937052/GitLab+Integration) -- [Deepfactor](https://docs.deepfactor.io/hc/en-us/articles/1500008981941) +- [Deepfactor](https://www.deepfactor.io/docs/integrate-deepfactor-scanner-in-your-ci-cd-pipelines/#gitlab) - [GrammaTech](https://www.grammatech.com/codesonar-gitlab-integration) - [Indeni](https://docs.cloudrail.app/#/integrations/gitlab) - [JScrambler](https://docs.jscrambler.com/code-integrity/documentation/gitlab-ci-integration) @@ -23,7 +23,7 @@ each security partner: - [Semgrep](https://semgrep.dev/for/gitlab) - [StackHawk](https://docs.stackhawk.com/continuous-integration/gitlab.html) - [Tenable](https://docs.tenable.com/tenableio/Content/ContainerSecurity/GetStarted.htm) -- [Venafi](https://marketplace.venafi.com/details/gitlab-ci-cd/) +- [Venafi](https://marketplace.venafi.com/xchange/620d2d6ed419fb06a5c5bd36/solution/6292c2ef7550f2ee553cf223) - [Veracode](https://community.veracode.com/s/knowledgeitem/gitlab-ci-MCEKSYPRWL35BRTGOVI55SK5RI4A) - [Fortify](https://www.microfocus.com/en-us/fortify-integrations/gitlab) diff --git a/doc/integration/slash_commands.md b/doc/integration/slash_commands.md index a8f31da940a..02c7debc6dc 100644 --- a/doc/integration/slash_commands.md +++ b/doc/integration/slash_commands.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/integration/sourcegraph.md b/doc/integration/sourcegraph.md index a40290683a5..d90a1fa2cca 100644 --- a/doc/integration/sourcegraph.md +++ b/doc/integration/sourcegraph.md @@ -89,23 +89,16 @@ When visiting one of these views, you can now hover over a code reference to see ## Sourcegraph for GitLab.com -Sourcegraph powered code intelligence is available for all public projects on GitLab.com. +Sourcegraph is available for all public projects on GitLab.com. +Private projects are not supported. +For more information, see [epic 2201](https://gitlab.com/groups/gitlab-org/-/epics/2201). -Support for private projects is not yet available for GitLab.com; -follow the epic [&2201](https://gitlab.com/groups/gitlab-org/-/epics/2201) -for updates. +## Sourcegraph and privacy + +See the [Sourcegraph browser extension documentation](https://docs.sourcegraph.com/integration/browser_extension/references/privacy). ## Troubleshooting ### Sourcegraph isn't working If you enabled Sourcegraph for your project but it isn't working, Sourcegraph may not have indexed the project yet. You can check if Sourcegraph is available for your project by visiting `https://sourcegraph.com/gitlab.com/`replacing `` with the path to your GitLab project. - -## Sourcegraph and Privacy - -From the Sourcegraph [extension documentation](https://docs.sourcegraph.com/integration/browser_extension#privacy) which is the -engine behind the native GitLab integration: - -> Sourcegraph integrations never send any logs, pings, usage statistics, or telemetry to Sourcegraph.com. -> They connect only to Sourcegraph.com as required to provide code intelligence or other functionality on public code. -> As a result, no private code, private repository names, usernames, or any other specific data is sent to Sourcegraph.com. diff --git a/doc/integration/trello_power_up.md b/doc/integration/trello_power_up.md index 9cca8e5485d..122400e8446 100644 --- a/doc/integration/trello_power_up.md +++ b/doc/integration/trello_power_up.md @@ -1,19 +1,19 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Trello Power-Ups **(FREE)** -You can use the Trello Power-Up for GitLab to attach +You can use Trello Power-Ups for GitLab to attach GitLab merge requests to Trello cards. -![GitLab Trello PowerUp - Trello card](img/trello_card_with_gitlab_powerup.png) +![GitLab Trello Power-Ups - Trello card](img/trello_card_with_gitlab_powerup.png) -## Configure the Power-Up +## Configure Power-Ups -To configure a Power-Up for a Trello board: +To configure Power-Ups for a Trello board: 1. Go to your Trello board. 1. Select **Power-Ups** and find the **GitLab** row. diff --git a/doc/integration/twitter.md b/doc/integration/twitter.md index f1bfc5a3662..0ccda59f9b3 100644 --- a/doc/integration/twitter.md +++ b/doc/integration/twitter.md @@ -62,7 +62,7 @@ Twitter. Twitter generates a client ID and secret key for you to use. sudo -u git -H editor config/gitlab.yml ``` -1. Edit the [common configuration file settings](omniauth.md#configure-common-settings) +1. Configure the [common settings](omniauth.md#configure-common-settings) to add `twitter` as a single sign-on provider. This enables Just-In-Time account provisioning for users who do not have an existing GitLab account. diff --git a/doc/integration/vault.md b/doc/integration/vault.md index 78456596723..c505097d65c 100644 --- a/doc/integration/vault.md +++ b/doc/integration/vault.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/legal/index.md b/doc/legal/index.md index ea0301284ea..17a17ba5509 100644 --- a/doc/legal/index.md +++ b/doc/legal/index.md @@ -2,7 +2,6 @@ stage: none group: unassigned info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false --- # Legal diff --git a/doc/operations/error_tracking.md b/doc/operations/error_tracking.md index 0be2f087c62..db68abf0778 100644 --- a/doc/operations/error_tracking.md +++ b/doc/operations/error_tracking.md @@ -6,12 +6,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Error Tracking **(FREE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/389991) in GitLab 15.9. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/389991) -for use in GitLab 15.9, and is planned for removal in GitLab 16.0. We are replacing this feature with functionality in the [GitLab Observability UI](https://gitlab.com/gitlab-org/opstrace/opstrace-ui). Please also reference our direction for [Observability](https://about.gitlab.com/direction/monitor/observability/) and [data visualization](https://about.gitlab.com/direction/monitor/observability/data-visualization/). - Error Tracking allows developers to discover and view errors generated by their application. Because error information is surfaced where the code is being developed, efficiency and awareness are increased. ## How error tracking works @@ -34,15 +28,15 @@ For error tracking to work, you need two pieces: ## Sentry error tracking -[Sentry](https://sentry.io/) is an open source error tracking system. GitLab allows administrators to connect Sentry to GitLab, to allow users to view a list of Sentry errors in GitLab. +[Sentry](https://sentry.io/) is an open source error tracking system. GitLab allows administrators to connect Sentry to GitLab so users can view a list of Sentry errors in GitLab. ### Deploying Sentry -You can sign up to the cloud hosted [Sentry](https://sentry.io) or deploy your own [on-premise instance](https://github.com/getsentry/onpremise/). +You can sign up to the cloud-hosted [Sentry](https://sentry.io) or deploy your own [on-premise instance](https://github.com/getsentry/onpremise/). ### Enabling Sentry -GitLab provides an easy way to connect Sentry to your project. You need at +GitLab provides a way to connect Sentry to your project. You need at least Maintainer [permissions](../user/permissions.md) to enable the Sentry integration. 1. Sign up to Sentry.io or [deploy your own](#deploying-sentry) Sentry instance. @@ -84,7 +78,7 @@ DSN in **Sentry.io > Project Settings > Client Keys (DSN) > Show deprecated DSN* ERROR: Sentry failure builds=0 error=raven: dsn missing private key ``` -## Error Tracking List +## Error Tracking list Users with at least Reporter [permissions](../user/permissions.md) can find the Error Tracking list at **Monitor > Error Tracking** in your project's sidebar. @@ -92,11 +86,11 @@ Here, you can filter errors by title or by status (one of Ignored , Resolved, or ![Error Tracking list](img/error_tracking_list_v12_6.png) -## Error Details +## Error details -From error list, users can navigate to the error details page by selecting the title of any error. +From error list, users can go to the error details page by selecting the title of any error. -This page has: +This page includes: - A link to the Sentry issue. - A link to the GitLab commit if the Sentry [release ID/version](https://docs.sentry.io/product/releases/?platform=javascript#configure-sdk) on the Sentry Issue's first release matches a commit SHA in your GitLab hosted project. @@ -108,26 +102,26 @@ By default, a **Create issue** button is displayed: ![Error Details without Issue Link](img/error_details_v12_7.png) If you create a GitLab issue from the error, the **Create issue** button changes to a **View issue** -button and a link to the GitLab issue displays within the error detail section. +button and a link to the GitLab issue displays in the error detail section. -## Taking Action on errors +## Taking action on errors -You can take action on Sentry Errors from within the GitLab UI. Marking errors ignored or resolved require at least Developer role. +You can take action on Sentry Errors in the GitLab UI. Marking errors as ignored or resolved requires at least Developer role. ### Ignoring errors > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/39665) in GitLab 12.7. -From within the [Error Details](#error-details) page you can ignore a Sentry error by selecting the **Ignore** button near the top of the page. +In the [Error Details](#error-details) page you can ignore a Sentry error by selecting **Ignore** near the top of the page. -Ignoring an error prevents it from appearing in the [Error Tracking List](#error-tracking-list), and silences notifications that were set up within Sentry. +Ignoring an error prevents it from appearing in the [Error Tracking List](#error-tracking-list), and silences notifications that were set up in Sentry. ### Resolving errors > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/39825) in GitLab 12.7. -From within the [Error Details](#error-details) page you can resolve a Sentry error by -selecting the **Resolve** button near the top of the page. +In the [Error Details](#error-details) page you can resolve a Sentry error by +selecting **Resolve** near the top of the page. Marking an error as resolved indicates that the error has stopped firing events. If a GitLab issue is linked to the error, then the issue closes. @@ -140,53 +134,225 @@ If another event occurs, the error reverts to unresolved. > - [Enabled on GitLab.com](https://gitlab.com/gitlab-com/gl-infra/production/-/issues/7586) in GitLab 15.6. NOTE: -Available only on GitLab.com. This feature is currently in [Open Beta](https://about.gitlab.com/handbook/product/gitlab-the-product/#open-beta). +Available only on GitLab.com. This feature is in [Open Beta](https://about.gitlab.com/handbook/product/gitlab-the-product/#open-beta). + +### Known limitations + +Only basic support is provided with `capture_exception` as the holding method. +Additional features requests (see this [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/340178)) are added on a case-by-case basis. + +### Debugging issues + +The majority of languages are supported by Sentry expose `debug` option as part of initialization. +It is also possible to output JSON before it is sent to the API. +See the [Go example](#go) below for a suggested solution. + +### Enabling error tracking + +Regardless of the programming language, you need to enable error tracking for your project. This doc assumes you already have a project for which you want to enable error tracking. +This example uses the `gitlab.com` instance. + +To enable error tracking, follow these steps: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Monitor**. +1. Expand the **Error Tracking** tab. +1. Under **Enable error tracking**, select the **Active** checkbox. +1. Under **Error tracking backend**, select **GitLab**. +1. Select **Save changes**. +1. Copy the DSN string to use later. + +### Listing captured errors + +Once your application has emitted errors to the Error Tracking API through the Sentry SDK, they should be available under **Monitor > Error Tracking** tab/section. + +For more details, refer to the [GitLab error tracking documentation](https://gitlab.com/help/operations/error_tracking#error-tracking-list). + +This process assumes the GDK feature flag `integrated_error_tracking` is enabled. If you are running GDK locally and you do not see the option for error tracking, you can enable it by running the following commands: + +```linux +cd +gdk rails console +Feature.enable(:integrated_error_tracking) +``` + +### Emitting errors + +#### Supported Sentry types + +According to the [data model](https://develop.sentry.dev/sdk/envelopes/#data-model), the available item types are: + +- [Event](https://develop.sentry.dev/sdk/event-payloads/) +- [Transactions](https://develop.sentry.dev/sdk/event-payloads/transaction/) +- Attachment +- [Session](https://develop.sentry.dev/sdk/sessions/) +- [Sessions](https://develop.sentry.dev/sdk/sessions/) +- [User feedback](https://develop.sentry.dev/sdk/envelopes/#user-feedback) (also known as user report) +- [Client report](https://develop.sentry.dev/sdk/client-reports/) + +Items of various types can be sent to the error tracking app, using either the Store endpoint, the envelope endpoint, or both. The following table lists all event types available through Sentry SDK. It also explains which endpoint can be used for ingestion and whether it is supported by GitLab Observability Backend. + +Event item types can contain various interfaces, such as exception, message, stack trace, and template. You can read more about the core data interfaces in [Sentry documentation](https://develop.sentry.dev/sdk/event-payloads/#core-interfaces). + +| Item type | Interface | Can be sent through the Store endpoint | Can be sent through the Envelope endpoint | Supported | +| --------------- | ----------- | -------------------------------------- | ----------------------------------------- | ---------------------- | +| `event` | exception | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | +| `event` | message | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | +| `event` | stack trace | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | +| `event` | template | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | +| `transaction` | N/A | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | +| `attachment` | N/A | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | +| `session` | N/A | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | +| `sessions` | N/A | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | +| `user_report` | N/A | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | +| `client_report` | N/A | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | + +#### Supported languages + +Each language shows a basic example of how to capture exceptions with the respective SDK. +For more in-depth documentation, see [documentation for Sentry SDK](https://docs.sentry.io/). You can also find information for additional programming languages. + +Only a subset of languages is supported. + +The following table lists them: + +| Sentry SDK | Supported? | +| ----------- | ----------- | +| Ruby | Yes | +| Go | Yes | +| JavaScript | Yes | +| Java | Yes | +| Python | Yes | +| PHP | Yes | +| .NET | Not tested | +| Android | Not tested | +| Apple | Not tested | +| Perl | Not tested | + +A more up-to-date version of [this matrix can be found in this doc](https://gitlab.com/gitlab-org/opstrace/opstrace/-/issues/1737). -Integrated error tracking is a lightweight alternative to Sentry backend. -You still use Sentry SDK with your application. But you don't need to deploy Sentry -or set up for cloud-hosted Sentry. Instead, you use GitLab as a backend for it. +#### Go -Sentry backend automatically assigns a Data Source Name (DSN) for every project you create. -GitLab does the same. You should be able to find a DSN for your project in the GitLab error tracking -settings. By using a GitLab-provided DSN, your application connects to GitLab to report an error. -Those errors are stored in the GitLab database and rendered by the GitLab UI, in the same way as -Sentry integration. +1. `chdir` into folder `docs/guides/user/error_tracking_examples/go/` +1. Install the dependencies with the following command: + + ```shell + go mod tidy + ``` + +1. Run the following command: + + ```shell + export SENTRY_DSN="" + go run main.go + ``` + +After you've run this program, there should be an error visible in the Error tracking tab from `Listing captured errors` section of this document. + +#### Ruby + +1. `chdir` into folder `docs/guides/user/error_tracking_examples/ruby/` +1. Install the dependencies with the following command: + + ```shell + gem install bundler + bundle install + ``` + +1. Execute the example with the following command: + + ```shell + export SENTRY_DSN="" + ruby app.rb + ``` + +After you've run this program, there should be an error visible in the Error tracking tab from `Listing captured errors` section of this document. + +#### PHP + +1. `chdir` into folder `docs/guides/user/error_tracking_examples/php/` + +1. Build and run the Docker container with the following commands: + +```shell +export SENTRY_DSN="" +docker build -t sentry-php . +docker run -e SENTRY_DSN --rm sentry-php +``` + +After you've run this program, there should be an error visible in the Error tracking tab from `Listing captured errors` section of this document. + +#### Python + +1. `chdir` into folder `docs/guides/user/error_tracking_examples/python/` + +1. Install the dependencies with the following commands: + + ```shell + virtualenv env + source env/bin/activate + pip install -r requirements.txt + ``` + +1. Run the following commands: + +```shell +export SENTRY_DSN="" +python send_exception.py +``` + +After you've run this program, there should be an error visible in the Error tracking tab from `Listing captured errors` section of this document. + +#### Java + +1. `chdir` into folder `docs/guides/user/error_tracking_examples/python/` + +1. Run the following command: + +```shell +export SENTRY_DSN="" +./gradlew run +``` -In GitLab 15.6 and later, the integrated error tracking -uses a new backend based on the ClickHouse database that enables better scalability. -Integrated error tracking remains limited in comparison to the Sentry backend, as only a small subset of the -Sentry API is implemented. +#### Node.js -Changing the GitLab error UI to use the GitLab Observability UI is tracked in epic [19](https://gitlab.com/groups/gitlab-org/opstrace/-/epics/32). +1. `chdir` into folder `docs/guides/user/error_tracking_examples/nodejs/` -### Project settings +1. Install the dependencies with the following command: -You can find the feature configuration at **Settings > Monitor > Error Tracking**. + ```shell + npm install --save @sentry/node @sentry/tracing + ``` -#### How to enable +1. Run the following command: -1. Select **GitLab** as the error tracking backend for your project: + ```shell + export SENTRY_DSN="" + node ./test.js + ``` - ![Error Tracking Settings](img/error_tracking_setting_v14_3.png) +After you've run this program, there should be an error visible in the Error tracking tab from `Listing captured errors` section of this document. -1. Select **Save changes**. After page reload you should see a text field with the DSN string. Copy it. +### Rotating Sentry DSN - ![Error Tracking Settings DSN](img/error_tracking_setting_dsn_v14_4.png) +The Sentry DSN (client key) is a secret and it should not be exposed to the public. If it's leaked, you can rotate the Sentry DSN with the following steps: -1. Take the DSN from the previous step and configure your Sentry SDK with it. Errors are now - reported to the GitLab collector and are visible in the [GitLab UI](#error-tracking-list). +1. [Create an access token](../user/profile/personal_access_tokens.md#create-a-personal-access-token) by clicking your profile picture in GitLab.com. Then choose Preferences,then Access Token. Make sure you add the API scope. +1. Using the [error tracking API](../api/error_tracking.md), create a new Sentry DSN with the following command: -#### Managing DSN + ```shell + curl --request POST --header "PRIVATE-TOKEN: " --header "Content-Type: application/json" \ + "https://gitlab.example.com/api/v4/projects//error_tracking/client_keys" + ``` -When you enable the feature you receive a DSN. It includes a hash used for authentication. This hash -is a client key. GitLab uses client keys to authenticate error tracking requests from your -application to the GitLab backend. +1. Get the available client keys (Sentry DSNs). Ensure that the newly created Sentry DSN is in place. Then note down the key ID of the old client key: -In some situations, you may want to create a new client key and remove an existing one. -You can do so by managing client keys with the [error tracking API](../api/error_tracking.md). + ```shell + curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects//error_tracking/client_keys" + ``` -#### Limitations +1. Delete the old client key with the following command: -The Integrated Error Tracking feature was built and tested with Sentry SDK for Ruby on Rails. -Support for other languages and frameworks is not guaranteed. For up-to-date information, see the -[compatibility issue](https://gitlab.com/gitlab-org/gitlab/-/issues/340178). + ```shell + curl --request DELETE --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects//error_tracking/client_keys/" + ``` diff --git a/doc/operations/feature_flags.md b/doc/operations/feature_flags.md index 68fc0fb9499..a3874b54ccd 100644 --- a/doc/operations/feature_flags.md +++ b/doc/operations/feature_flags.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -38,7 +38,7 @@ with GitLab, so it's up to developers to use a compatible client library and To create and enable a feature flag: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Deployments > Feature Flags**. +1. On the left sidebar, select **Deployments > Feature flags**. 1. Select **New feature flag**. 1. Enter a name that starts with a letter and contains only lowercase letters, digits, underscores (`_`), or dashes (`-`), and does not end with a dash (`-`) or underscore (`_`). @@ -85,7 +85,7 @@ and the supported strategies are: - [User List](#user-list) Strategies can be added to feature flags when [creating a feature flag](#create-a-feature-flag), -or by editing an existing feature flag after creation by navigating to **Deployments > Feature Flags** +or by editing an existing feature flag after creation by navigating to **Deployments > Feature flags** and selecting **Edit** (**{pencil}**). ### All users @@ -151,7 +151,7 @@ Enables the feature for a list of target users. It is implemented using the Unleash UserIDs (`userWithId`) activation [strategy](https://docs.getunleash.io/reference/activation-strategies#userids). Enter user IDs as a comma-separated list of values (for example, -`user@example.com, user2@example.com`, or `username1,username2,username3`, and so on). +`user@example.com, user2@example.com`, or `username1,username2,username3`, and so on). User IDs are identifiers for your application users. They do not need to be GitLab users. WARNING: @@ -181,7 +181,7 @@ For example: To create a user list: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Deployments > Feature Flags**. +1. On the left sidebar, select **Deployments > Feature flags**. 1. Select **View user lists** 1. Select **New user list**. 1. Enter a name for the list. @@ -197,7 +197,7 @@ When viewing a list, you can rename it by selecting **Edit** (**{pencil}**). To add users to a user list: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Deployments > Feature Flags**. +1. On the left sidebar, select **Deployments > Feature flags**. 1. Select **Edit** (**{pencil}**) next to the list you want to add users to. 1. Select **Add Users**. 1. Enter the user IDs as a comma-separated list of values. For example, @@ -211,7 +211,7 @@ To add users to a user list: To remove users from a user list: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Deployments > Feature Flags**. +1. On the left sidebar, select **Deployments > Feature flags**. 1. Select **Edit** (**{pencil}**) next to the list you want to change. 1. Select **Remove** (**{remove}**) next to the ID you want to remove. @@ -225,7 +225,7 @@ code so that you can clean it up when it's time to remove the feature flag. To search for code references of a feature flag: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Deployments > Feature Flags**. +1. On the left sidebar, select **Deployments > Feature flags**. 1. Edit the feature flag you want to remove. 1. Select **More actions** (**{ellipsis_v}**). 1. Select **Search code references**. @@ -236,7 +236,7 @@ In [GitLab 13.0 and earlier](https://gitlab.com/gitlab-org/gitlab/-/issues/8621) to disable a feature flag for a specific environment: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Deployments > Feature Flags**. +1. On the left sidebar, select **Deployments > Feature flags**. 1. For the feature flag you want to disable, select **Edit** (**{pencil}**). 1. To disable the flag: @@ -251,7 +251,7 @@ to disable a feature flag for a specific environment: To disable a feature flag for all environments: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Deployments > Feature Flags**. +1. On the left sidebar, select **Deployments > Feature flags**. 1. For the feature flag you want to disable, slide the Status toggle to **Disabled**. The feature flag is displayed on the **Disabled** tab. @@ -266,7 +266,7 @@ Then prepare your application with a client library. To get the access credentials that your application needs to communicate with GitLab: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Deployments > Feature Flags**. +1. On the left sidebar, select **Deployments > Feature flags**. 1. Select **Configure** to view the following: - **API URL**: URL where the client (application) connects to get a list of feature flags. - **Instance ID**: Unique token that authorizes the retrieval of the feature flags. @@ -295,14 +295,13 @@ Unleash currently [offers many SDKs for various languages and frameworks](https: For API content, see: - [Feature flags API](../api/feature_flags.md) -- [Feature flag specs API](../api/feature_flag_specs.md) (Deprecated and [scheduled for removal](https://gitlab.com/gitlab-org/gitlab/-/issues/213369) in GitLab 14.0.) - [Feature flag user lists API](../api/feature_flag_user_lists.md) -### Golang application example +### Go application example -Here's an example of how to integrate feature flags in a Golang application: +Here's an example of how to integrate feature flags in a Go application: -```golang +```go package main import ( diff --git a/doc/operations/img/copy-group-id.png b/doc/operations/img/copy-group-id.png deleted file mode 100644 index 7837f49c3e3..00000000000 Binary files a/doc/operations/img/copy-group-id.png and /dev/null differ diff --git a/doc/operations/img/create-gitlab-application.png b/doc/operations/img/create-gitlab-application.png deleted file mode 100644 index 430b830cdb2..00000000000 Binary files a/doc/operations/img/create-gitlab-application.png and /dev/null differ diff --git a/doc/operations/img/error_tracking_setting_dsn_v14_4.png b/doc/operations/img/error_tracking_setting_dsn_v14_4.png deleted file mode 100644 index b7ecaa047b2..00000000000 Binary files a/doc/operations/img/error_tracking_setting_dsn_v14_4.png and /dev/null differ diff --git a/doc/operations/img/error_tracking_setting_v14_3.png b/doc/operations/img/error_tracking_setting_v14_3.png deleted file mode 100644 index 14306130c97..00000000000 Binary files a/doc/operations/img/error_tracking_setting_v14_3.png and /dev/null differ diff --git a/doc/operations/img/listing_groups.png b/doc/operations/img/listing_groups.png deleted file mode 100644 index 1094bf4ff28..00000000000 Binary files a/doc/operations/img/listing_groups.png and /dev/null differ diff --git a/doc/operations/incident_management/alerts.md b/doc/operations/incident_management/alerts.md index 6d6fde45de7..74e0ed5f06a 100644 --- a/doc/operations/incident_management/alerts.md +++ b/doc/operations/incident_management/alerts.md @@ -205,3 +205,28 @@ You can manually create a [to-do item](../../user/todos.md) for yourself from an alert, and view it later on your **To-Do List**. To add a to-do item, on the right sidebar, select **Add a to do**. + +### Trigger actions from alerts **(ULTIMATE)** + +> - Introduced in GitLab 13.1: incidents are not created automatically by default. +> - Mapping common severity values from the alert payload [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50871) in GitLab 13.9. + +Turn on creating [incidents](incidents.md) automatically whenever an alert is triggered. + +Prerequisites: + +- You must have at least the Maintainer role for the project. + +To configure the actions: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Monitor**. +1. Expand the **Alerts** section, then select the **Alert settings** tab. +1. Select the **Create an incident** checkbox. +1. Optional. To customize the incident, from the **Incident template**, select a template to be + appended to the [incident summary](incidents.md#summary). + If the dropdown list is empty, + [create an issue template](../../user/project/description_templates.md#create-an-issue-template) first. +1. Optional. To send [an email notification](paging.md#email-notifications-for-alerts), select the + **Send a single email notification to Owners and Maintainers for new alerts** checkbox. +1. Select **Save changes**. diff --git a/doc/operations/incident_management/incident_timeline_events.md b/doc/operations/incident_management/incident_timeline_events.md index e79f36884cb..d509061eca0 100644 --- a/doc/operations/incident_management/incident_timeline_events.md +++ b/doc/operations/incident_management/incident_timeline_events.md @@ -83,6 +83,17 @@ of an incident. ![Incident timeline event for severity change](img/timeline_event_for_severity_change_v15_6.png) +### When labels change (Experiment) + +> [Introduced]([issue-link](https://gitlab.com/gitlab-org/gitlab/-/issues/365489)) in GitLab 15.3 [with a flag](../../administration/feature_flags.md) named `incident_timeline_events_from_labels`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available per project or for your entire instance, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `incident_timeline_events_from_labels`. +On GitLab.com, this feature is not available. +This feature is not ready for production use. + +A new timeline event is created when someone adds or removes [labels](../../user/project/labels.md) on an incident. + ## Delete an event > Ability to delete an event when editing it [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/372265) in GitLab 15.7. @@ -110,12 +121,10 @@ Alternatively: ## Incident tags -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/8741) in GitLab 15.9 [with a flag](../../administration/feature_flags.md) named `incident_event_tags`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `incident_event_tags`. -On GitLab.com, this feature is not available. -This feature is not ready for production use. +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/8741) in GitLab 15.9 [with a flag](../../administration/feature_flags.md) named `incident_event_tags`. Disabled by default. +> - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/387647) in GitLab 15.9. +> - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/387647) in GitLab 15.10. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/387647) in GitLab 15.11. Feature flag `incident_event_tags` removed. [When creating an event using the form](#using-the-form) or editing it, you can specify incident tags to capture relevant incident timestamps. diff --git a/doc/operations/incident_management/incidents.md b/doc/operations/incident_management/incidents.md index 45f1e10d2f1..5f1a2880c4b 100644 --- a/doc/operations/incident_management/incidents.md +++ b/doc/operations/incident_management/incidents.md @@ -70,7 +70,7 @@ To filter the incident list by author or assignee, enter these values in the sea ### Summary The summary section for incidents provides critical details about the -incident and the contents of the issue template (if [selected](../metrics/alerts.md#trigger-actions-from-alerts)). The highlighted +incident and the contents of the issue template (if [selected](alerts.md#trigger-actions-from-alerts)). The highlighted bar at the top of the incident displays from left to right: - The link to the original alert. @@ -87,14 +87,14 @@ Below the highlight bar, a summary includes the following fields: The incident summary can be further customized using [GitLab Flavored Markdown](../../user/markdown.md). -If an incident is [created from an alert](../metrics/alerts.md#trigger-actions-from-alerts) +If an incident is [created from an alert](alerts.md#trigger-actions-from-alerts) that provided Markdown for the incident, then the Markdown is appended to the summary. If an incident template is configured for the project, then the template content is appended at the end. Comments are displayed in threads, but can be displayed chronologically [by toggling on the recent updates view](#recent-updates-view). -When you make changes to an incident, GitLab creates system notes and +When you make changes to an incident, GitLab creates [system notes](../../user/project/system_notes.md) and displays them below the summary. ### Metrics **(PREMIUM)** @@ -169,14 +169,13 @@ label to the incident. ## Related topics - [Create an incident](manage_incidents.md#create-an-incident) -- [Create an incident automatically](../metrics/alerts.md#trigger-actions-from-alerts) +- [Create an incident automatically](alerts.md#trigger-actions-from-alerts) whenever an alert is triggered - [View incidents list](manage_incidents.md#view-incidents-list) - [Assign to a user](manage_incidents.md#assign-to-a-user) - [Change incident severity](manage_incidents.md#change-severity) - [Change incident status](manage_incidents.md#change-status) - [Change escalation policy](manage_incidents.md#change-escalation-policy) -- [Embed metrics](manage_incidents.md#embed-metrics) - [Close an incident](manage_incidents.md#close-an-incident) - [Automatically close incidents via recovery alerts](manage_incidents.md#automatically-close-incidents-via-recovery-alerts) - [Add a to-do item](../../user/todos.md#create-a-to-do-item) @@ -187,7 +186,7 @@ label to the incident. - [Toggle notifications](../../user/profile/notifications.md#edit-notification-settings-for-issues-merge-requests-and-epics) - [Track spent time](../../user/project/time_tracking.md) - [Add a Zoom meeting to an incident](../../user/project/issues/associate_zoom_meeting.md) the same - way you add it to an issue. + way you add it to an issue - [Linked resources in incidents](linked_resources.md) -- Create incidents and receive incident notifications [directly from Slack](slack.md). -- Use the [Issues API](../../api/issues.md) to interact with incidents. +- Create incidents and receive incident notifications [directly from Slack](slack.md) +- Use the [Issues API](../../api/issues.md) to interact with incidents diff --git a/doc/operations/incident_management/manage_incidents.md b/doc/operations/incident_management/manage_incidents.md index ad4de8641e5..187a398b1ee 100644 --- a/doc/operations/incident_management/manage_incidents.md +++ b/doc/operations/incident_management/manage_incidents.md @@ -70,7 +70,7 @@ You are then credited with the alert's status change. ### Automatically, when an alert is triggered **(ULTIMATE)** -In the project settings, you can turn on [creating an incident automatically](../metrics/alerts.md#trigger-actions-from-alerts) +In the project settings, you can turn on [creating an incident automatically](alerts.md#trigger-actions-from-alerts) whenever an alert is triggered. ### Using the PagerDuty webhook @@ -201,18 +201,14 @@ In GitLab 15.1 and earlier, the escalation policy for [incidents created from al reflects the alert's escalation policy and cannot be changed. In [GitLab 15.2 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/356057), the incident escalation policy is independent and can be changed. -## Embed metrics + -You can embed metrics anywhere [GitLab Flavored Markdown](../../user/markdown.md) is -used, like descriptions or comments. Embedding -metrics helps you share them when discussing incidents or performance issues. +## Embed metrics (removed) -To embed metrics in a Markdown text box in GitLab, -[paste the link to the dashboard](../metrics/embed.md#embedding-gitlab-managed-kubernetes-metrics). +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. -You can embed both [GitLab-hosted metrics](../metrics/embed.md) (deprecated) and -[Grafana metrics](../metrics/embed_grafana.md) in incidents and issue -templates. + ## Close an incident @@ -220,12 +216,16 @@ Prerequisites: - You must have at least the Reporter role for the project. -To close an incident, in the upper right, select **Close incident**. +To close an incident, in the upper-right corner, select **Close incident**. When you close an incident that is linked to an [alert](alerts.md), the linked alert's status changes to **Resolved**. You are then credited with the alert's status change. + +If you don't see this action at the top of an incident, your project or instance might have +enabled a feature flag for [moved actions](../../user/project/merge_requests/index.md#move-sidebar-actions) + ### Automatically close incidents via recovery alerts > [Introduced for HTTP integrations](https://gitlab.com/gitlab-org/gitlab/-/issues/13402) in GitLab 13.4. @@ -249,6 +249,22 @@ When GitLab receives a recovery alert, it closes the associated incident. This action is recorded as a system note on the incident indicating that it was closed automatically by the GitLab Alert bot. +## Delete an incident + +Prerequisites: + +- You must have the Owner role for a project. + +To delete an incident: + +1. In an incident, select **Incident actions** (**{ellipsis_v}**). +1. Select **Delete incident**. + +Alternatively: + +1. In an incident, select **Edit title and description** (**{pencil}**). +1. Select **Delete incident**. + ## Other actions Because incidents in GitLab are built on top of [issues](../../user/project/issues/index.md), diff --git a/doc/operations/incident_management/slack.md b/doc/operations/incident_management/slack.md index 434c481900c..1f6097ccbdb 100644 --- a/doc/operations/incident_management/slack.md +++ b/doc/operations/incident_management/slack.md @@ -4,13 +4,14 @@ group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Incident management for Slack **(FREE SAAS)** +# Incident management for Slack (Beta) **(FREE SAAS)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/344856) in GitLab 15.7 [with a flag](../../administration/feature_flags.md) named `incident_declare_slash_command`. Disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/344856) in GitLab 15.7 [with a flag](../../administration/feature_flags.md) named `incident_declare_slash_command`. Disabled by default. +> - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/378072) in GitLab 15.10 in [Beta](../../policy/alpha-beta-support.md#beta). FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `incident_declare_slash_command`. -On GitLab.com, this feature is not available. +On self-managed GitLab, this feature is not available. +On GitLab.com, this feature is available. The feature is not ready for production use. Many teams receive alerts and collaborate in real time during incidents in Slack. @@ -68,7 +69,7 @@ To declare a GitLab incident from Slack: - The project where the incident should be created. - The severity of the incident. - If there is an existing [incident template](../metrics/alerts.md#trigger-actions-from-alerts) for your + If there is an existing [incident template](alerts.md#trigger-actions-from-alerts) for your project, that template is automatically applied to the description text box. The template is applied only if the description text box is empty. diff --git a/doc/operations/incident_management/status_page.md b/doc/operations/incident_management/status_page.md index 5dd690a1c9f..fd37279806f 100644 --- a/doc/operations/incident_management/status_page.md +++ b/doc/operations/incident_management/status_page.md @@ -159,13 +159,13 @@ To publish comments to the Status Page Incident: - Create a comment on the incident issue. - When you're ready to publish the comment, mark the comment for publication by - adding a microphone [award emoji](../../user/award_emojis.md) + adding a microphone [emoji reaction](../../user/award_emojis.md) reaction (`:microphone:` 🎤) to the comment. - Any files attached to the comment (up to 5000 per issue) are also published. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/205166) in GitLab 13.1.) WARNING: -Anyone with access to view the Issue can add an emoji award to a comment, so +Anyone with access to view the Issue can add an emoji reaction to a comment, so consider limiting access to issues to team members only. ### Update the incident status diff --git a/doc/operations/index.md b/doc/operations/index.md index ff13c617ea7..922ec557c4c 100644 --- a/doc/operations/index.md +++ b/doc/operations/index.md @@ -9,40 +9,26 @@ info: To determine the technical writer assigned to the Stage/Group associated w GitLab provides a variety of tools to help operate and maintain your applications. -## Measure reliability and stability with metrics (DEPRECATED) + -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. +## Measure reliability and stability with metrics (removed) -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) -for use in GitLab 14.7, and is planned for removal in GitLab 16.0. +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. -Metrics help you understand the health and performance of your infrastructure, -applications, and systems by providing insights into your application's reliability, -stability, and performance. GitLab provides a default dashboard that you -can extend with custom metrics, and augment with additional custom dashboards. You -can track the metrics that matter most to your team, generate automated alerts when -performance degrades, and manage those alerts - all within GitLab. - -- Collect [Prometheus metrics](../user/project/integrations/prometheus_library/index.md). -- Monitor application status with the [out-of-the-box metrics dashboard](metrics/index.md), - which you can [customize](metrics/dashboards/settings.md). -- Create [custom performance alerts](metrics/alerts.md). -- Create [custom metrics](metrics/index.md#adding-custom-metrics) and - [custom dashboards](metrics/dashboards/index.md). + ## Manage alerts and incidents GitLab helps reduce alert fatigue for IT responders by providing tools to identify issues across multiple systems and aggregate alerts in a centralized place. Your -team needs a single, central interface where they can easily investigate alerts +team needs a single, central interface where they can investigate alerts and promote the critical alerts to incidents. -Are your alerts too noisy? Alerts configured on GitLab metrics can configured +Are your alerts too noisy? Alerts can be configured and fine-tuned in GitLab immediately following a fire-fight. - [Manage alerts and incidents](incident_management/index.md) in GitLab. -- [Configure alerts for metrics](metrics/alerts.md) in GitLab. (DEPRECATED) - Create a [status page](incident_management/status_page.md) to communicate efficiently to your users during an incident. @@ -76,4 +62,4 @@ an environment. - Deploy to different [environments](../ci/environments/index.md). - Connect your project to a [Kubernetes cluster](../user/infrastructure/clusters/index.md). -- Create, toggle, and remove [Feature Flags](feature_flags.md). +- Create, toggle, and remove [feature flags](feature_flags.md). diff --git a/doc/operations/metrics/alerts.md b/doc/operations/metrics/alerts.md index 44cd683bc4f..44a257f532b 100644 --- a/doc/operations/metrics/alerts.md +++ b/doc/operations/metrics/alerts.md @@ -2,81 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../index.md' --- -# Set up alerts for Prometheus metrics **(FREE)** +# Set up alerts for Prometheus metrics (removed) **(FREE)** -After [configuring metrics for your CI/CD environment](index.md), you can set up -alerting for Prometheus metrics, and -[trigger actions from alerts](#trigger-actions-from-alerts) to notify -your team when environment performance falls outside of the boundaries you set. - -## Prometheus cluster integrations - -Alerts are not supported for [Prometheus cluster integrations](../../user/clusters/integrations.md). - -## Trigger actions from alerts **(ULTIMATE)** - -> - Introduced in GitLab 13.1: incidents are not created automatically by default . -> - Mapping common severity values from the alert payload ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50871) in GitLab 13.9. - -Turn on creating [incidents](../incident_management/incidents.md) automatically whenever an alert is triggered. - -Prerequisites: - -- You must have at least the Maintainer role for the project. - -To configure the actions: - -1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Settings > Monitor**. -1. Expand the **Alerts** section, then select the **Alert settings** tab. -1. Select the **Create an incident** checkbox. -1. Optional. To customize the incident, from the **Incident template**, select a template to be - appended to the [incident summary](../incident_management/incidents.md#summary). - If the dropdown list is empty, - [create an issue template](../../user/project/description_templates.md#create-an-issue-template) first. -1. Optional. To send [an email notification](../incident_management/paging.md#email-notifications-for-alerts), select the - **Send a single email notification to Owners and Maintainers for new alerts** checkbox. -1. Select **Save changes**. - -### Fields in automatically created incidents - -Incidents [created automatically from an alert](#trigger-actions-from-alerts) are filled with -values extracted from the `alerts` field in the -[webhook payload](https://prometheus.io/docs/alerting/latest/configuration/#webhook_config): - -- Incident author: `GitLab Alert Bot` -- Incident title: Extracted from the alert payload fields `annotations/title`, `annotations/summary`, or `labels/alertname`. -- Incident description: Extracted from alert payload field `annotations/description`. -- Alert `Summary`: A list of properties from the alert's payload. - - `starts_at`: Alert start time from the payload's `startsAt` field - - `full_query`: Alert query extracted from the payload's `generatorURL` field - - Optional list of attached annotations extracted from `annotations/*` -- Alert [GLFM](../../user/markdown.md): GitLab Flavored Markdown from the payload's `annotations/gitlab_incident_markdown` field. -- Alert severity: - Extracted from the alert payload field `labels/severity`. Maps case-insensitive - value to [Alert's severity](../incident_management/alerts.md#alert-severity): - - | Alert payload | Mapped to alert severity | - | ------------- | --------------------------------------------------------------------------- | - | Critical | `critical`, `s1`, `p1`, `emergency`, `fatal`, or any value not in this list | - | High | `high`, `s2`, `p2`, `major`, `page` | - | Medium | `medium`, `s3`, `p3`, `error`, `alert` | - | Low | `low`, `s4`, `p4`, `warn`, `warning` | - | Info | `info`, `s5`, `p5`, `debug`, `information`, `notice` | - -To further customize the incident, you can add labels, mentions, or any other supported -[quick action](../../user/project/quick_actions.md) in the selected issue template, -which applies to all incidents. To limit quick actions or other information to -only specific types of alerts, use the `annotations/gitlab_incident_markdown` field. - -GitLab tags each incident issue with the `incident` label automatically. If the label -does not yet exist, it's created automatically. - -### Recovery alerts - -The alert in GitLab is automatically resolved when Prometheus -sends a payload with the field `status` set to `resolved`. - -You can also configure the associated [incident to be closed automatically](../incident_management/manage_incidents.md#automatically-close-incidents-via-recovery-alerts) when the alert resolves. +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/operations/metrics/dashboards/default.md b/doc/operations/metrics/dashboards/default.md index e22b1096023..28a3adc5051 100644 --- a/doc/operations/metrics/dashboards/default.md +++ b/doc/operations/metrics/dashboards/default.md @@ -2,42 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../../index.md' --- -# GitLab-defined metrics dashboards (DEPRECATED) **(FREE)** +# GitLab-defined metrics dashboards (removed) **(FREE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) -in GitLab 14.7, and is planned for removal in GitLab 16.0. - -GitLab provides some dashboards out-of-the-box for any project with -[Prometheus available](../../../user/project/integrations/prometheus.md). You can -[duplicate these GitLab-defined dashboards](index.md#duplicate-a-gitlab-defined-dashboard): - -- [Overview dashboard](#overview-dashboard). -- [Kubernetes pod health dashboard](#kubernetes-pod-health-dashboard). - -To learn about the components of a dashboard, read -[Metrics dashboard for your CI/CD environment](../index.md). - -## Overview dashboard - -This dashboard is the default metrics dashboard. It displays a large number of -metrics about the [deployed application](../index.md#configure-prometheus-to-gather-metrics). - -![Example of metrics dashboard](../img/example-dashboard_v13_3.png) - -## Kubernetes pod health dashboard - -This dashboard requires Kubernetes v1.14 or higher, due to the -[change in metric labels](https://github.com/kubernetes/kubernetes/pull/69099) -in Kubernetes 1.14. - -This dashboard displays CPU, memory, network and disk metrics for the pods in your -[connected Kubernetes cluster](../../../user/infrastructure/clusters/index.md). It provides a -[variable selector](templating_variables.md#metric_label_values-variable-type) -at the top of the dashboard to select which pod's metrics to display. - -![Kubernetes pod health dashboard](img/k8s_pod_health_dashboard_v13_3.png) +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/operations/metrics/dashboards/develop.md b/doc/operations/metrics/dashboards/develop.md index fe14ec1dc3d..b7912e164d7 100644 --- a/doc/operations/metrics/dashboards/develop.md +++ b/doc/operations/metrics/dashboards/develop.md @@ -2,38 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../../index.md' --- -# Developing templates for custom dashboards (DEPRECATED) **(FREE)** +# Developing templates for custom dashboards (removed) **(FREE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) -in GitLab 14.7, and is planned for removal in GitLab 16.0. - -GitLab provides a template to make it easier for you to create templates for -[custom dashboards](index.md). Templates provide helpful guidance and -commented-out examples you can use. - -## Apply a dashboard template - -Go to the browser-based editor of your choice: - -- In the **Repository view**: - - 1. Go to **Repository > Files**. - 1. Select **Add to tree** and select **New file**, - then select **Select a template type** to see a list of available templates: - ![Metrics dashboard template selection](img/metrics_dashboard_template_selection_v13_3.png) - -- In the **[Web IDE](../../../user/project/web_ide/index.md)**: - - 1. Select **Web IDE** when viewing your repository. - 1. Select **New file**, then select **Choose a template** to see a list of available templates: - ![Metrics dashboard template selection WebIDE](img/metrics_dashboard_template_selection_web_ide_v13_3.png) - -## Custom dashboard templates **(PREMIUM SELF)** - -To enable and use a custom dashboard templates on your GitLab instance, read the -[guide for creating custom templates](../../../user/admin_area/settings/instance_template_repository.md). +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/operations/metrics/dashboards/img/actions_menu_create_add_panel_v13_3.png b/doc/operations/metrics/dashboards/img/actions_menu_create_add_panel_v13_3.png deleted file mode 100644 index e03fbef3b35..00000000000 Binary files a/doc/operations/metrics/dashboards/img/actions_menu_create_add_panel_v13_3.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/actions_menu_create_new_dashboard_v13_3.png b/doc/operations/metrics/dashboards/img/actions_menu_create_new_dashboard_v13_3.png deleted file mode 100644 index ba4780b730b..00000000000 Binary files a/doc/operations/metrics/dashboards/img/actions_menu_create_new_dashboard_v13_3.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/dashboard_external_link_v13_1.png b/doc/operations/metrics/dashboards/img/dashboard_external_link_v13_1.png deleted file mode 100644 index 3e8d792c53e..00000000000 Binary files a/doc/operations/metrics/dashboards/img/dashboard_external_link_v13_1.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/dashboard_local_timezone_v13_1.png b/doc/operations/metrics/dashboards/img/dashboard_local_timezone_v13_1.png deleted file mode 100644 index 8d45607a940..00000000000 Binary files a/doc/operations/metrics/dashboards/img/dashboard_local_timezone_v13_1.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/external_dashboard_link.png b/doc/operations/metrics/dashboards/img/external_dashboard_link.png deleted file mode 100644 index 82c5e05e467..00000000000 Binary files a/doc/operations/metrics/dashboards/img/external_dashboard_link.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/heatmap_chart_too_much_data_v_13_2.png b/doc/operations/metrics/dashboards/img/heatmap_chart_too_much_data_v_13_2.png deleted file mode 100644 index c3a391b06c7..00000000000 Binary files a/doc/operations/metrics/dashboards/img/heatmap_chart_too_much_data_v_13_2.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/heatmap_panel_type.png b/doc/operations/metrics/dashboards/img/heatmap_panel_type.png deleted file mode 100644 index a2b3911ec68..00000000000 Binary files a/doc/operations/metrics/dashboards/img/heatmap_panel_type.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/k8s_pod_health_dashboard_v13_3.png b/doc/operations/metrics/dashboards/img/k8s_pod_health_dashboard_v13_3.png deleted file mode 100644 index dc0bc951500..00000000000 Binary files a/doc/operations/metrics/dashboards/img/k8s_pod_health_dashboard_v13_3.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/metrics_dashboard_annotations_ui_v13.0.png b/doc/operations/metrics/dashboards/img/metrics_dashboard_annotations_ui_v13.0.png deleted file mode 100644 index a042fbbcf4e..00000000000 Binary files a/doc/operations/metrics/dashboards/img/metrics_dashboard_annotations_ui_v13.0.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/metrics_dashboard_panel_preview_v13_3.png b/doc/operations/metrics/dashboards/img/metrics_dashboard_panel_preview_v13_3.png deleted file mode 100644 index 3c3203265e1..00000000000 Binary files a/doc/operations/metrics/dashboards/img/metrics_dashboard_panel_preview_v13_3.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/metrics_dashboard_template_selection_v13_3.png b/doc/operations/metrics/dashboards/img/metrics_dashboard_template_selection_v13_3.png deleted file mode 100644 index 1571ab9de90..00000000000 Binary files a/doc/operations/metrics/dashboards/img/metrics_dashboard_template_selection_v13_3.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/metrics_dashboard_template_selection_web_ide_v13_3.png b/doc/operations/metrics/dashboards/img/metrics_dashboard_template_selection_web_ide_v13_3.png deleted file mode 100644 index 650f66e9a30..00000000000 Binary files a/doc/operations/metrics/dashboards/img/metrics_dashboard_template_selection_web_ide_v13_3.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/panel_context_menu_v14_0.png b/doc/operations/metrics/dashboards/img/panel_context_menu_v14_0.png deleted file mode 100644 index 78cce5d30b7..00000000000 Binary files a/doc/operations/metrics/dashboards/img/panel_context_menu_v14_0.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/prometheus_dashboard_anomaly_panel_type.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_anomaly_panel_type.png deleted file mode 100644 index 5cba6fa9038..00000000000 Binary files a/doc/operations/metrics/dashboards/img/prometheus_dashboard_anomaly_panel_type.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/prometheus_dashboard_area_panel_type_v12_8.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_area_panel_type_v12_8.png deleted file mode 100644 index 8c5663fef12..00000000000 Binary files a/doc/operations/metrics/dashboards/img/prometheus_dashboard_area_panel_type_v12_8.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/prometheus_dashboard_bar_chart_panel_type_v12.10.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_bar_chart_panel_type_v12.10.png deleted file mode 100644 index 593e31477f4..00000000000 Binary files a/doc/operations/metrics/dashboards/img/prometheus_dashboard_bar_chart_panel_type_v12.10.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/prometheus_dashboard_column_panel_type.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_column_panel_type.png deleted file mode 100644 index 985f2b04ef3..00000000000 Binary files a/doc/operations/metrics/dashboards/img/prometheus_dashboard_column_panel_type.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/prometheus_dashboard_gauge_panel_type_v13_3.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_gauge_panel_type_v13_3.png deleted file mode 100644 index 547c565c6f9..00000000000 Binary files a/doc/operations/metrics/dashboards/img/prometheus_dashboard_gauge_panel_type_v13_3.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/prometheus_dashboard_label_variable_shorthand.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_label_variable_shorthand.png deleted file mode 100644 index 15111a97464..00000000000 Binary files a/doc/operations/metrics/dashboards/img/prometheus_dashboard_label_variable_shorthand.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/prometheus_dashboard_label_variables.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_label_variables.png deleted file mode 100644 index 9b94d0c6afa..00000000000 Binary files a/doc/operations/metrics/dashboards/img/prometheus_dashboard_label_variables.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/prometheus_dashboard_repeated_label.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_repeated_label.png deleted file mode 100644 index d43a890f0fa..00000000000 Binary files a/doc/operations/metrics/dashboards/img/prometheus_dashboard_repeated_label.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/prometheus_dashboard_single_stat_panel_type.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_single_stat_panel_type.png deleted file mode 100644 index 2d7dfb27b49..00000000000 Binary files a/doc/operations/metrics/dashboards/img/prometheus_dashboard_single_stat_panel_type.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/prometheus_dashboard_stacked_column_panel_type_v12_8.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_stacked_column_panel_type_v12_8.png deleted file mode 100644 index ba67509bcf3..00000000000 Binary files a/doc/operations/metrics/dashboards/img/prometheus_dashboard_stacked_column_panel_type_v12_8.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/prometheus_dashboard_yaml_validation_v13_1.png b/doc/operations/metrics/dashboards/img/prometheus_dashboard_yaml_validation_v13_1.png deleted file mode 100644 index 08d7d6603d2..00000000000 Binary files a/doc/operations/metrics/dashboards/img/prometheus_dashboard_yaml_validation_v13_1.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/img/related_links_v13_1.png b/doc/operations/metrics/dashboards/img/related_links_v13_1.png deleted file mode 100644 index 4dc141f0e7f..00000000000 Binary files a/doc/operations/metrics/dashboards/img/related_links_v13_1.png and /dev/null differ diff --git a/doc/operations/metrics/dashboards/index.md b/doc/operations/metrics/dashboards/index.md index 09bb8cedbf6..a4cf12cc64a 100644 --- a/doc/operations/metrics/dashboards/index.md +++ b/doc/operations/metrics/dashboards/index.md @@ -2,267 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../../index.md' --- -# Custom dashboards (DEPRECATED) **(FREE)** +# Custom dashboards (removed) **(FREE)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/59974) in GitLab 12.1. -> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) -in GitLab 14.7, and is planned for removal in GitLab 16.0. - -By default, all projects include a [GitLab-defined Prometheus dashboard](default.md), which -includes a few key metrics, but you can also define your own custom dashboards. - -You may create a [new dashboard from scratch](#add-a-new-dashboard-to-your-project) -or [duplicate a GitLab-defined Prometheus dashboard](#duplicate-a-gitlab-defined-dashboard). - -## Add a new dashboard to your project - -> UI option [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/228856) in GitLab 13.3. - -You can configure a custom dashboard by adding a new YAML file into your project's -`.gitlab/dashboards/` directory. For the dashboard to display on your project's -**Monitor > Metrics** page, the files must have a `.yml` -extension and be present in your project's **default** branch. - -To create a new dashboard from the GitLab user interface: - -1. Sign in to GitLab as a user with Maintainer or Owner - [permissions](../../../user/permissions.md#project-members-permissions). -1. Navigate to your dashboard at **Monitor > Metrics**. -1. In the upper-right corner of your dashboard, select the **{ellipsis_v}** **More actions** menu, - and select **Create new**: - ![Monitoring Dashboard actions menu with create new item](img/actions_menu_create_new_dashboard_v13_3.png) -1. In the modal window, select **Open Repository**, then follow the instructions - for creating a new dashboard from the command line. - -To create a new dashboard from the command line: - -1. Create `.gitlab/dashboards/prom_alerts.yml` under your repository's root - directory. Each YAML file should define the layout of the dashboard and the - Prometheus queries used to populate data. This example dashboard displays a - single area chart: - - ```yaml - dashboard: 'Dashboard Title' - panel_groups: - - group: 'Group Title' - panels: - - type: area-chart - title: 'Chart Title' - y_label: 'Y-Axis' - y_axis: - format: number - precision: 0 - metrics: - - id: my_metric_id - query_range: 'http_requests_total' - label: 'Instance: {{instance}}, method: {{method}}' - unit: 'count' - ``` - -1. Save the file, commit, and push to your repository. The file must be present in your **default** branch. -1. Navigate to your project's **Monitor > Metrics** and choose the custom - dashboard from the dropdown list. - -Your custom dashboard is available at `https://example.com/project/-/metrics/custom_dashboard_name.yml`. - -NOTE: -Configuration files nested under subdirectories of `.gitlab/dashboards` aren't -supported or available in the UI. - -## Add a new metrics panel to a dashboard - -> UI option [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/228761) in GitLab 13.3. - -The metrics dashboard supports various [multiple panel types](../../../operations/metrics/dashboards/panel_types.md). -You can quickly test how a panel configuration would display in your metrics dashboard -with the **Add Panel** page: - -1. Sign in to GitLab as a user with Maintainer or Owner - [permissions](../../../user/permissions.md#project-members-permissions). -1. Select **Add panel** in the **{ellipsis_v}** **More actions** menu. - - NOTE: - You can only add panels to custom dashboards. - - ![Monitoring Dashboard actions menu with add panel item](img/actions_menu_create_add_panel_v13_3.png) -1. In the **Define and preview panel** section, paste in the YAML you want to - preview in the **Panel YAML** field. -1. Select **Preview panel**, and GitLab displays a preview of the chart below the - `Define and preview panel` section: - ![Monitoring Dashboard Add Panel page](img/metrics_dashboard_panel_preview_v13_3.png) - -## Duplicate a GitLab-defined dashboard - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/37238) in GitLab 12.7. -> - [GitLab versions 12.8 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/39505), custom metrics are also duplicated when you duplicate a dashboard. - -You can save a complete copy of a GitLab-defined dashboard along with all custom metrics added to it. -The resulting `.yml` file can be customized and adapted to your project. -You can decide to save the dashboard `.yml` file in the project's **default** branch or in a -new branch. To duplicate a GitLab-defined dashboard: - -1. Select **Duplicate current dashboard** in the **{ellipsis_v}** **More actions** menu. -1. Enter the filename and other information, such as the new commit's message, and select **Duplicate**. -1. Select a branch to add your dashboard to: - - *If you select your **default** branch,* the new dashboard becomes immediately available. - - *If you select another branch,* this branch should be merged to your **default** branch first. - -Your custom dashboard is available at `https://example.com/project/-/metrics/custom_dashboard_name.yml`. - -## Manage the metrics dashboard settings - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/223204) in GitLab 13.2. - -Users with project Maintainer or Administrator -[permissions](../../../user/permissions.md#project-members-permissions) -can manage [the settings](settings.md) for your metrics dashboard. - -## Chart Context Menu - -You can take action related to a chart's data by selecting the -**{ellipsis_v}** **More actions** dropdown list above the upper right corner of -any chart on a dashboard: - -![Context Menu](img/panel_context_menu_v14_0.png) - -The options are: - -- **Expand panel** - Displays a larger version of a visualization. To return to - the dashboard, select the **Back** button in your browser, or press the Escape key. - ([Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3100) in GitLab 13.0.) -- **Download CSV** - Data from Prometheus charts on the metrics dashboard can be downloaded as CSV. -- [Copy link to chart](../embed.md#embedding-gitlab-managed-kubernetes-metrics) - -### Timeline zoom and URL sharing - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/198910) in GitLab 12.8. - -You can use the **Timeline zoom** function at the bottom of a chart to zoom in -on a date and time of your choice. When you select and drag the sliders to select -a different beginning or end date of data to display, GitLab adds your selected start -and end times to the URL, enabling you to share specific time frames more easily. - -## Dashboard Annotations - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211330) in GitLab 12.10 (enabled by feature flag `metrics_dashboard_annotations`). -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/215224) in GitLab 13.0. - -You can use **Metrics Dashboard Annotations** to mark any important events on -every metrics dashboard by adding annotations to it. While viewing a dashboard, -annotation entries assigned to the selected time range are automatically -fetched and displayed on every chart within that dashboard. On mouse hover, each -annotation presents additional details, including the exact time of an event and -its description. - -You can create annotations by making requests to the -[Metrics dashboard annotations API](../../../api/metrics_dashboard_annotations.md) - -![Annotations UI](img/metrics_dashboard_annotations_ui_v13.0.png) - -### Annotation retention policy - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211433) in GitLab 13.01. - -To avoid excessive storage space consumption by stale annotations, records attached -to time periods older than two weeks are removed daily. This recurring background -job runs at 1:00 a.m. local server time. - -## Add related links to custom dashboards - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216385) in GitLab 13.1. - -You can embed links to other dashboards or external services in your custom -dashboard by adding **Related links** to your dashboard's YAML file. Related links -open in the same tab as the dashboard. Related links can be displayed in the -following locations on your dashboard: - -- At the top of your dashboard as the top level [`links` dashboard property](../../../operations/metrics/dashboards/yaml.md#dashboard-top-level-properties). -- In charts context menus as the [`links` property of a panel](../../../operations/metrics/dashboards/yaml.md#panel-panels-properties). - -Related links can contain the following attributes: - -- `url`: The full URL to the link. Required. -- `title`: A phrase describing the link. Optional. If this attribute is not set, - the full URL is used for the link title. -- `type`: A string declaring the type of link. Optional. If set to `grafana`, the - dashboard's time range values are converted to the Grafana time range format and - appended to the `url`. - -The dashboard's time range is appended to the `url` as URL parameters. - -The following example shows two related links (`GitLab.com` and `GitLab Documentation`) -added to a dashboard: - -![Links UI](img/related_links_v13_1.png) - -### Links Syntax - -```yaml -links: - - title: GitLab.com - url: https://gitlab.com - - title: GitLab Documentation - url: https://docs.gitlab.com - - title: Public Grafana playground dashboard - url: https://play.grafana.org/d/000000012/grafana-play-home?orgId=1 - type: grafana -``` - -## Troubleshooting - -### Accessing the UI of Prometheus in Kubernetes - -When troubleshooting issues with an in-cluster Prometheus, it can help to -view the Prometheus UI. In the example below, we assume the Prometheus -server to be the pod `prometheus-prometheus-server` in the `gitlab-managed-apps` -namespace: - -1. Find the name of the Prometheus pod in the user interface of your Kubernetes - provider, such as GKE, or by running the following `kubectl` command in your - terminal. For example: - - ```shell - kubectl get pods -n gitlab-managed-apps | grep 'prometheus-prometheus-server' - ``` - - The command should return a result like the following example, where - `prometheus-prometheus-server-55b4bd64c9-dpc6b` is the name of the Prometheus pod: - - ```plaintext - gitlab-managed-apps prometheus-prometheus-server-55b4bd64c9-dpc6b 2/2 Running 0 71d - ``` - -1. Run a `kubectl port-forward` command. In the following example, `9090` is the - Prometheus server's listening port: - - ```shell - kubectl port-forward prometheus-prometheus-server-55b4bd64c9-dpc6b 9090:9090 -n gitlab-managed-apps - ``` - - The `port-forward` command forwards all requests sent to your system's `9090` port - to the `9090` port of the Prometheus pod. If the `9090` port on your system is used - by another application, you can change the port number before the colon to your - desired port. For example, to forward port `8080` of your local system, change the - command to: - - ```shell - kubectl port-forward prometheus-prometheus-server-55b4bd64c9-dpc6b 8080:9090 -n gitlab-managed-apps - ``` - -1. Open `localhost:9090` in your browser to display the Prometheus user interface. - -### "No data found" error on Metrics dashboard page - -If the "No data found" screen continues to appear, it could be due to: - -- No successful deployments have occurred to this environment. -- Prometheus does not have performance data for this environment, or the metrics - are not labeled correctly. To test this, connect to the Prometheus server and - [run a query](../../../user/project/integrations/prometheus_library/kubernetes.md#metrics-supported), replacing `$CI_ENVIRONMENT_SLUG` - with the name of your environment. -- You may need to re-add the GitLab predefined common metrics. This can be done by running the [import common metrics Rake task](../../../administration/raketasks/maintenance.md#import-common-metrics). +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/operations/metrics/dashboards/panel_types.md b/doc/operations/metrics/dashboards/panel_types.md index 177a55fb85b..c789e99052c 100644 --- a/doc/operations/metrics/dashboards/panel_types.md +++ b/doc/operations/metrics/dashboards/panel_types.md @@ -2,316 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../../index.md' --- -# Panel types for dashboards (DEPRECATED) **(FREE)** +# Panel types for dashboards (removed) **(FREE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) -in GitLab 14.7, and is planned for removal in GitLab 16.0. - -The below panel types are supported in monitoring dashboards. - -## Area or Line Chart - -To add an area chart panel type to a dashboard, look at the following sample dashboard file: - -```yaml -dashboard: 'Dashboard Title' -panel_groups: - - group: 'Group Title' - panels: - - type: area-chart # or line-chart - title: 'Area Chart Title' - y_label: 'Y-Axis' - y_axis: - format: number - precision: 0 - metrics: - - id: area_http_requests_total - query_range: 'http_requests_total' - label: 'Instance: {{instance}}, Method: {{method}}' - unit: "count" -``` - -Note the following properties: - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| `type` | string | no | Type of panel to be rendered. Optional for area panel types | -| `query_range` | string | required | For area panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) | - -![area panel chart](img/prometheus_dashboard_area_panel_type_v12_8.png) - -Starting in [version 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/202696), the y-axis values scale according to the data. Previously, it always started from 0. - -## Anomaly chart - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16530) in GitLab 12.5. - -To add an anomaly chart panel type to a dashboard, add a panel with *exactly* 3 metrics. - -The first metric represents the current state, and the second and third metrics represent the upper and lower limit respectively: - -```yaml -dashboard: 'Dashboard Title' -panel_groups: - - group: 'Group Title' - panels: - - type: anomaly-chart - title: 'Chart Title' - y_label: "Y-Axis" - metrics: - - id: anomaly_requests_normal - query_range: 'http_requests_total' - label: '# of Requests' - unit: 'count' - - id: anomaly_requests_upper_limit - query_range: 10000 - label: 'Max # of requests' - unit: 'count' - - id: anomaly_requests_lower_limit - query_range: 2000 - label: 'Min # of requests' - unit: 'count' -``` - -Note the following properties: - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| `type` | string | required | Must be `anomaly-chart` for anomaly panel types | -| `query_range` | yes | required | For anomaly panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) in every metric. | - -![anomaly panel type](img/prometheus_dashboard_anomaly_panel_type.png) - -## Bar chart - -To add a bar chart to a dashboard, look at the following sample dashboard file: - -```yaml -dashboard: 'Dashboard Title' -panel_groups: - - group: 'Group title' - panels: - - type: bar - title: 'HTTP Handlers' - x_label: 'Response Size' - y_axis: - name: 'Handlers' - metrics: - - id: prometheus_http_response_size_bytes_bucket - query_range: 'sum(increase(prometheus_http_response_size_bytes_bucket[1d])) by (handler)' - unit: 'Bytes' -``` - -Note the following properties: - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| `type` | string | yes | Type of panel to be rendered. For bar chart types, set to `bar` | -| `query_range` | yes | yes | For bar chart, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) - -![bar chart panel type](img/prometheus_dashboard_bar_chart_panel_type_v12.10.png) - -## Column chart - -To add a column panel type to a dashboard, look at the following sample dashboard file: - -```yaml -dashboard: 'Dashboard Title' -panel_groups: - - group: 'Group title' - panels: - - title: 'Column' - type: 'column' - metrics: - - id: 1024_memory - query: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}) without (job)) /1024/1024' - unit: MB - label: 'Memory Usage' -``` - -Note the following properties: - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| `type` | string | yes | Type of panel to be rendered. For column panel types, set to `column` | -| `query_range` | yes | yes | For column panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) | - -![anomaly panel type](img/prometheus_dashboard_column_panel_type.png) - -## Stacked column - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30583) in GitLab 12.8. - -To add a stacked column panel type to a dashboard, look at the following sample dashboard file: - -```yaml -dashboard: 'Dashboard title' -priority: 1 -panel_groups: - - group: 'Group Title' - priority: 5 - panels: - - type: 'stacked-column' - title: 'Stacked column' - y_label: 'y label' - x_label: 'x label' - metrics: - - id: memory_1 - query_range: 'memory_query' - label: 'memory query 1' - unit: 'count' - series_name: 'group 1' - - id: memory_2 - query_range: 'memory_query_2' - label: 'memory query 2' - unit: 'count' - series_name: 'group 2' -``` - -![stacked column panel type](img/prometheus_dashboard_stacked_column_panel_type_v12_8.png) - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| `type` | string | yes | Type of panel to be rendered. For stacked column panel types, set to `stacked-column` | -| `query_range` | yes | yes | For stacked column panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) | - -## Single Stat - -To add a single stat panel type to a dashboard, look at the following sample dashboard file: - -```yaml -dashboard: 'Dashboard Title' -panel_groups: - - group: 'Group Title' - panels: - - title: 'Single Stat' - type: 'single-stat' - metrics: - - id: 10 - query: 'max(go_memstats_alloc_bytes{job="prometheus"})' - unit: MB - label: 'Total' -``` - -Note the following properties: - -| Property | Type | Required | Description | -| ------- | ------ | ------ | ------ | -| `type` | string | yes | Type of panel to be rendered. For single stat panel types, set to `single-stat` | -| `field` | string | no | Panels display the value of a metric. For a panel to display the value of a label instead, put the name of the label in this key. | -| `query` | string | yes | For single stat panel types, you must use an [instant query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries). | - -![single stat panel type](img/prometheus_dashboard_single_stat_panel_type.png) - -## Percentile based results - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201946) in GitLab 12.8. - -Query results sometimes need to be represented as a percentage value out of 100. You can use the `max_value` property at the root of the panel definition: - -```yaml -dashboard: 'Dashboard Title' -panel_groups: - - group: 'Group Title' - panels: - - title: 'Single Stat' - type: 'single-stat' - max_value: 100 - metrics: - - id: 10 - query: 'max(go_memstats_alloc_bytes{job="prometheus"})' - unit: '%' - label: 'Total' -``` - -For example, if you have a query value of `53.6`, adding `%` as the unit results in a single stat value of `53.6%`, but if the maximum expected value of the query is `120`, the value would be `44.6%`. Adding the `max_value` causes the correct percentage value to display. - -## Gauge - -WARNING: -This panel type is an [Alpha](../../../policy/alpha-beta-support.md#alpha-features) feature, and is subject to change at any time -without prior notice! - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207044) in GitLab 13.3. - -To add a gauge panel type to a dashboard, look at the following sample dashboard file: - -```yaml -dashboard: 'Dashboard Title' -panel_groups: - - group: 'Group Title' - panels: - - title: 'Gauge' - type: 'gauge' - min_value: 0 - max_value: 1000 - split: 5 - thresholds: - values: [60, 90] - mode: 'percentage' - format: 'kilobytes' - metrics: - - id: 10 - query: 'floor(max(prometheus_http_response_size_bytes_bucket)/1000)' - unit: 'kb' -``` - -Note the following properties: - -| Property | Type | Required | Description | -| ------------ | ------ | ------ | ------ | -| `type` | string | yes | Type of panel to be rendered. For gauge panel types, set to `gauge`. | -| `min_value` | number | no, defaults to `0` | The minimum value of the gauge chart axis. If either of `min_value` or `max_value` are not set, they both get their default values. | -| `max_value` | number | no, defaults to `100` | The maximum value of the gauge chart axis. If either of `min_value` or `max_value` are not set, they both get their default values. | -| `split` | number | no, defaults to `10` | The amount of split segments on the gauge chart axis. | -| `thresholds` | object | no | Thresholds configuration for the gauge chart axis. | -| `format` | string | no, defaults to `engineering` | Unit format used. See the [full list of units](yaml_number_format.md). | -| `query` | string | yes | For gauge panel types, you must use an [instant query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries). | - -### Thresholds properties - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| values | array | no, defaults to 95% of the range between `min_value` and `max_value`| An array of gauge chart axis threshold values. | -| mode | string | no, defaults to `absolute` | The mode in which the thresholds are interpreted in relation to `min_value` and `max_value`. Can be either `percentage` or `absolute`. | - -![gauge panel type](img/prometheus_dashboard_gauge_panel_type_v13_3.png) - -## Heatmaps - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30581) in GitLab 12.5. - -To add a heatmap panel type to a dashboard, look at the following sample dashboard file: - -```yaml -dashboard: 'Dashboard Title' -panel_groups: - - group: 'Group Title' - panels: - - title: 'Heatmap' - type: 'heatmap' - metrics: - - id: 10 - query: 'sum(rate(nginx_upstream_responses_total{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[60m])) by (status_code)' - unit: req/sec - label: 'Status code' -``` - -Note the following properties: - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| `type` | string | yes | Type of panel to be rendered. For heatmap panel types, set to `heatmap` | -| `query_range` | yes | yes | For area panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) | - -![heatmap panel type](img/heatmap_panel_type.png) - -WARNING: -When a query returns too many data points, the heatmap data bucket dimensions tend downwards to 0, making the chart's data invisible, as shown in the image below. To fix this problem, limit the amount of data returned by changing the time range filter on the metrics dashboard UI, or adding the **step** property to your dashboard's YAML file. - -![heatmap chart with too much data](img/heatmap_chart_too_much_data_v_13_2.png) +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/operations/metrics/dashboards/settings.md b/doc/operations/metrics/dashboards/settings.md index 5fb0476e164..5572dfa360f 100644 --- a/doc/operations/metrics/dashboards/settings.md +++ b/doc/operations/metrics/dashboards/settings.md @@ -2,57 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../../index.md' --- -# Dashboard settings (DEPRECATED) **(FREE)** +# Dashboard settings (removed) **(FREE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) -in GitLab 14.7, and is planned for removal in GitLab 16.0. - -You can configure your [Monitoring dashboard](../index.md) to -display the time zone of your choice, and the links of your choice. - -To configure these settings you must have Manage Project -Operations [permissions](../../../user/permissions.md). - -## Change the dashboard time zone - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214370) in GitLab 13.1. - -By default, your monitoring dashboard displays dates and times in your local -time zone, but you can display dates and times in UTC format. To change the -time zone: - -1. Sign in as a user with Manage Project Operations [permissions](../../../user/permissions.md). -1. Navigate to **Settings > Monitor**. -1. Scroll to **Metrics Dashboard** and select **Expand**. -1. In the **Dashboard timezone** select box, select *User's local timezone* - or *UTC*: - - ![Dashboard timezone setting](img/dashboard_local_timezone_v13_1.png) -1. Select **Save changes**. - -## Link to an external dashboard - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/57171) in GitLab 12.0. - -You can add a button on your monitoring dashboard that links directly to your -existing external dashboards: - -1. Sign in as a user with Manage Project Operations [permissions](../../../user/permissions.md). -1. Navigate to **Settings > Monitor**. -1. Scroll to **Metrics Dashboard** and select **Expand**. -1. In **External dashboard URL**, provide the URL to your external dashboard: - - ![External Dashboard Setting](img/dashboard_external_link_v13_1.png) - -1. Select **Save changes**. - -GitLab displays a **View full dashboard** button in the upper-right corner of your -[monitoring dashboard](../../../ci/environments/index.md#monitor-environments) -which opens the URL you provided: - -![External Dashboard Link](img/external_dashboard_link.png) +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/operations/metrics/dashboards/templating_variables.md b/doc/operations/metrics/dashboards/templating_variables.md index a1c2ce063bc..a93c559c98c 100644 --- a/doc/operations/metrics/dashboards/templating_variables.md +++ b/doc/operations/metrics/dashboards/templating_variables.md @@ -2,132 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../../index.md' --- -# Templating variables for metrics dashboards (DEPRECATED) **(FREE)** +# Templating variables for metrics dashboards (removed) **(FREE)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214539) in GitLab 13.0. -> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) -in GitLab 14.7, and is planned for removal in GitLab 16.0. - -Templating variables can be used to make your metrics dashboard more versatile. - -`templating` is a top-level key in the -[dashboard YAML](yaml.md#dashboard-top-level-properties). -Define your variables in the `variables` key, under `templating`. The value of -the `variables` key should be a hash, and each key under `variables` -defines a templating variable on the dashboard, and may contain alphanumeric and underscore characters. - -A variable can be used in a Prometheus query in the same dashboard using the syntax -described [in Using Variables](variables.md). - -## `text` variable type - -WARNING: -This variable type is an [Alpha](../../../policy/alpha-beta-support.md#alpha-features) feature, and is subject to change at any time -without prior notice! - -For each `text` variable defined in the dashboard YAML, a free text field displays -on the dashboard UI, allowing you to enter a value for each variable. - -The `text` variable type supports a simple and a full syntax. - -### Simple syntax - -This example creates a variable called `variable1`, with a default value -of `default value`: - -```yaml -templating: - variables: - variable1: 'default value' # `text` type variable with `default value` as its default. -``` - -### Full syntax - -This example creates a variable called `variable1`, with a default value of `default`. -The label for the text box on the UI is the value of the `label` key: - -```yaml -templating: - variables: - variable1: # The variable name that can be used in queries. - label: 'Variable 1' # (Optional) label that will appear in the UI for this text box. - type: text - options: - default_value: 'default' # (Optional) default value. -``` - -## `custom` variable type - -WARNING: -This variable type is an [Alpha](../../../policy/alpha-beta-support.md#alpha-features) feature, and is subject to change at any time -without prior notice! - -Each `custom` variable defined in the dashboard YAML creates a dropdown list -selector on the dashboard UI, allowing you to select a value for each variable. - -The `custom` variable type supports a simple and a full syntax. - -### Simple syntax - -This example creates a variable called `variable1`, with a default value of `value1`. -The dashboard UI displays a dropdown list with `value1`, `value2` and `value3` -as the choices. - -```yaml -templating: - variables: - variable1: ['value1', 'value2', 'value3'] -``` - -### Full syntax - -This example creates a variable called `variable1`, with a default value of `value_option_2`. -The label for the text box on the UI is the value of the `label` key. -The dashboard UI displays a dropdown list with `Option 1` and `Option 2` -as the choices. - -If you select `Option 1` from the dropdown list, the variable is replaced with `value option 1`. -Similarly, if you select `Option 2`, the variable is replaced with `value_option_2`: - -```yaml -templating: - variables: - variable1: # The variable name that can be used in queries. - label: 'Variable 1' # (Optional) label that will appear in the UI for this dropdown. - type: custom - options: - values: - - value: 'value option 1' # The value that will replace the variable in queries. - text: 'Option 1' # (Optional) Text that will appear in the UI dropdown. - - value: 'value_option_2' - text: 'Option 2' - default: true # (Optional) This option should be the default value of this variable. -``` - -## `metric_label_values` variable type - -WARNING: -This variable type is an [Alpha](../../../policy/alpha-beta-support.md#alpha-features) feature, and is subject to change at any time -without prior notice! - -### Full syntax - -This example creates a variable called `variable2`. The values of the dropdown list are -all the different values of the `backend` label in the Prometheus series described by -`up{env="production"}`. - -```yaml -templating: - variables: - variable2: # The variable name that can be interpolated in queries. - label: 'Variable 2' # (Optional) label that will appear in the UI for this dropdown. - type: metric_label_values - options: - series_selector: 'up{env="production"}' - label: 'backend' -``` +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/operations/metrics/dashboards/variables.md b/doc/operations/metrics/dashboards/variables.md index 2881c084115..45e13aa731a 100644 --- a/doc/operations/metrics/dashboards/variables.md +++ b/doc/operations/metrics/dashboards/variables.md @@ -2,77 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../../index.md' --- -# Using variables (DEPRECATED) **(FREE)** +# Using variables (removed) **(FREE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) -in GitLab 14.7, and is planned for removal in GitLab 16.0. - -## Query variables - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20793) in GitLab 12.7. - -Variables can be specified using double curly braces, such as `"{{ci_environment_slug}}"`. - -Support for the `"%{ci_environment_slug}"` format was -[removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31581) in GitLab 13.0. -Queries that continue to use the old format display no data. - -## Predefined variables - -GitLab supports a limited set of [CI/CD variables](../../../ci/variables/index.md) -in the Prometheus query. This is particularly useful for identifying a specific -environment, for example with `ci_environment_slug`. Variables for Prometheus queries -must be lowercase. The supported variables are: - -- `environment_filter` -- `ci_environment_slug` -- `kube_namespace` -- `ci_project_name` -- `ci_project_namespace` -- `ci_project_path` -- `ci_environment_name` -- `__range` - -### `environment_filter` - -`environment_filter` is automatically expanded to `container_name!="POD",environment="ENVIRONMENT_NAME"` -where `ENVIRONMENT_NAME` is the name of the current environment. - -For example, a Prometheus query like `container_memory_usage_bytes{ {{environment_filter}} }` -becomes `container_memory_usage_bytes{ container_name!="POD",environment="production" }`. - -### `__range` - -The `__range` variable is useful in Prometheus -[range vector selectors](https://prometheus.io/docs/prometheus/latest/querying/basics/#range-vector-selectors). -Its value is the total number of seconds in the dashboard's time range. -For example, if the dashboard time range is set to 8 hours, the value of -`__range` is `28800s`. - -## User-defined variables - -[Variables can be defined](../../../operations/metrics/dashboards/yaml.md#templating-templating-properties) in a custom dashboard YAML file. - -Variable names are case-sensitive. - -## Query variables from URL - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214500) in GitLab 13.0. - -GitLab supports setting custom variables through URL parameters. Surround the variable -name with double curly braces (`{{example}}`) to interpolate the variable in a query: - -```plaintext -avg(sum(container_memory_usage_bytes{container_name!="{{pod}}"}) by (job)) without (job) /1024/1024/1024' -``` - -The URL for this query would be: - -```plaintext -https://gitlab.com///-/environments//metrics?dashboard=.gitlab%2Fdashboards%2Fcustom.yml&pod=POD -``` +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/operations/metrics/dashboards/yaml.md b/doc/operations/metrics/dashboards/yaml.md index 7b0a91dd18a..7807f713773 100644 --- a/doc/operations/metrics/dashboards/yaml.md +++ b/doc/operations/metrics/dashboards/yaml.md @@ -2,179 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../../index.md' --- -# Dashboard YAML properties (DEPRECATED) **(FREE)** +# Dashboard YAML properties (removed) **(FREE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) -in GitLab 14.7, and is planned for removal in GitLab 16.0. - -Dashboards have several components: - -- Templating variables. -- Panel groups, which consist of panels. -- Panels, which support one or more metrics. - -The following tables outline the details of expected properties. - -## Dashboard (top-level) properties - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| `dashboard` | string | yes | Heading for the dashboard. Only one dashboard should be defined per file. | -| `panel_groups` | array | yes | The panel groups which should be on the dashboard. | -| `templating` | hash | no | Top level key under which templating related options can be added. | -| `links` | array | no | Add links to display on the dashboard. | - -## Templating (`templating`) properties - -| Property | Type | Required | Description | -| -------- | ---- | -------- | ----------- | -| `variables` | hash | yes | Variables can be defined here. | - -Read the documentation on [templating](templating_variables.md). - -## Links (`links`) properties - -| Property | Type | Required | Description | -| -------- | ---- | -------- | ----------- | -| `url` | string | yes | The address of the link. | -| `title` | string | no | Display title for the link. | -| `type` | string | no | Type of the link. Specifies the link type, can be: `grafana` | - -Read the documentation on [links](index.md#add-related-links-to-custom-dashboards). - -## Panel group (`panel_groups`) properties - -Dashboards display panel groups in the order they are listed in the dashboard YAML file. - -In GitLab versions 13.3 and below, panel groups were ordered by a `priority` key, which -is no longer used. - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| `group` | string | required | Heading for the panel group. | -| `panels` | array | required | The panels which should be in the panel group. | - -Panels in a panel group are laid out in rows consisting of two panels per row. An exception to this rule are single panels on a row: these panels take the full width of their containing row. - -## Panel (`panels`) properties - -Dashboards display panels in the order they are listed in the dashboard YAML file. - -In GitLab versions 13.3 and below, panels were ordered by a `weight` key, which -is no longer used. - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------- | -| `type` | string | no, defaults to `area-chart` | Specifies the panel type to use, for example `area-chart`, `line-chart` or `anomaly-chart`. Only types listed among [all panel types](panel_types.md) are allowed. | -| `title` | string | yes | Heading for the panel. | -| `y_label` | string | no, but highly encouraged | Y-Axis label for the panel. | -| `y_axis` | string | no | Y-Axis configuration for the panel. | -| `max_value` | number | no | Denominator value used for calculating [percentile based results](panel_types.md#percentile-based-results) | -| `metrics` | array | yes | The metrics which should be displayed in the panel. Any number of metrics can be displayed when `type` is `area-chart` or `line-chart`, whereas only 3 can be displayed when `type` is `anomaly-chart`. | -| `links` | array | no | Add links to display on the chart's [context menu](index.md#chart-context-menu). | - -## Axis (`panels[].y_axis`) properties - -| Property | Type | Required | Description | -| ----------- | ------ | ----------------------------- | -------------------------------------------------------------------- | -| `name` | string | no, but highly encouraged | Y-Axis label for the panel. Replaces `y_label` if set. | -| `format` | string | no, defaults to `engineering` | Unit format used. See the [full list of units](yaml_number_format.md). | -| `precision` | number | no, defaults to `2` | Number of decimal places to display in the number. | | - -## Metrics (`metrics`) properties - -| Property | Type | Required | Description | -| ------ | ------ | ------ | ------ | -| `id` | string | no | Used for associating dashboard metrics with database records. Must be unique across dashboard configuration files. Required for [alerting](../alerts.md) (support not yet enabled, see [relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/27980)). | -| `unit` | string | yes | Defines the unit of the query's return data. | -| `label` | string | no, but highly encouraged | Defines the legend-label for the query. Should be unique within the panel's metrics. Can contain time series labels as interpolated variables. | -| `query` | string/number | yes if `query_range` is not defined | Defines the Prometheus query to be used to populate the chart/panel. If defined, the `query` endpoint of the [Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/) is used. | -| `query_range` | string/number | yes if `query` is not defined | Defines the Prometheus query to be used to populate the chart/panel. If defined, the `query_range` endpoint of the [Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/) is used. | -| `step` | number | no, value is calculated if not defined | Defines query resolution step width in float number of seconds. Metrics on the same panel should use the same `step` value. | - -## Dynamic labels - -Dynamic labels are useful when multiple time series are returned from a Prometheus query. - -When a static label is used and a query returns multiple time series, then all the legend items are labeled the same, which makes identifying each time series difficult: - -```yaml -metrics: - - id: my_metric_id - query_range: 'http_requests_total' - label: 'Time Series' - unit: 'count' -``` - -This may render a legend like this: - -![repeated legend label chart](img/prometheus_dashboard_repeated_label.png) - -For labels to be more explicit, using variables that reflect time series labels is a good practice. The variables are replaced by the values of the time series labels when the legend is rendered: - -```yaml -metrics: - - id: my_metric_id - query_range: 'http_requests_total' - label: 'Instance: {{instance}}, method: {{method}}' - unit: 'count' -``` - -The resulting rendered legend looks like this: - -![legend with label variables](img/prometheus_dashboard_label_variables.png) - -There is also a shorthand value for dynamic dashboard labels that make use of only one time series label: - -```yaml -metrics: - - id: my_metric_id - query_range: 'http_requests_total' - label: 'Method' - unit: 'count' -``` - -This works by converting the value of `label` to lower-case and, if there are more words separated by spaces, replacing those spaces with an underscore (`_`). The transformed value is then checked against the labels of the time series returned by the Prometheus query. If a time series label is found that is equal to the transformed value, then the label value renders in the legend like this: - -![legend with label shorthand variable](img/prometheus_dashboard_label_variable_shorthand.png) - -## Dashboard YAML syntax validation - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/33202) in GitLab 13.1. - -To confirm your dashboard definition contains valid YAML syntax: - -1. Go to **Repository > Files**. -1. Go to your dashboard file in your repository. -1. Review the information pane about the file, displayed above the file contents. - -Files with valid syntax display **Metrics Dashboard YAML definition is valid**, -and files with invalid syntax display **Metrics Dashboard YAML definition is invalid**. - -![Metrics dashboard YAML syntax validation](img/prometheus_dashboard_yaml_validation_v13_1.png) - -When **Metrics Dashboard YAML definition is invalid** at least one of the following messages is displayed: - -1. `dashboard: can't be blank` [learn more](#dashboard-top-level-properties) -1. `panel_groups: should be an array of panel_groups objects` [learn more](#dashboard-top-level-properties) -1. `group: can't be blank` [learn more](#panel-group-panel_groups-properties) -1. `panels: should be an array of panels objects` [learn more](#panel-group-panel_groups-properties) -1. `title: can't be blank` [learn more](#panel-panels-properties) -1. `metrics: should be an array of metrics objects` [learn more](#panel-panels-properties) -1. `query: can't be blank` [learn more](#metrics-metrics-properties) -1. `query_range: can't be blank` [learn more](#metrics-metrics-properties) -1. `unit: can't be blank` [learn more](#metrics-metrics-properties) -1. `YAML syntax: The parsed YAML is too big` - - This is displayed when the YAML file is larger than 1 MB. - -1. `YAML syntax: Invalid configuration format` - - This is displayed when the YAML file is empty or does not contain valid YAML. - -Metrics Dashboard YAML definition validation information is also available as a [GraphQL API field](../../../api/graphql/reference/index.md#metricsdashboard) +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/operations/metrics/dashboards/yaml_number_format.md b/doc/operations/metrics/dashboards/yaml_number_format.md index 99e6be96a3c..90e7f67b153 100644 --- a/doc/operations/metrics/dashboards/yaml_number_format.md +++ b/doc/operations/metrics/dashboards/yaml_number_format.md @@ -2,176 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../../index.md' --- -# Unit formats reference **(FREE)** +# Unit formats reference (removed) **(FREE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201999) in GitLab 12.9. - -Format the data in your dashboard panels. - -You can select units to format your charts by adding `format` to your -[axis configuration](yaml.md). - -## Internationalization and localization - -Currently, your [internationalization and localization options](https://en.wikipedia.org/wiki/Internationalization_and_localization) for number formatting are dependent on the system you are using (that is, your OS or browser). - -## Engineering Notation - -For generic or default data, numbers are formatted according to the current locale in [engineering notation](https://en.wikipedia.org/wiki/Engineering_notation). - -While an [engineering notation exists for the web](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/NumberFormat), GitLab uses a version based off the [scientific notation](https://en.wikipedia.org/wiki/Scientific_notation). GitLab formatting acts in accordance with SI prefixes. For example, using GitLab notation, `1500.00` becomes `1.5k` instead of `1.5E3`. Keep this distinction in mind when using the engineering notation for your metrics. - -Formats: `engineering` - -SI prefixes: - -| Name | Symbol | Value | -| ---------- | ------- | -------------------------- | -| `yotta` | Y | 1000000000000000000000000 | -| `zetta` | Z | 1000000000000000000000 | -| `exa` | E | 1000000000000000000 | -| `peta` | P | 1000000000000000 | -| `tera` | T | 1000000000000 | -| `giga` | G | 1000000000 | -| `mega` | M | 1000000 | -| `kilo` | k | 1000 | -| `milli` | m | 0.001 | -| `micro` | μ | 0.000001 | -| `nano` | n | 0.000000001 | -| `pico` | p | 0.000000000001 | -| `femto` | f | 0.000000000000001 | -| `atto` | a | 0.000000000000000001 | -| `zepto` | z | 0.000000000000000000001 | -| `yocto` | y | 0.000000000000000000000001 | - -**Examples:** - -| Data | Displayed | -| --------------------------------- | --------- | -| `0.000000000000000000000008` | 8y | -| `0.000000000000000000008` | 8z | -| `0.000000000000000008` | 8a | -| `0.000000000000008` | 8f | -| `0.000000000008` | 8p | -| `0.000000008` | 8n | -| `0.000008` | 8μ | -| `0.008` | 8m | -| `10` | 10 | -| `1080` | 1.08k | -| `18000` | 18k | -| `18888` | 18.9k | -| `188888` | 189k | -| `18888888` | 18.9M | -| `1888888888` | 1.89G | -| `1888888888888` | 1.89T | -| `1888888888888888` | 1.89P | -| `1888888888888888888` | 1.89E | -| `1888888888888888888888` | 1.89Z | -| `1888888888888888888888888` | 1.89Y | -| `1888888888888888888888888888` | 1.89e+27 | - -## Numbers - -For number data, numbers are formatted according to the current locale. - -Formats: `number` - -**Examples:** - -| Data | Displayed | -| ---------- | --------- | -| `10` | 1 | -| `1000` | 1,000 | -| `1000000` | 1,000,000 | - -## Percentage - -For percentage data, format numbers in the chart with a `%` symbol. - -Formats supported: `percent`, `percentHundred` - -**Examples:** - -| Format | Data | Displayed | -| ---------------- | ----- | --------- | -| `percent` | `0.5` | 50% | -| `percent` | `1` | 100% | -| `percent` | `2` | 200% | -| `percentHundred` | `50` | 50% | -| `percentHundred` | `100` | 100% | -| `percentHundred` | `200` | 200% | - -## Duration - -For time durations, format numbers in the chart with a time unit symbol. - -Formats supported: `milliseconds`, `seconds` - -**Examples:** - -| Format | Data | Displayed | -| -------------- | ------ | --------- | -| `milliseconds` | `10` | 10 ms | -| `milliseconds` | `500` | 100 ms | -| `milliseconds` | `1000` | 1000 ms | -| `seconds` | `10` | 10 s | -| `seconds` | `500` | 500 s | -| `seconds` | `1000` | 1000 s | - -## Digital (Metric) - -Converts a number of bytes using metric prefixes. It scales to -use the unit that's the best fit. - -Formats supported: - -- `decimalBytes` -- `kilobytes` -- `megabytes` -- `gigabytes` -- `terabytes` -- `petabytes` - -**Examples:** - -| Format | Data | Displayed | -| -------------- | --------- | --------- | -| `decimalBytes` | `1` | 1 B | -| `decimalBytes` | `1000` | 1 kB | -| `decimalBytes` | `1000000` | 1 MB | -| `kilobytes` | `1` | 1 kB | -| `kilobytes` | `1000` | 1 MB | -| `kilobytes` | `1000000` | 1 GB | -| `megabytes` | `1` | 1 MB | -| `megabytes` | `1000` | 1 GB | -| `megabytes` | `1000000` | 1 TB | - -## Digital (IEC) - -Converts a number of bytes using binary prefixes. It scales to -use the unit that's the best fit. - -Formats supported: - -- `bytes` -- `kibibytes` -- `mebibytes` -- `gibibytes` -- `tebibytes` -- `pebibytes` - -**Examples:** - -| Format | Data | Displayed | -| ----------- | ------------- | --------- | -| `bytes` | `1` | 1 B | -| `bytes` | `1024` | 1 KiB | -| `bytes` | `1024 * 1024` | 1 MiB | -| `kibibytes` | `1` | 1 KiB | -| `kibibytes` | `1024` | 1 MiB | -| `kibibytes` | `1024 * 1024` | 1 GiB | -| `mebibytes` | `1` | 1 MiB | -| `mebibytes` | `1024` | 1 GiB | -| `mebibytes` | `1024 * 1024` | 1 TiB | +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/operations/metrics/embed.md b/doc/operations/metrics/embed.md index f622780530a..00c145adee3 100644 --- a/doc/operations/metrics/embed.md +++ b/doc/operations/metrics/embed.md @@ -2,126 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../index.md' --- -# Embedding metric charts within GitLab Flavored Markdown **(FREE)** +# Embedding metric charts within GitLab Flavored Markdown (removed) **(FREE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) -for use in GitLab 14.7, and is planned for removal in GitLab 16.0. - -You can display metrics charts within -[GitLab Flavored Markdown (GLFM)](../../user/markdown.md) -fields such as issue or merge request descriptions. The maximum number of embedded -charts allowed in a GitLab Flavored Markdown field is 100. -Embedding charts is useful when sharing an application incident or performance -metrics to others, and you want to have relevant information directly available. - -## Embedding GitLab-managed Kubernetes metrics - -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/29691) in GitLab 12.2. - -This feature requires [Kubernetes](../../user/project/integrations/prometheus_library/kubernetes.md) metrics. - -NOTE: -In GitLab versions 13.3 and earlier, metrics dashboard links were in the form -`https:////-/environments//metrics`. These links -are still supported, and can be used to embed metric charts. - -To display metric charts, include a link of the form -`https:////-/metrics?environment=` in a field -that supports GitLab Flavored Markdown: - -```markdown -### Summary - -**Start time:** 2020-01-21T12:00:31+00:00 - -### Metrics - -https://gitlab.com/gitlab-org/monitor/tanuki-inc/-/metrics?environment=1118134 -``` - -GitLab unfurls the link as an embedded metrics panel: - -![Embedded Metrics Rendered](img/embedded_metrics_rendered_v13_4.png) - -You can also embed a single chart. To get a link to a chart, select -**{ellipsis_v}** **More actions** in the upper right corner of the chart, -and select **Copy link to chart**, as shown in this example: - -![Copy Link To Chart](img/copy_link_to_chart_v12_10.png) - -The following requirements must be met for the metric to unfurl: - -- The `` must correspond to a real environment. -- Prometheus must be monitoring the environment. -- The GitLab instance must be configured to receive data from the environment. -- The user must have at least the Reporter role for the monitoring dashboard for the environment. -- The dashboard must have data within the last 8 hours. - - If all of the above are true, then the metric unfurls as seen below: - -![Embedded Metrics](img/view_embedded_metrics_v12_10.png) - -Metric charts may also be hidden: - -![Show Hide](img/hide_embedded_metrics_v12_10.png) - -You can open the link directly into your browser for a -[detailed view of the data](dashboards/index.md#chart-context-menu). - -## Embedding metrics in issue templates - -You can also embed either the overview dashboard metrics or individual metrics in -issue templates. For charts to render side-by-side, separate links to the entire metrics -dashboard or individual metrics by either a comma or a space. - -![Embedded Metrics in issue templates](img/embed_metrics_issue_template.png) - -## Embedding metrics based on alerts in incident issues - -For [GitLab-managed alerting rules](alerts.md), the issue includes an embedded -chart for the query corresponding to the alert. The chart displays an hour of data -surrounding the starting point of the incident, 30 minutes before and after. - -For [manually configured Prometheus instances](../../user/project/integrations/prometheus.md#manual-configuration-of-prometheus), -a chart corresponding to the query can be included if these requirements are met: - -- The alert corresponds to an environment managed through GitLab. -- The alert corresponds to a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries). -- The alert contains the required attributes listed in the chart below. - -| Attributes | Required | Description | -| ---------- | -------- | ----------- | -| `annotations/gitlab_environment_name` | Yes | Name of the GitLab-managed environment corresponding to the alert | -| One of `annotations/title`, `annotations/summary`, `labels/alertname` | Yes | Used as the chart title | -| `annotations/gitlab_y_label` | No | Used as the chart's y-axis label | - -## Embedding cluster health charts - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/40997) in GitLab 12.9. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/208224) from GitLab Ultimate to GitLab Free in 13.2. - -[Cluster Health Metrics](../../user/infrastructure/clusters/manage/clusters_health.md) -can also be embedded in [GitLab Flavored Markdown](../../user/markdown.md). - -To embed a metric chart, include a link to that chart in the form -`https:////-/cluster/?` anywhere that -GitLab Flavored Markdown is supported. To generate and copy a link to the chart, -follow the instructions in the -[Cluster Health Metric documentation](../../user/infrastructure/clusters/manage/clusters_health.md). - -The following requirements must be met for the metric to unfurl: - -- The `` must correspond to a real cluster. -- Prometheus must be monitoring the cluster. -- The user must be allowed access to the project cluster metrics. -- The dashboards must be reporting data on the - [Cluster Health Page](../../user/infrastructure/clusters/manage/clusters_health.md) - - If the above requirements are met, then the metric unfurls as seen below. - -![Embedded Cluster Metric in issue descriptions](img/prometheus_cluster_health_embed_v12_9.png) +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/operations/metrics/embed_grafana.md b/doc/operations/metrics/embed_grafana.md index 7bc88165b95..e14b4b5a893 100644 --- a/doc/operations/metrics/embed_grafana.md +++ b/doc/operations/metrics/embed_grafana.md @@ -2,90 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../index.md' --- - -# Embed Grafana panels in Markdown (deprecated) **(FREE)** +# Embed Grafana panels in Markdown (removed) **(FREE)** -WARNING: -This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/110290) in GitLab 15.9 -and is planned for removal in 16.0. We intend to replace this feature with the ability to [embed charts](https://gitlab.com/groups/gitlab-org/opstrace/-/epics/33) with the [GitLab Observability UI](https://gitlab.com/gitlab-org/opstrace/opstrace-ui). -This change is a breaking change. - -Grafana panels can be embedded in [GitLab Flavored Markdown](../../user/markdown.md). You can -embed Grafana panels using either: - -- [Grafana-rendered images](#use-grafana-rendered-images). -- [Grafana API](#use-integration-with-grafana-api). - -## Use Grafana-rendered images - -You can embed live [Grafana](https://docs.gitlab.com/omnibus/settings/grafana.html) panels as -[a direct link](https://grafana.com/docs/grafana/v7.5/sharing/share-panel/#use-direct-link). -Your Grafana instance must: - -- Be available to the target user, either as a public dashboard or on the same network. -- Have [Grafana Image Renderer](https://grafana.com/grafana/plugins/grafana-image-renderer/) installed. - -To use Grafana-rendered images: - -1. Go to the dashboard containing the panel in Grafana. -1. From the panel's menu, select **Share**. -1. Select **Direct link rendered image**, which provides the link. -1. Copy the link and add an image tag as [inline HTML](../../user/markdown.md#inline-html) in your - Markdown in the format ``. You can tweak the query parameters to meet your needs, such as removing the `&from=` - and `&to=` parameters to display a live panel. - -## Use integration with Grafana API - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31376) in GitLab 12.5. - -Each project can support integration with one Grafana instance. This enables you to copy a link to a -panel in Grafana, then paste it into a GitLab Markdown field. The panel renders in the GitLab panel -format. To embed panels from a Grafana instance, the data source must be: - -- A Prometheus instance. -- Proxyable, so the **HTTP > Access** setting for the Grafana data source should be set to - **Server (default)**. - -### Set up Grafana integration - -To set up the Grafana API in Grafana: - -1. In Grafana, [generate an Admin-level API Token](https://grafana.com/docs/grafana/next/developers/http_api/auth/#create-api-token). -1. In your GitLab project, go to **Settings > Monitor** and expand the **Grafana authentication** - section. -1. To enable the integration, check the **Active** checkbox. -1. For **Grafana URL**, enter the base URL of the Grafana instance. -1. For **API Token**, enter the Administrator API token you just generated. -1. Select **Save Changes**. - -NOTE: -If the Grafana integration is enabled, users with the Reporter role on public -projects and the Guest role on non-public projects can query metrics from the -Prometheus instance. All requests proxied through GitLab are authenticated with -the same Grafana Administrator API token. - -### Generate a link to a panel - -To generate a link to a panel: - -1. In Grafana, go to the dashboard you wish to embed a panel from. -1. In the upper-left corner of the page, select a specific value for each variable required for the - queries in the dashboard. -1. In Grafana select a panel's title, then select **Share** to open the panel's sharing dialog to - the **Link** tab. - - If you select the dashboard's share button instead, GitLab attempts to embed the first supported - panel on the dashboard (if available). -1. If your Prometheus queries use the Grafana custom template variables, ensure the - **Template variables** option is set to on. Only the Grafana global template variables - `$__interval`, `$__from`, and `$__to` are supported. -1. Set the **Current time range** option to on, to specify the time range of the panel. Otherwise, - the default range is the last 8 hours. -1. Select **Copy** to copy the URL to the clipboard. -1. In GitLab, paste the URL into a Markdown field and save. The panel takes a few moments to render. - -See the following example of a rendered panel. - -![GitLab Rendered Grafana Panel](img/rendered_grafana_embed_v12_5.png) - +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. +Use [embed charts](https://gitlab.com/groups/gitlab-org/opstrace/-/epics/33) instead. diff --git a/doc/operations/metrics/img/copy_link_to_chart_v12_10.png b/doc/operations/metrics/img/copy_link_to_chart_v12_10.png deleted file mode 100644 index fc205240ea5..00000000000 Binary files a/doc/operations/metrics/img/copy_link_to_chart_v12_10.png and /dev/null differ diff --git a/doc/operations/metrics/img/embed_metrics_issue_template.png b/doc/operations/metrics/img/embed_metrics_issue_template.png deleted file mode 100644 index ca39a738d5f..00000000000 Binary files a/doc/operations/metrics/img/embed_metrics_issue_template.png and /dev/null differ diff --git a/doc/operations/metrics/img/embedded_metrics_rendered_v13_4.png b/doc/operations/metrics/img/embedded_metrics_rendered_v13_4.png deleted file mode 100644 index 876972dc721..00000000000 Binary files a/doc/operations/metrics/img/embedded_metrics_rendered_v13_4.png and /dev/null differ diff --git a/doc/operations/metrics/img/example-dashboard_v13_3.png b/doc/operations/metrics/img/example-dashboard_v13_3.png deleted file mode 100644 index 1178b4a9be7..00000000000 Binary files a/doc/operations/metrics/img/example-dashboard_v13_3.png and /dev/null differ diff --git a/doc/operations/metrics/img/hide_embedded_metrics_v12_10.png b/doc/operations/metrics/img/hide_embedded_metrics_v12_10.png deleted file mode 100644 index 1213029d1d1..00000000000 Binary files a/doc/operations/metrics/img/hide_embedded_metrics_v12_10.png and /dev/null differ diff --git a/doc/operations/metrics/img/prometheus_add_metric.png b/doc/operations/metrics/img/prometheus_add_metric.png deleted file mode 100644 index 32a7cbf3a73..00000000000 Binary files a/doc/operations/metrics/img/prometheus_add_metric.png and /dev/null differ diff --git a/doc/operations/metrics/img/prometheus_cluster_health_embed_v12_9.png b/doc/operations/metrics/img/prometheus_cluster_health_embed_v12_9.png deleted file mode 100644 index c669467757f..00000000000 Binary files a/doc/operations/metrics/img/prometheus_cluster_health_embed_v12_9.png and /dev/null differ diff --git a/doc/operations/metrics/img/prometheus_dashboard_edit_metric_link_v_12_9.png b/doc/operations/metrics/img/prometheus_dashboard_edit_metric_link_v_12_9.png deleted file mode 100644 index b66b1a9f39b..00000000000 Binary files a/doc/operations/metrics/img/prometheus_dashboard_edit_metric_link_v_12_9.png and /dev/null differ diff --git a/doc/operations/metrics/img/prometheus_monitoring_dashboard_v13_3.png b/doc/operations/metrics/img/prometheus_monitoring_dashboard_v13_3.png deleted file mode 100644 index 1178b4a9be7..00000000000 Binary files a/doc/operations/metrics/img/prometheus_monitoring_dashboard_v13_3.png and /dev/null differ diff --git a/doc/operations/metrics/img/rendered_grafana_embed_v12_5.png b/doc/operations/metrics/img/rendered_grafana_embed_v12_5.png deleted file mode 100644 index 6cabe4193bd..00000000000 Binary files a/doc/operations/metrics/img/rendered_grafana_embed_v12_5.png and /dev/null differ diff --git a/doc/operations/metrics/img/view_embedded_metrics_v12_10.png b/doc/operations/metrics/img/view_embedded_metrics_v12_10.png deleted file mode 100644 index 95bb148ba71..00000000000 Binary files a/doc/operations/metrics/img/view_embedded_metrics_v12_10.png and /dev/null differ diff --git a/doc/operations/metrics/index.md b/doc/operations/metrics/index.md index 11350d65237..98ed9aba0da 100644 --- a/doc/operations/metrics/index.md +++ b/doc/operations/metrics/index.md @@ -2,167 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../index.md' --- -# Monitor your environment's metrics **(FREE)** +# Monitor your environment's metrics (removed) **(FREE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) -for use in GitLab 14.7, and is planned for removal in GitLab 16.0. - -GitLab helps your team monitor the health and performance of your applications -and infrastructure by turning statistics and log files into charts and graphs -that are easy to understand, especially when time is short and decisions are -critical. For GitLab to display your information in charts, you must: - -1. **Instrument your application** - Collect accurate and complete measurements. - - For an overview, see [How to instrument Prometheus metrics in GitLab](https://www.youtube.com/watch?v=tuI2oJ3TTB4). -1. **Expose metrics for capture** - Make logs, metrics, and traces available for capture. -1. [**Configure Prometheus to gather metrics**](#configure-prometheus-to-gather-metrics) - - Use applications like Elasticsearch, Prometheus, and Jaeger to gather - the data you've exposed. -1. **GitLab collects metrics** - GitLab uses Prometheus to scrape the data you've - captured in your applications, and prepares the data for display. For more information, see - [Collect and process metrics](#collect-and-process-metrics). -1. **Display charts in the GitLab user interface** - GitLab converts your metrics - into easy-to-read charts on a default dashboard. You can create as many custom charts - and custom dashboards as needed so your team has full insight into your - application's health. - -## Configure Prometheus to gather metrics - -You must connect a Prometheus instance to GitLab to collect metrics. How you configure -your Prometheus integration depends on where your apps are running: - -- **For manually-configured Prometheus** - - [Specify your Prometheus server](../../user/project/integrations/prometheus.md#manual-configuration-of-prometheus), - and define at least one environment. -- **For a cluster integrated Prometheus** - GitLab can query - [an in-cluster Prometheus](../../user/clusters/integrations.md#prometheus-cluster-integration). - You must also complete a code deployment to your cluster for the **Monitor > Metrics** - page to contain data. You can do this using [Auto DevOps](../../topics/autodevops/cloud_deployments/auto_devops_with_gke.md). - -![Monitoring Dashboard](img/prometheus_monitoring_dashboard_v13_3.png) - -## Collect and process metrics - -After [configuring Prometheus for a cluster](../../user/project/integrations/prometheus.md), -GitLab attempts to retrieve performance metrics for any [environment](../../ci/environments/index.md) with -a successful deployment. - -GitLab scans the Prometheus server for metrics from known servers like Kubernetes -and NGINX, and attempts to identify individual environments. For more information about -the supported metrics and scan processes, see -[Prometheus Metrics library](../../user/project/integrations/prometheus_library/index.md). - -To view the [default metrics dashboard](dashboards/default.md) for an environment that is -[configured to gather metrics](#configure-prometheus-to-gather-metrics): - -1. *If the metrics dashboard is only visible to project members,* sign in to - GitLab as a member of a project. Learn more about [metrics dashboard visibility](#metrics-dashboard-visibility). -1. In your project, navigate to **Monitor > Metrics**. - -GitLab displays the [default metrics dashboard](dashboards/default.md) for the environment, -like the following example: - -![Example of metrics dashboard](img/example-dashboard_v13_3.png) - -The top of the dashboard contains a navigation bar. From left to right, the -navigation bar contains: - -- **Dashboard** - A dropdown list of all dashboards available for the project, - with starred dashboards listed first. -- **Environment** - A dropdown list of all [environments](../index.md) that searches - through all environments as you type. -- **Range** - The time period of data to display. -- **Refresh dashboard** **{retry}** - Reload the dashboard with current data. -- **Set refresh rate** - Set a time frame for refreshing the data displayed. -- **More actions** **{ellipsis_v}** - More dashboard actions - - **Add metric** - Adds a [custom metric](#adding-custom-metrics). Only available on GitLab-defined dashboards. - ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34779) in GitLab 12.5.) - - **Edit dashboard YAML** - Edit the source YAML file of a custom dashboard. Only available on - [custom dashboards](dashboards/index.md). - ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/34779) in GitLab 12.5.) - - **Duplicate current dashboard** - Save a [complete copy of a dashboard](dashboards/index.md#duplicate-a-gitlab-defined-dashboard). Only available on GitLab-defined dashboards. - - **Star dashboard** **{star-o}** - Mark a dashboard as a favorite. - Starred dashboards display a solid star **{star}** button, and display first - in the **Dashboard** dropdown list. - ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214582) in GitLab 13.0.) - - **Create new dashboard** - Create a [new custom dashboard for your project](dashboards/index.md#add-a-new-dashboard-to-your-project). - ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/228856) in GitLab 13.3.) -- **Metrics settings** - Configure the - [settings for this dashboard](dashboards/index.md#manage-the-metrics-dashboard-settings). - -## Customize your metrics dashboard - -After creating your dashboard, you can customize it to meet your needs: - -- **Add custom metrics**: In addition to the GitLab default metrics, you can - [create custom metrics](#adding-custom-metrics) and display them on your metrics dashboard. -- **Configure alerts for metrics**: [Configure custom alerts](alerts.md) for your team when - environment performance falls outside of the boundaries you set. -- **Trigger actions from alerts**: [Open new issues for your team](alerts.md#trigger-actions-from-alerts) **(ULTIMATE)** - when environment performance falls outside of the boundaries you set. - -## Metrics dashboard visibility - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/201924) in GitLab 13.0. - -You can set the visibility of the metrics dashboard to **Only Project Members** -or **Everyone With Access**. When set to **Everyone with Access**, the metrics -dashboard is visible to authenticated and non-authenticated users. - -## Adding custom metrics - -> [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28527) from GitLab Premium to GitLab Free in 12.10. - -Custom metrics can be monitored by adding them on the monitoring dashboard page. -After saving them, they display on the environment metrics dashboard provided that either: - -- A [connected Kubernetes cluster](../../user/clusters/agent/index.md) - with the matching [environment scope](../../ci/environments/index.md#limit-the-environment-scope-of-a-cicd-variable) is used and - [Prometheus installed on the cluster](../../user/project/integrations/prometheus.md#enabling-prometheus-integration). -- Prometheus is [manually configured](../../user/project/integrations/prometheus.md#manual-configuration-of-prometheus). - -![Add New Metric](img/prometheus_add_metric.png) - -A few fields are required: - -- **Name**: Chart title -- **Type**: Type of metric. Metrics of the same type are shown together. -- **Query**: Valid [PromQL query](https://prometheus.io/docs/prometheus/latest/querying/basics/). -- **Y-axis label**: Y axis title to display on the dashboard. -- **Unit label**: Query units, for example `req / sec`. Shown next to the value. - -Multiple metrics can be displayed on the same chart if the fields **Name**, **Type**, -and **Y-axis label** match between metrics. For example, a metric with **Name** -`Requests Rate`, **Type** `Business`, and **Y-axis label** `rec / sec` would display -on the same chart as a second metric with the same values. A **Legend label** is -suggested if this feature is used. - -## Editing additional metrics from the dashboard - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/208976) in GitLab 12.9. - -You can edit existing additional custom metrics for your dashboard by selecting the -**{ellipsis_v}** **More actions** dropdown list and selecting **Edit metric**. - -![Edit metric](img/prometheus_dashboard_edit_metric_link_v_12_9.png) - -## Keyboard shortcuts for charts - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/202146) in GitLab 13.2. - -You can use keyboard shortcuts to interact more quickly with your currently-focused -chart panel. To activate keyboard shortcuts, use keyboard tabs to highlight the -**{ellipsis_v}** **More actions** dropdown list, or hover over the dropdown list -with your mouse, then press the key corresponding to your desired action: - -- **Expand panel** - e -- **View logs** - l (lowercase 'L') -- **Download CSV** - d -- **Copy link to chart** - c -- **Alerts** - a +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/operations/quickstart-guide.md b/doc/operations/quickstart-guide.md deleted file mode 100644 index 91c5f25405c..00000000000 --- a/doc/operations/quickstart-guide.md +++ /dev/null @@ -1,229 +0,0 @@ ---- -stage: Monitor -group: Observability -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments ---- - -# GitLab Observability Quickstart - -You can try GitLab Observability by [cloning or forking the project](https://gitlab.com/gitlab-org/opstrace/opstrace.git) and creating a local installation. - -## Prerequisites and dependencies - -To install GitLab Observability Platform (GOP), install and configure the following third-party dependencies. You can do this manually, or [automatically by using asdf](#install-dependencies-using-asdf): - -- [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) for creating a local Kubernetes cluster. -- [Docker](https://docs.docker.com/install) - - [Docker Compose](https://docs.docker.com/compose/compose-v2/) is now part of the `docker` distribution. -- [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) for interacting with GitLab Observability. -- [Telepresence](https://www.telepresence.io/) allows you to code and test microservices locally against a remote Kubernetes cluster. -- [jq](https://stedolan.github.io/jq/download/) for some Makefile utilities. -- [Go 1.19](https://go.dev/doc/install). - -The current versions of these dependencies are pinned in the `.tool-versions` file in the project. - -You can run the following commands to check the availability and versions of these dependencies on your machine: - -```shell -kind --version -docker --version -kubectl version -telepresence version -jq --version -go version -``` - -### Run GOP on macOS - -If you're running GOP on macOS, ensure you have enough resources dedicated to Docker Desktop. The recommended minimum is: - -- CPUs: 4+ -- Memory: 8 GB+ -- Swap: 1 GB+ - -It's possible to run GOP with fewer resources, but this specification works. - -### Install dependencies using asdf - -If you install dependencies using [`asdf`](https://asdf-vm.com/#/core-manage-asdf), GOP manages them for you automatically. - -1. If you have not already done so, clone the `opstrace` repository into your preferred location: - - ```shell - git clone https://gitlab.com/gitlab-org/opstrace/opstrace.git - ``` - -1. Change into the project directory: - - ```shell - cd opstrace - ``` - -1. Optional. If you need to install `asdf`, run: - - ```shell - make install-asdf - ``` - -1. Install dependencies using `asdf`: - - ```shell - make bootstrap - ``` - -## Step 1: Create a local Kubernetes cluster with kind - -Make sure Docker Desktop is running. In the `opstrace` project you cloned, run the following command: - -```shell -make kind -``` - -Wait a few minutes while kind creates your Kubernetes cluster. When it's finished, you should see the following message: - -```plaintext -Traffic Manager installed successfully -``` - -Now deploy the scheduler by running the following command in the `opstrace` project: - -```shell -make deploy -``` - -This takes around 1 minute. - -## Step 2: Create a GitLab application for authentication - -You must create a GitLab application to use for authentication. - -In the GitLab instance you'd like to connect with GOP, [create an OAuth application](../integration/oauth_provider.md). -This application can be a user-owned, group-owned or instance-wide application. -In production, you would create a trusted instance-wide application so that users are explicitly authorized without the consent screen. -The following example shows how to configure the application. - -1. Select the API scope and enter `http://localhost/v1/auth/callback` as the redirect URI. - -1. Run the following command to create the secret that holds the authentication data: - - ```shell - kubectl create secret generic \ - --from-literal=gitlab_oauth_client_id= \ - --from-literal=gitlab_oauth_client_secret= \ - --from-literal=internal_endpoint_token= \ - dev-secret - ``` - -1. Replace `` and `` with the values from the GitLab application you just created. -Replace `` with any string if you do not plan to use error tracking. - -You can also view [this MR on how to get the token to test error tracking](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91928). -You must specify all the parameters when creating the secret. - -## Step 3: Create the cluster definition - -1. In your `opstrace` project, run the following command to create a `Cluster.yaml` manifest file: - - ```shell - cat < Cluster.yaml - apiVersion: opstrace.com/v1alpha1 - kind: Cluster - metadata: - name: dev-cluster - spec: - target: kind - goui: - image: "registry.gitlab.com/gitlab-org/opstrace/opstrace-ui/ gitlab-observability-ui:c9fb6e70" - dns: - acmeEmail: "" - dns01Challenge: {} - externalDNSProvider: {} - gitlab: - groupAllowedAccess: '*' - groupAllowedSystemAccess: "6543" - instanceUrl: https://gitlab.com - authSecret: - name: dev-secret - EOF - ``` - -1. Apply the file you just created with the following command: - - ```shell - kubectl apply -f Cluster.yaml - ``` - -1. Run the following command to wait for the cluster to be ready: - - ```shell - kubectl wait --for=condition=ready cluster/dev-cluster --timeout=600s - ``` - -After the previous command exits, the cluster is ready. - -## Step 4: Enable Observability on a GitLab namespace you own - -Go to a namespace you own in the connected GitLab instance and copy the Group ID below the group name. - -GOP can only be enabled for groups you own. -To list all the groups that your user owns, go to the menu in upper left corner and select `Groups`->`View all Groups`. You then see the **Your groups** tab. - -In your browser, go to `http://localhost/-/{GroupID}`. For example, `http://localhost/-/14485840`. - -Follow the on-screen instructions to enable observability for the namespace. -This can take a couple of minutes if it's the first time observability has been enabled for the root level namespace (GitLab.org in the previous example.) - -Once your namespace has been enabled and is ready, the page automatically redirects you to the GitLab Observability UI. - -## Step 5: Send traces to GOP - -[Follow this guide to send traces to your namespace and monitor them in the UI](https://gitlab.com/gitlab-org/opstrace/opstrace/-/blob/main/docs/guides/user/sending-traces-locally.md). - -## Step 6: Clean up your local GOP - -To tear down your locally running GOP instance, run the following command: - -```shell -make destroy -``` - -## Known issues - -### Incorrect architecture for `kind/node` image - -If your machine has an Apple silicon (M1/M2) chip, you might encounter an architecture problem with the `kind/node` image when running the `make kind` command. For more details, see [issue 1802](https://gitlab.com/gitlab-org/opstrace/opstrace/-/issues/1802). - -To fix this problem, you first need to create a Dockerfile. Then build and deploy the image: - -1. Create a new Dockerfile (without a file extension) and paste the following commands: - - ```Dockerfile - FROM --platform=arm64 kindest/node:v1.23.13 - RUN arch - ``` - -1. Save your Dockerfile, then build the image with the following command: - - ```shell - docker build -t tempkind . - ``` - - Do not forget the period at the end. - -1. Create a cluster using your new image with the following command: - - ```shell - kind create cluster --image tempkind - ``` - -### scheduler-controller-manager pod cannot start due to ImagePullBackOff - -If while executing `make deploy` in step 1, the `scheduler-controller-manager` pod cannot start due to `ImagePullBackOff`, you must set the `CI_COMMIT_TAG` to a non-dirty state. By setting the commit tag to the latest commit, you ensure the Docker image can be pulled from the container registry. - -Run the following command to set the commit tag: - -```shell -make kind -export CI_COMMIT_TAG=0.2.0-e1206acf -make deploy -``` diff --git a/doc/policy/alpha-beta-support.md b/doc/policy/alpha-beta-support.md index 1c7e9e77751..e142fe9e908 100644 --- a/doc/policy/alpha-beta-support.md +++ b/doc/policy/alpha-beta-support.md @@ -4,68 +4,62 @@ group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Support for Alpha, Beta, Limited Availability, and Generally Available features **(PREMIUM)** +# Support for Experiment, Beta, and Generally Available features **(PREMIUM)** -Some GitLab features are released as Alpha or Beta versions and are +Some GitLab features are released as Experiment or Beta versions and are [not fully supported](https://about.gitlab.com/support/statement-of-support/#alpha-beta-features). All other features are considered to be Generally Available (GA). -## Alpha features +## Experiment -Support is **not** provided for Alpha features and issues with them should be opened in the [GitLab issue tracker](https://gitlab.com/gitlab-org/gitlab/issues). - -Characteristics of Alpha features: +Support is not provided for features listed as "Experimental" or "Alpha" or any similar designation. Issues regarding such features should be opened in the GitLab issue tracker. - Not ready for production use. -- Unstable and can cause performance and stability issues. -- Configuration and dependencies are likely to change. -- Features and functions may be removed. Breaking changes may occur outside of major releases or with less notice than for Beta or Generally Available features. -- Data loss can occur (be that through bugs or updates). -- Documentation reflects the Alpha status. -- Behind flags that are off by default. -- Not announced in release posts. - -## Beta features - -Your Support Contract provides **commercially-reasonable effort** support for Beta features, with the expectation that issues require extra time and assistance from development to troubleshoot. - -### Closed Beta features - -Closed Beta features are available to selected users only. - -- Not ready for production use. -- Unstable and can cause performance and stability issues. -- Configuration and dependencies unlikely to change. -- Features and functions unlikely to change. However, breaking changes may occur outside of major releases or with less notice than for Generally Available features. -- Data loss less likely. -- Behind a feature flag that is off by default and the UI reflects Beta status. -- Documentation reflects Beta status. -- Can be announced in a release post that reflects Beta status. - -### Open Beta features - -- Not ready for production use. -- Unstable and can cause performance and stability issues. +- No support available. +- May be unstable or have performance issues. +- Can be removed at any time. +- Data loss may occur. +- Documentation may not exist or just be in a blog format. +- Behind a feature flag that is on by default and the [UI reflects Experiment status](https://design.gitlab.com/usability/feature-management#highlighting-feature-versions). +- Behind a toggle that is off by default and the [UI reflects Experiment status](https://design.gitlab.com/usability/feature-management#highlighting-feature-versions). +- Feedback issue to engage with team. +- UX not finalized, might be just quick action access. +- Not announced in a release post. + +## Beta + +Commercially-reasonable efforts are made to provide limited support for features designated as "Beta," with the expectation that issues require extra time and assistance from development to troubleshoot. + +- May not be ready for production use and the UI and documentation will reflect this status. +- May be unstable and can cause performance and stability issues. - Configuration and dependencies unlikely to change. - Features and functions unlikely to change. However, breaking changes may occur outside of major releases or with less notice than for Generally Available features. - Data loss not likely. - Support on a commercially-reasonable effort basis. - Documentation reflects Beta status. -- Behind a feature flag that is on by default and the UI reflects Beta status. -- Behind a toggle that is off by default and the UI reflects Beta status. +- UX complete or near completion. +- Behind a feature flag that is on by default and the [UI reflects Beta status](https://design.gitlab.com/usability/feature-management#highlighting-feature-versions). +- Behind a toggle that is off by default and the [UI reflects Beta status](https://design.gitlab.com/usability/feature-management#highlighting-feature-versions). - Can be announced in a release post that reflects Beta status. -## Limited Availability (LA) - -Characteristics of Limited Availability features: - -- Ready for production use by a small set of customers. -- Can be booked by Deal Desk as part of an order. -- Fully documented and [supported](https://about.gitlab.com/support/statement-of-support/#starter-premium-and-ultimate-users). - ## Generally Available (GA) Generally Available features means that they passed the [Production Readiness Review](https://gitlab.com/gitlab-com/gl-infra/readiness/-/blob/master/.gitlab/issue_templates/production_readiness.md) for GitLab.com, and are: - Ready for production use at any scale. - Fully documented and supported. +- UX complete and in line with GitLab design standards. + +## Never internal + +Features are never internal (GitLab team-members) only. +Our [mission is "everyone can contribute"](https://about.gitlab.com/company/mission/), and that is only possible if people outside the company can try a feature. +We will get higher quality (more diverse) feedback if people from different organizations try something. +We've also learned that internal only as a state slows us down more than it speeds us up. +Release the experiment instead of testing internally or waiting for the feature to be in a Beta state. +The experimental features are only shown when people/organizations opt-in to experiments, we are allowed to make mistakes here and literally experiment. + +## All features are in production + +All features that are available on GitLab.com are considered "in production." +Because all Experiment, Beta, and Generally Available features are available on GitLab.com, they are all considered to be in production. diff --git a/doc/policy/maintenance.md b/doc/policy/maintenance.md index fcb6e5c1b20..eed9f006bfa 100644 --- a/doc/policy/maintenance.md +++ b/doc/policy/maintenance.md @@ -110,9 +110,9 @@ The decision on whether backporting a change is performed is done at the discret [current release managers](https://about.gitlab.com/community/release-managers/), based on *all* of the following: -1. Estimated [severity](../development/contributing/issue_workflow.md#severity-labels) of the bug: +1. Estimated [severity](../development/labels/index.md#severity-labels) of the bug: Highest possible impact to users based on the current definition of severity. -1. Estimated [priority](../development/contributing/issue_workflow.md#priority-labels) of the bug: +1. Estimated [priority](../development/labels/index.md#priority-labels) of the bug: Immediate impact on all impacted users based on the above estimated severity. 1. Potentially incurring data loss and/or security breach. 1. Potentially affecting one or more strategic accounts due to a proven inability by the user to upgrade to the current stable version. @@ -122,7 +122,7 @@ the current stable release, and two previous monthly releases. In rare cases a r For instance, if we release `13.2.1` with a fix for a severe bug introduced in `13.0.0`, we could backport the fix to a new `13.0.x`, and `13.1.x` patch release. -Note that [severity](../development/contributing/issue_workflow.md#severity-labels) 3 and lower +Note that [severity](../development/labels/index.md#severity-labels) 3 and lower requests are automatically turned down. To request backporting to more than one stable release for consideration, raise an issue in the diff --git a/doc/raketasks/backup_gitlab.md b/doc/raketasks/backup_gitlab.md index d63ef61a99f..e04d63da2ad 100644 --- a/doc/raketasks/backup_gitlab.md +++ b/doc/raketasks/backup_gitlab.md @@ -4,7 +4,7 @@ group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Back up GitLab +# Back up GitLab **(FREE SELF)** GitLab provides a command line interface to back up your entire instance, including: @@ -262,6 +262,11 @@ For installations from source: sudo -u git -H bundle exec rake gitlab:backup:create SKIP=db,uploads RAILS_ENV=production ``` +`SKIP=` is also used to: + +- [Skip creation of the tar file](#skipping-tar-creation) (`SKIP=tar`). +- [Skip uploading the backup to remote storage](#skip-uploading-backups-to-remote-storage) (`SKIP=remote`). + ### Skipping tar creation NOTE: @@ -348,9 +353,17 @@ to create an incremental backup from: To create an incremental backup, run: -```shell -sudo gitlab-backup create INCREMENTAL=yes PREVIOUS_BACKUP= -``` +- In GitLab 15.0 or later: + + ```shell + sudo gitlab-backup create INCREMENTAL=yes PREVIOUS_BACKUP= + ``` + +- In GitLab 14.9 and 14.10: + + ```shell + sudo gitlab-backup create INCREMENTAL=yes BACKUP= + ``` To create an [untarred](#skipping-tar-creation) incremental backup from a tarred backup, use `SKIP=tar`: @@ -411,12 +424,9 @@ You can let the backup script upload (using the [Fog library](https://fog.io/)) the `.tar` file it creates. In the following example, we use Amazon S3 for storage, but Fog also lets you use [other storage providers](https://fog.io/storage/). GitLab also [imports cloud drivers](https://gitlab.com/gitlab-org/gitlab/-/blob/da46c9655962df7d49caef0e2b9f6bbe88462a02/Gemfile#L113) -for AWS, Google, OpenStack Swift, Rackspace, and Aliyun. A local driver is +for AWS, Google, and Aliyun. A local driver is [also available](#upload-to-locally-mounted-shares). -NOTE: -Support for Openstack Swift and Rackspace APIs will be removed in GitLab 15.10. See [issue #387976](https://gitlab.com/gitlab-org/gitlab/-/issues/387976) for more information. - [Read more about using object storage with GitLab](../administration/object_storage.md). #### Using Amazon S3 diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index a13d38a199d..18303a5f45c 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -665,11 +665,20 @@ You may need to reconfigure or restart GitLab for the changes to take effect. 1. Clear all the tokens for pending jobs: + For GitLab 15.3 and earlier: + ```sql -- Clear build tokens UPDATE ci_builds SET token = null, token_encrypted = null; ``` + For GitLab 15.4 and later: + + ```sql + -- Clear build tokens + UPDATE ci_builds SET token_encrypted = null; + ``` + A similar strategy can be employed for the remaining features. By removing the data that can't be decrypted, GitLab can be returned to operation, and the lost data can be manually replaced. @@ -788,7 +797,7 @@ Truncating filenames to resolve the error involves: #### Clean up remote uploaded files -A [known issue](https://gitlab.com/gitlab-org/gitlab-foss/issues/45425) caused object store uploads to remain after a parent resource was deleted. This issue was [resolved](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/18698). +A [known issue](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/45425) caused object store uploads to remain after a parent resource was deleted. This issue was [resolved](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/18698). To fix these files, you must clean up all remote uploaded files that are in the storage but not tracked in the `uploads` database table. diff --git a/doc/raketasks/cleanup.md b/doc/raketasks/cleanup.md index 073a0351ec5..11a6e06fcf5 100644 --- a/doc/raketasks/cleanup.md +++ b/doc/raketasks/cleanup.md @@ -111,6 +111,8 @@ I, [2018-07-27T12:08:33.755624 #89817] INFO -- : Did fix /opt/gitlab/embedded/s I, [2018-07-27T12:08:33.760257 #89817] INFO -- : Did move to lost and found /opt/gitlab/embedded/service/gitlab-rails/public/uploads/foo/bar/1dd6f0f7eefd2acc4c2233f89a0f7b0b/image.png -> /opt/gitlab/embedded/service/gitlab-rails/public/uploads/-/project-lost-found/foo/bar/1dd6f0f7eefd2acc4c2233f89a0f7b0b/image.png ``` +If using object storage, run the [All-in-one Rake task](../administration/raketasks/uploads/migrate.md#all-in-one-rake-task) to ensure all uploads are migrated to object storage and there are no files on disk in the uploads folder. + ### Clean up project upload files from object storage > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/20918) in GitLab 11.2. @@ -154,7 +156,7 @@ These commands don't work for artifacts stored on [object storage](../administration/object_storage.md). WARNING: -Prior to GitLab 14.9, this task incorrectly deletes [pipeline artifacts](../ci/pipelines/pipeline_artifacts.md) +Prior to GitLab 14.9, this task incorrectly deletes [pipeline artifacts](../ci/pipelines/pipeline_artifacts.md). [The bug fix](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81022) was also back-ported to 14.6.6, 14.7.5, and 14.8.3. Upgrade to a release with the bug fix to avoid data loss. diff --git a/doc/raketasks/generate_sample_prometheus_data.md b/doc/raketasks/generate_sample_prometheus_data.md index 7445065c77e..73c49bd2599 100644 --- a/doc/raketasks/generate_sample_prometheus_data.md +++ b/doc/raketasks/generate_sample_prometheus_data.md @@ -4,9 +4,9 @@ group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Generate sample Prometheus data Rake task **(FREE SELF)** +# Sample Prometheus data Rake task **(FREE SELF)** -This command runs Prometheus queries for each of the metrics of a specific environment +The Rake task runs Prometheus queries for each of the metrics of a specific environment for a series of time intervals to now: - 30 minutes @@ -16,12 +16,12 @@ for a series of time intervals to now: - 72 hours - 7 days -The results of each of query are stored under a `sample_metrics` directory as a YAML +The results of each query are stored under a `sample_metrics` directory as a YAML file named by the metric's `identifier`. When the environmental variable `USE_SAMPLE_METRICS` -is set, the Prometheus API query is re-routed to `Projects::Environments::SampleMetricsController` -which loads the appropriate data set if it is present within the `sample_metrics` directory. +is set, the Prometheus API query is re-routed to `Projects::Environments::SampleMetricsController`, +which loads the appropriate data set if it's present in the `sample_metrics` directory. -This command requires an ID from an environment with an available Prometheus installation. +The Rake task requires an ID from an environment with an available Prometheus installation. ## Example diff --git a/doc/raketasks/import.md b/doc/raketasks/import.md index 5ab62215570..d3b734eba12 100644 --- a/doc/raketasks/import.md +++ b/doc/raketasks/import.md @@ -2,175 +2,10 @@ stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' --- -# Import bare repositories (deprecated) **(FREE SELF)** +# Import bare repositories (removed) **(FREE SELF)** -WARNING: The Rake task for importing bare repositories was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/108507) -in GitLab 15.8 and will be removed in GitLab 16.0. - -Alternatives to using the `gitlab:import:repos` Rake task include: - -- Migrating projects using either [an export file](../user/project/settings/import_export.md) or [direct transfer](../user/group/import/index.md#migrate-groups-by-direct-transfer-recommended) migrate repositories as well. -- Importing a [repository by URL](../user/project/import/repo_by_url.md). -- Importing a [repositories from a non-GitLab source](../user/project/import/index.md). - -Rake tasks are available to import bare repositories into a GitLab instance. -When migrating from an existing GitLab instance, and to preserve ownership by users and their namespaces, -migrate projects using either: - -- [Direct transfer](../user/group/import/index.md#migrate-groups-by-direct-transfer-recommended). -- [An export file](../user/project/settings/import_export.md). - -When you import a repository: - -- The owner of the project is the first administrator. -- The groups are created as needed, including subgroups. -- The owner of the group is the first administrator. -- Existing projects are skipped. -- Projects in hashed storage may be skipped. For more information, see - [Importing bare repositories from hashed storage](#importing-bare-repositories-from-hashed-storage). -- The existing Git repositories are moved from disk (removed from the original path). -- You must manually [push Git LFS objects](#push-git-lfs-objects). - -To import bare repositories into a GitLab instance: - -1. Create a new folder to import your Git repositories from. - You can also import projects into a (sub)group's namespace, - instead of the administrator's namespace. To do so, create subfolders and - give ownership and read/write/execute permissions of those subfolders to the - `git` user and its group: - - ```shell - sudo -u git mkdir -p /var/opt/gitlab/git-data/repository-import-$(date "+%Y-%m-%d")// - ``` - -1. Copy your bare repositories inside this newly created folder. Note: - - - Any `.git` repositories found on any of the subfolders are imported as projects. - - Groups are created as needed, these could be nested folders. - - For example, if we copy the repositories to `/var/opt/gitlab/git-data/repository-import-2020-08-22`, - and repository `A` must be under the groups `G1` and `G2`, it must be created under those folders: - `/var/opt/gitlab/git-data/repository-import-2020-08-22/G1/G2/A.git`. - - ```shell - sudo cp -r /old/git/foo.git /var/opt/gitlab/git-data/repository-import-$(date "+%Y-%m-%d")// - - # Do this once when you are done copying git repositories - sudo chown -R git:git /var/opt/gitlab/git-data/repository-import-$(date "+%Y-%m-%d") - ``` - - `foo.git` must be owned by the `git` user and `git` users group. - - If you are using an installation from source, replace `/var/opt/gitlab/` with `/home/git`. - -1. Run the following command depending on your type of installation: - - - Omnibus Installation - - ```shell - sudo gitlab-rake gitlab:import:repos["/var/opt/gitlab/git-data/repository-import-$(date "+%Y-%m-%d")"] - ``` - - - Installation from source. Before running this command you must change to the directory where - your GitLab installation is located: - - ```shell - cd /home/git/gitlab - sudo -u git -H bundle exec rake gitlab:import:repos["/var/opt/gitlab/git-data/repository-import-$(date "+%Y-%m-%d")"] RAILS_ENV=production - ``` - -## Example output - -```plaintext -Processing /var/opt/gitlab/git-data/repository-import-1/a/b/c/blah.git - * Using namespace: a/b/c - * Created blah (a/b/c/blah) - * Skipping repo /var/opt/gitlab/git-data/repository-import-1/a/b/c/blah.wiki.git -Processing /var/opt/gitlab/git-data/repository-import-1/abcd.git - * Created abcd (abcd.git) -Processing /var/opt/gitlab/git-data/repository-import-1/group/xyz.git - * Using namespace: group (2) - * Created xyz (group/xyz.git) - * Skipping repo /var/opt/gitlab/git-data/repository-import-1/@shared/a/b/abcd.git -[...] -``` - -## Importing bare repositories from hashed storage - -Projects in legacy storage have a directory structure that mirrors their full -project path in GitLab, including their namespace structure. This information is -leveraged by the bare repository importer to import projects into their proper -locations. Each project and its parent namespaces are meaningfully named. - -However, the directory structure of projects in hashed storage do not contain -this information. This is beneficial for a variety of reasons, especially -improved performance and data integrity. See -[Repository Storage Types](../administration/repository_storage_types.md) for -more details. - -The repositories that are importable depends on the version of GitLab. - -### GitLab 10.3 or earlier - -Importing bare repositories from hashed storage is unsupported. - -### GitLab 10.4 and later - -To support importing bare repositories from hashed storage, GitLab 10.4 and -later stores the full project path with each repository, in a special section of -the Git repository's configuration file. This section is formatted as follows: - -```ini -[gitlab] - fullpath = gitlab-org/gitlab -``` - -However, existing repositories were not migrated to include this path. - -Bare repositories are importable if the following events occurred to the -repository in GitLab 10.4 and later: - -- Created -- Migrated to hashed storage -- Renamed -- Transferred to another namespace -- Ancestor renamed -- Ancestor transferred to another namespace - -Bare repositories are **not** importable by GitLab 10.4 to GitLab 11.6, if all the following are true about the repository: - -- It was created in GitLab 10.3 or earlier. -- It was not renamed, transferred, or migrated to [hashed storage](../administration/repository_storage_types.md#hashed-storage) in GitLab 10.4 to GitLab 11.6. -- Its ancestor namespaces were not renamed or transferred in GitLab 10.4 to GitLab 11.6. - -[In GitLab 11.6](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41776) and later, all -bare repositories are importable. - -To manually migrate repositories yourself (for GitLab 10.4 to GitLab 11.6), you can use the -[Rails console](../administration/operations/rails_console.md#starting-a-rails-console-session) -to do so. In a Rails console session, run the following to migrate a project: - -```ruby -project = Project.find_by_full_path('gitlab-org/gitlab') -project.set_full_path -``` - -In a Rails console session, run the following to migrate all of a namespace's -projects (this may take a while if there are thousands of projects in a namespace): - -```ruby -namespace = Namespace.find_by_full_path('gitlab-org') -namespace.send(:write_projects_repository_config) -``` - -## Push Git LFS objects - -The import task doesn't import Git LFS objects. You must manually push the LFS objects to the newly -created GitLab repository using the following command: - -```shell -git lfs push --all -``` +in GitLab 15.8 and [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118676) in GitLab 16.0. diff --git a/doc/raketasks/index.md b/doc/raketasks/index.md index b5a778d6b74..a44f053bc7b 100644 --- a/doc/raketasks/index.md +++ b/doc/raketasks/index.md @@ -2,7 +2,6 @@ stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false --- # Rake tasks **(FREE SELF)** @@ -12,7 +11,7 @@ processes. You can perform GitLab Rake tasks by using: -- `gitlab-rake ` for [Omnibus GitLab](https://docs.gitlab.com/omnibus/index.html) installations. +- `gitlab-rake ` for [Omnibus GitLab](https://docs.gitlab.com/omnibus/index.html) and [GitLab Helm chart](https://docs.gitlab.com/charts/troubleshooting/kubernetes_cheat_sheet.html#gitlab-specific-kubernetes-information) installations. - `bundle exec rake ` for [source](../install/installation.md) installations. ## Available Rake tasks @@ -28,7 +27,7 @@ The following Rake tasks are available for use with GitLab: | [General maintenance](../administration/raketasks/maintenance.md) | General maintenance and self-check tasks. | | [Geo maintenance](../administration/raketasks/geo.md) | [Geo](../administration/geo/index.md)-related maintenance. | | [GitHub import](../administration/raketasks/github_import.md) | Retrieve and import repositories from GitHub. | -| [Import large project exports](../development/import_project.md#importing-via-a-rake-task) | Import large GitLab [project exports](../user/project/settings/import_export.md). | +| [Import large project exports](../administration/raketasks/project_import_export.md#import-large-projects) | Import large GitLab [project exports](../user/project/settings/import_export.md). | | [Incoming email](../administration/raketasks/incoming_email.md) | Incoming email-related tasks. | | [Integrity checks](../administration/raketasks/check.md) | Check the integrity of repositories, files, LDAP, and more. | | [LDAP maintenance](../administration/raketasks/ldap.md) | [LDAP](../administration/auth/ldap/index.md)-related tasks. | @@ -56,6 +55,9 @@ To list all available Rake tasks: # Omnibus GitLab sudo gitlab-rake -vT +# GitLab Helm chart +gitlab-rake -vT + # Installations from source cd /home/git/gitlab sudo -u git -H bundle exec rake -vT RAILS_ENV=production diff --git a/doc/raketasks/migrate_snippets.md b/doc/raketasks/migrate_snippets.md index e51edc5c133..16f78d7fc71 100644 --- a/doc/raketasks/migrate_snippets.md +++ b/doc/raketasks/migrate_snippets.md @@ -1,6 +1,6 @@ --- stage: Create -group: Editor +group: IDE info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/raketasks/restore_gitlab.md b/doc/raketasks/restore_gitlab.md index c5bbb38cc37..c7abc7ac562 100644 --- a/doc/raketasks/restore_gitlab.md +++ b/doc/raketasks/restore_gitlab.md @@ -4,7 +4,7 @@ group: Geo info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Restore GitLab +# Restore GitLab **(FREE SELF)** GitLab provides a command line interface to restore your entire installation, and is flexible enough to fit your needs. @@ -29,15 +29,21 @@ restore. This is because the system user performing the restore actions (`git`) is usually not allowed to create or delete the SQL database needed to import data into (`gitlabhq_production`). All existing data is either erased (SQL) or moved to a separate directory (such as repositories and uploads). +Restoring SQL data skips views owned by PostgreSQL extensions. -To restore a backup, you must restore `/etc/gitlab/gitlab-secrets.json` -(for Omnibus packages) or `/home/git/gitlab/.secret` (for installations from -source). This file contains the database encryption key, -[CI/CD variables](../ci/variables/index.md), and +To restore a backup, **you must also restore the GitLab secrets**. +These include the database encryption key, [CI/CD variables](../ci/variables/index.md), and variables used for [two-factor authentication](../user/profile/account/two_factor_authentication.md). -If you fail to restore this encryption key file along with the application data -backup, users with two-factor authentication enabled and GitLab Runner -loses access to your GitLab server. +Without the keys, [multiple issues occur](backup_restore.md#when-the-secrets-file-is-lost), +including loss of access by users with [two-factor authentication enabled](../user/profile/account/two_factor_authentication.md), +and GitLab Runners cannot log in. + +Restore: + +- `/etc/gitlab/gitlab-secrets.json` (Linux package) +- `/home/git/gitlab/.secret` (self-compiled installations) +- Rails secret (cloud-native GitLab) + - [This can be converted to the Linux package format](https://docs.gitlab.com/charts/installation/migration/helm_to_package.html), if required. You may also want to restore your previous `/etc/gitlab/gitlab.rb` (for Omnibus packages) or `/home/git/gitlab/config/gitlab.yml` (for installations from source) and @@ -60,6 +66,9 @@ restoring the new data, which causes an error. Read more about [configuring NFS mounts](../administration/nfs.md) +Restoring a backup from an instance using local storage restores to local storage even if the target instance uses object storage. +Migrations to object storage must be done before or after restoration. + ## Restore for Omnibus GitLab installations This procedure assumes that: @@ -88,11 +97,15 @@ sudo gitlab-ctl stop sidekiq sudo gitlab-ctl status ``` +Next, ensure you have completed the [restore prerequisites](#restore-prerequisites) steps and have run `gitlab-ctl reconfigure` +after copying over the GitLab secrets file from the original installation. + Next, restore the backup, specifying the timestamp of the backup you wish to restore: ```shell # This command will overwrite the contents of your GitLab database! +# NOTE: "_gitlab_backup.tar" is omitted from the name sudo gitlab-backup restore BACKUP=11493107454_2018_04_25_10.6.4-ce ``` @@ -114,13 +127,9 @@ WARNING: The restore command requires [additional parameters](backup_restore.md#back-up-and-restore-for-installations-using-pgbouncer) when your installation is using PgBouncer, for either performance reasons or when using it with a Patroni cluster. -Next, restore `/etc/gitlab/gitlab-secrets.json` if necessary, -[as previously mentioned](#restore-prerequisites). - -Reconfigure, restart and [check](../administration/raketasks/maintenance.md#check-gitlab-configuration) GitLab: +Next, restart and [check](../administration/raketasks/maintenance.md#check-gitlab-configuration) GitLab: ```shell -sudo gitlab-ctl reconfigure sudo gitlab-ctl restart sudo gitlab-rake gitlab:check SANITIZE=true ``` @@ -155,9 +164,34 @@ the restore target directories are empty. For both these installation types, the backup tarball has to be available in the backup location (default location is `/var/opt/gitlab/backups`). -If you use Docker Swarm, [first disable the health check](#restore-gitlab-from-backup-using-docker-swarm). +### Restore for Helm chart installations + +The GitLab Helm chart uses the process documented in +[restoring a GitLab Helm chart installation](https://docs.gitlab.com/charts/backup-restore/restore.html#restoring-a-gitlab-installation) + +### Restore for Docker image installations -For Docker installations, the restore task can be run from host: +If you're using [Docker Swarm](../install/docker.md#install-gitlab-using-docker-swarm-mode), +the container might restart during the restore process because Puma is shut down, +and so the container health check fails. To work around this problem, +temporarily disable the health check mechanism. + +1. Edit `docker-compose.yml`: + + ```yaml + healthcheck: + disable: true + ``` + +1. Deploy the stack: + + ```shell + docker stack deploy --compose-file docker-compose.yml mystack + ``` + +For more information, see [issue 6846](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6846 "GitLab restore can fail owing to `gitlab-healthcheck`"). + +The restore task can be run from the host: ```shell # Stop the processes that are connected to the database @@ -177,31 +211,6 @@ docker restart docker exec -it gitlab-rake gitlab:check SANITIZE=true ``` -Users of GitLab 12.1 and earlier should use the command `gitlab-rake gitlab:backup:create` instead. - -WARNING: -`gitlab-rake gitlab:backup:restore` doesn't set the correct file system -permissions on your Registry directory. This is a [known issue](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/62759). -In GitLab 12.2 or later, you can use `gitlab-backup restore` to avoid this -issue. - -The GitLab Helm chart uses a different process, documented in -[restoring a GitLab Helm chart installation](https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/backup-restore/restore.md). - -### Restore GitLab from backup using Docker Swarm - -Docker Swarm might restart the container during the restore process because Puma is shut down, -and so the container health check fails. To work around this problem, disable the health check -mechanism in Docker compose file: - -```yaml -healthcheck: - disable: true -``` - -Read more in issue #6846, -[GitLab restore can fail owing to `gitlab-healthcheck`](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6846). - ## Restore for installation from source First, ensure your backup tar file is in the backup directory described in the @@ -273,9 +282,7 @@ project or group from there: the backed-up instance from which you want to restore. 1. [Restore the backup](#restore-gitlab) into this new instance, then export your [project](../user/project/settings/import_export.md) - or [group](../user/group/settings/import_export.md). Be sure to read the - **Important Notes** on either export feature's documentation to understand - what is and isn't exported. + or [group](../user/group/import/index.md#migrate-groups-by-uploading-an-export-file-deprecated). For more information about what is and isn't exported, see the export feature's documentation. 1. After the export is complete, go to the old instance and then import it. 1. After importing the projects or groups that you wanted is complete, you may delete the new, temporary GitLab instance. diff --git a/doc/raketasks/user_management.md b/doc/raketasks/user_management.md index 7756774e432..bb3f1f404b4 100644 --- a/doc/raketasks/user_management.md +++ b/doc/raketasks/user_management.md @@ -180,4 +180,4 @@ sudo /etc/init.d/gitlab start ## Related topics -- [Reset a user's password](../security/reset_user_password.md#use-a-rake-task). +- [Reset a user's password](../security/reset_user_password.md#use-a-rake-task) diff --git a/doc/security/email_verification.md b/doc/security/email_verification.md new file mode 100644 index 00000000000..b87fd28dbed --- /dev/null +++ b/doc/security/email_verification.md @@ -0,0 +1,46 @@ +--- +stage: Anti-Abuse +group: Anti-Abuse +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Account email verification **(FREE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86352) in GitLab 15.2 [with a flag](../administration/feature_flags.md) named `require_email_verification`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `require_email_verification`. On GitLab.com, this feature is not available. + +Account email verification provides an additional layer of GitLab account security. +When certain conditions are met, an account is locked. If your account is locked, +you must verify your identity or reset your password to sign in to GitLab. + + +For a demo, see [Require email verification - demo](https://www.youtube.com/watch?v=wU6BVEGB3Y0). + +## Accounts without two-factor authentication (2FA) + +An account is locked when either: + +- There are three or more failed sign-in attempts in 24 hours. +- A user attempts to sign in from a new IP address and the + `check_ip_address_for_email_verification` feature flag is enabled. + +A locked account without 2FA is not unlocked automatically. + +After a successful sign in, an email with a six-digit verification code is sent. +The verification code expires after 60 minutes. + +To unlock your account, sign in and enter the verification code. You can also +[reset your password](https://gitlab.com/users/password/new). + +## Accounts with 2FA or OAuth + +An account is locked when there are five or more failed sign-in attempts in 10 minutes. + +Accounts with 2FA or OAuth are automatically unlocked after 10 minutes. To unlock an account manually, +reset your password. + +## Related topics + +- [Locked and blocked account support](https://about.gitlab.com/handbook/support/workflows/reinstating-blocked-accounts.html) diff --git a/doc/security/index.md b/doc/security/index.md index 38eb5337f5a..7a78461d717 100644 --- a/doc/security/index.md +++ b/doc/security/index.md @@ -2,7 +2,6 @@ stage: Manage group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false type: index --- @@ -13,7 +12,7 @@ type: index - [Generated passwords for users created through integrated authentication](passwords_for_integrated_authentication_methods.md) - [Restrict SSH key technologies and minimum length](ssh_keys_restrictions.md) - [Rate limits](rate_limits.md) -- [Webhooks and insecure internal web services](webhooks.md) +- [Filtering outbound requests](webhooks.md) - [Information exclusivity](information_exclusivity.md) - [Reset user password](reset_user_password.md) - [Unlock a locked user](unlock_user.md) diff --git a/doc/security/rate_limits.md b/doc/security/rate_limits.md index a9ccbccaa90..d835d8eb08c 100644 --- a/doc/security/rate_limits.md +++ b/doc/security/rate_limits.md @@ -44,6 +44,7 @@ You can set these rate limits in the Admin Area of your instance: - [GitLab Pages rate limits](../administration/pages/index.md#rate-limits) - [Pipeline rate limits](../user/admin_area/settings/rate_limit_on_pipelines_creation.md) - [Incident management rate limits](../user/admin_area/settings/incident_management_rate_limits.md) +- [Unauthenticated access to Projects List API rate limits](../user/admin_area/settings/rate_limit_on_projects_api.md) You can set these rate limits using the Rails console: @@ -137,17 +138,18 @@ The **rate limit** is 20 calls per minute per IP address. ### Project Jobs API endpoint -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104912) in GitLab 15.7 [with a flag](../administration/feature_flags.md) named `ci_enforce_rate_limits_jobs_api`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, -ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `ci_enforce_rate_limits_jobs_api`. -The feature is not ready for production use. - There is a rate limit for the endpoint `project/:id/jobs`, which is enforced to reduce timeouts when retrieving jobs. The **rate limit** is 600 calls per minute per authenticated user. +### AI action + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118010) in GitLab 16.0. + +There is a rate limit for the GraphQL `aiAction` mutation, which is enforced to prevent from abusing this endpoint. + +The **rate limit** is 160 calls per 8 hours per authenticated user. + ## Troubleshooting ### Rack Attack is denylisting the load balancer @@ -193,7 +195,7 @@ To remove a blocked IP: keys *rack::attack* ``` - By default, the [`keys` command is disabled](https://docs.gitlab.com/omnibus/settings/redis.html#renamed-commands). +By default, the [`keys` command is disabled](https://docs.gitlab.com/omnibus/settings/redis.html#renamed-commands). 1. Optionally, add [the IP to the allowlist](https://docs.gitlab.com/omnibus/settings/configuration.html#configuring-rack-attack) to prevent it being denylisted again. diff --git a/doc/security/reset_user_password.md b/doc/security/reset_user_password.md index 38c52912d5c..f8ba3953379 100644 --- a/doc/security/reset_user_password.md +++ b/doc/security/reset_user_password.md @@ -96,7 +96,7 @@ If you know the username, user ID, or email address, you can use the Rails conso user.password = new_password user.password_confirmation = new_password ``` - + To set a specific value for the new password: ```ruby @@ -113,9 +113,9 @@ If you know the username, user ID, or email address, you can use the Rails conso 1. Save the changes: - ```ruby - user.save! - ``` + ```ruby + user.save! + ``` 1. Exit the console: @@ -145,10 +145,10 @@ attempt to fix this issue in a Rails console. For example, if a new `root` passw 1. Start a [Rails console](../administration/operations/rails_console.md). 1. Find the user and skip reconfirmation: - ```ruby - user = User.find(1) - user.skip_reconfirmation! - ``` + ```ruby + user = User.find(1) + user.skip_reconfirmation! + ``` 1. Attempt to sign in again. diff --git a/doc/security/token_overview.md b/doc/security/token_overview.md index a36d72f128d..8acd4a125ce 100644 --- a/doc/security/token_overview.md +++ b/doc/security/token_overview.md @@ -77,7 +77,7 @@ Project maintainers and owners can add or enable a deploy key for a project repo ## Runner registration tokens (deprecated) WARNING: -The ability to pass a runner registration token was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/380872) in GitLab 15.6 and is +The ability to pass a runner registration token has been [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/380872) and is planned for removal in 17.0, along with support for certain configuration arguments. This change is a breaking change. GitLab plans to introduce a new [GitLab Runner token architecture](../architecture/blueprints/runner_tokens/index.md), which introduces a new method for registering runners and eliminates the @@ -89,7 +89,7 @@ You can use the runner registration token to add runners that execute jobs in a ## Runner authentication tokens (also called runner tokens) -After registration, the runner receives an authentication token, which it uses to authenticate with GitLab when picking up jobs from the job queue. The authentication token is stored locally in the runner's [`config.toml`](https://docs.gitlab.com/runner/configuration/advanced-configuration.html) file. +Once created, the runner receives an authentication token, which it uses to authenticate with GitLab when picking up jobs from the job queue. The authentication token is stored locally in the runner's [`config.toml`](https://docs.gitlab.com/runner/configuration/advanced-configuration.html) file. After authentication with GitLab, the runner receives a [job token](../ci/jobs/ci_job_token.md), which it uses to execute the job. @@ -97,6 +97,17 @@ In case of Docker Machine/Kubernetes/VirtualBox/Parallels/SSH executors, the exe Malicious access to a runner's file system may expose the `config.toml` file and thus the authentication token, allowing an attacker to [clone the runner](https://docs.gitlab.com/runner/security/#cloning-a-runner). +In GitLab 16.0 and later, you can use an authentication token to register runners instead of a +registration token. Runner registration tokens have been [deprecated](../update/deprecations.md#registration-tokens-and-server-side-runner-arguments-in-gitlab-runner-register-command). + +To generate an authentication token, you create a runner in the GitLab UI and use the authentication token +instead of the registration token. + +| Process | Registration command | +| ------------------ | --------------------- | +| Registration token (deprecated) | `gitlab-runner register --registration-token $RUNNER_REGISTRATION_TOKEN ` | +| Authentication token | `gitlab-runner register --token $RUNNER_AUTHENTICATION_TOKEN` | + ## CI/CD job tokens The [CI/CD](../ci/jobs/ci_job_token.md) job token diff --git a/doc/security/two_factor_authentication.md b/doc/security/two_factor_authentication.md index a419ca4f3eb..9ac799610bb 100644 --- a/doc/security/two_factor_authentication.md +++ b/doc/security/two_factor_authentication.md @@ -11,7 +11,10 @@ Two-factor authentication (2FA) provides an additional level of security to your users' GitLab account. When enabled, users are prompted for a code generated by an application in addition to supplying their username and password to sign in. -Read more about [two-factor authentication (2FA)](../user/profile/account/two_factor_authentication.md) +NOTE: +If you are [using and enforcing SSO](../user/group/saml_sso/index.md#sso-enforcement), you might already be enforcing 2FA on the identity provider (IDP) side. Enforcing 2FA on GitLab as well might be unnecessary. + +Read more about [two-factor authentication (2FA)](../user/profile/account/two_factor_authentication.md). ## Enforce 2FA for all users **(FREE SELF)** diff --git a/doc/security/user_email_confirmation.md b/doc/security/user_email_confirmation.md index c3f19c92f91..962f6f0fd61 100644 --- a/doc/security/user_email_confirmation.md +++ b/doc/security/user_email_confirmation.md @@ -5,7 +5,7 @@ group: Authentication and Authorization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# User email confirmation at sign-up **(FREE SELF)** +# Make new users confirm email **(FREE SELF)** GitLab can be configured to require confirmation of a user's email address when the user signs up. When this setting is enabled, the user is unable to sign in until diff --git a/doc/security/user_file_uploads.md b/doc/security/user_file_uploads.md index db2948a8bd5..498245c4efb 100644 --- a/doc/security/user_file_uploads.md +++ b/doc/security/user_file_uploads.md @@ -7,20 +7,27 @@ info: To determine the technical writer assigned to the Stage/Group associated w # User file uploads **(FREE)** +Users can upload files to: + +- Issues or merge requests in a project. +- Epics in a group. + +GitLab generates direct URLs for these images with a random 32-character ID to prevent unauthorized users from guessing the URLs. This randomization offers some security for images containing sensitive information. + +## Access control for uploaded files + > - Enforced authorization checks [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80117) in GitLab 14.8 [with a flag](../administration/feature_flags.md) named `enforce_auth_checks_on_uploads`. Disabled by default. > - Enforced authorization checks became [generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/352291) in GitLab 15.3. Feature flag `enforce_auth_checks_on_uploads` removed. > - Project settings in the user interface [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/88567) in GitLab 15.3. -In private or internal projects, GitLab restricts access to uploaded files (such as PDFs) -to authenticated users only. By default, image files are not subject to the same -restriction, and unauthenticated users can use the URL to view the -file. If you enable authorization checks for all media files, images -receive the same protection and are viewable only by authenticated users. +Access to non-image files uploaded to: -Users can upload files to issues, merge requests, or comments in a project. Direct URLs -to these images in GitLab contain a random 32-character ID to help prevent -unauthorized users from guessing image URLs. This randomization provides some protection -if an image contains sensitive information. +- Issues or merge requests is determined by the project visibility. +- Group epics is determined by the group visibility. + +For public projects or groups, anyone can access these files through the direct attachment URL, even if the issue, merge request, or epic is confidential. +For private and internal projects, GitLab ensures only authenticated project members can access non-image file uploads, such as PDFs. +By default, image files do not have the same restriction, and anyone can view them using the URL. To protect image files, [enable authorization checks for all media files](#enable-authorization-checks-for-all-media-files), making them viewable only by authenticated users. Authentication checks for images can cause display issues in the body of notification emails. Emails are frequently read from clients (such as Outlook, Apple Mail, or your mobile device) @@ -29,8 +36,9 @@ the client is not authorized to GitLab. ## Enable authorization checks for all media files -Non-image attachments (including PDFs) always require authentication to be viewed. -You can use this setting to extend this protection to image files. +Only authenticated project members can view non-image attachments (including PDFs) in private and internal projects. + +To apply authentication requirements to image files in private or internal projects: Prerequisite: @@ -43,7 +51,34 @@ To configure authentication settings for all media files: 1. On the left sidebar, select **Settings > General**. 1. Expand **Visibility, project features, permissions**. 1. Scroll to **Project visibility** and select **Require authentication to view media files**. - You cannot select this option for projects with **Public** visibility. + +NOTE: +You cannot select this option for public projects. + +## Delete uploaded files + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/92791) in GitLab 15.3. + +You should delete an uploaded file when that file contains sensitive or confidential information. When you have deleted that file, users cannot access the file and the direct URL returns a 404 error. + +Project Owners and Maintainers can use the [interactive GraphiQL explorer](../api/graphql/index.md#graphiql) to access a [GraphQL endpoint](../api/graphql/reference/index.md#mutationuploaddelete) and delete an uploaded file. + +For example: + +```graphql +mutation{ + uploadDelete(input: { projectPath: "", secret: "<32-character-id>" , filename: "" }) { + upload { + id + size + path + } + errors + } +} +``` + +Project members that do not have the Owner or Maintainer role cannot access this GraphQL endpoint. +To work around this problem, add `customers.gitlab.com:443` to the +[allowlist](#allow-outbound-requests-to-certain-ip-addresses-and-domains). diff --git a/doc/subscriptions/bronze_starter.md b/doc/subscriptions/bronze_starter.md index 91da875a049..b076987c0ae 100644 --- a/doc/subscriptions/bronze_starter.md +++ b/doc/subscriptions/bronze_starter.md @@ -18,7 +18,7 @@ the tiers are no longer mentioned in GitLab documentation: - [Activate GitLab EE with a license](../user/admin_area/license.md) - [Add a help message to the sign-in page](../user/admin_area/settings/help_page.md#add-a-help-message-to-the-sign-in-page) - [Burndown and burnup charts](../user/project/milestones/burndown_and_burnup_charts.md) in the [Milestone View](../user/project/milestones/index.md#burndown-charts), -- [Code owners](../user/project/code_owners.md) +- [Code owners](../user/project/codeowners/index.md) - Description templates: - [Setting a default template for merge requests and issues](../user/project/description_templates.md#set-a-default-template-for-merge-requests-and-issues) - [Email from GitLab](../user/admin_area/email_from_gitlab.md) @@ -71,7 +71,7 @@ the tiers are no longer mentioned in GitLab documentation: - [Code Owners as eligible approvers](../user/project/merge_requests/approvals/rules.md#code-owners-as-eligible-approvers) - [Approval rules](../user/project/merge_requests/approvals/rules.md) features - [Restricting push and merge access to certain users](../user/project/protected_branches.md) - - [Visual Reviews](../ci/review_apps/index.md#visual-reviews) + - [Visual Reviews (deprecated)](../ci/review_apps/index.md#visual-reviews-deprecated) - Metrics and analytics: - [Contribution Analytics](../user/group/contribution_analytics/index.md) - [Merge Request Analytics](../user/analytics/merge_request_analytics.md) @@ -106,7 +106,7 @@ the tiers are no longer mentioned in GitLab documentation: - Search: - [Filtering merge requests](../user/project/merge_requests/index.md#filter-the-list-of-merge-requests) by approvers - [Filtering merge requests](../user/project/merge_requests/index.md#filter-the-list-of-merge-requests) by "approved by" - - [Advanced Search (Elasticsearch)](../user/search/advanced_search.md) + - [Advanced search (Elasticsearch)](../user/search/advanced_search.md) - [Service Desk](../user/project/service_desk.md) - [Storage usage statistics](../user/usage_quotas.md#storage-usage-statistics) @@ -128,7 +128,7 @@ Bronze-level subscribers: - [Group iterations API](../api/group_iterations.md) - Project milestones API: [Get all burndown chart events for a single milestone](../api/milestones.md#get-all-burndown-chart-events-for-a-single-milestone) - [Project iterations API](../api/iterations.md) - - Fields in the [Search API](../api/search.md) available only to [Advanced Search (Elasticsearch)](../integration/advanced_search/elasticsearch.md) users + - Fields in the [Search API](../api/search.md) available only to [advanced search (Elasticsearch)](../integration/advanced_search/elasticsearch.md) users - Fields in the [Merge requests API](../api/merge_requests.md) for [merge request approvals](../user/project/merge_requests/approvals/index.md) - Fields in the [Protected branches API](../api/protected_branches.md) that specify users or groups allowed to merge - [Merge request approvals API](../api/merge_request_approvals.md) diff --git a/doc/subscriptions/choosing_subscription.md b/doc/subscriptions/choosing_subscription.md new file mode 100644 index 00000000000..07b04c83bd7 --- /dev/null +++ b/doc/subscriptions/choosing_subscription.md @@ -0,0 +1,61 @@ +--- +stage: Fulfillment +group: Purchase +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Choosing a GitLab subscription + +To choose the right GitLab subscription, choose an offering and a tier. + +## Choose a subscription + +Choose which GitLab subscription suits your needs: + +- [GitLab SaaS](gitlab_com/index.md): The GitLab software-as-a-service offering. + You don't need to install anything to use GitLab SaaS, you only need to + [sign up](https://gitlab.com/users/sign_up) and start using GitLab straight away. +- [GitLab Dedicated](gitlab_dedicated/index.md): A single-tenant SaaS service for highly regulated and large enterprises. +- [GitLab self-managed](self_managed/index.md): Install, administer, and maintain + your own GitLab instance. + +On a GitLab self-managed instance, a GitLab subscription provides the same set of +features for _all_ users. On GitLab SaaS, you can apply a subscription to a group +namespace. You cannot apply a subscription to a personal namespace. + +NOTE: +Subscriptions cannot be transferred between GitLab SaaS and GitLab self-managed. +A new subscription must be purchased and applied as needed. + +## Choose a GitLab tier + +Pricing is [tier-based](https://about.gitlab.com/pricing/), allowing you to choose +the features which fit your budget. For information on what features are available +at each tier for each product, see the [GitLab self-managed feature comparison](https://about.gitlab.com/pricing/feature-comparison/). + +## Find your subscription + +The following chart should help you determine your subscription model. Select +the list item to go to the respective help page. + +```mermaid +graph TD + +A(Is your user account on GitLab.com?) +A --> B(Yes) +A --> C(No) +B --> D(fa:fa-link View your subscription on GitLab.com) +C --> E(fa:fa-link View your self-hosted subscription) + +click D "./gitlab_com/index.html#view-your-gitlabcom-subscription" +click E "./self_managed/index.html#view-your-subscription" +``` + +## Contact Support + +- See the tiers of [GitLab Support](https://about.gitlab.com/support/). +- [Submit a request](https://support.gitlab.com/hc/en-us/requests/new) through the Support Portal. + +We also encourage all users to search our project trackers for known issues and existing feature requests in the [GitLab project](https://gitlab.com/gitlab-org/gitlab/-/issues/). + +These issues are the best avenue for getting updates on specific product plans and for communicating directly with the relevant GitLab team members. diff --git a/doc/subscriptions/community_programs.md b/doc/subscriptions/community_programs.md new file mode 100644 index 00000000000..36a5788a3fa --- /dev/null +++ b/doc/subscriptions/community_programs.md @@ -0,0 +1,80 @@ +--- +stage: Fulfillment +group: Purchase +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Community programs + +GitLab provides the following community program subscriptions. + +## GitLab for Education + +For qualifying non-profit educational institutions, the [GitLab for Education Program](https://about.gitlab.com/solutions/education/) provides GitLab Ultimate, plus 50,000 CI/CD minutes per month. The subscription granted under GitLab for Education can only be used for instructional use or non-commercial academic research. For more information—including instructions for applying to the program and renewing program membership—see the [GitLab for Education Program page](https://about.gitlab.com/solutions/education/) and the [GitLab handbook](https://about.gitlab.com/handbook/marketing/community-relations/community-programs/education-program/). + +## GitLab for Open Source + +For qualifying open source projects, the [GitLab for Open Source Program](https://about.gitlab.com/solutions/open-source/) provides GitLab Ultimate, plus 50,000 CI/CD minutes per month. For more information—including instructions for applying to the program and renewing program membership—see the [GitLab for Open Source Program page](https://about.gitlab.com/solutions/open-source/) and the [GitLab handbook](https://about.gitlab.com/handbook/marketing/community-relations/community-programs/opensource-program/). + +### Meeting GitLab for Open Source Program requirements + +To meet GitLab for Open Source Program requirements, first add an OSI-approved open source license to all projects in your namespace. + +To add a license to a project: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the overview page, select **Add LICENSE**. If the license you want is not available as a license template, manually copy the entire, unaltered [text of your chosen license](https://opensource.org/licenses/) into the `LICENSE` file. GitLab defaults to **All rights reserved** if users do not perform this action. + +![Add license](img/add-license.png) + +Applicants must add the correct license to each project in their respective groups or namespaces. When you're sure you're using OSI-approved licenses for your projects, you can take your screenshots. + +NOTE: +GitLab for Open Source Program benefits apply to an entire GitLab namespace. To qualify for the GitLab for Open Source Program, all projects in an applicant's namespace must meet program requirements. Applicants submit materials related to one project in the applying namespace, and the open source program team uses that project to verify eligibility of the entire namespace. + +### Verification for Open Source Program + +Next, take screenshots of your project to confirm that project's eligibility. You must upload three screenshots: + +- [OSI-approved license overview](#screenshot-1-license-overview) +- [OSI-approved license contents](#screenshot-2-license-contents) +- [Publicly visible settings](#screenshot-3-publicly-visible-settings) + +NOTE: +Benefits of the GitLab Open Source Program apply to all projects in a GitLab namespace. All projects in an eligible namespace must meet program requirements. + +#### Screenshot 1: License overview + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select your project avatar. If you haven't specified an avatar for your project, the avatar displays as a single letter. +1. Take a screenshot of the project overview that clearly displays the license you've chosen for your project. + +![License overview](img/license-overview.png) + +#### Screenshot 2: License contents + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Repository** and locate the project's `LICENSE` file. +1. Take a screenshot of the contents of the file. Make sure the screenshot includes the title of the license. + +![License file](img/license-file.png) + +#### Screenshot 3: Publicly visible settings + +To be eligible for the GitLab Open Source Program, projects must be publicly visible. To check your project's public visibility settings: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. From the left sidebar, select **Settings > General**. +1. Expand **Visibility, project features, permissions**. +1. From the **Project visibility** dropdown list, select **Public**. +1. Select the **Users can request access** checkbox. +1. Take a screenshot of this view. Include as much of the publicly visible settings as possible. Make sure to include your project's name in the upper-left of the screenshot. + +![Publicly visible setting](img/publicly-visible.png) + +NOTE: +Exceptions to this public visibility requirement apply in select circumstances (for example, in cases where a project in an applicant's namespace may hold sensitive data). Email `opensource@gitlab.com` with details of your use case to request written permission for exceptions. + +## GitLab for Startups + +For qualifying startups, the [GitLab for Startups](https://about.gitlab.com/solutions/startups/) program provides GitLab Ultimate, plus 50,000 CI/CD minutes per month for 12 months. For more information—including instructions for applying to the program and renewing program membership—see the [GitLab for Startups Program page](https://about.gitlab.com/solutions/startups/) and the [GitLab handbook](https://about.gitlab.com/handbook/marketing/community-relations/community-programs/startups-program/). diff --git a/doc/subscriptions/customers_portal.md b/doc/subscriptions/customers_portal.md new file mode 100644 index 00000000000..07e6e952419 --- /dev/null +++ b/doc/subscriptions/customers_portal.md @@ -0,0 +1,104 @@ +--- +stage: Fulfillment +group: Purchase +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# The Customers Portal + +For some management tasks for your subscription and account, you use the Customers Portal. + +If you made your purchase through an authorized reseller, you must contact them directly to make changes to your subscription (your subscriptions are read-only). + +You can also specifically manage your [GitLab SaaS subscription](gitlab_com/index.md) +or [self-managed subscription](self_managed/index.md). + +## Change account owner information + +Account owner personal details are used on invoices. The account owner email address is used for the Customers Portal legacy login and license-related email. + +If you have registered a Customers Portal account through a GitLab.com account, the GitLab.com account is used for login. + +To change account owner information, including name, billing address, and email address: + +1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). +1. Select **My account > Account details**. +1. Expand the **Personal details** section. +1. Edit the personal details. +1. Select **Save changes**. + +If you want to transfer ownership of the Customers Portal account +to another person, after you enter that person's personal details, you must also: + +- [Change the Customers Portal account password](#change-customers-portal-account-password). +- [Change the linked GitLab.com account](#change-the-linked-account), if you have one linked. + +## Change your company details + +To change your company details, including company name and VAT number: + +1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). +1. Select **My account > Account details**. +1. Expand the **Company details** section. +1. Edit the company details. +1. Select **Save changes**. + +## Change your payment method + +Purchases in the Customers Portal require a credit card on record as a payment method. You can add +multiple credit cards to your account, so that purchases for different products are charged to the +correct card. + +If you would like to use an alternative method to pay, please +[contact our Sales team](https://about.gitlab.com/sales/). + +To change your payment method: + +1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). +1. Select **My account > Payment methods**. +1. **Edit** an existing payment method's information or **Add new payment method**. +1. Select **Save Changes**. + +### Set a default payment method + +Automatic renewal of a subscription is charged to your default payment method. To mark a payment +method as the default: + +1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). +1. Select **My account > Payment methods**. +1. **Edit** the selected payment method and check the **Make default payment method** checkbox. +1. Select **Save Changes**. + +## Link a GitLab.com account + +Follow this guideline if you have a legacy Customers Portal account and use an email and password to log in. + +To link a GitLab.com account to your Customers Portal account: + +1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in?legacy=true) using email and password. +1. On the Customers Portal page, select **My account > Account details**. +1. Under **Your GitLab.com account**, select **Link account**. +1. Log in to the [GitLab.com](https://gitlab.com/users/sign_in) account you want to link to the Customers Portal account. + +## Change the linked account + +Customers are required to use their GitLab.com account to register for a new Customers Portal account. + +If you have a legacy Customers Portal account that is not linked to a GitLab.com account, you may still [sign in](https://customers.gitlab.com/customers/sign_in?legacy=true) using an email and password. However, you should [create](https://gitlab.com/users/sign_up) and [link a GitLab.com account](#change-the-linked-account) to ensure continued access to the Customers Portal. + +To change the GitLab.com account linked to your Customers Portal account: + +1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). +1. In a separate browser tab, go to [GitLab.com](https://gitlab.com/users/sign_in) and ensure you are not logged in. +1. On the Customers Portal page, select **My account > Account details**. +1. Under **Your GitLab.com account**, select **Change linked account**. +1. Log in to the [GitLab.com](https://gitlab.com/users/sign_in) account you want to link to the Customers Portal account. + +## Change Customers Portal account password + +To change the password for this customers portal account: + +1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). +1. Select the **My account** dropdown list and select **Account details**. +1. Make the required changes to the **Your password** section. +1. Select **Save changes**. diff --git a/doc/subscriptions/gitlab_com/index.md b/doc/subscriptions/gitlab_com/index.md index 632581e66d3..ff93d1c903a 100644 --- a/doc/subscriptions/gitlab_com/index.md +++ b/doc/subscriptions/gitlab_com/index.md @@ -70,8 +70,9 @@ The following information is displayed: A GitLab SaaS subscription uses a concurrent (_seat_) model. You pay for a subscription according to the maximum number of users assigned to the top-level group or its children during the billing period. You can -add and remove users during the subscription period, as long as the total users -at any given time doesn't exceed the subscription count. +add and remove users during the subscription period without incurring additional charges, as long as the total users +at any given time doesn't exceed the subscription count. If the total users exceeds your subscription count, you will incur an overage +which must be paid at your next [reconciliation](../quarterly_reconciliation.md). A top-level group can be [changed](../../user/group/manage.md#change-a-groups-path) like any other group. @@ -163,20 +164,20 @@ Seats owed = 12 - 10 (Maximum users - users in subscription) ### Free Guest users **(ULTIMATE)** In the **Ultimate** tier, users who are assigned the Guest role do not consume a seat. -The user must not be assigned any other role, anywhere in the instance. +The user must not be assigned any other role, anywhere in the instance or in the namespace for GitLab SaaS. - If your project is private or internal, a user with the Guest role has [a set of permissions](../../user/permissions.md#project-members-permissions). - If your project is public, all users, including those with the Guest role can access your project. -### Add users to your subscription +### Add seats to your subscription Your subscription cost is based on the maximum number of seats you use during the billing period. Even if you reach the number of seats in your subscription, you can continue to add users. GitLab [bills you for the overage](../quarterly_reconciliation.md). -To add users to a subscription: +To add seats to a subscription: 1. Log in to the [Customers Portal](https://customers.gitlab.com/). 1. Navigate to the **Manage Purchases** page. @@ -236,17 +237,25 @@ amounts at which the alert displays. To change the namespace linked to a subscription: 1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in) with a - [linked](../index.md#change-the-linked-account) GitLab.com account. + [linked](../customers_portal.md#link-a-gitlabcom-account) GitLab.com account. 1. Go to the **Manage Purchases** page. 1. Select **Change linked namespace**. -1. Select the desired group from the **This subscription is for** dropdown list. For a group to appear here, you must have the Owner role for that group. -1. Select **Proceed to checkout**. +1. Select the desired group from the **New Namespace** dropdown list. For a group to appear here, you must have the Owner role for that group. +1. If the [total number of users](#view-seat-usage) in your group exceeds the number of seats in your subscription, + you are prompted to pay for the additional users. Subscription charges are calculated based on + the total number of users in a group, including its subgroups and nested projects. + + If you purchased your subscription through an authorized reseller, you are unable to pay for additional users. + You can either: + + - Remove additional users, so that no overage is detected. + - Contact the partner to purchase additional seats now or at the end of your subscription term. + +1. Select **Confirm changes**. For a demo, see [Linking GitLab Subscription to the Namespace](https://youtu.be/qAq8pyFP-a0). -Subscription charges are calculated based on the total number of users in a group, including its subgroups and nested projects. If the [total number of users](#view-seat-usage) exceeds the number of seats in your subscription, your account is charged for the additional users and you need to pay for the overage before you can change the linked namespace. - Only one namespace can be linked to a subscription. ## Upgrade your GitLab SaaS subscription tier @@ -311,7 +320,7 @@ For details on upgrading your subscription tier, see ### Automatic subscription renewal When a subscription is set to auto-renew, it renews automatically on the -expiration date without a gap in available service. Subscriptions purchased through Customers Portal or GitLab.com are set to auto-renew by default. The number of seats is adjusted to fit the [number of billable users in your group](#view-seat-usage) at the time of renewal. You can view and download your renewal invoice on the Customers Portal [View invoices](https://customers.gitlab.com/receipts) page. If your account has a [saved credit card](../index.md#change-your-payment-method), the card is charged for the invoice amount. If we are unable to process a payment or the auto-renewal fails for any other reason, you have 14 days to renew your subscription, after which your access is downgraded. +expiration date without a gap in available service. Subscriptions purchased through Customers Portal or GitLab.com are set to auto-renew by default. The number of seats is adjusted to fit the [number of billable users in your group](#view-seat-usage) at the time of renewal. You can view and download your renewal invoice on the Customers Portal [View invoices](https://customers.gitlab.com/receipts) page. If your account has a [saved credit card](../customers_portal.md#change-your-payment-method), the card is charged for the invoice amount. If we are unable to process a payment or the auto-renewal fails for any other reason, you have 14 days to renew your subscription, after which your access is downgraded. #### Email notifications @@ -326,7 +335,7 @@ expiration date without a gap in available service. Subscriptions purchased thro To view or change automatic subscription renewal (at the same tier as the previous period), sign in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in), and: -- If a **Resume subscription** button is displayed, your subscription was canceled +- If a **Turn on auto-renew** button is displayed, your subscription was canceled previously. Select it to resume automatic renewal. - If a **Cancel subscription** button is displayed, your subscription is set to automatically renew at the end of the subscription period. Select it to cancel automatic renewal. diff --git a/doc/subscriptions/gitlab_dedicated/index.md b/doc/subscriptions/gitlab_dedicated/index.md index 2d684edb992..0c2b6375d7a 100644 --- a/doc/subscriptions/gitlab_dedicated/index.md +++ b/doc/subscriptions/gitlab_dedicated/index.md @@ -20,19 +20,97 @@ It's the offering of choice for enterprises and organizations in highly regulate ## Available features -- Authentication: Support for instance-level [SAML OmniAuth](../../integration/saml.md) functionality. GitLab Dedicated acts as the service provider, and you must provide the necessary [configuration](../../integration/saml.md#configure-saml-support-in-gitlab) in order for GitLab to communicate with your IdP. This is provided during onboarding. - - SAML [request signing](../../integration/saml.md#sign-saml-authentication-requests-optional), [group sync](../../user/group/saml_sso/group_sync.md#configure-saml-group-sync), and [SAML groups](../../integration/saml.md#configure-users-based-on-saml-group-membership) are supported. -- Networking: - - Public connectivity with support for IP Allowlists. During onboarding, you can optionally specify a list of IP addresses that can access your GitLab Dedicated instance. Subsequently, when an IP not on the allowlist tries to access your instance the connection is refused. - - Optional. Private connectivity via [AWS PrivateLink](https://aws.amazon.com/privatelink/). - You can specify an AWS IAM Principal and preferred Availability Zones during onboarding to enable this functionality. Both Ingress and Egress PrivateLinks are supported. When connecting to an internal service running in your VPC over HTTPS via PrivateLink, GitLab Dedicated supports the ability to use a private SSL certificate, which can be provided during onboarding. -- Upgrades: - - Monthly upgrades tracking one release behind the latest (n-1), with the latest security release. - - Out of band security patches provided for high severity releases. -- Backups: Regular backups taken and tested. -- Choice of cloud region: Upon onboarding, choose the cloud region where you want to deploy your instance. Some AWS regions have limited features and as a result, we are not able to deploy production instances to those regions. See below for the [full list of regions](#aws-regions-not-supported) not currently supported. -- Security: Data encrypted at rest and in transit using latest encryption standards. -- Application: Self-managed [Ultimate feature set](https://about.gitlab.com/pricing/feature-comparison/) with the exception of the unsupported features [listed below](#features-that-are-not-available). +### Data residency + +GitLab Dedicated allows you to select the cloud region where your data will be stored. Upon [onboarding](../../administration/dedicated/index.md#onboarding), choose the cloud region where you want to deploy your Dedicated instance. Some AWS regions have limited features and as a result, we are not able to deploy production instances to those regions. See below for the [full list of regions](#aws-regions-not-supported) not supported. + +### Availability and scalability + +GitLab Dedicated leverages the GitLab [Cloud Native Hybrid reference architectures](../../administration/reference_architectures/index.md#cloud-native-hybrid) with high availability enabled. When [onboarding](../../administration/dedicated/index.md#onboarding), GitLab will match you to the closest reference architecture size based on your number of users. Learn about the [current Service Level Objective](https://about.gitlab.com/handbook/engineering/infrastructure/team/gitlab-dedicated/slas/#current-service-level-objective). + +#### Disaster Recovery + +When [onboarding](../../administration/dedicated/index.md#onboarding) to GitLab Dedicated, you can provide a Secondary AWS region in which your data is stored. This region is used to recover your GitLab Dedicated instance in case of a disaster. Regular backups of all GitLab Dedicated datastores (including Database and Git repositories) are taken and tested regularly and stored in your desired secondary region. GitLab Dedicated also provides the ability to store copies of these backups in a separate cloud region of choice for greater redundancy. + +For more information, read about the [recovery plan for GitLab Dedicated](https://about.gitlab.com/handbook/engineering/infrastructure/team/gitlab-dedicated/slas/#disaster-recovery-plan) as well as RPO and RTO targets. + +### Security + +#### Authentication and authorization + +GitLab Dedicated supports instance-level [SAML OmniAuth](../../integration/saml.md) functionality. Your GitLab Dedicated instance acts as the service provider, and you must provide the necessary [configuration](../../integration/saml.md#configure-saml-support-in-gitlab) in order for GitLab to communicate with your IdP. For more information, see how to [configure SAML](../../administration/dedicated/index.md#saml) for your instance. + +- SAML [request signing](../../integration/saml.md#sign-saml-authentication-requests-optional), [group sync](../../user/group/saml_sso/group_sync.md#configure-saml-group-sync), and [SAML groups](../../integration/saml.md#configure-users-based-on-saml-group-membership) are supported. + +#### Secure networking + +GitLab Dedicated offers public connectivity by default with support for IP allowlists. You can [optionally specify a list of IP addresses](../../administration/dedicated/index.md#ip-allowlist) that can access your GitLab Dedicated instance. Subsequently, when an IP not on the allowlist tries to access your instance the connection is refused. + +Private connectivity via [AWS PrivateLink](https://aws.amazon.com/privatelink/) is also offered as an option. Both [inbound](../../administration/dedicated/index.md#inbound-private-link) and [outbound](../../administration/dedicated/index.md#outbound-private-link) PrivateLinks are supported. When connecting to an internal service running in your VPC over HTTPS via PrivateLink, GitLab Dedicated supports the ability to use a private SSL certificate, which can be provided when [updating your instance configuration](../../administration/dedicated/index.md#custom-certificates). + +#### Encryption + +Data is encrypted at rest and in transit using the latest encryption standards. + +#### Bring your own key encryption + +During onboarding, you can specify an AWS KMS encryption key stored in your own AWS account that GitLab uses to encrypt the data for your Dedicated instance. This gives you full control over the data you store in GitLab. + +### Compliance + +#### Certifications + +GitLab Dedicated offers the following [compliance certifications](https://about.gitlab.com/security/): + +- SOC 2 Type 1 Report (Security and Confidentiality criteria) +- ISO/IEC 27001:2013 +- ISO/IEC 27017:2015 +- ISO/IEC 27018:2019 + +#### Isolation + +As a single-tenant SaaS service, GitLab Dedicated provides infrastructure-level isolation of your GitLab environment. Your environment is placed into a separate AWS account from other tenants. This AWS account contains all of the underlying infrastructure necessary to host the GitLab application and your data stays within the account boundary. You administer the application while GitLab manages the underlying infrastructure. Tenant environments are also completely isolated from GitLab.com. + +#### Access controls + +GitLab Dedicated adheres to the [principle of least privilege](https://about.gitlab.com/handbook/security/access-management-policy.html#principle-of-least-privilege) to control access to customer tenant environments. Tenant AWS accounts live under a top-level GitLab Dedicated AWS parent organization. Access to the AWS Organization is restricted to select GitLab team members. All user accounts within the AWS Organization follow the overall GitLab Access Management Policy [outlined here](https://about.gitlab.com/handbook/security/access-management-policy.html). Direct access to customer tenant environments is restricted to a single Hub account. The GitLab Dedicated Control Plane uses the Hub account to perform automated actions over tenant accounts when managing environments. Similarly, GitLab Dedicated engineers do not have direct access to customer tenant environments. In break glass situations, where access to resources in the tenant environment is required to address a high-severity issue, GitLab engineers must go through the Hub account to manage those resources. This is done via an approval process, and after permission is granted, the engineer will assume an IAM role on a temporary basis to access tenant resources through the Hub account. All actions within the hub account and tenant account are logged to CloudTrail. + +Inside tenant accounts, GitLab leverages Intrusion Detection and Malware Scanning capabilities from AWS GuardDuty. Infrastructure logs are monitored by the GitLab Security Incident Response Team to detect anomalous events. + +#### Audit and observability + +GitLab Dedicated provides access to [audit and system logs](../../administration/dedicated/index.md#access-to-application-logs) generated by the application. + +### Maintenance + +GitLab leverages [weekly maintenance windows](../../administration/dedicated/index.md#maintenance-window) to keep your instance up to date, fix security issues, and ensure the overall reliability and performance of your environment. + +#### Upgrades + +GitLab performs monthly upgrades to your instance with the latest security release during your preferred [maintenance window](../../administration/dedicated/index.md#maintenance-window) tracking one release behind the latest GitLab release. For example, if the latest version of GitLab available is 15.8, GitLab Dedicated runs on 15.7. + +#### Unscheduled maintenance + +GitLab may conduct unscheduled maintenance to address high-severity issues affecting the security, availability, or reliability of your instance. + +### Application + +GitLab Dedicated comes with the self-managed [Ultimate feature set](https://about.gitlab.com/pricing/feature-comparison/) with the exception of the unsupported features [listed below](#features-that-are-not-available). + +#### GitLab Runners + +With GitLab Dedicated, you must [install the GitLab Runner application](https://docs.gitlab.com/runner/install/index.html) on infrastructure that you own or manage. If hosting GitLab Runners on AWS, you can avoid having requests from the Runner fleet route through the public internet by setting up a secure connection from the Runner VPC to the GitLab Dedicated endpoint via AWS Private Link. Learn more about [networking options](#secure-networking). + +#### Migration + +To help you migrate your data to GitLab Dedicated, you can choose from the following options: + +1. When migrating from another GitLab instance, you can either: + - Use the UI, including [group import](../../user/group/import/index.md) and [project import](../../user/project/settings/import_export.md). + - Use APIs, including the [group import API](../../api/group_import_export.md) and [project import API](../../api/project_import_export.md). + - Note: Import functionality behind a feature flag (such as `bulk_import_project`) is not supported in GitLab Dedicated. +1. When migrating from third-party services, you can use [the GitLab importers](../../user/project/import/index.md#available-project-importers). +1. You can perform a fully-automated migration through the [Congregate Automation Tool](../../user/project/import/index.md#automate-group-and-project-import), which supports migrating from existing GitLab instances as well as third-party services. ## Features that are not available @@ -42,7 +120,7 @@ The following GitLab application features are not available: - LDAP, Smartcard, or Kerberos authentication - Multiple login providers -- Advanced Search +- Advanced search - GitLab Pages - FortiAuthenticator, or FortiToken 2FA - Reply-by email @@ -53,14 +131,13 @@ The following GitLab application features are not available: The following features will not be supported: - Mattermost -- Server-side Git hooks +- Server-side Git hooks. Use [push rules](../../user/project/repository/push_rules.md) instead. ### GitLab Dedicated service features The following operational features are not available: - Custom domains -- Bring Your Own Key (BYOK) encryption - Multiple Geo secondaries (Geo replicas) beyond the secondary site included by default - Self-serve purchasing and configuration - Multiple login providers diff --git a/doc/subscriptions/index.md b/doc/subscriptions/index.md index 2c9f93886bf..9f329700c74 100644 --- a/doc/subscriptions/index.md +++ b/doc/subscriptions/index.md @@ -5,250 +5,18 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: index, reference --- -# GitLab subscription **(PREMIUM)** - -GitLab offers tiers of features. Your subscription determines which tier you -have access to. Subscriptions are valid for 12 months. - -GitLab provides special subscriptions to participants in: - -- [Education](#gitlab-for-education) -- [Open Source](#gitlab-for-open-source) - -## Choose a GitLab subscription - -When choosing a subscription, there are two factors to consider: - -- [GitLab SaaS or GitLab self-managed](#choose-between-gitlab-saas-or-gitlab-self-managed) -- [GitLab tier](#choose-a-gitlab-tier) - -### Choose between GitLab SaaS or GitLab self-managed - -There are some differences in how a subscription applies, depending if you use -GitLab SaaS or GitLab self-managed: - -- [GitLab SaaS](gitlab_com/index.md): The GitLab software-as-a-service offering. - You don't need to install anything to use GitLab SaaS, you only need to - [sign up](https://gitlab.com/users/sign_up) and start using GitLab straight away. -- [GitLab Dedicated](gitlab_dedicated/index.md): a single-tenant SaaS service for highly regulated and large enterprises. -- [GitLab self-managed](self_managed/index.md): Install, administer, and maintain - your own GitLab instance. - -On a GitLab self-managed instance, a GitLab subscription provides the same set of -features for _all_ users. On GitLab SaaS, you can apply a subscription to a group -namespace. You cannot apply a subscription to a personal namespace. - -NOTE: -Subscriptions cannot be transferred between GitLab SaaS and GitLab self-managed. -A new subscription must be purchased and applied as needed. - -### Choose a GitLab tier - -Pricing is [tier-based](https://about.gitlab.com/pricing/), allowing you to choose -the features which fit your budget. For information on what features are available -at each tier for each product, see: [GitLab self-managed feature comparison](https://about.gitlab.com/pricing/feature-comparison/) - -## Find your subscription - -The following chart should help you determine your subscription model. Select -the list item to go to the respective help page. - -```mermaid -graph TD - -A(Is your user account on GitLab.com?) -A --> B(Yes) -A --> C(No) -B --> D(fa:fa-link View your subscription on GitLab.com) -C --> E(fa:fa-link View your self-hosted subscription) - -click D "./gitlab_com/index.html#view-your-gitlabcom-subscription" -click E "./self_managed/index.html#view-your-subscription" -``` - -## Customers Portal - -With the [Customers Portal](https://customers.gitlab.com/) you can: - -- [Change account owner information](#change-account-owner-information) -- [Change your company details](#change-your-company-details) -- [Change your payment method](#change-your-payment-method) -- [Change the linked account](#change-the-linked-account) -- [Change the namespace the subscription is linked to](gitlab_com/index.md#change-the-linked-namespace) -- [Change customers portal account password](#change-customers-portal-account-password) - -The Customers Portal is available only to customers who purchased their -subscription from GitLab. If you made your purchase through a partner or -reseller, you must contact them directly for assistance with your subscription. - -### Change account owner information - -Account owner personal details are used on invoices. The account owner email address is used for the Customers Portal legacy login and license-related email. - -If you have registered a Customers Portal account through a GitLab.com account, the GitLab.com account is used for login. - -To change account owner information, including name, billing address, and email address: - -1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). -1. Select **My account > Account details**. -1. Expand the **Personal details** section. -1. Edit the personal details. -1. Select **Save changes**. - -If you want to transfer ownership of the Customers Portal account -to another person, after you enter that person's personal details, you must also: - -- [Change the Customers Portal account password](#change-customers-portal-account-password). -- [Change the linked GitLab.com account](#change-the-linked-account), if you have one linked. - -### Change your company details - -To change your company details, including company name and VAT number: - -1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). -1. Select **My account > Account details**. -1. Expand the **Company details** section. -1. Edit the company details. -1. Select **Save changes**. - -### Change your payment method - -Purchases in the Customers Portal require a credit card on record as a payment method. You can add -multiple credit cards to your account, so that purchases for different products are charged to the -correct card. - -If you would like to use an alternative method to pay, please -[contact our Sales team](https://about.gitlab.com/sales/). - -To change your payment method: - -1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). -1. Select **My account > Payment methods**. -1. **Edit** an existing payment method's information or **Add new payment method**. -1. Select **Save Changes**. - -#### Set a default payment method - -Automatic renewal of a subscription is charged to your default payment method. To mark a payment -method as the default: - -1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). -1. Select **My account > Payment methods**. -1. **Edit** the selected payment method and check the **Make default payment method** checkbox. -1. Select **Save Changes**. - -### Change the linked account - -To change the GitLab.com account linked to your Customers Portal account: - -1. Log in to the - [Customers Portal](https://customers.gitlab.com/customers/sign_in). -1. In a separate browser tab, go to [GitLab.com](https://gitlab.com/users/sign_in) and ensure you - are not logged in. -1. On the Customers Portal page, select **My account > Account details**. -1. Under **Your GitLab.com account**, select **Change linked account**. -1. Log in to the [GitLab.com](https://gitlab.com/users/sign_in) account you want to link to the Customers Portal - account. - -### Change Customers Portal account password - -To change the password for this customers portal account: - -1. Log in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in). -1. Select the **My account** dropdown list and select **Account details**. -1. Make the required changes to the **Your password** section. -1. Select **Save changes**. - -## Community program subscriptions - -### GitLab for Education - -For qualifying non-profit educational institutions, the [GitLab for Education Program](https://about.gitlab.com/solutions/education/) provides GitLab Ultimate, plus 50,000 CI/CD minutes per month. The subscription granted under GitLab for Education can only be used for instructional use or non-commercial academic research. For more information—including instructions for applying to the program and renewing program membership—see the [GitLab for Education Program page](https://about.gitlab.com/solutions/education/) and the [GitLab handbook](https://about.gitlab.com/handbook/marketing/community-relations/community-programs/education-program/). - -### GitLab for Open Source - -For qualifying open source projects, the [GitLab for Open Source Program](https://about.gitlab.com/solutions/open-source/) provides GitLab Ultimate, plus 50,000 CI/CD minutes per month. For more information—including instructions for applying to the program and renewing program membership—see the [GitLab for Open Source Program page](https://about.gitlab.com/solutions/open-source/) and the [GitLab handbook](https://about.gitlab.com/handbook/marketing/community-relations/community-programs/opensource-program/). - -#### Meeting GitLab for Open Source Program requirements - -To meet GitLab for Open Source Program requirements, first add an OSI-approved open source license to all projects in your namespace. - -To add a license to a project: - -1. On the top bar, select **Main menu > Projects** and find your project. -1. On the overview page, select **Add LICENSE**. If the license you want is not available as a license template, manually copy the entire, unaltered [text of your chosen license](https://opensource.org/licenses/alphabetical) into the `LICENSE` file. Note that GitLab defaults to **All rights reserved** if users do not perform this action. - -![Add license](img/add-license.png) - -Applicants must add the correct license to each project in their respective groups or namespaces. When you're sure you're using OSI-approved licenses for your projects, you can take your screenshots. - -NOTE: -GitLab for Open Source Program benefits apply to an entire GitLab namespace. To qualify for the GitLab for Open Source Program, all projects in an applicant's namespace must meet program requirements. Applicants submit materials related to one project in the applying namespace, and the open source program team uses that project to verify eligibility of the entire namespace. - -#### Verification for Open Source Program - -Next, take screenshots of your project to confirm that project's eligibility. You must upload three screenshots: - -- [OSI-approved license overview](#screenshot-1-license-overview) -- [OSI-approved license contents](#screenshot-2-license-contents) -- [Publicly visible settings](#screenshot-3-publicly-visible-settings) - -NOTE: -Benefits of the GitLab Open Source Program apply to all projects in a GitLab namespace. All projects in an eligible namespace must meet program requirements. - -##### Screenshot 1: License overview - -1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select your project avatar. If you haven't specified an avatar for your project, the avatar displays as a single letter. -1. Take a screenshot of the project overview that clearly displays the license you've chosen for your project. - -![License overview](img/license-overview.png) - -##### Screenshot 2: License contents - -1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Repository** and locate the project's `LICENSE` file. -1. Take a screenshot of the contents of the file. Make sure the screenshot includes the title of the license. - -![License file](img/license-file.png) - -##### Screenshot 3: Publicly visible settings - -To be eligible for the GitLab Open Source Program, projects must be publicly visible. To check your project's public visibility settings: - -1. On the top bar, select **Main menu > Projects** and find your project. -1. From the left sidebar, select **Settings > General**. -1. Expand **Visibility, project features, permissions**. -1. From the **Project visibility** dropdown list, select **Public**. -1. Select the **Users can request access** checkbox. -1. Take a screenshot of this view. Include as much of the publicly visible settings as possible. Make sure to include your project's name in the upper-left of the screenshot. - -![Publicly visible setting](img/publicly-visible.png) - -NOTE: -Exceptions to this public visibility requirement apply in select circumstances (for example, in cases where a project in an applicant's namespace may hold sensitive data). Email `opensource@gitlab.com` with details of your use case to request written permission for exceptions. - -### GitLab for Startups - -For qualifying startups, the [GitLab for Startups](https://about.gitlab.com/solutions/startups/) program provides GitLab Ultimate, plus 50,000 CI/CD minutes per month for 12 months. For more information—including instructions for applying to the program and renewing program membership—see the [GitLab for Startups Program page](https://about.gitlab.com/solutions/startups/) and the [GitLab handbook](https://about.gitlab.com/handbook/marketing/community-relations/community-programs/startups-program/). - -## Contact Support - -- See the tiers of [GitLab Support](https://about.gitlab.com/support/). -- [Submit a request](https://support.gitlab.com/hc/en-us/requests/new) through the Support Portal. - -We also encourage all users to search our project trackers for known issues and existing feature requests in the [GitLab project](https://gitlab.com/gitlab-org/gitlab/-/issues/). - -These issues are the best avenue for getting updates on specific product plans and for communicating directly with the relevant GitLab team members. - - +# Subscribe to GitLab **(PREMIUM)** + +Choose and manage the subscription that's right for you and your organization. + +- [Choose a subscription](choosing_subscription.md) +- [Compare self-managed to SaaS](../install/migrate/compare_sm_to_saas.md) +- [GitLab SaaS](gitlab_com/index.md) +- [GitLab self-managed](self_managed/index.md) +- [GitLab Dedicated](gitlab_dedicated/index.md) +- [Community programs](community_programs.md) +- [Customers Portal](customers_portal.md) +- [Quarterly reconciliation](quarterly_reconciliation.md) +- [Storage usage quota](../user/usage_quotas.md) +- [CI/CD minutes quota](../ci/pipelines/cicd_minutes.md) +- [Features available to Starter and Bronze subscribers](../subscriptions/bronze_starter.md) diff --git a/doc/subscriptions/quarterly_reconciliation.md b/doc/subscriptions/quarterly_reconciliation.md index 126a34334c4..16f1828f2c3 100644 --- a/doc/subscriptions/quarterly_reconciliation.md +++ b/doc/subscriptions/quarterly_reconciliation.md @@ -91,7 +91,7 @@ sent and subject to your payment terms. - You purchased your subscription from a reseller or another channel partner. - You purchased a multi-year subscription. - You purchased your subscription with a purchasing order. -- You are a pubic sector customer. +- You are a public sector customer. - You have an offline environment and used a license file to activate your subscription. - You are enrolled in a program that provides a free tier such as the GitLab for Education, GitLab for Open Source Program, or GitLab for Startups. diff --git a/doc/subscriptions/self_managed/index.md b/doc/subscriptions/self_managed/index.md index b5436b6cb72..ece1484b107 100644 --- a/doc/subscriptions/self_managed/index.md +++ b/doc/subscriptions/self_managed/index.md @@ -7,39 +7,22 @@ type: index, reference # GitLab self-managed subscription **(PREMIUM SELF)** -You can install, administer, and maintain your own GitLab instance. +After you subscribe to GitLab, you can manage the details of your self-managed subscription. -This page covers the details of your GitLab self-managed subscription. +## Obtain a self-managed subscription -GitLab subscription management requires access to the Customers Portal. +To subscribe to GitLab for a GitLab self-managed installation: -## Customers Portal - -GitLab provides the [Customers Portal](../index.md#customers-portal) where you can -manage your subscriptions and your account details. - -Customers are required to use their GitLab.com account to register for a new Customers Portal account. - -If you have a legacy Customers Portal account that is not linked to a GitLab.com account, you may still [sign in](https://customers.gitlab.com/customers/sign_in?legacy=true) using an email and password. However, you should [create](https://gitlab.com/users/sign_up) and [link a GitLab.com account](../index.md#change-the-linked-account) to ensure continued access to the Customers Portal. - -Customers of resellers do not have access to this portal and should contact their reseller for any -changes to their subscription. - -## Subscription - -The cost of a GitLab self-managed subscription is determined by the following: - -- [GitLab tier](https://about.gitlab.com/pricing/) -- [Subscription seats](#subscription-seats) - -## Choose a GitLab tier +1. Go to the [Customers Portal](https://customers.gitlab.com/) and purchase a GitLab self-managed plan. +1. After purchase, an activation code is sent to the email address associated with the Customers Portal account. + You must [add this code to your GitLab instance](../../user/admin_area/license.md). -Pricing is [tier-based](https://about.gitlab.com/pricing/), so you can choose -the features that fit your budget. For information on the features available -for each tier, see the -[GitLab self-managed feature comparison](https://about.gitlab.com/pricing/feature-comparison/). +NOTE: +If you're purchasing a subscription for an existing **Free** GitLab self-managed +instance, ensure you're purchasing enough seats to +[cover your users](../../user/admin_area/index.md#administering-users). -## Subscription seats +### Subscription seats A GitLab self-managed subscription uses a hybrid model. You pay for a subscription according to the [maximum number](#maximum-users) of users enabled during the subscription period. @@ -49,7 +32,7 @@ simultaneous users in the GitLab self-managed installation is checked each quart If an instance is unable to generate a quarterly usage report, the existing [true up model](#users-over-subscription) is used. Prorated charges are not possible without a quarterly usage report. -### View user totals +## View user totals You can view users for your license and determine if you've gone over your subscription. @@ -58,7 +41,7 @@ You can view users for your license and determine if you've gone over your subsc The lists of users are displayed. -#### Billable users +### Billable users A _billable user_ counts against the number of subscription seats. Every user is considered a billable user, with the following exceptions: @@ -67,7 +50,8 @@ billable user, with the following exceptions: [blocked users](../../user/admin_area/moderate_users.md#block-a-user) don't count as billable users in the current subscription. When they are either deactivated or blocked they release a _billable user_ seat. However, they may count toward overages in the subscribed seat count. - Users who are [pending approval](../../user/admin_area/moderate_users.md#users-pending-approval). -- Members with the [Guest role on an Ultimate subscription](#free-guest-users). +- Users with only the [Minimal Access role](../../user/permissions.md#users-with-minimal-access) on self-managed Ultimate subscriptions or any GitLab.com subscriptions. +- Users with only the [Guest or Minimal Access roles on an Ultimate subscription](#free-guest-users). - Users without project or group memberships on an Ultimate subscription. - GitLab-created service accounts: - [Ghost User](../../user/profile/account/delete_account.md#associated-records). @@ -75,14 +59,15 @@ billable user, with the following exceptions: - [Support Bot](../../user/project/service_desk.md#support-bot-user). - [Bot users for projects](../../user/project/settings/project_access_tokens.md#bot-users-for-projects). - [Bot users for groups](../../user/group/settings/group_access_tokens.md#bot-users-for-groups). + - Other [internal users](../../development/internal_users.md#internal-users). **Billable users** as reported in the `/admin` section is updated once per day. -#### Maximum users +### Maximum users The number of _maximum users_ reflects the highest number of billable users for the current license period. -#### Users over subscription +### Users over subscription The number of _users over subscription_ shows how many users are in excess of the number allowed by the subscription. This number reflects the current subscription period. @@ -101,7 +86,7 @@ If you add more users to your GitLab instance than you are licensed for, payment If you do not add these users during the renewal process, your license key will not work. -#### Free Guest users **(ULTIMATE)** +### Free Guest users **(ULTIMATE)** In the **Ultimate** tier, users who are assigned the Guest role do not consume a seat. The user must not be assigned any other role, anywhere in the instance. @@ -117,7 +102,7 @@ If a user creates a project, they are assigned the Maintainer or Owner role. To prevent a user from creating projects, as an administrator, you can mark the user as [external](../../user/admin_area/external_users.md). -### Tips for managing users and subscription seats +## Tips for managing users and subscription seats Managing the number of users against the number of subscription seats can be a challenge: @@ -237,19 +222,6 @@ You can manually synchronize your subscription details at any time. A job is queued. When the job finishes, the subscription details are updated. -## Obtain a subscription - -To subscribe to GitLab through a GitLab self-managed installation: - -1. Go to the [Customers Portal](https://customers.gitlab.com/) and purchase a GitLab self-managed plan. -1. After purchase, an activation code is sent to the email address associated with the Customers Portal account. - You must [add this code to your GitLab instance](../../user/admin_area/license.md). - -NOTE: -If you're purchasing a subscription for an existing **Free** GitLab self-managed -instance, ensure you're purchasing enough seats to -[cover your users](../../user/admin_area/index.md#administering-users). - ## View your subscription If you are an administrator, you can view the status of your subscription: @@ -280,7 +252,7 @@ If you are an administrator, you can export your license usage into a CSV: 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Subscription**. -1. In the upper right, select **Export license usage file**. +1. In the upper-right corner, select **Export license usage file**. This file contains the information GitLab uses to manually process quarterly reconciliations or renewals. If your instance is firewalled or an offline environment, you must provide GitLab with this information. @@ -404,7 +376,7 @@ Before auto-renewal you should [prepare for the renewal](#prepare-for-renewal-by you must have enabled the [synchronization of subscription data](#subscription-data-synchronization). You can view and download your renewal invoice on the Customers Portal -[View invoices](https://customers.gitlab.com/receipts) page. If your account has a [saved credit card](../index.md#change-your-payment-method), the card is charged for the invoice amount. If we are unable to process a payment or the auto-renewal fails for any other reason, you have 14 days to renew your subscription, after which your GitLab tier is downgraded. +[View invoices](https://customers.gitlab.com/receipts) page. If your account has a [saved credit card](../customers_portal.md#change-your-payment-method), the card is charged for the invoice amount. If we are unable to process a payment or the auto-renewal fails for any other reason, you have 14 days to renew your subscription, after which your GitLab tier is downgraded. #### Email notifications @@ -419,7 +391,7 @@ You can view and download your renewal invoice on the Customers Portal To view or change automatic subscription renewal (at the same tier as the previous period), sign in to the [Customers Portal](https://customers.gitlab.com/customers/sign_in), and: -- If a **Resume subscription** button is displayed, your subscription was canceled +- If a **Turn on auto-renew** button is displayed, your subscription was canceled previously. Select it to resume automatic renewal. - If a **Cancel subscription** button is displayed, your subscription is set to automatically renew at the end of the subscription period. Select it to cancel automatic renewal. diff --git a/doc/topics/authentication/index.md b/doc/topics/authentication/index.md index c1d0a69e1f4..a7611784284 100644 --- a/doc/topics/authentication/index.md +++ b/doc/topics/authentication/index.md @@ -6,18 +6,18 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Authentication **(FREE)** -This page gathers all the resources for the topic **Authentication** within GitLab. +This page gathers all the resources for the topic **Authentication** in GitLab. ## GitLab users - [SSH](../../user/ssh.md) - [Two-factor authentication](../../user/profile/account/two_factor_authentication.md) -- [Why do you keep getting signed out?](../../user/profile/index.md#why-do-you-keep-getting-signed-out) +- [Stay signed in indefinitely](../../user/profile/index.md#stay-signed-in-indefinitely) - **Articles:** - [Support for Universal 2nd Factor Authentication - YubiKeys](https://about.gitlab.com/blog/2016/06/22/gitlab-adds-support-for-u2f/) - [Security Webcast with Yubico](https://about.gitlab.com/blog/2016/08/31/gitlab-and-yubico-security-webcast/) - **Integrations:** - - [GitLab as OAuth2 authentication service provider](../../integration/oauth_provider.md) + - [GitLab as OAuth 2.0 authentication service provider](../../integration/oauth_provider.md) - [GitLab as OpenID Connect identity provider](../../integration/openid_connect_provider.md) ## GitLab administrators @@ -38,7 +38,7 @@ This page gathers all the resources for the topic **Authentication** within GitL ## API -- [OAuth 2 Tokens](../../api/rest/index.md#oauth2-tokens) +- [OAuth 2.0 tokens](../../api/rest/index.md#oauth-20-tokens) - [Personal access tokens](../../api/rest/index.md#personalprojectgroup-access-tokens) - [Project access tokens](../../api/rest/index.md#personalprojectgroup-access-tokens) - [Group access tokens](../../api/rest/index.md#personalprojectgroup-access-tokens) diff --git a/doc/topics/autodevops/cicd_variables.md b/doc/topics/autodevops/cicd_variables.md index b22b4677f24..529f449e452 100644 --- a/doc/topics/autodevops/cicd_variables.md +++ b/doc/topics/autodevops/cicd_variables.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -62,8 +62,7 @@ Use these variables to customize and deploy your build. ## Database variables WARNING: -The default value of `true` for `POSTGRES_ENABLED` was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387766) -in GitLab 15.8. In [GitLab 16.0](https://gitlab.com/gitlab-org/gitlab/-/issues/343988), the default value will change to `false`. +From [GitLab 16.0](https://gitlab.com/gitlab-org/gitlab/-/issues/343988), `POSTGRES_ENABLED` is no longer set by default. Use these variables to integrate CI/CD with PostgreSQL databases. @@ -71,7 +70,7 @@ Use these variables to integrate CI/CD with PostgreSQL databases. |-----------------------------------------|------------------------------------| | `DB_INITIALIZE` | Used to specify the command to run to initialize the application's PostgreSQL database. Runs inside the application pod. | | `DB_MIGRATE` | Used to specify the command to run to migrate the application's PostgreSQL database. Runs inside the application pod. | -| `POSTGRES_ENABLED` | Whether PostgreSQL is enabled. Defaults to `true` until GitLab 16.0, when the default will change to `false`. Set to `false` to disable the automatic deployment of PostgreSQL. | +| `POSTGRES_ENABLED` | Whether PostgreSQL is enabled. Set to `true` to enable the automatic deployment of PostgreSQL. | | `POSTGRES_USER` | The PostgreSQL user. Defaults to `user`. Set it to use a custom username. | | `POSTGRES_PASSWORD` | The PostgreSQL password. Defaults to `testing-password`. Set it to use a custom password. | | `POSTGRES_DB` | The PostgreSQL database name. Defaults to the value of [`$CI_ENVIRONMENT_SLUG`](../../ci/variables/index.md#predefined-cicd-variables). Set it to use a custom database name. | @@ -87,45 +86,39 @@ Use these variables to disable CI/CD jobs. | **Job name** | **CI/CD variable** | **GitLab version** | **Description** | |----------------------------------------|---------------------------------|-----------------------|-----------------| -| `.fuzz_base` | `COVFUZZ_DISABLED` | [From GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34984) | [Read more](../../user/application_security/coverage_fuzzing/index.md) about how `.fuzz_base` provide capability for your own jobs. If the variable is present, your jobs aren't created. | -| `apifuzzer_fuzz` | `API_FUZZING_DISABLED` | [From GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39135) | If the variable is present, the job isn't created. | +| `.fuzz_base` | `COVFUZZ_DISABLED` | [From GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/34984) | [Read more](../../user/application_security/coverage_fuzzing/index.md) about how `.fuzz_base` provide capability for your own jobs. The job isn't created if the value is `"true"`. | +| `apifuzzer_fuzz` | `API_FUZZING_DISABLED` | [From GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39135) | The job isn't created if the value is `"true"`. | | `build` | `BUILD_DISABLED` | | If the variable is present, the job isn't created. | | `build_artifact` | `BUILD_DISABLED` | | If the variable is present, the job isn't created. | -| `bandit-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `brakeman-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | +| `brakeman-sast` | `SAST_DISABLED` | | The job isn't created if the value is `"true"`. | | `canary` | `CANARY_ENABLED` | | This manual job is created if the variable is present. | | `code_intelligence` | `CODE_INTELLIGENCE_DISABLED` | From GitLab 13.6 | If the variable is present, the job isn't created. | -| `code_quality` | `CODE_QUALITY_DISABLED` | | If the variable is present, the job isn't created. | -| `container_scanning` | `CONTAINER_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | -| `dast` | `DAST_DISABLED` | | If the variable is present, the job isn't created. | -| `dast_environment_deploy` | `DAST_DISABLED_FOR_DEFAULT_BRANCH` or `DAST_DISABLED` | [From GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17789) | If either variable is present, the job isn't created. | -| `dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | -| `eslint-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `flawfinder-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `gemnasium-dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | -| `gemnasium-maven-dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | -| `gemnasium-python-dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | -| `gosec-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `kubesec-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | +| `code_quality` | `CODE_QUALITY_DISABLED` | | The job isn't created if the value is `"true"`. | +| `container_scanning` | `CONTAINER_SCANNING_DISABLED` | | The job isn't created if the value is `"true"`. | +| `dast` | `DAST_DISABLED` | | The job isn't created if the value is `"true"`. | +| `dast_environment_deploy` | `DAST_DISABLED_FOR_DEFAULT_BRANCH` or `DAST_DISABLED` | [From GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17789) | The job isn't created if the value is `"true"`. | +| `dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | The job isn't created if the value is `"true"`. | +| `flawfinder-sast` | `SAST_DISABLED` | | The job isn't created if the value is `"true"`. | +| `gemnasium-dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | The job isn't created if the value is `"true"`. | +| `gemnasium-maven-dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | The job isn't created if the value is `"true"`. | +| `gemnasium-python-dependency_scanning` | `DEPENDENCY_SCANNING_DISABLED` | | The job isn't created if the value is `"true"`. | +| `kubesec-sast` | `SAST_DISABLED` | | The job isn't created if the value is `"true"`. | | `license_management` | `LICENSE_MANAGEMENT_DISABLED` | GitLab 12.7 and earlier | If the variable is present, the job isn't created. Job deprecated [from GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22773) | -| `license_scanning` | `LICENSE_MANAGEMENT_DISABLED` | [From GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22773) | If the variable is present, the job isn't created. | +| `license_scanning` | `LICENSE_MANAGEMENT_DISABLED` | [From GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22773) | The job isn't created if the value is `"true"`.| | `load_performance` | `LOAD_PERFORMANCE_DISABLED` | From GitLab 13.2 | If the variable is present, the job isn't created. | -| `nodejs-scan-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | +| `nodejs-scan-sast` | `SAST_DISABLED` | | The job isn't created if the value is `"true"`. | | `performance` | `PERFORMANCE_DISABLED` | GitLab 13.12 and earlier | Browser performance. If the variable is present, the job isn't created. Replaced by `browser_performance`. | | `browser_performance` | `BROWSER_PERFORMANCE_DISABLED` | From GitLab 14.0 | Browser performance. If the variable is present, the job isn't created. Replaces `performance`. | -| `phpcs-security-audit-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `pmd-apex-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | +| `phpcs-security-audit-sast` | `SAST_DISABLED` | | The job isn't created if the value is `"true"`. | +| `pmd-apex-sast` | `SAST_DISABLED` | | The job isn't created if the value is `"true"`. | | `review` | `REVIEW_DISABLED` | | If the variable is present, the job isn't created. | | `review:stop` | `REVIEW_DISABLED` | | Manual job. If the variable is present, the job isn't created. | -| `sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `sast:container` | `CONTAINER_SCANNING_DISABLED` | | If the variable is present, the job isn't created. | -| `secret_detection` | `SECRET_DETECTION_DISABLED` | From GitLab 13.1 | If the variable is present, the job isn't created. | -| `secret_detection_default_branch` | `SECRET_DETECTION_DISABLED` | [From GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22773) | If the variable is present, the job isn't created. | -| `security-code-scan-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `secrets-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `sobelaw-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | -| `stop_dast_environment` | `DAST_DISABLED_FOR_DEFAULT_BRANCH` or `DAST_DISABLED` | [From GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17789) | If either variable is present, the job isn't created. | -| `spotbugs-sast` | `SAST_DISABLED` | | If the variable is present, the job isn't created. | +| `secret_detection` | `SECRET_DETECTION_DISABLED` | From GitLab 13.1 | The job isn't created if the value is `"true"`. | +| `secret_detection_default_branch` | `SECRET_DETECTION_DISABLED` | [From GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22773) | The job isn't created if the value is `"true"`. | +| `semgrep-sast` | `SAST_DISABLED` | | The job isn't created if the value is `"true"`. | +| `sobelow-sast` | `SAST_DISABLED` | | The job isn't created if the value is `"true"`. | +| `stop_dast_environment` | `DAST_DISABLED_FOR_DEFAULT_BRANCH` or `DAST_DISABLED` | [From GitLab 12.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/17789) | The job isn't created if the value is `"true"`. | +| `spotbugs-sast` | `SAST_DISABLED` | | The job isn't created if the value is `"true"`. | | `test` | `TEST_DISABLED` | | If the variable is present, the job isn't created. | | `staging` | `STAGING_ENABLED` | | The job is created if the variable is present. | | `stop_review` | `REVIEW_DISABLED` | | If the variable is present, the job isn't created. | diff --git a/doc/topics/autodevops/cloud_deployments/auto_devops_with_ec2.md b/doc/topics/autodevops/cloud_deployments/auto_devops_with_ec2.md index 6bebe9eb9e3..381326d68ea 100644 --- a/doc/topics/autodevops/cloud_deployments/auto_devops_with_ec2.md +++ b/doc/topics/autodevops/cloud_deployments/auto_devops_with_ec2.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/topics/autodevops/cloud_deployments/auto_devops_with_ecs.md b/doc/topics/autodevops/cloud_deployments/auto_devops_with_ecs.md index 76fd6ad82d8..0d865b5cf0d 100644 --- a/doc/topics/autodevops/cloud_deployments/auto_devops_with_ecs.md +++ b/doc/topics/autodevops/cloud_deployments/auto_devops_with_ecs.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/topics/autodevops/cloud_deployments/auto_devops_with_eks.md b/doc/topics/autodevops/cloud_deployments/auto_devops_with_eks.md new file mode 100644 index 00000000000..32a47bb790b --- /dev/null +++ b/doc/topics/autodevops/cloud_deployments/auto_devops_with_eks.md @@ -0,0 +1,301 @@ +--- +stage: Deploy +group: Environments +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Use Auto DevOps to deploy an application to Amazon Elastic Kubernetes Service (EKS) + +In this tutorial, we'll help you to get started with [Auto DevOps](../index.md) +through an example of how to deploy an application to Amazon Elastic Kubernetes Service (EKS). + +The tutorial uses the GitLab native Kubernetes integration, so you don't need +to create a Kubernetes cluster manually using the AWS console. + +You can also follow this tutorial on a self-managed instance. +Ensure your own [runners are configured](../../../ci/runners/index.md). + +To deploy a project to EKS: + +1. [Configure your Amazon account](#configure-your-amazon-account) +1. [Create a Kubernetes cluster and deploy the agent](#create-a-kubernetes-cluster) +1. [Create a new project from a template](#create-an-application-project-from-a-template) +1. [Configure the agent](#configure-the-agent) +1. [Install Ingress](#install-ingress) +1. [Configure Auto DevOps](#configure-auto-devops) +1. [Enable Auto DevOps and run the pipeline](#enable-auto-devops-and-run-the-pipeline) +1. [Deploy the application](#deploy-the-application) + +## Configure your Amazon account + +Before you create and connect your Kubernetes cluster to your GitLab project, +you need an [Amazon Web Services account](https://aws.amazon.com/). +Sign in with an existing Amazon account or create a new one. + +## Create a Kubernetes cluster + +To create an new cluster on Amazon EKS: + +- Follow the steps in [Create an Amazon EKS cluster](../../../user/infrastructure/clusters/connect/new_eks_cluster.md). + +If you prefer, you can also create a cluster manually using `eksctl`. + +## Create an application project from a template + +Use a GitLab project template to get started. As the name suggests, +those projects provide a bare-bones application built on some well-known frameworks. + +WARNING: +Create the application project in the group hierarchy at the same level or below the project for cluster management. Otherwise, it fails to [authorize the agent](../../../user/clusters/agent/ci_cd_workflow.md#authorize-the-agent). + +1. On the top bar in GitLab, select the plus icon (**{plus-square}**), and select + **New project/repository**. +1. Select **Create from template**. +1. Select the **Ruby on Rails** template. +1. Give your project a name, optionally a description, and make it public so that + you can take advantage of the features available in the + [GitLab Ultimate plan](https://about.gitlab.com/pricing/). +1. Select **Create project**. + +Now you have an application project you are going to deploy to the EKS cluster. + +## Configure the agent + +Next, we'll configure the GitLab agent for Kubernetes so we can use it to deploy the application project. + +1. Go to the project [we created to manage the cluster](#create-a-kubernetes-cluster). +1. Navigate to the [agent configuration file](../../../user/clusters/agent/install/index.md#create-an-agent-configuration-file) (`.gitlab/agents/eks-agent/config.yaml`) and edit it. +1. Configure `ci_access:projects` attribute. Use the application project path as `id`: + +```yaml +ci_access: + projects: + - id: path/to/application-project +``` + +## Install Ingress + +After your cluster is running, you must install NGINX Ingress Controller as a +load balancer to route traffic from the internet to your application. +Install the NGINX Ingress Controller +through the GitLab [Cluster management project template](../../../user/clusters/management_project_template.md), +or manually via the command line: + +1. Ensure you have `kubectl` and Helm installed on your machine. +1. Create an IAM role to access the cluster. +1. Create an access token to access the cluster. +1. Use `kubectl` to connect to your cluster: + + ```shell + helm upgrade --install ingress-nginx ingress-nginx \ + --repo https://kubernetes.github.io/ingress-nginx \ + --namespace gitlab-managed-apps --create-namespace + + # Check that the ingress controller is installed successfully + kubectl get service ingress-nginx-controller -n gitlab-managed-apps + ``` + +## Configure Auto DevOps + +Follow these steps to configure the base domain and other settings required for Auto DevOps. + +1. A few minutes after you install NGINX, the load balancer obtains an IP address, and you can + get the external IP address with the following command: + + ```shell + kubectl get all -n gitlab-managed-apps --selector app.kubernetes.io/instance=ingress-nginx + ``` + + Replace `gitlab-managed-apps` if you have overwritten your namespace. + + Next, find the actual external IP address for your cluster with the following command: + + ```shell + nslookup [External IP] + ``` + + Where the `[External IP]` is the hostname found with the previous command. + + The IP address might be listed in the `Non-authoritative answer:` section of the response. + + Copy this IP address, as you need it in the next step. + +1. Go back to the application project. +1. On the left sidebar, select **Settings > CI/CD** and expand **Variables**. + - Add a key called `KUBE_INGRESS_BASE_DOMAIN` with the application deployment domain as the value. For this example, use the domain `.nip.io`. + - Add a key called `KUBE_NAMESPACE` with a value of the Kubernetes namespace for your deployments to target. You can use different namespaces per environment. Configure the environment, use the environment scope. + - Add a key called `KUBE_CONTEXT` with a value like `path/to/agent/project:eks-agent`. Select the environment scope of your choice. + - Select **Save changes**. + +## Enable Auto DevOps and run the pipeline + +While Auto DevOps is enabled by default, Auto DevOps can be disabled at both +the instance level (for self-managed instances) and the group level. Complete +these steps to enable Auto DevOps if it's disabled: + +1. On the top bar, select **Main menu > Projects** and find the application project. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **Auto DevOps**. +1. Select **Default to Auto DevOps pipeline** to display more options. +1. In **Deployment strategy**, select your desired [continuous deployment strategy](../requirements.md#auto-devops-deployment-strategy) + to deploy the application to production after the pipeline successfully runs on the default branch. +1. Select **Save changes**. +1. Edit `.gitlab-ci.yml` file to include the Auto DevOps template and commit the change to the default branch: + + ```yaml + include: + - template: Auto-DevOps.gitlab-ci.yml + ``` + +The commit should trigger a pipeline. In the next section, we explain what each job does in the pipeline. + +## Deploy the application + +When your pipeline runs, what is it doing? + +To view the jobs in the pipeline, select the pipeline's status badge. The +**{status_running}** icon displays when pipeline jobs are running, and updates +without refreshing the page to **{status_success}** (for success) or +**{status_failed}** (for failure) when the jobs complete. + +The jobs are separated into stages: + +![Pipeline stages](img/guide_pipeline_stages_v13_0.png) + +- **Build** - The application builds a Docker image and uploads it to your project's + [Container Registry](../../../user/packages/container_registry/index.md) ([Auto Build](../stages.md#auto-build)). +- **Test** - GitLab runs various checks on the application, but all jobs except `test` + are allowed to fail in the test stage: + + - The `test` job runs unit and integration tests by detecting the language and + framework ([Auto Test](../stages.md#auto-test-deprecated)) + - The `code_quality` job checks the code quality and is allowed to fail + ([Auto Code Quality](../stages.md#auto-code-quality)) + - The `container_scanning` job checks the Docker container if it has any + vulnerabilities and is allowed to fail ([Auto Container Scanning](../stages.md#auto-container-scanning)) + - The `dependency_scanning` job checks if the application has any dependencies + susceptible to vulnerabilities and is allowed to fail + ([Auto Dependency Scanning](../stages.md#auto-dependency-scanning)) + - Jobs suffixed with `-sast` run static analysis on the current code to check for potential + security issues, and are allowed to fail ([Auto SAST](../stages.md#auto-sast)) + - The `secret-detection` job checks for leaked secrets and is allowed to fail ([Auto Secret Detection](../stages.md#auto-secret-detection)) + - The `license_scanning` job searches the application's dependencies to determine each of their + licenses and is allowed to fail + ([Auto License Compliance](../stages.md#auto-license-compliance)) + +- **Review** - Pipelines on the default branch include this stage with a `dast_environment_deploy` job. + To learn more, see [Dynamic Application Security Testing (DAST)](../../../user/application_security/dast/index.md). + +- **Production** - After the tests and checks finish, the application deploys in + Kubernetes ([Auto Deploy](../stages.md#auto-deploy)). + +- **Performance** - Performance tests are run on the deployed application + ([Auto Browser Performance Testing](../stages.md#auto-browser-performance-testing)). + +- **Cleanup** - Pipelines on the default branch include this stage with a `stop_dast_environment` job. + +After running a pipeline, you should view your deployed website and learn how +to monitor it. + +### Monitor your project + +After successfully deploying your application, you can view its website and check +on its health on the **Environments** page by navigating to +**Deployments > Environments**. This page displays details about +the deployed applications, and the right-hand column displays icons that link +you to common environment tasks: + +![Environments](img/guide_environments_v12_3.png) + +- **Open live environment** (**{external-link}**) - Opens the URL of the application deployed in production +- **Monitoring** (**{chart}**) - Opens the metrics page where Prometheus collects data + about the Kubernetes cluster and how the application + affects it in terms of memory usage, CPU usage, and latency +- **Deploy to** (**{play}** **{chevron-lg-down}**) - Displays a list of environments you can deploy to +- **Terminal** (**{terminal}**) - Opens a [web terminal](../../../ci/environments/index.md#web-terminals-deprecated) + session inside the container where the application is running +- **Re-deploy to environment** (**{repeat}**) - For more information, see + [Retrying and rolling back](../../../ci/environments/index.md#retry-or-roll-back-a-deployment) +- **Stop environment** (**{stop}**) - For more information, see + [Stopping an environment](../../../ci/environments/index.md#stopping-an-environment) + +GitLab displays the [deploy board](../../../user/project/deploy_boards.md) below the +environment's information, with squares representing pods in your +Kubernetes cluster, color-coded to show their status. Hovering over a square on +the deploy board displays the state of the deployment, and selecting the square +takes you to the pod's logs page. + +Although the example shows only one pod hosting the application at the moment, you can add +more pods by defining the [`REPLICAS` CI/CD variable](../cicd_variables.md) +in **Settings > CI/CD > Variables**. + +### Work with branches + +Following the [GitLab flow](../../gitlab_flow.md#working-with-feature-branches), +you should next create a feature branch to add content to your application: + +1. In your project's repository, go to the following file: `app/views/welcome/index.html.erb`. + This file should only contain a paragraph: `

You're on Rails!

`. +1. Open the GitLab [Web IDE](../../../user/project/web_ide/index.md) to make the change. +1. Edit the file so it contains: + + ```html +

You're on Rails! Powered by GitLab Auto DevOps.

+ ``` + +1. Stage the file. Add a commit message, then create a new branch and a merge request + by selecting **Commit**. + + ![Web IDE commit](img/guide_ide_commit_v12_3.png) + +After submitting the merge request, GitLab runs your pipeline, and all the jobs +in it, as [described previously](#deploy-the-application), in addition to +a few more that run only on branches other than the default branch. + +After a few minutes a test fails, which means a test was +'broken' by your change. Select the failed `test` job to see more information +about it: + +```plaintext +Failure: +WelcomeControllerTest#test_should_get_index [/app/test/controllers/welcome_controller_test.rb:7]: + expected but was +.. +Expected 0 to be >= 1. + +bin/rails test test/controllers/welcome_controller_test.rb:4 +``` + +To fix the broken test: + +1. Return to your merge request. +1. In the upper right corner, select **Code**, then select **Open in Gitpod**. +1. In the left-hand directory of files, find the `test/controllers/welcome_controller_test.rb` + file, and select it to open it. +1. Change line 7 to say `You're on Rails! Powered by GitLab Auto DevOps.` +1. Select **Commit**. +1. In the left-hand column, under **Unstaged changes**, select the checkmark icon + (**{stage-all}**) to stage the changes. +1. Write a commit message, and select **Commit**. + +Return to the **Overview** page of your merge request, and you should not only +see the test passing, but also the application deployed as a +[review application](../stages.md#auto-review-apps). You can visit it by selecting +the **View app** **{external-link}** button to see your changes deployed. + +After merging the merge request, GitLab runs the pipeline on the default branch, +and then deploys the application to production. + +## Conclusion + +After implementing this project, you should have a solid understanding of the basics of Auto DevOps. +You started from building and testing, to deploying and monitoring an application +all in GitLab. Despite its automatic nature, Auto DevOps can also be configured +and customized to fit your workflow. Here are some helpful resources for further reading: + +1. [Auto DevOps](../index.md) +1. [Multiple Kubernetes clusters](../multiple_clusters_auto_devops.md) +1. [Incremental rollout to production](../cicd_variables.md#incremental-rollout-to-production) +1. [Disable jobs you don't need with CI/CD variables](../cicd_variables.md) +1. [Use your own buildpacks to build your application](../customize.md#custom-buildpacks) +1. [Prometheus monitoring](../../../user/project/integrations/prometheus.md) diff --git a/doc/topics/autodevops/cloud_deployments/auto_devops_with_gke.md b/doc/topics/autodevops/cloud_deployments/auto_devops_with_gke.md index 9bd1d30e1b1..1a03a66d17a 100644 --- a/doc/topics/autodevops/cloud_deployments/auto_devops_with_gke.md +++ b/doc/topics/autodevops/cloud_deployments/auto_devops_with_gke.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -221,7 +221,7 @@ you to common environment tasks: - **Re-deploy to environment** (**{repeat}**) - For more information, see [Retrying and rolling back](../../../ci/environments/index.md#retry-or-roll-back-a-deployment) - **Stop environment** (**{stop}**) - For more information, see - [Stopping an environment](../../../ci/environments/index.md#stop-an-environment) + [Stopping an environment](../../../ci/environments/index.md#stopping-an-environment) GitLab displays the [deploy board](../../../user/project/deploy_boards.md) below the environment's information, with squares representing pods in your @@ -274,7 +274,7 @@ bin/rails test test/controllers/welcome_controller_test.rb:4 To fix the broken test: 1. Return to your merge request. -1. In the upper right corner, select **Code**, then select **Open in Gitpod**. +1. In the upper-right corner, select **Code**, then select **Open in Gitpod**. 1. In the left-hand directory of files, find the `test/controllers/welcome_controller_test.rb` file, and select it to open it. 1. Change line 7 to say `You're on Rails! Powered by GitLab Auto DevOps.` diff --git a/doc/topics/autodevops/customize.md b/doc/topics/autodevops/customize.md index b8ea242df1a..e7fe4ce6e0b 100644 --- a/doc/topics/autodevops/customize.md +++ b/doc/topics/autodevops/customize.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index 588be855659..87810193a8d 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -185,6 +185,7 @@ and clear the **Default to Auto DevOps pipeline** checkbox. ### Deploy your app to a cloud provider - [Use Auto DevOps to deploy to a Kubernetes cluster on Google Kubernetes Engine (GKE)](cloud_deployments/auto_devops_with_gke.md) +- [Use Auto DevOps to deploy to a Kubernetes cluster on Amazon Elastic Kubernetes Service (EKS)](cloud_deployments/auto_devops_with_eks.md) - [Use Auto DevOps to deploy to EC2](cloud_deployments/auto_devops_with_ec2.md) - [Use Auto DevOps to deploy to ECS](cloud_deployments/auto_devops_with_ecs.md) diff --git a/doc/topics/autodevops/multiple_clusters_auto_devops.md b/doc/topics/autodevops/multiple_clusters_auto_devops.md index 3411beedefb..0b1699d8230 100644 --- a/doc/topics/autodevops/multiple_clusters_auto_devops.md +++ b/doc/topics/autodevops/multiple_clusters_auto_devops.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/topics/autodevops/prepare_deployment.md b/doc/topics/autodevops/prepare_deployment.md index ebba20ca821..6f434e8fb84 100644 --- a/doc/topics/autodevops/prepare_deployment.md +++ b/doc/topics/autodevops/prepare_deployment.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/topics/autodevops/requirements.md b/doc/topics/autodevops/requirements.md index a409c6f1520..dc126edf1aa 100644 --- a/doc/topics/autodevops/requirements.md +++ b/doc/topics/autodevops/requirements.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -63,8 +63,8 @@ To define the base domain, either: - In the project or group level: add it as an environment variable: `KUBE_INGRESS_BASE_DOMAIN`. - In the instance level: go to **Main menu > Admin > Settings > CI/CD > Continuous Integration and Delivery** and add it there. -The base domain variable `KUBE_INGRESS_BASE_DOMAIN` follows the same order of precedence -as other environment [variables](../../ci/variables/index.md#cicd-variable-precedence). +The base domain variable `KUBE_INGRESS_BASE_DOMAIN` follows the same order of +[precedence as other environment variables](../../ci/variables/index.md#cicd-variable-precedence). If you don't specify the base domain in your projects and groups, Auto DevOps uses the instance-wide **Auto DevOps domain**. @@ -108,10 +108,7 @@ To make full use of Auto DevOps with Kubernetes, you need: or manually by using the [`ingress-nginx`](https://github.com/kubernetes/ingress-nginx/tree/master/charts/ingress-nginx) Helm chart. - NOTE: - For metrics to appear when using the [Prometheus cluster integration](../../user/clusters/integrations.md#prometheus-cluster-integration), you must [enable Prometheus metrics](https://github.com/kubernetes/ingress-nginx/tree/master/charts/ingress-nginx#prometheus-metrics). - - When deploying [using custom charts](customize.md#custom-helm-chart), you must also + When deploying [using custom charts](customize.md#custom-helm-chart), you must [annotate](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) the Ingress manifest to be scraped by Prometheus using `prometheus.io/scrape: "true"` and `prometheus.io/port: "10254"`. @@ -142,21 +139,6 @@ To make full use of Auto DevOps with Kubernetes, you need: for the entire GitLab instance, or [project runners](../../ci/runners/runners_scope.md#project-runners) that are assigned to specific projects. -- **Prometheus** (for [Auto Monitoring](stages.md#auto-monitoring)) - - To enable Auto Monitoring, you need Prometheus installed either inside or - outside your cluster, and configured to scrape your Kubernetes cluster. - If you've configured the GitLab integration with Kubernetes, you can - instruct GitLab to query an in-cluster Prometheus by enabling - the [Prometheus cluster integration](../../user/clusters/integrations.md#prometheus-cluster-integration). - - The [Prometheus integration](../../user/project/integrations/prometheus.md) - integration must be activated for the project, or activated at the group or instance level. - For more information, see [Project integration management](../../user/admin_area/settings/project_integration_management.md). - - To get response metrics (in addition to system metrics), you must - [configure Prometheus to monitor NGINX](../../user/project/integrations/prometheus_library/nginx_ingress.md#configuring-nginx-ingress-monitoring). - - **cert-manager** (optional, for TLS/HTTPS) To enable HTTPS endpoints for your application, you can [install cert-manager](https://cert-manager.io/docs/installation/supported-releases/), diff --git a/doc/topics/autodevops/stages.md b/doc/topics/autodevops/stages.md index 0c5c87cdf0f..c95426b03e6 100644 --- a/doc/topics/autodevops/stages.md +++ b/doc/topics/autodevops/stages.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -491,7 +491,7 @@ deletion). You can configure database initialization and migrations for PostgreSQL to run within the application pod by setting the project CI/CD variables `DB_INITIALIZE` and -`DB_MIGRATE` respectively. +`DB_MIGRATE`. If present, `DB_INITIALIZE` is run as a shell command within an application pod as a Helm post-install hook. As some applications can't run without a successful @@ -614,8 +614,7 @@ To use Auto Monitoring: 1. Select **Run pipeline**. 1. After the pipeline finishes successfully, open the [monitoring dashboard for a deployed environment](../../ci/environments/index.md#monitor-environments) - to view the metrics of your deployed application. To view the metrics of the - whole Kubernetes cluster, on the left sidebar, select **Monitor > Metrics**. + to view the metrics of your deployed application. ![Auto Metrics](img/auto_monitoring.png) diff --git a/doc/topics/autodevops/troubleshooting.md b/doc/topics/autodevops/troubleshooting.md index dd715e95512..21a74cca07a 100644 --- a/doc/topics/autodevops/troubleshooting.md +++ b/doc/topics/autodevops/troubleshooting.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/topics/autodevops/upgrading_auto_deploy_dependencies.md b/doc/topics/autodevops/upgrading_auto_deploy_dependencies.md index 858562eef48..0c31ccbfe32 100644 --- a/doc/topics/autodevops/upgrading_auto_deploy_dependencies.md +++ b/doc/topics/autodevops/upgrading_auto_deploy_dependencies.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: reference --- @@ -52,7 +52,7 @@ The following table explains the version compatibility between GitLab and Auto D | GitLab version | `auto-deploy-image` version | Notes | |------------------|-----------------------------|-------| | v10.0 to v14.0 | v0.1.0 to v2.0.0 | v0 and v1 auto-deploy-image are backwards compatible. | -| v13.4 and higher | v2.0.0 and higher | v2 auto-deploy-image contains breaking changes, as explained in the [upgrade guide](#upgrade-deployments-to-the-v2-auto-deploy-image). | +| v13.4 and later | v2.0.0 and later | v2 auto-deploy-image contains breaking changes, as explained in the [upgrade guide](#upgrade-deployments-to-the-v2-auto-deploy-image). | You can find the current stable version of auto-deploy-image in the [Auto Deploy stable template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml). @@ -179,7 +179,7 @@ used the `v0.17.0` chart, add `AUTO_DEVOPS_FORCE_DEPLOY_V2`. ## Early adopters -If you want to use the latest [Beta](../../policy/alpha-beta-support.md#beta-features) or unstable version of `auto-deploy-image`, include +If you want to use the latest [Beta](../../policy/alpha-beta-support.md#beta) or unstable version of `auto-deploy-image`, include the latest Auto Deploy template into your `.gitlab-ci.yml`: ```yaml @@ -189,7 +189,7 @@ include: ``` WARNING: -Using a [Beta](../../policy/alpha-beta-support.md#beta-features) or unstable `auto-deploy-image` could cause unrecoverable damage to +Using a [Beta](../../policy/alpha-beta-support.md#beta) or unstable `auto-deploy-image` could cause unrecoverable damage to your environments. Do not test it with important projects or environments. The next stable template update is [planned for GitLab v14.0](https://gitlab.com/gitlab-org/gitlab/-/issues/232788). diff --git a/doc/topics/autodevops/upgrading_postgresql.md b/doc/topics/autodevops/upgrading_postgresql.md index f18d5c49be5..ffb72b802f7 100644 --- a/doc/topics/autodevops/upgrading_postgresql.md +++ b/doc/topics/autodevops/upgrading_postgresql.md @@ -1,13 +1,13 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Upgrading PostgreSQL for Auto DevOps **(FREE)** -Auto DevOps provides an [in-cluster PostgreSQL database](customize.md#postgresql-database-support) -for your application. +When `POSTGRES_ENABLED` is `true`, Auto DevOps provides an +[in-cluster PostgreSQL database](customize.md#postgresql-database-support) for your application. The version of the chart used to provision PostgreSQL: @@ -49,12 +49,12 @@ being modified after the database dump is created. 1. Get the Kubernetes namespace for the environment. It typically looks like `--`. In our example, the namespace is called `minimal-ruby-app-4349298-production`. - ```shell - $ kubectl get ns + ```shell + $ kubectl get ns - NAME STATUS AGE - minimal-ruby-app-4349298-production Active 7d14h - ``` + NAME STATUS AGE + minimal-ruby-app-4349298-production Active 7d14h + ``` 1. For ease of use, export the namespace name: @@ -64,20 +64,20 @@ being modified after the database dump is created. 1. Get the deployment name for your application with the following command. In our example, the deployment name is `production`. - ```shell - $ kubectl get deployment --namespace "$APP_NAMESPACE" - NAME READY UP-TO-DATE AVAILABLE AGE - production 2/2 2 2 7d21h - production-postgres 1/1 1 1 7d21h - ``` + ```shell + $ kubectl get deployment --namespace "$APP_NAMESPACE" + NAME READY UP-TO-DATE AVAILABLE AGE + production 2/2 2 2 7d21h + production-postgres 1/1 1 1 7d21h + ``` 1. To prevent the database from being modified, set replicas to 0 for the deployment with the following command. We use the deployment name from the previous step (`deployments/`). - ```shell - $ kubectl scale --replicas=0 deployments/production --namespace "$APP_NAMESPACE" - deployment.extensions/production scaled - ``` + ```shell + $ kubectl scale --replicas=0 deployments/production --namespace "$APP_NAMESPACE" + deployment.extensions/production scaled + ``` 1. You must also set replicas to zero for workers if you have any. @@ -85,26 +85,26 @@ being modified after the database dump is created. 1. Get the service name for PostgreSQL. The name of the service should end with `-postgres`. In our example the service name is `production-postgres`. - ```shell - $ kubectl get svc --namespace "$APP_NAMESPACE" - NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE - production-auto-deploy ClusterIP 10.30.13.90 5000/TCP 7d14h - production-postgres ClusterIP 10.30.4.57 5432/TCP 7d14h - ``` + ```shell + $ kubectl get svc --namespace "$APP_NAMESPACE" + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + production-auto-deploy ClusterIP 10.30.13.90 5000/TCP 7d14h + production-postgres ClusterIP 10.30.4.57 5432/TCP 7d14h + ``` 1. Get the pod name for PostgreSQL with the following command. In our example, the pod name is `production-postgres-5db86568d7-qxlxv`. - ```shell - $ kubectl get pod --namespace "$APP_NAMESPACE" -l app=production-postgres - NAME READY STATUS RESTARTS AGE - production-postgres-5db86568d7-qxlxv 1/1 Running 0 7d14h - ``` + ```shell + $ kubectl get pod --namespace "$APP_NAMESPACE" -l app=production-postgres + NAME READY STATUS RESTARTS AGE + production-postgres-5db86568d7-qxlxv 1/1 Running 0 7d14h + ``` 1. Connect to the pod with: - ```shell - kubectl exec -it production-postgres-5db86568d7-qxlxv --namespace "$APP_NAMESPACE" -- bash - ``` + ```shell + kubectl exec -it production-postgres-5db86568d7-qxlxv --namespace "$APP_NAMESPACE" -- bash + ``` 1. Once, connected, create a dump file with the following command. @@ -114,20 +114,20 @@ being modified after the database dump is created. - When prompted for the database password, the default is `testing-password`. - ```shell - ## Format is: - # pg_dump -h SERVICE_NAME -U USERNAME DATABASE_NAME > /tmp/backup.sql + ```shell + ## Format is: + # pg_dump -h SERVICE_NAME -U USERNAME DATABASE_NAME > /tmp/backup.sql - pg_dump -h production-postgres -U user production > /tmp/backup.sql - ``` + pg_dump -h production-postgres -U user production > /tmp/backup.sql + ``` 1. Once the backup dump is complete, exit the Kubernetes exec process with `Control-D` or `exit`. 1. Download the dump file with the following command: - ```shell - kubectl cp --namespace "$APP_NAMESPACE" production-postgres-5db86568d7-qxlxv:/tmp/backup.sql backup.sql - ``` + ```shell + kubectl cp --namespace "$APP_NAMESPACE" production-postgres-5db86568d7-qxlxv:/tmp/backup.sql backup.sql + ``` ## Retain persistent volumes @@ -184,8 +184,7 @@ You can also 1. Set `AUTO_DEVOPS_POSTGRES_DELETE_V1` to a non-empty value. This flag is a safeguard to prevent accidental deletion of databases. -1. If you have a `POSTGRES_VERSION` set, make sure it is set to `9.6.16` *or -higher*. This is the +1. If you have a `POSTGRES_VERSION` set, make sure it is set to `9.6.16` *or later*. This is the minimum PostgreSQL version supported by Auto DevOps. See also the list of [tags available](https://hub.docker.com/r/bitnami/postgresql/tags). 1. Set `PRODUCTION_REPLICAS` to `0`. For other environments, use @@ -205,11 +204,11 @@ higher*. This is the 1. Get the pod name for the new PostgreSQL, in our example, the pod name is `production-postgresql-0`: - ```shell - $ kubectl get pod --namespace "$APP_NAMESPACE" -l app=postgresql - NAME READY STATUS RESTARTS AGE - production-postgresql-0 1/1 Running 0 19m - ```` + ```shell + $ kubectl get pod --namespace "$APP_NAMESPACE" -l app=postgresql + NAME READY STATUS RESTARTS AGE + production-postgresql-0 1/1 Running 0 19m + ```` 1. Copy the dump file from the backup steps to the pod: diff --git a/doc/topics/awesome_co.md b/doc/topics/awesome_co.md deleted file mode 100644 index ff3c81a1b90..00000000000 --- a/doc/topics/awesome_co.md +++ /dev/null @@ -1,143 +0,0 @@ ---- -stage: Manage -group: Foundations -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments -description: AwesomeCo test data harness created by the Test Data Working Group https://about.gitlab.com/company/team/structure/working-groups/demo-test-data/ -comments: false ---- - -# AwesomeCo - -AwesomeCo is a test data seeding harness, that can seed test data into a user or group namespace. - -AwesomeCo uses FactoryBot in the backend which makes maintenance extremely easy. When a Model is changed, -FactoryBot will already be reflected to account for the change. - -## Docker Setup - -See [AwesomeCo Docker Demo](https://gitlab.com/-/snippets/2390362) - -## GDK Setup - -```shell -$ gdk start db -ok: run: services/postgresql: (pid n) 0s, normally down -ok: run: services/redis: (pid n) 74s, normally down -$ bundle install -Bundle complete! -$ bundle exec rake db:migrate -main: migrated -ci: migrated -``` - -### Run - -The `ee:gitlab:seed:awesome_co` Rake task takes two arguments. `:name` and `:namespace_id`. - -```shell -$ bundle exec rake "ee:gitlab:seed:awesome_co[awesome_co,1]" -Seeding AwesomeCo for Administrator -``` - -#### `:name` - -Where `:name` is the name of the AwesomeCo. (This will reflect .rb files located in db/seeds/awesome_co/*.rb) - -#### `:namespace_id` - -Where `:namespace_id` is the ID of the User or Group Namespace - -## List of Awesome Companies - -Each company (i.e. test data template) is represented as a Ruby file (.rb) in `db/seeds/awesome_co`. - -### AwesomeCo (db/seeds/awesome_co/awesome_co.rb) - -```shell -$ bundle exec rake "ee:gitlab:seed:awesome_co[awesome_co,:namespace_id]" -Seeding AwesomeCo for :namespace_id -``` - -AwesomeCo is an automated seeding of [this demo repository](https://gitlab.com/tech-marketing/demos/gitlab-agile-demo/awesome-co). - -## Develop - -AwesomeCo seeding uses FactoryBot definitions from `spec/factories` which ... - -1. Saves time on development -1. Are easy-to-read -1. Are easy to maintain -1. Do not rely on an API that may change in the future -1. Are always up-to-date -1. Execute on the lowest-level (`ActiveRecord`) possible to create data as quickly as possible - -> From the [FactoryBot README](https://github.com/thoughtbot/factory_bot#readme_) : `factory_bot` is a fixtures replacement with a straightforward definition syntax, support for multiple build -> strategies (saved instances, unsaved instances, attribute hashes, and stubbed objects), and support for multiple factories for the same class, including factory -> inheritance - -Factories reside in `spec/factories/*` and are fixtures for Rails models found in `app/models/*`. For example, For a model named `app/models/issue.rb`, the factory will -be named `spec/factories/issues.rb`. For a model named `app/models/project.rb`, the factory will be named `app/models/projects.rb`. - -### Taxonomy of a Factory - -Factories consist of three main parts - the **Name** of the factory, the **Traits** and the **Attributes**. - -Given: `create(:iteration, :with_title, :current, title: 'My Iteration')` - -||| -|:-|:-| -| **:iteration** | This is the **Name** of the factory. The file name will be the plural form of this **Name** and reside under either `spec/factories/iterations.rb` or `ee/spec/factories/iterations.rb`. | -| **:with_title** | This is a **Trait** of the factory. [See how it's defined](https://gitlab.com/gitlab-org/gitlab/-/blob/9c2a1f98483921dd006d70fdaed316e21fc5652f/ee/spec/factories/iterations.rb#L21-23). | -| **:current** | This is a **Trait** of the factory. [See how it's defined](https://gitlab.com/gitlab-org/gitlab/-/blob/9c2a1f98483921dd006d70fdaed316e21fc5652f/ee/spec/factories/iterations.rb#L29-31). | -| **title: 'My Iteration'** | This is an **Attribute** of the factory that will be passed to the Model for creation. | - -### Examples - -In these examples, you will see an instance variable `@owner`. This is the `root` user (`User.first`). - -#### Create a Group - -```ruby -my_group = create(:group, name: 'My Group', path: 'my-group-path') -``` - -#### Create a Project - -```ruby -# create a Project belonging to a Group -my_project = create(:project, :public, name: 'My Project', namespace: my_group, creator: @owner) -``` - -#### Create an Issue - -```ruby -# create an Issue belonging to a Project -my_issue = create(:issue, title: 'My Issue', project: my_project, weight: 2) -``` - -#### Create an Iteration - -```ruby -# create an Iteration under a Group -my_iteration = create(:iteration, :with_title, :current, title: 'My Iteration', group: my_group) -``` - -### Frequently encountered issues - -#### ActiveRecord::RecordInvalid: Validation failed: Email has already been taken, Username has already been taken - -This is because, by default, our factories are written to backfill any data that is missing. For instance, when a project -is created, the project must have somebody that created it. If the owner is not specified, the factory attempts to create it. - -**How to fix** - -Check the respective Factory to find out what key is required. Usually `:author` or `:owner`. - -```ruby -# This throws ActiveRecord::RecordInvalid -create(:project, name: 'Throws Error', namespace: create(:group, name: 'Some Group')) - -# Specify the user where @owner is a [User] record -create(:project, name: 'No longer throws error', owner: @owner, namespace: create(:group, name: 'Some Group')) -create(:epic, group: create(:group), author: @owner) -``` diff --git a/doc/topics/data_seeder.md b/doc/topics/data_seeder.md new file mode 100644 index 00000000000..19c0e05d8ed --- /dev/null +++ b/doc/topics/data_seeder.md @@ -0,0 +1,331 @@ +--- +stage: Manage +group: Foundations +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +description: Data Seeder test data harness created by the Test Data Working Group https://about.gitlab.com/company/team/structure/working-groups/demo-test-data/ +--- + +# GitLab Data Seeder + +GitLab Data Seeder (GDS) is a test data seeding harness, that can seed test data into a user or group namespace. + +The Data Seeder uses FactoryBot in the backend which makes maintenance extremely easy. When a Model is changed, +FactoryBot will already be reflected to account for the change. + +## Docker Setup + +See [Data Seeder Docker Demo](https://gitlab.com/-/snippets/2390362) + +## GDK Setup + +```shell +$ gdk start db +ok: run: services/postgresql: (pid n) 0s, normally down +ok: run: services/redis: (pid n) 74s, normally down +$ bundle install +Bundle complete! +$ bundle exec rake db:migrate +main: migrated +ci: migrated +``` + +### Run + +The `ee:gitlab:seed:data_seeder` Rake task takes two arguments. `:name` and `:namespace_id`. + +```shell +$ bundle exec rake "ee:gitlab:seed:data_seeder[data_seeder,1]" +Seeding Data for Administrator +``` + +#### `:name` + +Where `:name` is the file name. (This will reflect relative `.rb`, `.yml`, or `.json` files located in `ee/db/seeds/data_seeder`, or absolute paths to seed files) + +#### `:namespace_id` + +Where `:namespace_id` is the ID of the User or Group Namespace + +## Develop + +The Data Seeder uses FactoryBot definitions from `spec/factories` which ... + +1. Saves time on development +1. Are easy-to-read +1. Are easy to maintain +1. Do not rely on an API that may change in the future +1. Are always up-to-date +1. Execute on the lowest-level (`ActiveRecord`) possible to create data as quickly as possible + +> From the [FactoryBot README](https://github.com/thoughtbot/factory_bot#readme_) : `factory_bot` is a fixtures replacement with a straightforward definition syntax, support for multiple build +> strategies (saved instances, unsaved instances, attribute hashes, and stubbed objects), and support for multiple factories for the same class, including factory +> inheritance + +Factories reside in `spec/factories/*` and are fixtures for Rails models found in `app/models/*`. For example, For a model named `app/models/issue.rb`, the factory will +be named `spec/factories/issues.rb`. For a model named `app/models/project.rb`, the factory will be named `app/models/projects.rb`. + +There are currently three parsers that the GitLab Data Seeder supports. Ruby, YAML, and JSON. + +### Ruby + +All Ruby Seeds must define a `DataSeeder` class with a `#seed` instance method. You may structure your Ruby class as you wish. All FactoryBot [methods](https://www.rubydoc.info/gems/factory_bot/FactoryBot/Syntax/Methods) (`create`, `build`, `create_list`) will be included in the class automatically and may be called. + +The `DataSeeder` class will have the following instance variables defined upon seeding: + +- `@seed_file` - The `File` object. +- `@owner` - The owner of the seed data. +- `@name` - The name of the seed. This will be the seed file name without the extension. +- `@group` - The root group that all seeded data will be created under. + +```ruby +# frozen_string_literal: true + +class DataSeeder + def seed + my_group = create(:group, name: 'My Group', path: 'my-group-path', parent: @group) + my_project = create(:project, :public, name: 'My Project', namespace: my_group, creator: @owner) + end +end +``` + +### YAML + +The YAML Parser is a DSL that supports Factory definitions and allows you to seed data using a human-readable format. + +```yaml +name: My Seeder +groups: + - _id: my_group + name: My Group + path: my-group-path + +projects: + - _id: my_project + name: My Project + namespace_id: <%= groups.my_group.id %> + creator_id: <%= @owner.id %> + traits: + - public +``` + +### JSON + +The JSON Parser allows you to house seed files in JSON format. + +```json +{ + "name": "My Seeder", + "groups": [ + { "_id": "my_group", "name": "My Group", "path": "my-group-path" } + ], + "projects": [ + { + "_id": "my_project", + "name": "My Project", + "namespace_id": "<%= groups.my_group.id %>", + "creator_id": "<%= @owner.id %>", + "traits": ["public"] + } + ] +} +``` + +### Taxonomy of a Factory + +Factories consist of three main parts - the **Name** of the factory, the **Traits** and the **Attributes**. + +Given: `create(:iteration, :with_title, :current, title: 'My Iteration')` + +||| +|:-|:-| +| **:iteration** | This is the **Name** of the factory. The file name will be the plural form of this **Name** and reside under either `spec/factories/iterations.rb` or `ee/spec/factories/iterations.rb`. | +| **:with_title** | This is a **Trait** of the factory. [See how it's defined](https://gitlab.com/gitlab-org/gitlab/-/blob/9c2a1f98483921dd006d70fdaed316e21fc5652f/ee/spec/factories/iterations.rb#L21-23). | +| **:current** | This is a **Trait** of the factory. [See how it's defined](https://gitlab.com/gitlab-org/gitlab/-/blob/9c2a1f98483921dd006d70fdaed316e21fc5652f/ee/spec/factories/iterations.rb#L29-31). | +| **title: 'My Iteration'** | This is an **Attribute** of the factory that will be passed to the Model for creation. | + +### Examples + +In these examples, you will see an instance variable `@owner`. This is the `root` user (`User.first`). + +#### Create a Group + +```ruby +my_group = create(:group, name: 'My Group', path: 'my-group-path') +``` + +#### Create a Project + +```ruby +# create a Project belonging to a Group +my_project = create(:project, :public, name: 'My Project', namespace: my_group, creator: @owner) +``` + +#### Create an Issue + +```ruby +# create an Issue belonging to a Project +my_issue = create(:issue, title: 'My Issue', project: my_project, weight: 2) +``` + +#### Create an Iteration + +```ruby +# create an Iteration under a Group +my_iteration = create(:iteration, :with_title, :current, title: 'My Iteration', group: my_group) +``` + +### Frequently encountered issues + +#### ActiveRecord::RecordInvalid: Validation failed: Email has already been taken, Username has already been taken + +This is because, by default, our factories are written to backfill any data that is missing. For instance, when a project +is created, the project must have somebody that created it. If the owner is not specified, the factory attempts to create it. + +**How to fix** + +Check the respective Factory to find out what key is required. Usually `:author` or `:owner`. + +```ruby +# This throws ActiveRecord::RecordInvalid +create(:project, name: 'Throws Error', namespace: create(:group, name: 'Some Group')) + +# Specify the user where @owner is a [User] record +create(:project, name: 'No longer throws error', owner: @owner, namespace: create(:group, name: 'Some Group')) +create(:epic, group: create(:group), author: @owner) +``` + +#### `parsing id "my id" as "my_id"` + +See [specifying variables](#specify-a-variable) + +#### `id is invalid` + +Given that non-Ruby parsers parse IDs as Ruby Objects, the [naming conventions](https://docs.ruby-lang.org/en/2.0.0/syntax/methods_rdoc.html#label-Method+Names) of Ruby must be followed when specifying an ID. + +Examples of invalid IDs: + +- IDs that start with a number +- IDs that have special characters (-, !, $, @, `, =, <, >, ;, :) + +#### ActiveRecord::AssociationTypeMismatch: Model expected, got ... which is an instance of String + +This is currently a limitation for the seeder. + +See the issue for [allowing parsing of raw Ruby objects](https://gitlab.com/gitlab-org/gitlab/-/issues/403079). + +## YAML Factories + +### Generator to generate _n_ amount of records + +### [Group Labels](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/factories/labels.rb) + +```yaml +group_labels: + # Group Label with Name and a Color + - name: Group Label 1 + group_id: <%= @group.id %> + color: "#FF0000" +``` + +### [Group Milestones](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/factories/milestones.rb) + +```yaml +group_milestones: + # Past Milestone + - name: Past Milestone + group_id: <%= @group.id %> + group: + start_date: <%= 1.month.ago %> + due_date: <%= 1.day.ago %> + + # Ongoing Milestone + - name: Ongoing Milestone + group_id: <%= @group.id %> + group: + start_date: <%= 1.day.ago %> + due_date: <%= 1.month.from_now %> + + # Future Milestone + - name: Ongoing Milestone + group_id: <%= @group.id %> + group: + start_date: <%= 1.month.from_now %> + due_date: <%= 2.months.from_now %> +``` + +#### Quirks + +- You _must_ specify `group:` and have it be empty. This is because the Milestones factory will manipulate the factory in an `after(:build)`. If this is not present, the Milestone will not be associated properly with the Group. + +### [Epics](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/spec/factories/epics.rb) + +```yaml +epics: + # Simple Epic + - title: Simple Epic + group_id: <%= @group.id %> + author_id: <%= @owner.id %> + + # Epic with detailed Markdown description + - title: Detailed Epic + group_id: <%= @group.id %> + author_id: <%= @owner.id %> + description: | + # Markdown + + **Description** + + # Epic with dates + - title: Epic with dates + group_id: <%= @group.id %> + author_id: <%= @owner.id %> + start_date: <%= 1.day.ago %> + due_date: <%= 1.month.from_now %> +``` + +## Variables + +Each created factory can be assigned an identifier to be used in future seeding. + +You can specify an ID for any created factory that you may use later in the seed file. + +### Specify a variable + +You may pass an `_id` attribute on any factory to refer back to it later in non-Ruby parsers. + +Variables are under the factory definitions that they reside in. + +```yaml +--- +group_labels: + - _id: my_label #=> group_labels.my_label + +projects: + - _id: my_project #=> projects.my_project +``` + +Variables: + +NOTE: +It is not advised, but you may specify variables with spaces. These variables may be referred back to with underscores. + +### Referencing a variable + +Given a YAML seed file: + +```yaml +--- +group_labels: + - _id: my_group_label #=> group_labels.my_group_label + name: My Group Label + color: "#FF0000" + - _id: my_other_group_label #=> group_labels.my_other_group_label + color: <%= group_labels.my_group_label.color %> + +projects: + - _id: my_project #=> projects.my_project + name: My Project +``` + +When referring to a variable, the variable refers to the _already seeded_ models. In other words, the model's `id` attribute will +be populated. diff --git a/doc/topics/git/bisect.md b/doc/topics/git/bisect.md index a7f8b2a846c..eaf619ce36f 100644 --- a/doc/topics/git/bisect.md +++ b/doc/topics/git/bisect.md @@ -1,76 +1,11 @@ --- -stage: Create -group: Source Code -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false +redirect_to: 'index.md' +remove_date: '2023-06-09' --- -# Bisect **(FREE)** +This document was moved to [another location](index.md). -- Find a commit that introduced a bug -- Works through a process of elimination -- Specify a known good and bad revision to begin - -## Bisect sample workflow - -1. Start the bisect process -1. Enter the bad revision (usually latest commit) -1. Enter a known good revision (commit/branch) -1. Run code to see if bug still exists -1. Tell bisect the result -1. Repeat the previous 2 items until you find the offending commit - -## Setup - -```shell - mkdir bisect-ex - cd bisect-ex - touch index.html - git add -A - git commit -m "starting out" - vi index.html - # Add all good - git add -A - git commit -m "second commit" - vi index.html - # Add all good 2 - git add -A - git commit -m "third commit" - vi index.html -``` - -```shell - # Add all good 3 - git add -A - git commit -m "fourth commit" - vi index.html - # This looks bad - git add -A - git commit -m "fifth commit" - vi index.html - # Really bad - git add -A - git commit -m "sixth commit" - vi index.html - # again just bad - git add -A - git commit -m "seventh commit" -``` - -## Commands - -```shell - git bisect start - # Test your code - git bisect bad - git bisect next - # Say yes to the warning - # Test - git bisect good - # Test - git bisect bad - # Test - git bisect good - # done - git bisect reset -``` + + + + diff --git a/doc/topics/git/cherry_picking.md b/doc/topics/git/cherry_picking.md index ce2a1d4b954..bd589562ed1 100644 --- a/doc/topics/git/cherry_picking.md +++ b/doc/topics/git/cherry_picking.md @@ -2,7 +2,6 @@ stage: Create group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false --- # Cherry-pick a Git commit **(FREE)** diff --git a/doc/topics/git/feature_branch_development.md b/doc/topics/git/feature_branch_development.md deleted file mode 100644 index 4125d8e8fdb..00000000000 --- a/doc/topics/git/feature_branch_development.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'index.md' -remove_date: '2023-03-31' ---- - -This document was moved to [another location](index.md). - - - - - diff --git a/doc/topics/git/feature_branching.md b/doc/topics/git/feature_branching.md index 73de351fde2..eaf619ce36f 100644 --- a/doc/topics/git/feature_branching.md +++ b/doc/topics/git/feature_branching.md @@ -1,31 +1,11 @@ --- -stage: Create -group: Source Code -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false +redirect_to: 'index.md' +remove_date: '2023-06-09' --- -# Feature branching **(FREE)** +This document was moved to [another location](index.md). -- Efficient parallel workflow for teams -- Develop each feature in a branch -- Keeps changes isolated -- Consider a 1-to-1 link to issues -- Push branches to the server frequently - - Hint: Pushing branches is a cheap backup for your work-in-progress code. - -## Feature branching sample workflow - -1. Create a new feature branch called `squash_some_bugs` -1. Edit '`bugs.rb`' and remove all the bugs. -1. Commit -1. Push - -```shell -git checkout -b squash_some_bugs -# Edit `bugs.rb` -git status -git add bugs.rb -git commit -m 'Fix some buggy code' -git push origin squash_some_bugs -``` + + + + diff --git a/doc/topics/git/getting_started.md b/doc/topics/git/getting_started.md deleted file mode 100644 index 790fd3aa6c0..00000000000 --- a/doc/topics/git/getting_started.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../../tutorials/make_your_first_git_commit.md' -remove_date: '2023-04-23' ---- - -This document was moved to [another location](../../tutorials/make_your_first_git_commit.md). - - - - - diff --git a/doc/topics/git/git_add.md b/doc/topics/git/git_add.md index 722701dbb49..1089202bbd3 100644 --- a/doc/topics/git/git_add.md +++ b/doc/topics/git/git_add.md @@ -2,7 +2,6 @@ stage: Create group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false --- # Git add **(FREE)** diff --git a/doc/topics/git/git_log.md b/doc/topics/git/git_log.md index e2e9c0e8804..eaf619ce36f 100644 --- a/doc/topics/git/git_log.md +++ b/doc/topics/git/git_log.md @@ -1,60 +1,11 @@ --- -stage: Create -group: Source Code -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false +redirect_to: 'index.md' +remove_date: '2023-06-09' --- -# Git Log **(FREE)** +This document was moved to [another location](index.md). -Git log lists commit history. It allows searching and filtering. - -- Initiate log: - - ```shell - git log - ``` - -- Retrieve set number of records: - - ```shell - git log -n 2 - ``` - -- Search commits by author. Allows user name or a regular expression. - - ```shell - git log --author="user_name" - ``` - -- Search by comment message: - - ```shell - git log --grep="" - ``` - -- Search by date: - - ```shell - git log --since=1.month.ago --until=3.weeks.ago - ``` - -## Git Log Workflow - -1. Change to workspace directory -1. Clone the multi runner projects -1. Change to project dir -1. Search by author -1. Search by date -1. Combine - -## Commands - -```shell -cd ~/workspace -git clone git@gitlab.com:gitlab-org/gitlab-runner.git -cd gitlab-runner -git log --author="Travis" -git log --since=1.month.ago --until=3.weeks.ago -git log --since=1.month.ago --until=1.day.ago --author="Travis" -``` + + + + diff --git a/doc/topics/git/git_rebase.md b/doc/topics/git/git_rebase.md index ec28d0b6431..bc9337481d4 100644 --- a/doc/topics/git/git_rebase.md +++ b/doc/topics/git/git_rebase.md @@ -226,5 +226,4 @@ git push --force origin my-feature ## Related topics - [Numerous undo possibilities in Git](numerous_undo_possibilities_in_git/index.md#undo-staged-local-changes-without-modifying-history) -for a deeper look into interactive rebases. -- [Git documentation for branches and rebases](https://git-scm.com/book/en/v2/Git-Branching-Rebasing). +- [Git documentation for branches and rebases](https://git-scm.com/book/en/v2/Git-Branching-Rebasing) diff --git a/doc/topics/git/index.md b/doc/topics/git/index.md index b47a34fa7b2..d5aa74935f2 100644 --- a/doc/topics/git/index.md +++ b/doc/topics/git/index.md @@ -26,13 +26,12 @@ The following resources can help you get started with Git: - [Git-ing started with Git](https://www.youtube.com/watch?v=Ce5nz5n41z4), a video introduction to Git. -- [Make your first Git commit](../../tutorials/make_your_first_git_commit.md) +- [Make your first Git commit](../../tutorials/make_first_git_commit/index.md) - [Git Basics](https://git-scm.com/book/en/v2/Getting-Started-Git-Basics) - [Git on the Server - GitLab](https://git-scm.com/book/en/v2/Git-on-the-Server-GitLab) - [How to install Git](how_to_install_git/index.md) -- [Git terminology](terminology.md) +- [Git concepts](terminology.md) - [Start using Git on the command line](../../gitlab-basics/start-using-git.md) -- [Edit files through the command line](../../gitlab-basics/command-line-commands.md) - [GitLab Git Cheat Sheet (download)](https://about.gitlab.com/images/press/git-cheat-sheet.pdf) - Commits: - [Revert a commit](../../user/project/merge_requests/revert_changes.md#revert-a-commit) @@ -43,7 +42,7 @@ The following resources can help you get started with Git: - [Git stash](stash.md) - [Git file blame](../../user/project/repository/git_blame.md) - [Git file history](../../user/project/repository/git_history.md) -- [Git tags](tags.md) +- [Git tags](../../user/project/repository/tags/index.md) ### Concepts @@ -58,16 +57,11 @@ The following are resources on version control concepts: You can do many Git tasks from the command line: -- [Bisect](bisect.md). - [Cherry-pick](cherry_picking.md). -- [Feature branching](feature_branching.md). -- [Getting started with Git](getting_started.md). +- [Getting started with Git](../../tutorials/make_first_git_commit/index.md). - [Git add](git_add.md). -- [Git log](git_log.md). - [Git stash](stash.md). -- [Merge conflicts](merge_conflicts.md). - [Rollback commits](rollback_commits.md). -- [Subtree](subtree.md). - [Unstage](unstage.md). ## Git tips @@ -88,7 +82,6 @@ If you have problems with Git, the following may help: ## Branching strategies - [Feature branch workflow](../../gitlab-basics/feature_branch_workflow.md) -- [Develop on a feature branch](feature_branch_development.md) - [GitLab Flow](../gitlab_flow.md) - [Git Branching - Branches in a Nutshell](https://git-scm.com/book/en/v2/Git-Branching-Branches-in-a-Nutshell) - [Git Branching - Branching Workflows](https://git-scm.com/book/en/v2/Git-Branching-Branching-Workflows) diff --git a/doc/topics/git/lfs/index.md b/doc/topics/git/lfs/index.md index cac203ffac0..4f0d1bfc5e6 100644 --- a/doc/topics/git/lfs/index.md +++ b/doc/topics/git/lfs/index.md @@ -3,7 +3,6 @@ stage: Create group: Source Code info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments" type: reference, howto -disqus_identifier: 'https://docs.gitlab.com/ee/workflow/lfs/lfs/index.html' --- # Git Large File Storage (LFS) **(FREE)** @@ -27,10 +26,10 @@ instructions from where to fetch or where to push the large file. Documentation for GitLab instance administrators is under [LFS administration doc](../../../administration/lfs/index.md). -## Requirements +## Prerequisites - Git LFS must be [enabled in project settings](../../../user/project/settings/index.md#configure-project-visibility-features-and-permissions). -- [Git LFS client](https://git-lfs.com/) version 1.0.1 or higher must be installed. +- [Git LFS client](https://git-lfs.com/) version 1.0.1 or later must be installed. ## Known limitations diff --git a/doc/topics/git/lfs/migrate_to_git_lfs.md b/doc/topics/git/lfs/migrate_to_git_lfs.md index 07c89e52653..d14d0db6087 100644 --- a/doc/topics/git/lfs/migrate_to_git_lfs.md +++ b/doc/topics/git/lfs/migrate_to_git_lfs.md @@ -35,7 +35,7 @@ The method described on this guide rewrites Git history. Make sure to back up your repository before beginning and use it at your own risk. -## Requirements +## Prerequisites Before beginning, make sure: @@ -161,7 +161,7 @@ Consider an example upstream project, `git@gitlab.com:gitlab-tests/test-git-lfs- expand **Protected branches**. 1. Select the default branch from the **Branch** dropdown list, and set up the - **Allowed to push** and **Allowed to merge** rules. + **Allowed to push and merge** and **Allowed to merge** rules. 1. Select **Protect**. - - - diff --git a/doc/topics/git/partial_clone.md b/doc/topics/git/partial_clone.md index f5f11b17675..de0547ae49d 100644 --- a/doc/topics/git/partial_clone.md +++ b/doc/topics/git/partial_clone.md @@ -94,9 +94,7 @@ Updating files: 100% (28/28), done. $ cd www-gitlab-com -$ git sparse-checkout init --cone - -$ git sparse-checkout add data +$ git sparse-checkout set data --cone remote: Enumerating objects: 301, done. remote: Counting objects: 100% (301/301), done. remote: Compressing objects: 100% (292/292), done. diff --git a/doc/topics/git/rollback_commits.md b/doc/topics/git/rollback_commits.md index 56d740f3d1b..d1f34c7cf9c 100644 --- a/doc/topics/git/rollback_commits.md +++ b/doc/topics/git/rollback_commits.md @@ -2,7 +2,6 @@ stage: Create group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false --- # Roll back commits **(FREE)** diff --git a/doc/topics/git/stash.md b/doc/topics/git/stash.md index ee931bbbb7c..9f621308a1d 100644 --- a/doc/topics/git/stash.md +++ b/doc/topics/git/stash.md @@ -2,7 +2,6 @@ stage: Create group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false --- # Git stash **(FREE)** diff --git a/doc/topics/git/subtree.md b/doc/topics/git/subtree.md index a8a665d4e13..eaf619ce36f 100644 --- a/doc/topics/git/subtree.md +++ b/doc/topics/git/subtree.md @@ -1,50 +1,11 @@ --- -stage: Create -group: Source Code -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false +redirect_to: 'index.md' +remove_date: '2023-06-09' --- -# Subtree **(FREE)** +This document was moved to [another location](index.md). -- Used when there are nested repositories. -- Not recommended when the amount of dependencies is too large. -- For these cases we need a dependency control system. -- Command are painfully long so aliases are necessary. - -## Subtree Aliases - -- Add: `git subtree add --prefix --squash` -- Pull: `git subtree pull --prefix --squash` -- Push: `git subtree push --prefix ` - -```shell - # Add an alias - # Add - git config alias.sba 'subtree add --prefix st / - git@gitlab.com:balameb/subtree-nested-example.git master --squash' - # Pull - git config alias.sbpl 'subtree pull --prefix st / - git@gitlab.com:balameb/subtree-nested-example.git master --squash' - # Push - git config alias.sbph 'subtree push --prefix st / - git@gitlab.com:balameb/subtree-nested-example.git master' - - # Adding this subtree adds a st dir with a readme - git sba - vi st/README.md - # Edit file - git status shows differences - -``` - -```shell - # Adding, or committing won't change the sub repo at remote - # even if we push - git add -A - git commit -m "Adding to subtree readme" - - # Push to subtree repo - git sbph - # now we can check our remote sub repo -``` + + + + diff --git a/doc/topics/git/tags.md b/doc/topics/git/tags.md index ab196f409f7..c66c97717f2 100644 --- a/doc/topics/git/tags.md +++ b/doc/topics/git/tags.md @@ -1,41 +1,11 @@ --- -stage: Create -group: Source Code -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../../user/project/repository/tags/index.md' +remove_date: '2023-05-27' --- -# Tags **(FREE)** +This document was moved to [another location](../../user/project/repository/tags/index.md). -Tags help you mark certain deployments and releases for later -reference. Git supports two types of tags: - -- Annotated tags: An unchangeable part of Git history. -- Lightweight (soft) tags: Tags that can be set and removed as needed. - -Many projects combine an annotated release tag with a stable branch. Consider -setting deployment or release tags automatically. - -## Tags sample workflow - -1. Create a lightweight tag. -1. Create an annotated tag. -1. Push the tags to the remote repository. - -```shell -git checkout master - -# Lightweight tag -git tag my_lightweight_tag - -# Annotated tag -git tag -a v1.0 -m 'Version 1.0' - -# Show list of the existing tags -git tag - -git push origin --tags -``` - -## Related topics - -- [Tagging](https://git-scm.com/book/en/v2/Git-Basics-Tagging) Git reference page + + + + diff --git a/doc/topics/git/terminology.md b/doc/topics/git/terminology.md index ac097396bef..cef9b7cc35b 100644 --- a/doc/topics/git/terminology.md +++ b/doc/topics/git/terminology.md @@ -4,9 +4,9 @@ group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Git terminology +# Git concepts -The following are commonly-used Git terms. +The following are commonly-used Git concepts. ## Repository @@ -27,7 +27,7 @@ In GitLab, a repository is contained in a **project**. ## Fork When you want to contribute to someone else's repository, you make a copy of it. -This copy is called a [**fork**](../../user/project/repository/forking_workflow.md#creating-a-fork). +This copy is called a [**fork**](../../user/project/repository/forking_workflow.md#create-a-fork). The process is called "creating a fork." When you fork a repository, you create a copy of the project in your own diff --git a/doc/topics/git/troubleshooting_git.md b/doc/topics/git/troubleshooting_git.md index a116f6cc978..502acf5f7b4 100644 --- a/doc/topics/git/troubleshooting_git.md +++ b/doc/topics/git/troubleshooting_git.md @@ -75,7 +75,7 @@ Configuring both the client and the server is unnecessary. ``` - On Windows, if you are using PuTTY, go to your session properties, then - navigate to "Connection" and under "Sending of null packets to keep + go to "Connection" and under "Sending of null packets to keep session active", set `Seconds between keepalives (0 to turn off)` to `60`. **To configure SSH on the server side**, edit `/etc/ssh/sshd_config` and add: @@ -186,7 +186,7 @@ fatal: early EOF fatal: index-pack failed ``` -This is a common problem with Git itself, due to its inability to handle large files or large quantities of files. +This problem is common in Git itself, due to its inability to handle large files or large quantities of files. [Git LFS](https://about.gitlab.com/blog/2017/01/30/getting-started-with-git-lfs-tutorial/) was created to work around this problem; however, even it has limitations. It's usually due to one of these reasons: - The number of files in the repository. @@ -221,11 +221,17 @@ apply more than one: 1. Modify the GitLab instance's [`gitlab.rb`](https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/13.5.1+ee.0/files/gitlab-config-template/gitlab.rb.template#L1435-1455) file: - ```shell - gitaly['gitconfig'] = [ - # Set the http.postBuffer size, in bytes - {key: "http.postBuffer", value: "524288000"}, - ] + ```ruby + gitaly['configuration'] = { + # ... + git: { + # ... + config: [ + # Set the http.postBuffer size, in bytes + {key: "http.postBuffer", value: "524288000"}, + ], + }, + } ``` 1. After applying this change, apply the configuration change: @@ -282,3 +288,105 @@ The bug was reported [in this issue](https://gitlab.com/gitlab-org/gitlab/-/issu If you receive an `HTTP Basic: Access denied` error when using Git over HTTP(S), refer to the [two-factor authentication troubleshooting guide](../../user/profile/account/two_factor_authentication.md#troubleshooting). + +## `401` errors logged during successful `git clone` + +When cloning a repository via HTTP, the +[`production_json.log`](../../administration/logs/index.md#production_jsonlog) file +may show an initial status of `401` (unauthorized), quickly followed by a `200`. + +```json +{ + "method":"GET", + "path":"/group/project.git/info/refs", + "format":"*/*", + "controller":"Repositories::GitHttpController", + "action":"info_refs", + "status":401, + "time":"2023-04-18T22:55:15.371Z", + "remote_ip":"x.x.x.x", + "ua":"git/2.39.2", + "correlation_id":"01GYB98MBM28T981DJDGAD98WZ", + "duration_s":0.03585 +} +{ + "method":"GET", + "path":"/group/project.git/info/refs", + "format":"*/*", + "controller":"Repositories::GitHttpController", + "action":"info_refs", + "status":200, + "time":"2023-04-18T22:55:15.714Z", + "remote_ip":"x.x.x.x", + "user_id":1, + "username":"root", + "ua":"git/2.39.2", + "correlation_id":"01GYB98MJ0CA3G9K8WDH7HWMQX", + "duration_s":0.17111 +} +``` + +You should expect this initial `401` log entry for each Git operation performed over HTTP, +due to how [HTTP Basic authentication](https://en.wikipedia.org/wiki/Basic_access_authentication) works. + +When the Git client initiates a clone, the initial request sent to GitLab does not provide +any authentication details. GitLab returns a `401 Unauthorized` result for that request. +A few milliseconds later, the Git client sends a follow-up request containing authentication +details. This second request should succeed, and result in a `200 OK` log entry. + +If a `401` log entry lacks a corresponding `200` log entry, the Git client is likely using either: + +- An incorrect password. +- An expired or revoked token.an incorrect + +If not rectified, you could encounter +[`403` (Forbidden) errors](#403-error-when-performing-git-operations-over-http) +instead. + +## `403` error when performing Git operations over HTTP + +When performing Git operations over HTTP, a `403` (Forbidden) error indicates that +your IP address has been blocked by the failed-authentication ban: + +```plaintext +fatal: unable to access 'https://gitlab.com/group/project.git/': The requested URL returned error: 403 +``` + +The `403` can be seen in the [`production_json.log`](../../administration/logs/index.md#production_jsonlog): + +```json +{ + "method":"GET", + "path":"/group/project.git/info/refs", + "format":"*/*", + "controller":"Repositories::GitHttpController", + "action":"info_refs", + "status":403, + "time":"2023-04-19T22:14:25.894Z", + "remote_ip":"x.x.x.x", + "user_id":1, + "username":"root", + "ua":"git/2.39.2", + "correlation_id":"01GYDSAKAN2SPZPAMJNRWW5H8S", + "duration_s":0.00875 +} +``` + +If your IP address has been blocked, a corresponding log entry exists in the +[`auth_json.log`](../../administration/logs/index.md#auth_jsonlog): + +```json +{ + "severity":"ERROR", + "time":"2023-04-19T22:14:25.893Z", + "correlation_id":"01GYDSAKAN2SPZPAMJNRWW5H8S", + "message":"Rack_Attack", + "env":"blocklist", + "remote_ip":"x.x.x.x", + "request_method":"GET", + "path":"/group/project.git/info/refs?service=git-upload-pack"} +``` + +The failed authentication ban limits differ depending if you are using a +[self-managed instance](../../security/rate_limits.md#failed-authentication-ban-for-git-and-container-registry) +or [GitLab.com](../../user/gitlab_com/index.md#ip-blocks). diff --git a/doc/topics/git/unstage.md b/doc/topics/git/unstage.md index 142a80a9ad9..3f509cdacef 100644 --- a/doc/topics/git/unstage.md +++ b/doc/topics/git/unstage.md @@ -2,18 +2,22 @@ stage: Create group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false --- -# Unstage **(FREE)** +# Unstage a file in Git **(FREE)** -- To remove files from stage use reset HEAD where HEAD is the last commit of the current branch. This unstages the file but maintain the modifications. +When you _stage_ a file in Git, you instruct Git to track changes to the file in +preparation for a commit. To instruct Git to disregard changes to a file, and not +include it in your next commit, _unstage_ the file. + +- To remove files from stage use `reset HEAD`, where HEAD is the last commit of + the current branch. This unstages the file but maintains the modifications. ```shell git reset HEAD ``` -- To revert the file back to the state it was in before the changes we can use: +- To revert the file back to the state it was in before the changes: ```shell git checkout -- @@ -26,7 +30,8 @@ comments: false git rm -r ``` -- If we want to remove a file from the repository but keep it on disk, say we forgot to add it to our `.gitignore` file then use `--cache`: +- To keep a file on disk but remove it from the repository (such as a file you want + to add to `.gitignore`), use the `rm` command with the `--cache` flag: ```shell git rm --cache diff --git a/doc/topics/git/useful_git_commands.md b/doc/topics/git/useful_git_commands.md index 235412f511a..22548be2e8b 100644 --- a/doc/topics/git/useful_git_commands.md +++ b/doc/topics/git/useful_git_commands.md @@ -130,12 +130,6 @@ Use this to check the Git history of the file: git log -- ``` -### Find the tags that contain a particular SHA - -```shell -git tag --contains -``` - ### Check the content of each change to a file ```shell diff --git a/doc/topics/gitlab_flow.md b/doc/topics/gitlab_flow.md index 0bf2e1fcc20..eb298841247 100644 --- a/doc/topics/gitlab_flow.md +++ b/doc/topics/gitlab_flow.md @@ -2,25 +2,94 @@ stage: Create group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -disqus_identifier: 'https://docs.gitlab.com/ee/workflow/gitlab_flow.html' --- -# Introduction to GitLab Flow **(FREE)** +# Introduction to Git workflows **(FREE)** With Git, you can use a variety of branching strategies and workflows. +Having a structured workflow for collaboration in complex projects is +crucial for several reasons: -Because the default workflow is not specifically defined, many organizations -end up with workflows that are too complicated, not clearly defined, or -not integrated with their issue tracking systems. +- **Code organization**: Keep the codebase organized, prevent + overlapping work, and ensure focused efforts towards a common goal. + +- **Version control**: Allow simultaneous work on different features + without conflicts, maintaining code stability. + +- **Code quality**: A code review and approval process helps maintain high + code quality and adherence to coding standards. + +- **Traceability and accountability**: Enable tracking of changes and their authors, + simplifying issue identification and responsibility assignment. + +- **Easier onboarding**: Help new team members quickly grasp the + development process, and start contributing effectively. + +- **Time and resource management**: Enable better planning, resource + allocation, and meeting deadlines, ensuring an efficient development + process. + +- **CI/CD**: Incorporate automated testing and deployment + processes, streamlining the release cycle and delivering high-quality + software consistently. + +A structured workflow promotes organization, efficiency, and code +quality, leading to a more successful and streamlined development process. + +If the default workflow is not specifically defined, many organizations +end up with workflows that are: + +- Too complicated. +- Not clearly defined. +- Not integrated with their issue tracking systems. Your organization can use GitLab with any workflow you choose. -However, if you are looking for guidance on best practices, you can use -the GitLab Flow. This workflow combines [feature-driven development](https://en.wikipedia.org/wiki/Feature-driven_development) -and [feature branches](https://martinfowler.com/bliki/FeatureBranch.html) with issue tracking. +## Workflow types -While this workflow used at GitLab, you can choose whichever workflow -suits your organization best. +Here are some of the most common Git workflows. + +### Centralized workflow + +Best suited for small teams transitioning from a centralized version +control system like SVN. All team members work on a single branch, +usually `main`, and push their changes directly to the central +repository. + +### Feature branch workflow + +Developers create separate branches for each feature or bugfix, +keeping the 'main' branch stable. When a feature is complete, the +developer submits a merge request to integrate the +changes back into `main` after a code review. + +### Forking workflow + +Commonly used in open-source projects, this workflow allows external +contributors to work without direct access to the main repository. +Developers create a fork (personal copy) of the main repository and +make changes in it. They then submit a merge request to have those changes +integrated into the main repository. + +### Git flow workflow + +This workflow is best for projects with a structured release cycle. +It introduces two long-lived branches: `main` for production-ready +code and `develop` for integrating features. Additional branches like +`feature`, `release`, and `hotfix` are used for specific purposes, +ensuring a strict and organized development process. + +### GitLab/GitHub flow + +A simplified workflow primarily used for web development and +continuous deployment. It combines aspects of the Feature branch +workflow and the Git flow workflow. Developers create feature branches +from `main`, and after the changes are complete, they are merged back +into the `main` branch, which is then immediately deployed. + +Each of these Git workflows has its advantages and is suited to +different project types and team structures. Below the most popular +workflows are reviewed in more details. ## Git workflow @@ -97,12 +166,22 @@ graph TD ``` This flow is clean and straightforward, and many organizations have adopted it with great success. -Atlassian recommends [a similar strategy](https://www.atlassian.com/blog/git/simple-git-workflow-is-simple), although they rebase feature branches. +Another way to integrate change from one branch to another is [rebasing](https://git-scm.com/book/en/v2/Git-Branching-Rebasing). Merging everything into the `main` branch and frequently deploying means you minimize the amount of unreleased code. This approach is in line with lean and continuous delivery best practices. However, this flow still leaves a lot of questions unanswered regarding deployments, environments, releases, and integrations with issues. -With GitLab flow, we offer additional guidance for these questions. -## Production branch with GitLab flow +## Introduction to GitLab Flow **(FREE)** + +However, if you are looking for guidance on best practices, you can use +the GitLab Flow. This workflow combines [feature-driven development](https://en.wikipedia.org/wiki/Feature-driven_development) +and [feature branches](https://martinfowler.com/bliki/FeatureBranch.html) with issue tracking. + +While this workflow used at GitLab, you can choose whichever workflow +suits your organization best. + +With GitLab Flow, we offer additional guidance for these questions. + +## Production branch with GitLab Flow GitHub flow assumes you can deploy to production every time you merge a feature branch. While this is possible in some cases, such as SaaS applications, there are some cases where this is not possible, such as: @@ -114,7 +193,8 @@ While this is possible in some cases, such as SaaS applications, there are some In these cases, you can create a production branch that reflects the deployed code. You can deploy a new version by merging `main` into the `production` branch. -While not shown in the graph below, the work on the `main` branch works just like in GitHub flow, i.e. with feature-branches being merged into `main`. +While not shown in the graph below, the work on the `main` branch works just like in GitHub flow: +with feature branches being merged into `main`. ```mermaid graph TD @@ -136,7 +216,7 @@ This time is pretty accurate if you automatically deploy your production branch. If you need a more exact time, you can have your deployment script create a tag on each deployment. This flow prevents the overhead of releasing, tagging, and merging that happens with Git flow. -## Environment branches with GitLab flow +## Environment branches with GitLab Flow It might be a good idea to have an environment that is automatically updated to the `staging` branch. Only, in this case, the name of this environment might differ from the branch name. @@ -170,12 +250,12 @@ In this case, deploy the `staging` branch to your staging environment. To deploy to pre-production, create a merge request from the `staging` branch to the `pre-prod` branch. Go live by merging the `pre-prod` branch into the `production` branch. This workflow, where commits only flow downstream, ensures that everything is tested in all environments. -If you need to cherry-pick a commit with a hotfix, it is common to develop it on a feature branch and merge it into `production` with a merge request. +To cherry-pick a commit with a hotfix, develop it on a feature branch and merge it into `production` with a merge request. In this case, do not delete the feature branch yet. If `production` passes automatic testing, you then merge the feature branch into the other branches. If this is not possible because more manual testing is required, you can send merge requests from the feature branch to the downstream branches. -## Release branches with GitLab flow +## Release branches with GitLab Flow You should work with release branches only if you need to release software to the outside world. In this case, each branch contains a minor version, such as @@ -202,13 +282,13 @@ Create stable branches using `main` as a starting point, and branch as late as p By doing this, you minimize the length of time during which you have to apply bug fixes to multiple branches. After announcing a release branch, only add serious bug fixes to the branch. If possible, first merge these bug fixes into `main`, and then cherry-pick them into the release branch. -If you start by merging into the release branch, you might forget to cherry-pick them into `main`, and then you'd encounter the same bug in subsequent releases. +If you initially merged into the release branch and then forgot to cherry-pick to `main`, you'd encounter the same bug in subsequent releases. Merging into `main` and then cherry-picking into release is called an "upstream first" policy, which is also practiced by [Google](https://www.chromium.org/chromium-os/chromiumos-design-docs/upstream-first/) and [Red Hat](https://www.redhat.com/en/blog/a-community-for-using-openstack-with-red-hat-rdo). Every time you include a bug fix in a release branch, increase the patch version (to comply with [Semantic Versioning](https://semver.org/)) by setting a new tag. Some projects also have a stable branch that points to the same commit as the latest released branch. In this flow, it is not common to have a production branch (or Git flow `main` branch). -## Merge/pull requests with GitLab flow +## Merge/pull requests with GitLab Flow ![Merge request with inline comments](img/gitlab_flow_mr_inline_comments.png) @@ -226,12 +306,17 @@ The merge request serves as a code review tool, and no separate code review tool If the review reveals shortcomings, anyone can commit and push a fix. Usually, the person to do this is the creator of the merge request. The diff in the merge request automatically updates when new commits are pushed to the branch. +In GitLab Flow, you can configure your pipeline to run every time you commit changes to a branch. This type of pipeline is called a branch pipeline. Alternatively, you can configure your pipeline to run every time you make changes to the source branch for a merge request. This type of pipeline is called a [merge request pipeline](../ci/pipelines/merge_request_pipelines.md). In GitLab Flow, you can also take advantage of our [Review Apps](../ci/review_apps/index.md) capability, which are a collaboration tool that provide an environment to showcase product changes. Review Apps provide an automatic live preview of changes made in a feature branch by spinning up a dynamic environment for your merge requests allowing stakeholders to see your changes without needing to check out your branch and run your changes in a sandbox environment. When your changes are merged, Review Apps clean up the dynamic environment and related resources preventing environment sprawl. -When you are ready for your feature branch to be merged, assign the merge request to the person who knows most about the codebase you are changing. +When you are ready to merge your feature branch, assign the merge request to a maintainer for the project. Also, mention any other people from whom you would like feedback. After the assigned person feels comfortable with the result, they can merge the branch. +In GitLab Flow, a [merged results pipeline](../ci/pipelines/merged_results_pipelines.md) runs against the results of the source and target branches merged together. If the assigned person does not feel comfortable, they can request more changes or close the merge request without merging. +NOTE: +In your pipelines, you can include any automated CI tests for unit, security vulnerabilities, code quality, performance, dependency, etc. Information related to the pipeline runs as well as results of tests are all displayed as widgets in the merge request window, so that you can access and visualize all these from a central location. + In GitLab, it is common to protect the long-lived branches, such as the `main` branch, so [most developers can't modify them](../user/permissions.md). So, if you want to merge into a protected branch, assign your merge request to someone with the Maintainer role. @@ -246,9 +331,9 @@ When you reopen an issue you need to create a new merge request. ![Remove checkbox for branch in merge requests](img/gitlab_flow_remove_checkbox.png) -## Issue tracking with GitLab flow +## Issue tracking with GitLab Flow -GitLab flow is a way to make the relation between the code and the issue tracker more transparent. +GitLab Flow is a way to make the relation between the code and the issue tracker more transparent. Any significant change to the code should start with an issue that describes the goal. Having a reason for every code change helps to inform the rest of the team and to keep the scope of a feature branch small. @@ -256,7 +341,8 @@ In GitLab, each change to the codebase starts with an issue in the issue trackin If there is no issue yet, create the issue if the change requires more than an hour's work. In many organizations, raising an issue is part of the development process because they are used in sprint planning. The issue title should describe the desired state of the system. -For example, the issue title "As an administrator, I want to remove users without receiving an error" is better than "Administrators can't remove users." +For example, the issue title `As an administrator, I want to remove users without receiving an error` +is better than "Administrators can't remove users." When you are ready to code, create a branch for the issue from the `main` branch. This branch is the place for any work related to this change. @@ -268,7 +354,7 @@ When you are done or want to discuss the code, open a merge request. A merge request is an online place to discuss the change and review the code. If you open the merge request but do not assign it to anyone, it is a [draft merge request](../user/project/merge_requests/drafts.md). -These are used to discuss the proposed implementation but are not ready for inclusion in the `main` branch yet. +Drafts are used to discuss the proposed implementation but are not ready for inclusion in the `main` branch yet. Start the title of the merge request with `[Draft]`, `Draft:` or `(Draft)` to prevent it from being merged before it's ready. When you think the code is ready, assign the merge request to a reviewer. @@ -284,6 +370,8 @@ In this case, it is no problem to reuse the same branch name, because the first At any time, there is at most one branch for every issue. It is possible that one feature branch solves more than one issue. +In GitLab Flow, you can create a merge request from the issue itself. When you do it this way, a feature branch and its related merge request are automatically created and associated to each other and the merge request is automatically related to the issue. In addition, when the merge request is merged the issue is automatically closed for you. + ## Linking and closing issues from merge requests Link to issues by mentioning them in commit messages or the description of a merge request, for example, "Fixes #16" or "Duck typing is preferred. See #12." @@ -291,6 +379,8 @@ GitLab then creates links to the mentioned issues and creates comments in the is To automatically close linked issues, mention them with the words "fixes" or "closes," for example, "fixes #14" or "closes #67." GitLab closes these issues when the code is merged into the default branch. +Like mentioned in the previous section, in GitLab Flow, you can create a merge request from the issue itself. When you do it this way, a feature branch and its related merge request are automatically created and associated to each other and the merge request is automatically related to the issue. In addition, when the merge request is merged the issue is automatically closed for you. + If you have an issue that spans across multiple repositories, create an issue for each repository and link all issues to a parent issue. ## Squashing commits with rebase @@ -354,12 +444,16 @@ However, as discussed in [the section about rebasing](#squashing-commits-with-re Rebasing could create more work, as every time you rebase, you may need to resolve the same conflicts. Sometimes you can reuse recorded resolutions (`rerere`), but merging is better, because you only have to resolve conflicts once. -Atlassian has [a more thorough explanation of the tradeoffs between merging and rebasing](https://www.atlassian.com/blog/git/git-team-workflows-merge-or-rebase) on their blog. +You can read a more thorough explanation of the tradeoffs between merging and rebasing [here](https://git-scm.com/book/en/v2/Git-Branching-Rebasing#:~:text=Final%20commit%20history-,The,-Perils%20of%20Rebasing). A good way to prevent creating many merge commits is to not frequently merge `main` into the feature branch. -There are three reasons to merge in `main`: utilizing new code, resolving merge conflicts, and updating long-running branches. +Three reasons to merge in `main`: -If you need to use some code that was introduced in `main` after you created the feature branch, you can often solve this by just cherry-picking a commit. +1. Utilizing new code. +1. Resolving merge conflicts. +1. Updating long-running branches. + +To use some code that was introduced in `main` after you created the feature branch, cherry-pick a commit. If your feature branch has a merge conflict, creating a merge commit is a standard way of solving this. @@ -375,7 +469,7 @@ If your feature branches often take more than a day of work, try to split your f If you need to keep a feature branch open for more than a day, there are a few strategies to keep it up-to-date. One option is to use continuous integration (CI) to merge in `main` at the start of the day. Another option is to only merge in from well-defined points in time, for example, a tagged release. -You could also use [feature toggles](https://martinfowler.com/bliki/FeatureToggle.html) to hide incomplete features so you can still merge back into `main` every day. +You could also use [feature toggles](https://martinfowler.com/bliki/FeatureToggle.html) or [feature flags](../operations/feature_flags.md) to hide incomplete features so you can still merge back into `main` every day. NOTE: Don't confuse automatic branch testing with continuous integration. @@ -387,7 +481,7 @@ In conclusion, you should try to prevent merge commits, but not eliminate them. Your codebase should be clean, but your history should represent what actually happened. Developing software happens in small, messy steps, and it is OK to have your history reflect this. You can use tools to view the network graphs of commits and understand the messy history that created your code. -If you rebase code, the history is incorrect, and there is no way for tools to remedy this because they can't deal with changing commit identifiers. +If you rebase code, the commit history changes. Because of changed commit identifiers, tools can't restore the commit history. ## Commit often and push frequently @@ -419,8 +513,8 @@ The words "change," "improve," "fix," and "refactor" don't add much information For more information, see Tim Pope's excellent [note about formatting commit messages](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html). To add more context to a commit message, consider adding information regarding the -origin of the change. For example, the URL of a GitLab issue, or a Jira issue number, -containing more information for users who need in-depth context about the change. +origin of the change, such the GitLab issue URL or Jira issue number. That way, you provide +more information for users who need in-depth context about the change. For example: @@ -434,9 +528,9 @@ Issue: gitlab.com/gitlab-org/gitlab/-/issues/1 In old workflows, the continuous integration (CI) server commonly ran tests on the `main` branch only. Developers had to ensure their code did not break the `main` branch. -When using GitLab flow, developers create their branches from this `main` branch, so it is essential that it never breaks. +When using GitLab Flow, developers create their branches from this `main` branch, so it is essential that it never breaks. Therefore, each merge request must be tested before it is accepted. -CI software like Travis CI and GitLab CI/CD show the build results right in the merge request itself to simplify the process. +CI software like GitLab CI/CD shows the build results right in the merge request itself to simplify the process. There is one drawback to testing merge requests: the CI server only tests the feature branch itself, not the merged result. Ideally, the server could also test the `main` branch after each change. @@ -445,6 +539,8 @@ Because feature branches should be short-lived, testing just the branch is an ac If new commits in `main` cause merge conflicts with the feature branch, merge `main` back into the branch to make the CI server re-run the tests. As said before, if you often have feature branches that last for more than a few days, you should make your issues smaller. +In GitLab Flow, your can include automated CI tests in your branch or merge request pipelines, which can run when you commit changes to a branch. + ## Working with feature branches When creating a feature branch, always branch from an up-to-date `main`. diff --git a/doc/topics/offline/quick_start_guide.md b/doc/topics/offline/quick_start_guide.md index 80ce703f7db..57c7a80fa3e 100644 --- a/doc/topics/offline/quick_start_guide.md +++ b/doc/topics/offline/quick_start_guide.md @@ -23,7 +23,7 @@ For a video walkthrough of this process, see [Offline GitLab Installation: Downl You should [manually download the GitLab package](../../update/package/index.md#upgrade-using-a-manually-downloaded-package) and relevant dependencies using a server of the same operating system type that has access to the Internet. -If your offline environment has no local network access, you must manually transport across the relevant package files through physical media, such as a USB drive, or writable DVD. +If your offline environment has no local network access, you must manually transport the relevant package through physical media, such as a USB drive. In Ubuntu, this can be performed on a server with Internet access using the following commands: @@ -71,7 +71,7 @@ sudo EXTERNAL_URL="http://my-host.internal" dpkg -i .deb ## Enabling SSL Follow these steps to enable SSL for your fresh instance. These steps reflect those for -[manually configuring SSL in Omnibus's NGINX configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#configure-https-manually): +[manually configuring SSL in Omnibus's NGINX configuration](https://docs.gitlab.com/omnibus/settings/ssl/index.html#configure-https-manually): 1. Make the following changes to `/etc/gitlab/gitlab.rb`: @@ -200,7 +200,7 @@ done. ### Disable Version Check and Service Ping -The Version Check and Service Ping services improve the GitLab user experience and ensure that +Version Check and Service Ping improve the GitLab user experience and ensure that users are on the most up-to-date instances of GitLab. These two services can be turned off for offline environments so that they do not attempt and fail to reach out to GitLab services. @@ -214,3 +214,89 @@ and Praefect servers so they can use an accessible NTP server. On offline instances, the [GitLab Geo check Rake task](../../administration/geo/replication/troubleshooting.md#can-geo-detect-the-current-site-correctly) always fails because it uses `pool.ntp.org`. This error can be ignored but you can [read more about how to work around it](../../administration/geo/replication/troubleshooting.md#message-machine-clock-is-synchronized--exception). + +## Enabling the package metadata database + +Enabling the package metadata database is required to enable [license scanning of CycloneDX files](../../user/compliance/license_scanning_of_cyclonedx_files). +This process requires usage of the GitLab License Database, which is licensed under the [EE License](https://storage.googleapis.com/prod-export-license-bucket-1a6c642fc4de57d4/v1/LICENSE). +Note the following in relation to use of the License Database: + +- We may change or discontinue all or any part of the License Database, at any time and without notice, at our sole discretion. +- The License Database may contain links to third-party websites or resources. We provide these links only as a convenience and are not responsible for any third-party data, content, products, or services from those websites or resources or links displayed on such websites. +- The License Database is based in part on information made available by third parties, and GitLab is not responsible for the accuracy or completeness of content made available. + +### Using the gsutil tool to download the package metadata exports + +1. Install the [`gsutil`](https://cloud.google.com/storage/docs/gsutil_install) tool. +1. Find the root of the GitLab Rails directory. + + ```shell + export GITLAB_RAILS_ROOT_DIR="$(gitlab-rails runner 'puts Rails.root.to_s')" + echo $GITLAB_RAILS_ROOT_DIR + ``` + +1. Download the package metadata exports. + + ```shell + # To download the package metadata exports, an outbound connection to Google Cloud Storage bucket must be allowed. + mkdir $GITLAB_RAILS_ROOT_DIR/vendor/package_metadata_db/ + gsutil -m rsync -r -d gs://prod-export-license-bucket-1a6c642fc4de57d4 $GITLAB_RAILS_ROOT_DIR/vendor/package_metadata_db/ + + # Alternatively, if the GitLab instance is not allowed to connect to the Google Cloud Storage bucket, the package metadata + # exports can be downloaded using a machine with the allowed access, and then copied to the root of the GitLab Rails directory. + rsync rsync://example_username@gitlab.example.com/package_metadata_db $GITLAB_RAILS_ROOT_DIR/vendor/package_metadata_db/ + ``` + +### Using the Google Cloud Storage REST API to download the package metadata exports + +The package metadata exports can also be downloaded using the Google Cloud Storage API. The contents are available at [https://storage.googleapis.com/storage/v1/b/prod-export-license-bucket-1a6c642fc4de57d4/o](https://storage.googleapis.com/storage/v1/b/prod-export-license-bucket-1a6c642fc4de57d4/o). The following is an example of how this can be downloaded using [cURL](https://curl.se/) and [jq](https://stedolan.github.io/jq/). + +```shell +#!/bin/bash + +set -euo pipefail + +GITLAB_RAILS_ROOT_DIR="$(gitlab-rails runner 'puts Rails.root.to_s')" +PKG_METADATA_DIR="$GITLAB_RAILS_ROOT_DIR/vendor/package_metadata_db" +PKG_METADATA_MANIFEST_OUTPUT_FILE="/tmp/license_db_export_manifest.json" +PKG_METADATA_DOWNLOADS_OUTPUT_FILE="/tmp/license_db_object_links.tsv" + +# Download the contents of the bucket +curl --silent --show-error --request GET "https://storage.googleapis.com/storage/v1/b/prod-export-license-bucket-1a6c642fc4de57d4/o" > "$PKG_METADATA_MANIFEST_OUTPUT_FILE" + +# Parse the links and names for the bucket objects and output them into a tsv file +jq -r '.items[] | [.name, .mediaLink] | @tsv' "$PKG_METADATA_MANIFEST_OUTPUT_FILE" > "$PKG_METADATA_DOWNLOADS_OUTPUT_FILE" + +echo -e "Saving package metadata exports to $PKG_METADATA_DIR\n" + +# Track how many objects will be downloaded +INDEX=1 +TOTAL_OBJECT_COUNT="$(wc -l $PKG_METADATA_DOWNLOADS_OUTPUT_FILE | awk '{print $1}')" + +# Download the objects +while IFS= read -r line; do + FILE="$(echo -n $line | awk '{print $1}')" + URL="$(echo -n $line | awk '{print $2}')" + OUTPUT_DIR="$(dirname $PKG_METADATA_DIR/$FILE)" + OUTPUT_PATH="$PKG_METADATA_DIR/$FILE" + + echo "Downloading $FILE" + + curl --progress-bar --create-dirs --output "$OUTPUT_PATH" --request "GET" "$URL" + + echo -e "$INDEX of $TOTAL_OBJECT_COUNT objects downloaded\n" + + let INDEX=(INDEX+1) +done < "$PKG_METADATA_DOWNLOADS_OUTPUT_FILE" + +echo "All objects saved to $PKG_METADATA_DIR" +``` + +### Automatic synchronization + +Your GitLab instance is synchronized [every hour](https://gitlab.com/gitlab-org/gitlab/-/blob/d4331343d26d6e2a81fadd8f7ecd72f7cb74d04d/config/initializers/1_settings.rb#L831-832) with the contents of the `package_metadata_db` directory. +To automatically update your local copy with the upstream changes, a cron job can be added to periodically download new exports. For example, the following crontabs can be added to setup a cron job that runs every 30 minutes. + +```plaintext +*/30 * * * * gsutil -m rsync -r -d gs://prod-export-license-bucket-1a6c642fc4de57d4 $GITLAB_RAILS_ROOT_DIR/vendor/package_metadata_db/ +``` diff --git a/doc/topics/release_your_application.md b/doc/topics/release_your_application.md index 819ed0a1224..e8dba553cab 100644 --- a/doc/topics/release_your_application.md +++ b/doc/topics/release_your_application.md @@ -26,6 +26,7 @@ release features incrementally. - [Auto Deploy](autodevops/stages.md#auto-deploy) is the DevOps stage dedicated to software deployment using GitLab CI/CD. Auto Deploy has built-in support for EC2 and ECS deployments. - Deploy to Kubernetes clusters by using the [GitLab agent](../user/clusters/agent/install/index.md). +- View an example of [how to structure a GitOps deployment repository](../user/clusters/agent/gitops/example_repository_structure.md). - Use Docker images to run AWS commands from GitLab CI/CD, and a template to facilitate [deployment to AWS](../ci/cloud_deployment). - Use GitLab CI/CD to target any type of infrastructure accessible by GitLab Runner. diff --git a/doc/topics/set_up_organization.md b/doc/topics/set_up_organization.md index 855b4962960..cf0a0ae4ab7 100644 --- a/doc/topics/set_up_organization.md +++ b/doc/topics/set_up_organization.md @@ -11,7 +11,7 @@ and give everyone access to the projects they need. - [Namespaces](../user/namespace/index.md) - [Members](../user/project/members/index.md) -- [Organization](../user/workspace/index.md) _(In development)_ +- [Organization](../user/organization/index.md) _(In development)_ - [Groups](../user/group/index.md) - [User account options](../user/profile/index.md) - [SSH keys](../user/ssh.md) diff --git a/doc/topics/your_work.md b/doc/topics/your_work.md index 862f9ae8430..c57c9b52604 100644 --- a/doc/topics/your_work.md +++ b/doc/topics/your_work.md @@ -1,18 +1,11 @@ --- -stage: Manage -group: Foundations -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../tutorials/left_sidebar/index.md' +remove_date: '2023-08-03' --- -# Your work sidebar +This document was moved to [another location](../tutorials/left_sidebar/index.md). -- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/384342) in GitLab 15.9. - -The **Your work** left sidebar provides access to your: - -- [Projects](../user/project/working_with_projects.md#view-projects) -- [Groups](../user/group/index.md) -- [Issues](../user/project/issues/index.md) -- [Merge requests](../user/project/merge_requests/index.md) -- [To-do List](../user/todos.md) -- [Milestones](../user/project/milestones/index.md) + + + + diff --git a/doc/tutorials/agile_sprint.md b/doc/tutorials/agile_sprint.md index ff0b17d7eb0..84927fe0a66 100644 --- a/doc/tutorials/agile_sprint.md +++ b/doc/tutorials/agile_sprint.md @@ -1,101 +1,11 @@ --- -stage: none -group: Tutorials -info: For assistance with this tutorial, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. +redirect_to: 'agile_sprint/index.md' +remove_date: '2023-07-21' --- -# Tutorial: Use GitLab to run an Agile iteration +This document was moved to [another location](agile_sprint/index.md). -To run an Agile development iteration in GitLab, you use multiple GitLab features -that work together. - -To run an Agile iteration from GitLab: - -1. Create a group. -1. Create a project. -1. Set up an iteration cadence. -1. Create scoped labels. -1. Create your epics and issues. -1. Create an issue board. - -After you've created these core components, you can begin running your iterations. - -## Create a group - -Iteration cadences are created at the group level, so start by -[creating one](../user/group/manage.md#create-a-group) if you don't have one already. - -You use groups to manage one or more related projects at the same time. -You add your users as members in the group, and assign them a role. Roles determine -the [level of permissions](../user/permissions.md) each user has on the projects in the group. -Membership automatically cascades down to all subgroups and projects. - -## Create a project - -Now [create one or more projects](../user/project/index.md#create-a-project) in your group. -There are several different ways to create a project. A project contains -your code and pipelines, but also the issues that are used for planning your upcoming code changes. - -## Set up an iteration cadence - -Before you start creating epics or issues, create an -[iteration cadence](../user/group/iterations/index.md#iteration-cadences). -Iteration cadences contain the individual, sequential iteration timeboxes for planning and reporting -on your issues. - -When creating an iteration cadence, you can decide whether to automatically manage the iterations or -disable the automated scheduling to -[manually manage the iterations](../user/group/iterations/index.md#manual-iteration-management). - -Similar to membership, iterations cascade down your group, subgroup, and project hierarchy. If your -team works across many groups, subgroups, and projects, create the iteration cadence in the top-most -group shared by all projects that contain the team's issues as illustrated by the diagram below. - -```mermaid -graph TD - Group --> SubgroupA --> Project1 - Group --> SubgroupB --> Project2 - Group --> IterationCadence -``` - -## Create scoped labels - -You should also [create scoped labels](../user/project/labels.md) in the same group where you created -your iteration cadence. Labels help you -organize your epics, issues, and merge requests, as well as help you -to visualize the flow of issues in boards. For example, you can use scoped labels like -`workflow::planning`, `workflow::ready for development`, `workflow::in development`, and `workflow::complete` -to indicate the status of an issue. You can also leverage scoped labels to denote the type of issue -or epic such as `type::feature`, `type::defect`, and `type::maintenance`. - -## Create your epics and issues - -Now you can get started planning your iterations. Start by creating [epics](../user/group/epics/index.md) -in the group where you created your iteration cadence, -then create child [issues](../user/project/issues/index.md) in one or more of your projects. -Add labels to each as needed. - -## Create an issue board - -[Issue boards](../user/project/issue_board.md) help you plan your upcoming iterations or visualize -the workflow of the iteration currently in progress. List columns can be created based on label, -assignee, iteration, or milestone. You can also filter the board by multiple attributes and group -issues by their epic. - -In the group where you created your iteration cadence and labels, -[create an issue board](../user/project/issue_board.md#create-an-issue-board) and name it -"Iteration Planning." Then, create lists for each of your iterations. You can then drag issues from -the "Open" list into iteration lists to schedule them for upcoming iterations. - -To visualize the workflow for issues in the current iteration, create another issue board called -"Current Iteration." When you're creating the board: - -1. Select **Edit board**. -1. Next to **Iteration**, select **Edit**. -1. From the dropdown list, select **Current iteration**. -1. Select **Save changes**. - -Your board will now only ever show issues that are in the current iteration. -You can start adding lists for each of the `workflow::...` labels you created previously. - -Now you're ready to start development. + + + + \ No newline at end of file diff --git a/doc/tutorials/agile_sprint/index.md b/doc/tutorials/agile_sprint/index.md new file mode 100644 index 00000000000..0ce100df65e --- /dev/null +++ b/doc/tutorials/agile_sprint/index.md @@ -0,0 +1,101 @@ +--- +stage: none +group: Tutorials +info: For assistance with this tutorial, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. +--- + +# Tutorial: Use GitLab to run an Agile iteration + +To run an Agile development iteration in GitLab, you use multiple GitLab features +that work together. + +To run an Agile iteration from GitLab: + +1. Create a group. +1. Create a project. +1. Set up an iteration cadence. +1. Create scoped labels. +1. Create your epics and issues. +1. Create an issue board. + +After you've created these core components, you can begin running your iterations. + +## Create a group + +Iteration cadences are created at the group level, so start by +[creating one](../../user/group/manage.md#create-a-group) if you don't have one already. + +You use groups to manage one or more related projects at the same time. +You add your users as members in the group, and assign them a role. Roles determine +the [level of permissions](../../user/permissions.md) each user has on the projects in the group. +Membership automatically cascades down to all subgroups and projects. + +## Create a project + +Now [create one or more projects](../../user/project/index.md#create-a-project) in your group. +There are several different ways to create a project. A project contains +your code and pipelines, but also the issues that are used for planning your upcoming code changes. + +## Set up an iteration cadence + +Before you start creating epics or issues, create an +[iteration cadence](../../user/group/iterations/index.md#iteration-cadences). +Iteration cadences contain the individual, sequential iteration timeboxes for planning and reporting +on your issues. + +When creating an iteration cadence, you can decide whether to automatically manage the iterations or +disable the automated scheduling to +[manually manage the iterations](../../user/group/iterations/index.md#manual-iteration-management). + +Similar to membership, iterations cascade down your group, subgroup, and project hierarchy. If your +team works across many groups, subgroups, and projects, create the iteration cadence in the top-most +group shared by all projects that contain the team's issues as illustrated by the diagram below. + +```mermaid +graph TD + Group --> SubgroupA --> Project1 + Group --> SubgroupB --> Project2 + Group --> IterationCadence +``` + +## Create scoped labels + +You should also [create scoped labels](../../user/project/labels.md) in the same group where you created +your iteration cadence. Labels help you +organize your epics, issues, and merge requests, as well as help you +to visualize the flow of issues in boards. For example, you can use scoped labels like +`workflow::planning`, `workflow::ready for development`, `workflow::in development`, and `workflow::complete` +to indicate the status of an issue. You can also leverage scoped labels to denote the type of issue +or epic such as `type::feature`, `type::defect`, and `type::maintenance`. + +## Create your epics and issues + +Now you can get started planning your iterations. Start by creating [epics](../../user/group/epics/index.md) +in the group where you created your iteration cadence, +then create child [issues](../../user/project/issues/index.md) in one or more of your projects. +Add labels to each as needed. + +## Create an issue board + +[Issue boards](../../user/project/issue_board.md) help you plan your upcoming iterations or visualize +the workflow of the iteration currently in progress. List columns can be created based on label, +assignee, iteration, or milestone. You can also filter the board by multiple attributes and group +issues by their epic. + +In the group where you created your iteration cadence and labels, +[create an issue board](../../user/project/issue_board.md#create-an-issue-board) and name it +"Iteration Planning." Then, create lists for each of your iterations. You can then drag issues from +the "Open" list into iteration lists to schedule them for upcoming iterations. + +To visualize the workflow for issues in the current iteration, create another issue board called +"Current Iteration." When you're creating the board: + +1. Select **Edit board**. +1. Next to **Iteration**, select **Edit**. +1. From the dropdown list, select **Current iteration**. +1. Select **Save changes**. + +Your board will now only ever show issues that are in the current iteration. +You can start adding lists for each of the `workflow::...` labels you created previously. + +Now you're ready to start development. diff --git a/doc/tutorials/boards_for_teams/index.md b/doc/tutorials/boards_for_teams/index.md new file mode 100644 index 00000000000..e37bf5a2d31 --- /dev/null +++ b/doc/tutorials/boards_for_teams/index.md @@ -0,0 +1,202 @@ +--- +stage: Plan +group: Project Management +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Tutorial: Set up issue boards for team hand-off **(PREMIUM)** + + + +This tutorial shows you how to set up [issue boards](../../user/project/issue_board.md) and [scoped labels](../../user/project/labels.md#scoped-labels) for two teams that work on issues in sequence. + +In this example, you'll create two issue boards for the UX and Frontend teams. +Using the following steps, you can create issue boards and workflows for more sub-teams, like Backend +or Quality Assurance. +To learn how we use workflow labels at GitLab, see [Product Development Flow](https://about.gitlab.com/handbook/product-development-flow). + +To set up issue boards for multiple teams: + +1. [Create a group](#create-a-group) +1. [Create a project](#create-a-project) +1. [Create labels](#create-labels) +1. [Create team issue boards](#create-team-issue-boards) +1. [Create issues for features](#create-issues-for-features) + +## The goal workflow + +After you set up everything, the two teams will be able to hand off issues from one board to another, for example, like this: + +1. The project lead adds the `Workflow::Ready for design` and `Frontend` labels to a feature issue called **Redesign user profile page**. +1. A product designer on the UX team: + 1. Checks the `Workflow::Ready for design` list on the **UX workflow** board and decides to work on the profile page redesign. + + + + 1. Assigns themselves to the issue. + 1. Drags the issue card to the `Workflow::Design` list. The previous workflow label is automatically removed. + 1. Creates the ✨new designs✨. + 1. [Adds the designs to the issue](../../user/project/issues/design_management.md). + 1. Drags the issue card to the `Workflow::Ready for development` list, which adds this label and removes any other `Workflow::` label. + 1. Unassigns themselves from the issue. +1. A developer on the Frontend team: + 1. Checks the `Workflow::Ready for development` list on the **Frontend workflow** board and chooses an issue to work on. + + + + 1. Assigns themselves to the issue. + 1. Drags the issue card to the `Workflow::In development` list. The previous workflow label is automatically removed. + 1. Adds the frontend code in a [merge request](../../user/project/merge_requests/index.md). + 1. Adds the `Workflow::Complete` label. + +## Create a group + +To prepare for when your project grows, start by creating a group. +You use groups to manage one or more related projects at the same time. +You add your users as members in the group, and assign them a role. + +Prerequisites: + +- If you're using an existing group for this tutorial, make sure you have at least the Reporter role + for the group. + +To create a group: + +1. On the top bar, select **Create new... > New group**. +1. Select **Create group**. +1. Complete the fields. Name your group `Paperclip Software Factory`. +1. Select **Create group**. + +You've created an empty group. Next, you'll create a project that will store your issues and code. + +## Create a project + +The main code development work happens in projects and their repositories. +A project contains your code and pipelines, but also the issues that are used for planning your +upcoming code changes. + +Prerequisites: + +- If you're using an existing project for this tutorial, make sure you have at least the Reporter role + for the project. + +To create a blank project: + +1. In your group, on the right of the page, select **New project**. +1. Select **Create blank project**. +1. Enter the project details: + - In the **Project name** field, name your project `Paperclip Assistant`. +1. Select **Create project**. + +## Create labels + +You need a team label and a set of workflow labels to show where in the development cycle an issue is. + +You could create these labels in your `Paperclip Assistant` project, but it's better to create them +in the `Paperclip Software Factory` group. This way, these labels will also be available in all the other +projects you create later. + +To create each label: + +1. On the top bar, select **Main menu > Group** and find your **Paperclip Software Factory** group. +1. On the left sidebar, select **Group information > Labels**. +1. Select **New label**. +1. In the **Title** field, enter the name of the label. Start with `Frontend`. +1. Optional. Select a color by selecting from the available colors, or enter a hex color value for + a specific color in the **Background color** field. +1. Select **Create label**. + +Repeat these steps to create all the labels you'll need: + +- `Frontend` +- `Workflow::Ready for design` +- `Workflow::Design` +- `Workflow::Ready for development` +- `Workflow::In development` +- `Workflow::Complete` + +## Create team issue boards + +Like with labels, you could create your issue boards in the **Paperclip Assistant** project, +but it can be better to have them in the **Paperclip Software Factory** group. This way, you'll be able +to manage issues from all the projects that you might create later in this group. + +To create a new group issue board: + +1. On the top bar, select **Main menu > Group** and find your **Paperclip Software Factory** group. +1. On the left sidebar, select **Issues > Boards**. +1. Create the UX workflow and Frontend workflow boards. + +To create the **UX workflow** issue board: + +1. In the upper-left corner of the issue board page, select the dropdown list with the current board name. +1. Select **Create new board**. +1. In the **Title field**, enter `UX workflow`. +1. Clear the **Show the Open list** and **Show the Closed list** checkboxes. +1. Select **Create board**. You should see an empty board. + + + +1. Create a list for the `Workflow::Ready for design` label: + 1. In the upper-left corner of the issue board page, select **Create list**. + 1. In the column that appears, from the **Value** dropdown list, select the `Workflow::Ready for design` label. + 1. Select **Add to board**. +1. Repeat the previous step for labels `Workflow::Design` and `Workflow::Ready for development`. + +To create the **Frontend workflow** board: + +1. In the upper-left corner of the issue board page, select the dropdown list with the current board name. +1. Select **Create new board**. +1. In the **Title field**, enter `Frontend workflow`. +1. Clear the **Show the Open list** and **Show the Closed list** checkboxes. +1. Expand **Scope**. +1. Next to **Labels**, select **Edit** and select the `Frontend` label. +1. Select **Create board**. +1. Create a list for the `Workflow::Ready for development` label: + 1. In the upper-left corner of the issue board page, select **Create list**. + 1. In the column that appeared, from the **Value** dropdown list, select the `Workflow::Ready for development` label. + 1. Select **Add to board**. +1. Repeat the previous step for labels `Workflow::In development` and `Workflow::Complete`. + +For now, lists in both your boards should be empty. Next, you'll populate them with some issues. + +## Create issues for features + +To track upcoming features, enhancements, and bugs, you must create some issues. +Issues belong in projects, but you can also create them directly from your issue board. + +To create an issue from your board: + +1. In the upper-left corner of the issue board page, select the dropdown list with the current board name. +1. Select **UX workflow**. +1. On the `Workflow::Ready for development` list, select **List actions** (**{ellipsis_v}**) **> Create new issue**. +1. Complete the fields: + 1. Under **Title**, enter `Redesign user profile page`. + 1. Under **Projects**, select **Paperclip Software Factory / Paperclip Assistant**. +1. Select **Create issue**. Because you created the new issue in the label list, it gets created + with this label. +1. Add the `Frontend` label, because only issues with this label appear on the Frontend team's board: + 1. Select the issue card (not its title), and a sidebar appears on the right. + 1. In the **Labels** section of the sidebar, select **Edit**. + 1. From the **Assign labels** dropdown list, select the `Workflow::Ready for design` and + `Frontend` labels. The selected labels are marked with a checkmark. + 1. To apply your changes to labels, select **X** next to **Assign labels** or select any area + outside the label section. + +Repeat these steps to create a few more issues with the same labels. + +You should now see at least one issue there, ready for your product designers to start working on! + + + +Congratulations! Now your teams can start collaborating on amazing software. + +## Learn more about project management in GitLab + +Find other tutorials about project management on the [tutorials page](../plan_and_track.md). diff --git a/doc/tutorials/build_application.md b/doc/tutorials/build_application.md new file mode 100644 index 00000000000..2e0130e46ca --- /dev/null +++ b/doc/tutorials/build_application.md @@ -0,0 +1,32 @@ +--- +stage: none +group: Tutorials +info: For assistance with this tutorials page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. +--- + +# Build your application + +## Learn about CI/CD pipelines + +Use CI/CD pipelines to automatically build, test, and deploy your code. + +| Topic | Description | Good for beginners | +|-------|-------------|--------------------| +| [Create and run your first GitLab CI/CD pipeline](../ci/quick_start/index.md) | Create a `.gitlab-ci.yml` file and start a pipeline. | **{star}** | +| [Create a complex pipeline](../ci/quick_start/tutorial.md) | Learn about the most commonly used GitLab CI/CD keywords by building an increasingly complex pipeline. | | +| [Get started: Learn about CI/CD](https://www.youtube.com/watch?v=sIegJaLy2ug) (9m 02s) | Learn about the `.gitlab-ci.yml` file and how it's used. | **{star}** | +| [GitLab CI/CD](https://levelup.gitlab.com/courses/continuous-integration-and-delivery-ci-cd-with-gitlab) | Learn about GitLab CI/CD and build a pipeline in this self-paced course. | **{star}** | +| [CI deep dive](https://www.youtube.com/watch?v=ZVUbmVac-m8&list=PL05JrBw4t0KorkxIFgZGnzzxjZRCGROt_&index=27) (22m 51s) | Take a closer look at pipelines and continuous integration concepts. | | +| [Set up CI/CD in the cloud](../ci/examples/index.md#cicd-in-the-cloud) | Learn how to set up CI/CD in different cloud-based environments. | | +| [Find CI/CD examples and templates](../ci/examples/index.md#cicd-examples) | Use these examples and templates to set up CI/CD for your use case. | | +| [Understand CI/CD rules](https://www.youtube.com/watch?v=QjQc-zeL16Q) (8m 56s) | Learn more about how to use CI/CD rules. | | +| [Use Auto DevOps to deploy an application](../topics/autodevops/cloud_deployments/auto_devops_with_gke.md) | Deploy an application to Google Kubernetes Engine (GKE). | | + +## Publish a static website + +Use GitLab Pages to publish a static website directly from your project. + +| Topic | Description | Good for beginners | +|-------|-------------|--------------------| +| [Create a Pages website from a CI/CD template](../user/project/pages/getting_started/pages_ci_cd_template.md) | Quickly generate a Pages website for your project using a CI/CD template for a popular Static Site Generator (SSG). | **{star}** | +| [Create a Pages website from scratch](../user/project/pages/getting_started/pages_from_scratch.md) | Create all the components of a Pages website from a blank project. | | diff --git a/doc/tutorials/compliance_pipeline/index.md b/doc/tutorials/compliance_pipeline/index.md new file mode 100644 index 00000000000..2dab7a7ecb1 --- /dev/null +++ b/doc/tutorials/compliance_pipeline/index.md @@ -0,0 +1,177 @@ +--- +stage: Govern +group: Compliance +info: For assistance with this tutorial, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. +--- + +# Tutorial: Create a compliance pipeline **(ULTIMATE)** + +You can use [compliance pipelines](../../user/group/compliance_frameworks.md#compliance-pipelines) to ensure specific +compliance-related jobs are run on pipelines for all projects in a group. Compliance pipelines are applied +to projects through [compliance frameworks](../../user/group/compliance_frameworks.md). + +In this tutorial, you: + +1. Create a [new group](#create-a-new-group). +1. Create a [new project for the compliance pipeline configuration](#create-a-new-compliance-pipeline-project). +1. Configure a [compliance framework](#configure-compliance-framework) to apply to other projects. +1. Create a [new project and apply the compliance framework](#create-a-new-project-and-apply-the-compliance-framework) to it. +1. Combine [compliance pipeline configuration and regular pipeline configuration](#combine-pipeline-configurations). + +Prerequisites: + +- Permission to create new top-level groups. + +## Create a new group + +Compliance frameworks are configured in top-level groups. In this tutorial, you create a top-level group that: + +- Contains two projects: + - The compliance pipeline project to store the compliance pipeline configuration. + - Another project that must run a job in its pipeline that is defined by the compliance pipeline configuration. +- Has the compliance framework to apply to projects. + +To create the new group: + +1. On the top bar, select **Create new... > New group**. +1. Select **Create group**. +1. In the **Group name** field, enter `Tutorial group`. +1. Select **Create group**. + +## Create a new compliance pipeline project + +Now you're ready to create a compliance pipeline project. This project contains the +[compliance pipeline configuration](../../user/group/compliance_frameworks.md#example-configuration) to apply to all +projects with the compliance framework applied. + +To create the compliance pipeline project: + +1. On the top bar, select **Main menu > Groups** and find the `Tutorial group` group. +1. Select **New project**. +1. Select **Create blank project**. +1. In the **Project name** field, enter `Tutorial compliance project`. +1. Select **Create project**. + +To add compliance pipeline configuration to `Tutorial compliance project`: + +1. On the top bar, select **Main menu > Projects** and find the `Tutorial compliance project` project. +1. On the left sidebar, select **CI/CD > Editor**. +1. Select **Configure pipeline**. +1. In the pipeline editor, replace the default configuration with: + + ```yaml + --- + compliance-job: + script: + - echo "Running compliance job required for every project in this group..." + ``` + +1. Select **Commit changes**. + +## Configure compliance framework + +The compliance framework is configured in the [new group](#create-a-new-group). + +To configure the compliance framework: + +1. On the top bar, select **Main menu > Groups** and find the `Tutorial group` group. +1. On the left sidebar, select **Settings > General**. +1. Expand **Compliance frameworks**. +1. Select **Add framework**. +1. In the **Name** field, enter `Tutorial compliance framework`. +1. In the **Description** field, enter `Compliance framework for tutorial`. +1. In the **Compliance pipeline configuration (optional)** field, enter + `.gitlab-ci.yml@tutorial-group/tutorial-compliance-project`. +1. In the **Background color** field, select a color of your choice. +1. Select **Add framework**. + +For convenience, make the new compliance framework the default for all new projects in the group: + +1. On the top bar, select **Main menu > Groups** and find the `Tutorial group` group. +1. On the left sidebar, select **Settings > General**. +1. Expand **Compliance frameworks**. +1. In the row for `Tutorial compliance framework`, select **Options** (**{ellipsis_v}**). +1. Select **Set default**. + +## Create a new project and apply the compliance framework + +Your compliance framework is ready, so you can now create projects in the group and they automatically run the +compliance pipeline configuration in their pipelines. + +To create a new project for running the compliance pipeline configuration: + +1. On the top bar, select **Main menu > Groups** and find the `Tutorial group` group. +1. Select **New project**. +1. Select **Create blank project**. +1. In the **Project name** field, enter `Tutorial project`. +1. Select **Create project**. + +On the project page, notice the `Tutorial compliance framework` label appears because that was set as the default +compliance framework for the group. + +Without any other pipeline configuration, `Tutorial project` can run the jobs defined in the compliance +pipeline configuration in `Tutorial compliance project`. + +To run the compliance pipeline configuration in `Tutorial project`: + +1. On the top bar, select **Main menu > Projects** and find the `Tutorial project` project. +1. Select **CI/CD Pipelines**. +1. Select **Run pipeline**. +1. On the **Run pipeline** page, select **Run pipeline**. + +Notice the pipeline runs a job called `compliance-job` in a **test** stage. Nice work, you've run your first compliance +job! + +## Combine pipeline configurations + +If you want projects to run their own jobs as well as the compliance pipeline jobs, you must combine the compliance +pipeline configuration and the regular pipeline configuration of the project. + +To combine the pipeline configurations, you must define the regular pipeline configuration and then update the +compliance pipeline configuration to refer to it. + +To create the regular pipeline configuration: + +1. On the top bar, select **Main menu > Projects** and find the `Tutorial project` project. +1. On the left sidebar, select **CI/CD > Editor**. +1. Select **Configure pipeline**. +1. In the pipeline editor, replace the default configuration with: + + ```yaml + --- + project-job: + script: + - echo "Running project job..." + ``` + +1. Select **Commit changes**. + +To combine the new project pipeline configuration with the compliance pipeline configuration: + +1. On the top bar, select **Main menu > Projects** and find the `Tutorial compliance project` project. +1. On the left sidebar, select **CI/CD > Editor**. +1. In the existing configuration, add: + + ```yaml + include: + - project: 'tutorial-group/tutorial-project' + file: '.gitlab-ci.yml' + ``` + +1. Select **Commit changes**. + +To confirm the regular pipeline configuration is combined with the compliance pipeline configuration: + +1. On the top bar, select **Main menu > Projects** and find the `Tutorial project` project. +1. Select **CI/CD Pipelines**. +1. Select **Run pipeline**. +1. On the **Run pipeline** page, select **Run pipeline**. + +Notice the pipeline runs two jobs in a **test** stage: + +- `compliance-job`. +- `project-job`. + +Congratulations, you've created and configured a compliance pipeline! + +See more [example compliance pipeline configurations](../../user/group/compliance_frameworks.md#example-configuration). diff --git a/doc/tutorials/convert_personal_namespace_into_group.md b/doc/tutorials/convert_personal_namespace_into_group.md new file mode 100644 index 00000000000..c1b3b58efb8 --- /dev/null +++ b/doc/tutorials/convert_personal_namespace_into_group.md @@ -0,0 +1,11 @@ +--- +redirect_to: 'convert_personal_namespace_to_group/index.md' +remove_date: '2023-07-21' +--- + +This document was moved to [another location](convert_personal_namespace_to_group/index.md). + + + + + \ No newline at end of file diff --git a/doc/tutorials/convert_personal_namespace_to_group/index.md b/doc/tutorials/convert_personal_namespace_to_group/index.md new file mode 100644 index 00000000000..53ad46effd9 --- /dev/null +++ b/doc/tutorials/convert_personal_namespace_to_group/index.md @@ -0,0 +1,95 @@ +--- +stage: Data Stores +group: Tenant Scale +info: For assistance with this tutorial, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. +--- + +# Tutorial: Convert a personal namespace into a group **(FREE SAAS)** + +If you've started out on GitLab with a personal [namespace](../../user/namespace/index.md), but now find +that you've outgrown its capabilities and its limitations hinder the collaboration on your projects, +you might want to switch to a group namespace instead. +A group namespace allows you to create multiple subgroups, and manage their members and permissions. + +You don't have to start from scratch - you can create a new group +and move your existing projects to the group to get the added benefits. +To find out how, see [Tutorial: Move your personal project to a group](../move_personal_project_to_group/index.md). + +But you can go one step further and convert your personal namespace into a group namespace, +so you get to keep the existing username and URL. For example, if your username is `alex`, +you can continue using the `https://gitlab.example.com/alex` URL for your group. + +This tutorial shows you how to convert your personal namespace into a group namespace +using the following steps: + +1. [Create a group](#create-a-group). +1. [Transfer projects from the personal namespace to the group](#transfer-projects-from-the-personal-namespace-to-the-group). +1. [Rename the original username](#rename-the-original-username). +1. [Rename the new group namespace to the original username](#rename-the-new-group-namespace-to-the-original-username). + +For example, if your username for a personal namespace is `alex`, first create a group namespace named `alex-group`. +Then, move all projects from the `alex` to the `alex-group` namespace. Finally, +rename the `alex` namespace to `alex-user`, and `alex-group` namespace to the now available `alex` username. + +## Create a group + +1. On the top bar, select **Main menu > Groups > View all groups**. +1. On the right of the page, select **New group**. +1. In **Group name**, enter a name for the group. +1. In **Group URL**, enter a path for the group, which is used as the namespace. + Don't worry about the actual path, this is only temporary. You'll change this URL to the username of the personal namespace in the [final step](#rename-the-new-group-namespace-to-the-original-username). +1. Choose the [visibility level](../../user/public_access.md). +1. Optional. Fill in information to personalize your experience. +1. Select **Create group**. + +## Transfer projects from the personal namespace to the group + +Next, you must transfer your projects from the personal namespace to the new group. +You can transfer only one project at a time, so if you want to transfer multiple projects, +you must perform the steps below for each project. + +Before you start the transfer process, make sure you: + +- Have the Owner role for the project. +- Remove [container images](../../user/packages/container_registry/index.md#move-or-rename-container-registry-repositories). + You can't transfer a project that contains container images. +- Remove npm packages. You can't update the root namespace of a project that contains npm packages. + +To transfer a project to a group: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > General**. +1. Expand **Advanced**. +1. Under **Transfer project**, choose the group to transfer the project to. +1. Select **Transfer project**. +1. Enter the project's name and select **Confirm**. + +## Rename the original username + +Next, rename the original username of the personal namespace, so that the username becomes available for the new group namespace. +You can keep on using the personal namespace for other personal projects, or [delete that user account](../../user/profile/account/delete_account.md) + +From the moment you rename the personal namespace, the username becomes available, so it's possible that someone else registers an account with it. To avoid this, you should [rename the new group](#rename-the-new-group-namespace-to-the-original-username) as soon as possible. + +To [change a user's username](../../user/profile/index.md#change-your-username): + +1. On the top bar, in the top-right corner, select your avatar. +1. Select **Edit profile**. +1. On the left sidebar, select **Account**. +1. In the **Change username** section, enter a new username as the path. +1. Select **Update username**. + +## Rename the new group namespace to the original username + +Finally, rename the new group's URL to the username of the original personal namespace. + +To [change your group path](../../user/group/manage.md#change-a-groups-path) (group URL): + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Settings > General page**. +1. Expand the **Advanced** section. +1. Under **Change group URL**, enter the user's original username. +1. Select **Change group URL**. + +That's it! You have now converted a personal namespace into a group, which opens up new possibilities of +working on projects and collaborating with more members. diff --git a/doc/tutorials/create_compliance_pipeline.md b/doc/tutorials/create_compliance_pipeline.md new file mode 100644 index 00000000000..ac5550ad6a4 --- /dev/null +++ b/doc/tutorials/create_compliance_pipeline.md @@ -0,0 +1,11 @@ +--- +redirect_to: 'compliance_pipeline/index.md' +remove_date: '2023-07-21' +--- + +This document was moved to [another location](compliance_pipeline/index.md). + + + + + \ No newline at end of file diff --git a/doc/tutorials/develop.md b/doc/tutorials/develop.md new file mode 100644 index 00000000000..08497a09830 --- /dev/null +++ b/doc/tutorials/develop.md @@ -0,0 +1,17 @@ +--- +stage: none +group: Tutorials +info: For assistance with this tutorials page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. +--- + +# Develop with GitLab + +## Integrate with GitLab + +GitLab [integrates](../user/project/integrations/index.md) with a number of third-party services, +enabling you to work with those services directly from GitLab. + +| Topic | Description | Good for beginners | +|-------|-------------|--------------------| +| [Integrate with Jira](https://about.gitlab.com/blog/2021/04/12/gitlab-jira-integration-selfmanaged/) | Configure the Jira integration, so you can work with Jira issues from GitLab. | | +| [Integrate with Gitpod](https://about.gitlab.com/blog/2021/07/19/teams-gitpod-integration-gitlab-speed-up-development/) | Integrate with Gitpod, to help speed up your development. | | diff --git a/doc/tutorials/fuzz_testing/index.md b/doc/tutorials/fuzz_testing/index.md new file mode 100644 index 00000000000..1d4985f9003 --- /dev/null +++ b/doc/tutorials/fuzz_testing/index.md @@ -0,0 +1,243 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Tutorial: Perform fuzz testing in GitLab **(ULTIMATE)** + +[Coverage-guided fuzz testing](../../user/application_security/coverage_fuzzing/index.md#coverage-guided-fuzz-testing-process) sends unexpected, malformed, or random data to your application, and then monitors +your application for unstable behaviors and crashes. + +This helps you discover bugs and potential security issues that other QA processes may miss. + +You should use fuzz testing in addition to other security scanners and your own test processes. +If you're using GitLab CI/CD, you can run fuzz tests as part your CI/CD workflow. + +To set up, configure, and perform coverage-guided fuzz testing +using JavaScript in this tutorial, you: + +1. [Fork the project template](#fork-the-project-template) to create a project + to run the fuzz tests in. +1. [Create the fuzz targets](#create-the-fuzz-targets). +1. [Enable coverage-guided fuzz testing](#enable-coverage-guided-fuzz-testing) + in your forked project. +1. [Run the fuzz test](#run-the-fuzz-test) to identify security vulnerabilities. +1. [Fix any vulnerabilities](#fix-the-vulnerabilities) identified by the fuzz test. + +## Fork the project template + +First, to create a project to try out fuzz testing in, you must fork the `fuzz-testing` +project template: + +1. Open the [`fuzz-testing` project template](https://gitlab.com/gitlab-org/tutorial-project-templates/fuzz-testing). +1. [Fork the project template](../../user/project/repository/forking_workflow.md). +1. When forking the project template: + - Name the forked project `fuzz-testing-demo`. + - Select an appropriate [namespace](../../user/namespace/index.md). + - Set [project visibility](../../user/public_access.md) to **Private**. + +You have successfully forked the `fuzz-testing` project template. Before you can +start fuzz testing, remove the relationship between the project template and the fork: + +1. On the left sidebar, select **Settings > General**. +1. Expand **Advanced**. +1. In the **Remove fork relationship** section, select **Remove fork relationship**. + Enter the name of the project when prompted. + +Your project is ready and you can now create the fuzz test. Next you will create +the fuzz targets. + +## Create the fuzz targets + +Now you have a project for fuzz testing, you create the fuzz targets. A fuzz target +is a function or program that, given an input, makes a call to the application +being tested. + +In this tutorial, the fuzz targets call a function of the `my-tools.js` file using +a random buffer as a parameter. + +To create the two fuzz target files: + +1. On the top bar, select **Main menu > Projects** and select the `fuzz-testing-demo` project. +1. Create a file in the root directory of the project. +1. Name the file `fuzz-sayhello.js` and add the following code: + + ```javascript + let tools = require('./my-tools') + + function fuzz(buf) { + const text = buf.toString() + tools.sayHello(text) + } + + module.exports = { + fuzz + } + ``` + + You can also copy this code from the `fuzz-testing-demo/fuzzers/fuzz-sayhello.js` + project file. + +1. Name the **Target Branch** `add-fuzz-test` and write a descriptive commit message. + - Do not select the **Start a new merge request with these changes** checkbox yet. +1. Select **Commit changes**. +1. Return to the root directory of the project. +1. Make sure you are in the `add-fuzz-test` branch. +1. Create the second file named `fuzz-readme.js` and add the following code: + + ```javascript + let tools = require('./my-tools') + function fuzz(buf) { + const text = buf.toString() + tools.readmeContent(text) + } + module.exports = { + fuzz + } + ``` + + You can also copy this code from the `fuzz-testing-demo/fuzzers/fuzz-readme.js` + project file. + +1. Write a descriptive commit message. +1. Make sure the **Target Branch** is `add-fuzz-test`. +1. Select **Commit changes**. + +You now have two fuzz targets that can make calls to the application being tested. +Next you will enable the fuzz testing. + +## Enable coverage-guided fuzz testing + +To enable coverage-guided fuzz testing, create a CI/CD pipeline running +the `gitlab-cov-fuzz` CLI to execute the fuzz test on the two fuzz targets. + +To create the pipeline file: + +1. Make sure you are in the `add-fuzz-test` branch. +1. In the root directory of the `fuzz-testing-demo` project, create a new file. +1. Name the file `.gitlab-ci.yml` and add the following code: + + ```yaml + image: node:18 + + stages: + - fuzz + + include: + - template: Coverage-Fuzzing.gitlab-ci.yml + + readme_fuzz_target: + extends: .fuzz_base + tags: [saas-linux-large-amd64] # Optional + variables: + COVFUZZ_ADDITIONAL_ARGS: '--fuzzTime=60' + script: + - npm config set @gitlab-org:registry https://gitlab.com/api/v4/packages/npm/ && npm i -g @gitlab-org/jsfuzz + - ./gitlab-cov-fuzz run --engine jsfuzz -- fuzz-readme.js + + hello_fuzzing_target: + extends: .fuzz_base + tags: [saas-linux-large-amd64] # Optional + variables: + COVFUZZ_ADDITIONAL_ARGS: '--fuzzTime=60' + script: + - npm config set @gitlab-org:registry https://gitlab.com/api/v4/packages/npm/ && npm i -g @gitlab-org/jsfuzz + - ./gitlab-cov-fuzz run --engine jsfuzz -- fuzz-sayhello.js + ``` + + This step adds the following to your pipeline: + - A `fuzz` stage using a template. + - Two jobs, `readme_fuzz_target` and `hello_fuzzing_target`. Each job runs using + the `jsfuzz` engine, which reports unhandled exceptions as crashes. + + You can also copy this code from the `fuzz-testing-demo/fuzzers/fuzzers.yml` + project file. + +1. Write a descriptive commit message. +1. Make sure the **Target Branch** is `add-fuzz-test`. +1. Select **Commit changes**. + +You have successfully enabled coverage-guided fuzz testing. Next you will run the +fuzz test using the pipeline you've just created. + +## Run the fuzz test + +To run the fuzz test: + +1. On the left sidebar, select **Merge requests**. +1. Select **New merge request**. +1. In the **Source branch** section, select the `add-fuzz-test` branch. +1. In the **Target branch** section, make sure that your namespace and the `main` branch are selected. +1. Select **Compare branches and continue**. +1. [Create the merge request](../../user/project/merge_requests/creating_merge_requests.md). + +Creating the merge request triggers a new pipeline, which runs the fuzz test. +When the pipeline is finished running, you should see a security vulnerability +alert on the merge request page. + +To see more information on each vulnerability, select the individual **Uncaught-exception** links. + +You have successfully run the fuzz test and identified vulnerabilities to fix. + +## Fix the vulnerabilities + +The fuzz test identified two security vulnerabilities. To fix those +vulnerabilities, you use the `my-tools.js` library. + +To create the `my-tools.js` file: + +1. Make sure you are in the `add-fuzz-test` branch of the project. +1. Go to the root directory of your project and open the `my-tools.js` file. +1. Replace the contents of this file with the following code: + + ```javascript + const fs = require('fs') + + function sayHello(name) { + if(name.includes("z")) { + //throw new Error("😡 error name: " + name) + console.log("😡 error name: " + name) + } else { + return "😀 hello " + name + } + } + + function readmeContent(name) { + + let fileName = name => { + if(name.includes("w")) { + return "./README.txt" + } else { + return "./README.md" + } + } + + //const data = fs.readFileSync(fileName(name), 'utf8') + try { + const data = fs.readFileSync(fileName(name), 'utf8') + return data + } catch (err) { + console.error(err.message) + return "" + } + + } + + module.exports = { + sayHello, readmeContent + } + ``` + + You can also copy the code from the `fuzz-testing-demo/javascript/my-tools.js` + project file. + +1. Select **Commit changes**. This triggers another pipeline to run another fuzz test. +1. When the pipeline is finished, check the merge request **Overview** page. You + should see that the security scan detected no new potential vulnerabilities. +1. Merge your changes. + +Congratulations, you've successfully run a fuzz test and fixed the identified +security vulnerabilities! + +For more information, see [coverage-guided fuzz testing](../../user/application_security/coverage_fuzzing/index.md). diff --git a/doc/tutorials/fuzz_testing_tutorial.md b/doc/tutorials/fuzz_testing_tutorial.md new file mode 100644 index 00000000000..74b24005077 --- /dev/null +++ b/doc/tutorials/fuzz_testing_tutorial.md @@ -0,0 +1,11 @@ +--- +redirect_to: 'fuzz_testing/index.md' +remove_date: '2023-07-21' +--- + +This document was moved to [another location](fuzz_testing/index.md). + + + + + \ No newline at end of file diff --git a/doc/tutorials/gitlab_navigation.md b/doc/tutorials/gitlab_navigation.md new file mode 100644 index 00000000000..6003fbf67c0 --- /dev/null +++ b/doc/tutorials/gitlab_navigation.md @@ -0,0 +1,21 @@ +--- +stage: none +group: Tutorials +info: For assistance with this tutorials page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. +--- + +# Find your way around GitLab + +Get to know the features of GitLab and where to find them so you can get up +and running quickly. + +| Topic | Description | Good for beginners | +|-------|-------------|--------------------| +| [Introduction to GitLab](https://youtu.be/_4SmIyQ5eis?t=90) (59m 51s) | Walk through recommended processes and example workflows for using GitLab. | **{star}** | +| [GitLab with Git Essentials](https://levelup.gitlab.com/courses/gitlab-with-git-essentials) | Learn the basics of Git and GitLab in this self-paced course. | **{star}** | +| [Use GitLab for DevOps](https://www.youtube.com/watch?v=7q9Y1Cv-ib0) (12m 34s) | Use GitLab through the entire DevOps lifecycle, from planning to monitoring. | **{star}** | +| [Use the left sidebar to navigate GitLab](left_sidebar/index.md) | Start navigating the GitLab UI. | **{star}** | +| [Use Markdown at GitLab](../user/markdown.md) | GitLab Flavored Markdown (GLFM) is used in many areas of GitLab, for example, in merge requests. | **{star}** | +| [Learn GitLab project walkthrough](https://www.youtube.com/watch?v=-oaI2WEKdI4&list=PL05JrBw4t0KofkHq4GZJ05FnNGa11PQ4d) (59m 2s) | Step through the tutorial-style issues in the **Learn GitLab** project. If you don't have this project, download [the export file](https://gitlab.com/gitlab-org/gitlab/-/blob/master/vendor/project_templates/learn_gitlab_ultimate.tar.gz) and [import it to a new project](../user/project/settings/import_export.md#import-a-project-and-its-data). | | +| [GitLab Continuous Delivery overview](https://www.youtube.com/watch?v=g-gO93PMZvk&list=PLFGfElNsQthYDx0A_FaNNfUm9NHsK6zED&index=134) (17m 2s) | Learn how to use GitLab features to continuously build, test, and deploy iterative code changes. | | +| [Productivity tips](https://about.gitlab.com/blog/2021/02/18/improve-your-gitlab-productivity-with-these-10-tips/) | Get tips to help make you a productive GitLab user. | | diff --git a/doc/tutorials/img/branches_dropdown_v14_10.png b/doc/tutorials/img/branches_dropdown_v14_10.png deleted file mode 100644 index 926baff0ff8..00000000000 Binary files a/doc/tutorials/img/branches_dropdown_v14_10.png and /dev/null differ diff --git a/doc/tutorials/img/clone_project_v14_9.png b/doc/tutorials/img/clone_project_v14_9.png deleted file mode 100644 index 98666c95ba3..00000000000 Binary files a/doc/tutorials/img/clone_project_v14_9.png and /dev/null differ diff --git a/doc/tutorials/img/commit_message_v14_10.png b/doc/tutorials/img/commit_message_v14_10.png deleted file mode 100644 index 5636a135b4e..00000000000 Binary files a/doc/tutorials/img/commit_message_v14_10.png and /dev/null differ diff --git a/doc/tutorials/index.md b/doc/tutorials/index.md index c1b538bafbe..b06cd224d0c 100644 --- a/doc/tutorials/index.md +++ b/doc/tutorials/index.md @@ -8,117 +8,11 @@ info: For assistance with this tutorials page, see https://about.gitlab.com/hand These tutorials can help you learn how to use GitLab. -## Find your way around GitLab - -Get to know the features of GitLab and where to find them so you can get up -and running quickly. - -| Topic | Description | Good for beginners | -|-------|-------------|--------------------| -| [Introduction to GitLab](https://youtu.be/_4SmIyQ5eis?t=90) (59m 51s) | Walk through recommended processes and example workflows for using GitLab. | **{star}** | -| [GitLab 101](https://levelup.gitlab.com/learn/course/gitlab101) | Learn the basics of GitLab in this certification course. | **{star}** | -| [Use GitLab for DevOps](https://www.youtube.com/watch?v=7q9Y1Cv-ib0) (12m 34s) | Use GitLab through the entire DevOps lifecycle, from planning to monitoring. | **{star}** | -| [Use Markdown at GitLab](../user/markdown.md) | GitLab Flavored Markdown (GLFM) is used in many areas of GitLab, for example, in merge requests. | **{star}** | -| [Learn GitLab project walkthrough](https://www.youtube.com/watch?v=-oaI2WEKdI4&list=PL05JrBw4t0KofkHq4GZJ05FnNGa11PQ4d) (59m 2s) | Step through the tutorial-style issues in the **Learn GitLab** project. If you don't have this project, download [the export file](https://gitlab.com/gitlab-org/gitlab/-/blob/master/vendor/project_templates/learn_gitlab_ultimate.tar.gz) and [import it to a new project](../user/project/settings/import_export.md#import-a-project-and-its-data). | | -| [Productivity tips](https://about.gitlab.com/blog/2021/02/18/improve-your-gitlab-productivity-with-these-10-tips/) | Get tips to help make you a productive GitLab user. | | - -## Use Git - -GitLab is a Git-based platform, so understanding Git is important to get -the most out of GitLab. - -| Topic | Description | Good for beginners | -|-------|-------------|--------------------| -| [Make your first Git commit](make_your_first_git_commit.md) | Create a project, edit a file, and commit changes to a Git repository from the command line. | **{star}** | -| [Start using Git on the command line](../gitlab-basics/start-using-git.md) | Learn how to set up Git, clone repositories, and work with branches. | **{star}** | -| [Take advantage of Git rebase](https://about.gitlab.com/blog/2022/10/06/take-advantage-of-git-rebase/)| Learn how to use the `rebase` command in your workflow. | | -| [Git cheat sheet](https://about.gitlab.com/images/press/git-cheat-sheet.pdf) | Download a PDF of common Git commands. | | - -## Plan your work in projects - -Your work takes place in a project, from creating code, to planning, -collaborating, and more. - -| Topic | Description | Good for beginners | -|-------|-------------|--------------------| -| [Create a project from a template](https://gitlab.com/projects/new#create_from_template) | Choose a project template and create a project with files to get you started. | | -| [Migrate to GitLab](../user/project/import/index.md) | If you are coming to GitLab from another platform, you can import or convert your projects. | | -| [Run an agile iteration](agile_sprint.md) | Use group, projects, and iterations to run an agile development iteration. | - -## Use CI/CD pipelines - -CI/CD pipelines are used to automatically build, test, and deploy your code. - -| Topic | Description | Good for beginners | -|-------|-------------|--------------------| -| [Create and run your first GitLab CI/CD pipeline](../ci/quick_start/index.md) | Create a `.gitlab-ci.yml` file and start a pipeline. | **{star}** | -| [Get started: Learn about CI/CD](https://www.youtube.com/watch?v=sIegJaLy2ug) (9m 02s) | Learn about the `.gitlab-ci.yml` file and how it's used. | **{star}** | -| [CI deep dive](https://www.youtube.com/watch?v=ZVUbmVac-m8&list=PL05JrBw4t0KorkxIFgZGnzzxjZRCGROt_&index=27) (22m 51s) | Take a closer look at pipelines and continuous integration concepts. | | -| [CD deep dive](https://www.youtube.com/watch?v=Cn0rzND-Yjw&list=PL05JrBw4t0KorkxIFgZGnzzxjZRCGROt_&index=10) (47m 54s) | Learn about deploying in GitLab. | | -| [Set up CI/CD in the cloud](../ci/examples/index.md#cicd-in-the-cloud) | Learn how to set up CI/CD in different cloud-based environments. | | -| [Find CI/CD examples and templates](../ci/examples/index.md#cicd-examples) | Use these examples and templates to set up CI/CD for your use case. | | -| [Understand CI/CD rules](https://www.youtube.com/watch?v=QjQc-zeL16Q) (8m 56s) | Learn more about how to use CI/CD rules. | | - -## Configure your applications and infrastructure - -Use GitLab configuration features to reduce the effort needed to -configure the infrastructure for your application. - -| Topic | Description | Good for beginners | -|-------|-------------|--------------------| -| [Use GitOps with GitLab](https://about.gitlab.com/blog/2022/04/07/the-ultimate-guide-to-gitops-with-gitlab/) | Learn how to provision and manage a base infrastructure, connect to a Kubernetes cluster, deploy third-party applications, and carry out other infrastructure automation tasks. | | -| [Use Auto DevOps to deploy an application](../topics/autodevops/cloud_deployments/auto_devops_with_gke.md) | Deploy an application to Google Kubernetes Engine (GKE). | | - -## Publish a static website - -Use GitLab Pages to publish a static website directly from your project. - -| Topic | Description | Good for beginners | -|-------|-------------|--------------------| -| [Create a Pages website from a CI/CD template](../user/project/pages/getting_started/pages_ci_cd_template.md) | Quickly generate a Pages website for your project using a CI/CD template for a popular Static Site Generator (SSG). | **{star}** | -| [Create a Pages website from scratch](../user/project/pages/getting_started/pages_from_scratch.md) | Create all the components of a Pages website from a blank project. | | - -## Secure your application - -GitLab can check your application for security vulnerabilities. - -| Topic | Description | Good for beginners | -|-------|-------------|--------------------| -| [Set up dependency scanning](https://about.gitlab.com/blog/2021/01/14/try-dependency-scanning/) | Try out dependency scanning, which checks for known vulnerabilities in dependencies. | **{star}** | -| [Get started with GitLab application security](../user/application_security/get-started-security.md) | Follow recommended steps to set up security tools. | | - -## Work with a self-managed instance - -If you're an administrator of a self-managed instance of GitLab, these tutorials -can help you manage and configure your instance. - -| Topic | Description | Good for beginners | -|-------|-------------|--------------------| -| [Install GitLab](../install/install_methods.md) | Install GitLab according to your requirements.| | -| [Get started administering GitLab](../administration/get_started.md) | Configure your organization and its authentication, then secure, monitor, and back up GitLab. | | - -## Integrate with GitLab - -GitLab [integrates](../user/project/integrations/index.md) with a number of third-party services, -enabling you to work with those services directly from GitLab. - -| Topic | Description | Good for beginners | -|-------|-------------|--------------------| -| [Integrate with Jira](https://about.gitlab.com/blog/2021/04/12/gitlab-jira-integration-selfmanaged/) | Configure the Jira integration, so you can work with Jira issues from GitLab. | | -| [Integrate with Gitpod](https://about.gitlab.com/blog/2021/07/19/teams-gitpod-integration-gitlab-speed-up-development/) | Integrate with Gitpod, to help speed up your development. | | - -## Find more tutorial content - -If you're learning about GitLab, here are some ways you can find more tutorial -content: - -- Find learning tracks and certification options at [GitLab Learn](https://about.gitlab.com/learn/). - GitLab learning platform login required (email and password for non-GitLab team members). - -- Find recent tutorials on the GitLab blog by [searching by the `tutorial` tag](https://about.gitlab.com/blog/tags.html#tutorial). - -- Browse the **Learn@GitLab** [playlist on YouTube](https://www.youtube.com/playlist?list=PLFGfElNsQthYDx0A_FaNNfUm9NHsK6zED) - to find video tutorials. - -If you find an article, video, or other resource that would be a -great addition to this page, add it in a [merge request](../development/documentation/index.md). +- [Find your way around GitLab](gitlab_navigation.md) +- [Learn Git](learn_git.md) +- [Plan and track your work](plan_and_track.md) +- [Build your application](build_application.md) +- [Secure your application and check compliance](secure_application.md) +- [Manage your infrastructure](infrastructure.md) +- [Develop with GitLab](develop.md) +- [Find more tutorials](more_tutorials.md) diff --git a/doc/tutorials/infrastructure.md b/doc/tutorials/infrastructure.md new file mode 100644 index 00000000000..e9035461596 --- /dev/null +++ b/doc/tutorials/infrastructure.md @@ -0,0 +1,15 @@ +--- +stage: none +group: Tutorials +info: For assistance with this tutorials page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. +--- + +# Manage your infrastructure + +Use GitLab configuration features to reduce the effort needed to +configure the infrastructure for your application. + +| Topic | Description | Good for beginners | +|-------|-------------|--------------------| +| [Use GitOps with GitLab](https://about.gitlab.com/blog/2022/04/07/the-ultimate-guide-to-gitops-with-gitlab/) | Learn how to provision and manage a base infrastructure, connect to a Kubernetes cluster, deploy third-party applications, and carry out other infrastructure automation tasks. | | +| [Set up Flux for GitOps](../user/clusters/agent/gitops/flux_tutorial.md) | Learn how to set up Flux for GitOps in a sample project. | | diff --git a/doc/tutorials/learn_git.md b/doc/tutorials/learn_git.md new file mode 100644 index 00000000000..a5b52ef73e9 --- /dev/null +++ b/doc/tutorials/learn_git.md @@ -0,0 +1,17 @@ +--- +stage: none +group: Tutorials +info: For assistance with this tutorials page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. +--- + +# Learn Git + +GitLab is a Git-based platform, so understanding Git is important to get +the most out of GitLab. + +| Topic | Description | Good for beginners | +|-------|-------------|--------------------| +| [Make your first Git commit](make_first_git_commit/index.md) | Create a project, edit a file, and commit changes to a Git repository from the command line. | **{star}** | +| [Start using Git on the command line](../gitlab-basics/start-using-git.md) | Learn how to set up Git, clone repositories, and work with branches. | **{star}** | +| [Take advantage of Git rebase](https://about.gitlab.com/blog/2022/10/06/take-advantage-of-git-rebase/)| Learn how to use the `rebase` command in your workflow. | | +| [Git cheat sheet](https://about.gitlab.com/images/press/git-cheat-sheet.pdf) | Download a PDF of common Git commands. | | diff --git a/doc/tutorials/left_sidebar/img/admin_area_v16_0.png b/doc/tutorials/left_sidebar/img/admin_area_v16_0.png new file mode 100644 index 00000000000..7cc4ddedea0 Binary files /dev/null and b/doc/tutorials/left_sidebar/img/admin_area_v16_0.png differ diff --git a/doc/tutorials/left_sidebar/img/explore_v16_0.png b/doc/tutorials/left_sidebar/img/explore_v16_0.png new file mode 100644 index 00000000000..3cbe94e5de4 Binary files /dev/null and b/doc/tutorials/left_sidebar/img/explore_v16_0.png differ diff --git a/doc/tutorials/left_sidebar/img/pin_v16_0.png b/doc/tutorials/left_sidebar/img/pin_v16_0.png new file mode 100644 index 00000000000..17dbcac4caf Binary files /dev/null and b/doc/tutorials/left_sidebar/img/pin_v16_0.png differ diff --git a/doc/tutorials/left_sidebar/img/pinned_v16_0.png b/doc/tutorials/left_sidebar/img/pinned_v16_0.png new file mode 100644 index 00000000000..1c8d0bdd2cd Binary files /dev/null and b/doc/tutorials/left_sidebar/img/pinned_v16_0.png differ diff --git a/doc/tutorials/left_sidebar/img/project_selected_v16_0.png b/doc/tutorials/left_sidebar/img/project_selected_v16_0.png new file mode 100644 index 00000000000..534b06ac5de Binary files /dev/null and b/doc/tutorials/left_sidebar/img/project_selected_v16_0.png differ diff --git a/doc/tutorials/left_sidebar/img/search_projects_v16_0.png b/doc/tutorials/left_sidebar/img/search_projects_v16_0.png new file mode 100644 index 00000000000..12182f24009 Binary files /dev/null and b/doc/tutorials/left_sidebar/img/search_projects_v16_0.png differ diff --git a/doc/tutorials/left_sidebar/img/shortcuts_v16_0.png b/doc/tutorials/left_sidebar/img/shortcuts_v16_0.png new file mode 100644 index 00000000000..07094898117 Binary files /dev/null and b/doc/tutorials/left_sidebar/img/shortcuts_v16_0.png differ diff --git a/doc/tutorials/left_sidebar/img/your_work_v16_0.png b/doc/tutorials/left_sidebar/img/your_work_v16_0.png new file mode 100644 index 00000000000..f7b5ed4217d Binary files /dev/null and b/doc/tutorials/left_sidebar/img/your_work_v16_0.png differ diff --git a/doc/tutorials/left_sidebar/index.md b/doc/tutorials/left_sidebar/index.md new file mode 100644 index 00000000000..7de50afbe77 --- /dev/null +++ b/doc/tutorials/left_sidebar/index.md @@ -0,0 +1,73 @@ +--- +stage: none +group: Tutorials +info: For assistance with this tutorial, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. +--- + +# Tutorial: Use the left sidebar to navigate GitLab **(FREE)** + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/9044) in GitLab 16.0. + +Follow this tutorial to learn how to use the new left sidebar to navigate the UI. + +Provide feedback in +[issue 409005](https://gitlab.com/gitlab-org/gitlab/-/issues/409005). + +## Enable the new left sidebar + +To view the new sidebar: + +1. On the top bar, in the upper-right corner, select your avatar. +1. Turn on the **New navigation** toggle. + +To turn off this sidebar, return to your avatar and turn off the toggle. + +## Find your project + +Let's get started exploring the GitLab UI and left sidebar. + +1. Start by finding the project you want to work on. + To explore all available projects, on the left sidebar, select **Explore**: + + ![Explore](img/explore_v16_0.png) + +1. On the right, above the list of projects, type search criteria. + The search finds projects with a matching description. + + ![Search projects](img/search_projects_v16_0.png) + +1. When you find the project you want, select the project name. + The left sidebar now shows project-specific options. + + ![Project-specific options](img/project_selected_v16_0.png) + +Your issues, merge requests, and to-do items are listed in the shortcuts +at the top: + +![shortcuts](img/shortcuts_v16_0.png) + +## Customize the sidebar + +You can pin menu items if you tend to use them frequently. + +1. Expand the sections until you are viewing the item you want to pin. +1. Hover over and select the pin (**{thumbtack}**). + + ![pin](img/pin_v16_0.png) + +The item is displayed in the **Pinned** section: + +![pinned item](img/pinned_v16_0.png) + +## Use a more focused view + +On the left sidebar, you can also choose a more focused view into the areas you have access to. +Change the view to **Your work**: + +![Your work](img/your_work_v16_0.png) + +## Go to the Admin Area + +The Admin Area is also available on the left sidebar: + +![Admin Area](img/admin_area_v16_0.png) diff --git a/doc/tutorials/make_first_git_commit/img/branches_dropdown_v14_10.png b/doc/tutorials/make_first_git_commit/img/branches_dropdown_v14_10.png new file mode 100644 index 00000000000..926baff0ff8 Binary files /dev/null and b/doc/tutorials/make_first_git_commit/img/branches_dropdown_v14_10.png differ diff --git a/doc/tutorials/make_first_git_commit/img/clone_project_v14_9.png b/doc/tutorials/make_first_git_commit/img/clone_project_v14_9.png new file mode 100644 index 00000000000..98666c95ba3 Binary files /dev/null and b/doc/tutorials/make_first_git_commit/img/clone_project_v14_9.png differ diff --git a/doc/tutorials/make_first_git_commit/img/commit_message_v14_10.png b/doc/tutorials/make_first_git_commit/img/commit_message_v14_10.png new file mode 100644 index 00000000000..5636a135b4e Binary files /dev/null and b/doc/tutorials/make_first_git_commit/img/commit_message_v14_10.png differ diff --git a/doc/tutorials/make_first_git_commit/index.md b/doc/tutorials/make_first_git_commit/index.md new file mode 100644 index 00000000000..794b9d1f4b5 --- /dev/null +++ b/doc/tutorials/make_first_git_commit/index.md @@ -0,0 +1,271 @@ +--- +stage: none +group: Tutorials +info: For assistance with this tutorial, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. +--- + +# Tutorial: Make your first Git commit + +This tutorial is going to teach you a little bit about how Git works. It walks +you through the steps of creating your own project, editing a file, and +committing changes to a Git repository from the command line. + +When you're done, you'll have a project where you can practice using Git. + +## What you need + +Before you begin: + +- [Install Git on your local machine](../../topics/git/how_to_install_git/index.md). +- Ensure you can sign in to an instance of GitLab. If your organization doesn't + have GitLab, create an account on GitLab.com. +- [Create SSH keys and add them to GitLab](../../user/ssh.md). SSH keys are how you + securely communicate between your computer and GitLab. + +## What is Git? + +Before we jump into steps, let's go over some basic Git concepts. + +Git is a version control system. It's used to track changes to files. + +You store files, like code or documents, in a Git *repository*. When you want to edit the files, you +*clone* the repository to your computer, make the changes, and *push* your changes +back to the repository. In GitLab, a Git repository is located in +a *project*. + +Each time you push a change, Git records it as a unique *commit*. These commits make up +the history of when and how a file changed, and who changed it. + +```mermaid +graph LR + subgraph Repository commit history + direction LR + A(Author: Alex
Date: 3 Jan at 1PM
Commit message: Added sales figures
Commit ID: 123abc12) ---> B + B(Author: Sam
Date: 4 Jan at 10AM
Commit message: Removed old info
Commit ID: aabb1122) ---> C + C(Author: Zhang
Date: 5 Jan at 3PM
Commit message: Added invoices
Commit ID: ddee4455) + end +``` + +When you work in a Git repository, you work in *branches*. By default, the contents +of a repository are in a default branch. To make changes, you: + +1. Create your own branch, which is a snapshot of the default branch at the time + you create it. +1. Make changes and push them to your branch. Each push creates a commit. +1. When you're ready, *merge* your branch into the default branch. + +```mermaid +flowchart LR + subgraph Default branch + A[Commit] --> B[Commit] --> C[Commit] --> D[Commit] + end + subgraph My branch + B --1. Create my branch--> E(Commit) + E --2. Add my commit--> F(Commit) + F --3. Merge my branch to default--> D + end +``` + +If this all feels a bit overwhelming, hang in there. You're about to see these concepts in action. + +## Steps + +Here's an overview of what we're going to do: + +1. [Create a sample project](#create-a-sample-project). +1. [Clone the repository](#clone-the-repository). +1. [Create a branch and make your changes](#create-a-branch-and-make-changes). +1. [Commit and push your changes](#commit-and-push-your-changes). +1. [Merge your changes](#merge-your-changes). +1. [View your changes in GitLab](#view-your-changes-in-gitlab). + +### Create a sample project + +To start, create a sample project in GitLab. + +1. In GitLab, on the top bar, select **Main menu > Projects > View all projects**. +1. On the right of the page, select **New project**. +1. For **Project name**, enter `My sample project`. The project slug is generated for you. + This slug is the URL you can use to access the project after it's created. +1. Ensure **Initialize repository with a README** is selected. + How you complete the other fields is up to you. +1. Select **Create project**. + +### Clone the repository + +Now you can clone the repository in your project. *Cloning* a repository means you're creating +a copy on your computer, or wherever you want to store and work with the files. + +1. On your project page, select **Clone**. Copy the URL for **Clone with SSH**. + + ![Clone a project with SSH](img/clone_project_v14_9.png) + +1. Open a terminal on your computer and go to the directory + where you want to clone the files. + +1. Enter `git clone` and paste the URL: + + ```shell + git clone git@gitlab.com:gitlab-example/my-sample-project.git + ``` + +1. Go to the directory: + + ```shell + cd my-sample-project + ``` + +1. By default, you've cloned the default branch for the repository. Usually this + branch is `main`. To be sure, get the name of the default branch: + + ```shell + git branch + ``` + + The branch you're on is marked with an asterisk. + Press `Q` on your keyboard to return to the main terminal + window. + +### Create a branch and make changes + +Now that you have a copy of the repository, create your own branch so you can +work on your changes independently. + +1. Create a new branch called `example-tutorial-branch`. + + ```shell + git checkout -b example-tutorial-branch + ``` + +1. In a text editor like Visual Studio Code, Sublime, `vi`, or any other editor, + open the README.md file and add this text: + + ```plaintext + Hello world! I'm using Git! + ``` + +1. Save the file. + +1. Git keeps track of changed files. To confirm which files have changed, get + the status. + + ```shell + git status + ``` + + You should get output similar to the following: + + ```shell + On branch example-tutorial-branch + Changes not staged for commit: + (use "git add ..." to update what will be committed) + (use "git restore ..." to discard changes in working directory) + modified: README.md + + no changes added to commit (use "git add" and/or "git commit -a") + ``` + +### Commit and push your changes + +You've made changes to a file in your repository. Now it's time to record +those changes by making your first commit. + +1. Add the `README.md` file to the *staging* area. The staging area is where you + put files before you commit them. + + ```shell + git add README.md + ``` + +1. Confirm the file is staged: + + ```shell + git status + ``` + + You should get output similar to the following, and the filename should be in + green text. + + ```shell + On branch example-tutorial-branch + Changes to be committed: + (use "git restore --staged ..." to unstage) + modified: README.md + ``` + +1. Now commit the staged file, and include a message + that describes the change you made. Make sure you surround the message in double + quotes ("). + + ```shell + git commit -m "I added text to the README file" + ``` + +1. The change has been committed to your branch, but your branch and its commits + are still only available on your computer. No one else has access to them yet. + Push your branch to GitLab: + + ```shell + git push origin example-tutorial-branch + ``` + +Your branch is now available on GitLab and visible to other users in your project. + +![Branches dropdown list](img/branches_dropdown_v14_10.png) + +### Merge your changes + +Now you're ready to merge the changes from your `example-tutorial-branch` branch +to the default branch (`main`). + +1. Check out the default branch for your repository. + + ```shell + git checkout main + ``` + +1. Merge your branch into the default branch. + + ```shell + git merge example-tutorial-branch + ``` + +1. Push the changes. + + ```shell + git push + ``` + +NOTE: +For this tutorial, you merge your branch directly to the default branch for your +repository. In GitLab, you typically use a [merge request](../../user/project/merge_requests/index.md) +to merge your branch. + +### View your changes in GitLab + +You did it! You updated the `README.md` file in your branch, and you merged those changes +into the `main` branch. + +Let's look in the UI and confirm your changes. Go to your project. + +- Scroll down and view the contents of the `README.md` file. + Your changes should be visible. +- Above the `README.md` file, view the text in the **Last commit** column. + Your commit message is displayed in this column: + + ![Commit message](img/commit_message_v14_10.png) + +Now you can return to the command line and change back to your personal branch +(`git checkout example-tutorial-branch`). You can continue updating files or +creating new ones. Type `git status` to view the status +of your changes and commit with abandon. + +Don't worry if you mess things up. Everything in Git can be reverted, and if you +find you can't recover, you can always create a new branch and start again. + +Nice work. + +## Find more Git learning resources + +- Get a complete introduction to Git in the [Git for GitLab](https://www.youtube.com/watch?v=4lxvVj7wlZw) beginner's course (1h 33m). +- Find other tutorials about Git and GitLab on the [tutorials page](../index.md). diff --git a/doc/tutorials/make_your_first_git_commit.md b/doc/tutorials/make_your_first_git_commit.md index 3df65389c76..04c66a953af 100644 --- a/doc/tutorials/make_your_first_git_commit.md +++ b/doc/tutorials/make_your_first_git_commit.md @@ -1,271 +1,11 @@ --- -stage: none -group: Tutorials -info: For assistance with this tutorial, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. +redirect_to: 'make_first_git_commit/index.md' +remove_date: '2023-07-21' --- -# Tutorial: Make your first Git commit +This document was moved to [another location](make_first_git_commit/index.md). -This tutorial is going to teach you a little bit about how Git works. It walks -you through the steps of creating your own project, editing a file, and -committing changes to a Git repository from the command line. - -When you're done, you'll have a project where you can practice using Git. - -## What you need - -Before you begin: - -- [Install Git on your local machine](../topics/git/how_to_install_git/index.md). -- Ensure you can sign in to an instance of GitLab. If your organization doesn't - have GitLab, create an account on GitLab.com. -- [Create SSH keys and add them to GitLab](../user/ssh.md). SSH keys are how you - securely communicate between your computer and GitLab. - -## What is Git? - -Before we jump into steps, let's go over some basic Git concepts. - -Git is a version control system. It's used to track changes to files. - -You store files, like code or documents, in a Git *repository*. When you want to edit the files, you -*clone* the repository to your computer, make the changes, and *push* your changes -back to the repository. In GitLab, a Git repository is located in -a *project*. - -Each time you push a change, Git records it as a unique *commit*. These commits make up -the history of when and how a file changed, and who changed it. - -```mermaid -graph LR - subgraph Repository commit history - direction LR - A(Author: Alex
Date: 3 Jan at 1PM
Commit message: Added sales figures
Commit ID: 123abc12) ---> B - B(Author: Sam
Date: 4 Jan at 10AM
Commit message: Removed old info
Commit ID: aabb1122) ---> C - C(Author: Zhang
Date: 5 Jan at 3PM
Commit message: Added invoices
Commit ID: ddee4455) - end -``` - -When you work in a Git repository, you work in *branches*. By default, the contents -of a repository are in a default branch. To make changes, you: - -1. Create your own branch, which is a snapshot of the default branch at the time - you create it. -1. Make changes and push them to your branch. Each push creates a commit. -1. When you're ready, *merge* your branch into the default branch. - -```mermaid -flowchart LR - subgraph Default branch - A[Commit] --> B[Commit] --> C[Commit] --> D[Commit] - end - subgraph My branch - B --1. Create my branch--> E(Commit) - E --2. Add my commit--> F(Commit) - F --3. Merge my branch to default--> D - end -``` - -If this all feels a bit overwhelming, hang in there. You're about to see these concepts in action. - -## Steps - -Here's an overview of what we're going to do: - -1. [Create a sample project](#create-a-sample-project). -1. [Clone the repository](#clone-the-repository). -1. [Create a branch and make your changes](#create-a-branch-and-make-changes). -1. [Commit and push your changes](#commit-and-push-your-changes). -1. [Merge your changes](#merge-your-changes). -1. [View your changes in GitLab](#view-your-changes-in-gitlab). - -### Create a sample project - -To start, create a sample project in GitLab. - -1. In GitLab, on the top bar, select **Main menu > Projects > View all projects**. -1. On the right of the page, select **New project**. -1. For **Project name**, enter `My sample project`. The project slug is generated for you. - This slug is the URL you can use to access the project after it's created. -1. Ensure **Initialize repository with a README** is selected. - How you complete the other fields is up to you. -1. Select **Create project**. - -### Clone the repository - -Now you can clone the repository in your project. *Cloning* a repository means you're creating -a copy on your computer, or wherever you want to store and work with the files. - -1. On your project page, select **Clone**. Copy the URL for **Clone with SSH**. - - ![Clone a project with SSH](img/clone_project_v14_9.png) - -1. Open a terminal on your computer and go to the directory - where you want to clone the files. - -1. Enter `git clone` and paste the URL: - - ```shell - git clone git@gitlab.com:gitlab-example/my-sample-project.git - ``` - -1. Go to the directory: - - ```shell - cd my-sample-project - ``` - -1. By default, you've cloned the default branch for the repository. Usually this - branch is `main`. To be sure, get the name of the default branch: - - ```shell - git branch - ``` - - The branch you're on is marked with an asterisk. - Press `Q` on your keyboard to return to the main terminal - window. - -### Create a branch and make changes - -Now that you have a copy of the repository, create your own branch so you can -work on your changes independently. - -1. Create a new branch called `example-tutorial-branch`. - - ```shell - git checkout -b example-tutorial-branch - ``` - -1. In a text editor like Visual Studio Code, Sublime, `vi`, or any other editor, - open the README.md file and add this text: - - ```plaintext - Hello world! I'm using Git! - ``` - -1. Save the file. - -1. Git keeps track of changed files. To confirm which files have changed, get - the status. - - ```shell - git status - ``` - - You should get output similar to the following: - - ```shell - On branch example-tutorial-branch - Changes not staged for commit: - (use "git add ..." to update what will be committed) - (use "git restore ..." to discard changes in working directory) - modified: README.md - - no changes added to commit (use "git add" and/or "git commit -a") - ``` - -### Commit and push your changes - -You've made changes to a file in your repository. Now it's time to record -those changes by making your first commit. - -1. Add the `README.md` file to the *staging* area. The staging area is where you - put files before you commit them. - - ```shell - git add README.md - ``` - -1. Confirm the file is staged: - - ```shell - git status - ``` - - You should get output similar to the following, and the filename should be in - green text. - - ```shell - On branch example-tutorial-branch - Changes to be committed: - (use "git restore --staged ..." to unstage) - modified: README.md - ``` - -1. Now commit the staged file, and include a message - that describes the change you made. Make sure you surround the message in double - quotes ("). - - ```shell - git commit -m "I added text to the README file" - ``` - -1. The change has been committed to your branch, but your branch and its commits - are still only available on your computer. No one else has access to them yet. - Push your branch to GitLab: - - ```shell - git push origin example-tutorial-branch - ``` - -Your branch is now available on GitLab and visible to other users in your project. - -![Branches dropdown list](img/branches_dropdown_v14_10.png) - -### Merge your changes - -Now you're ready to merge the changes from your `example-tutorial-branch` branch -to the default branch (`main`). - -1. Check out the default branch for your repository. - - ```shell - git checkout main - ``` - -1. Merge your branch into the default branch. - - ```shell - git merge example-tutorial-branch - ``` - -1. Push the changes. - - ```shell - git push - ``` - -NOTE: -For this tutorial, you merge your branch directly to the default branch for your -repository. In GitLab, you typically use a [merge request](../user/project/merge_requests/index.md) -to merge your branch. - -### View your changes in GitLab - -You did it! You updated the `README.md` file in your branch, and you merged those changes -into the `main` branch. - -Let's look in the UI and confirm your changes. Go to your project. - -- Scroll down and view the contents of the `README.md` file. - Your changes should be visible. -- Above the `README.md` file, view the text in the **Last commit** column. - Your commit message is displayed in this column: - - ![Commit message](img/commit_message_v14_10.png) - -Now you can return to the command line and change back to your personal branch -(`git checkout example-tutorial-branch`). You can continue updating files or -creating new ones. Type `git status` to view the status -of your changes and commit with abandon. - -Don't worry if you mess things up. Everything in Git can be reverted, and if you -find you can't recover, you can always create a new branch and start again. - -Nice work. - -## Find more Git learning resources - -- Get a complete introduction to Git in the [Git for GitLab](https://www.youtube.com/watch?v=4lxvVj7wlZw) beginner's course (1h 33m). -- Find other tutorials about Git and GitLab on the [tutorials page](index.md). + + + + \ No newline at end of file diff --git a/doc/tutorials/more_tutorials.md b/doc/tutorials/more_tutorials.md new file mode 100644 index 00000000000..c52de180bff --- /dev/null +++ b/doc/tutorials/more_tutorials.md @@ -0,0 +1,20 @@ +--- +stage: none +group: Tutorials +info: For assistance with this tutorials page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. +--- + +# Find more tutorial content + +If you're learning about GitLab, to find more tutorial content: + +- Find learning tracks and certification options at [Level Up](https://levelup.gitlab.com/). + GitLab learning platform login required (email and password for non-GitLab team members). + +- Find recent tutorials on the GitLab blog by [searching by the `tutorial` tag](https://about.gitlab.com/blog/tags.html#tutorial). + +- Browse the **Learn@GitLab** [playlist on YouTube](https://www.youtube.com/playlist?list=PLFGfElNsQthYDx0A_FaNNfUm9NHsK6zED) + to find video tutorials. + +If you find an article, video, or other resource that would be a +great addition to this page, add it in a [merge request](../development/documentation/index.md). diff --git a/doc/tutorials/move_personal_project_to_a_group.md b/doc/tutorials/move_personal_project_to_a_group.md index e3ab1e8fbd8..361181fdde6 100644 --- a/doc/tutorials/move_personal_project_to_a_group.md +++ b/doc/tutorials/move_personal_project_to_a_group.md @@ -1,89 +1,11 @@ --- -stage: none -group: Tutorials -info: For assistance with this tutorial, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. +redirect_to: 'move_personal_project_to_group/index.md' +remove_date: '2023-07-21' --- -# Tutorial: Move your personal project to a group **(FREE SAAS)** +This document was moved to [another location](move_personal_project_to_group/index.md). -If you created a project under a [personal namespace](../user/namespace/index.md), -you can perform common tasks, like managing issue and merge requests, -and using source control and CI/CD. - -However, at some point you might outgrow your personal project and -want to move your project to a group namespace instead. With a group namespace, you can: - -- Give a group of users access to your project, rather than adding users one-by-one. -- View all issues and merge requests for all projects in the group. -- View all unique users in the group namespace, across all projects. -- Manage usage quotas. -- Start a trial or upgrade to a paid subscription tier. This option is important if you're - impacted by the [changes to user limits](https://about.gitlab.com/blog/2022/03/24/efficient-free-tier/), - and need more users. - -This tutorial shows you how to move your project from a personal namespace -to a group namespace. - -## Steps - -Here's an overview of the steps: - -1. [Create a group](#create-a-group). -1. [Move your project to a group](#move-your-project-to-a-group). -1. [Work with your group](#work-with-your-group). - -### Create a group - -To begin, make sure you have a suitable group to move your project to. -The group must allow the creation of projects, and you must have at least the -Maintainer role for the group. - -If you don't have a group, create one: - -1. On the top bar, select **Main menu > Groups > View all groups**. -1. On the right of the page, select **New group**. -1. In **Group name**, enter a name for the group. -1. In **Group URL**, enter a path for the group, which is used as the namespace. -1. Choose the [visibility level](../user/public_access.md). -1. Optional. Fill in information to personalize your experience. -1. Select **Create group**. - -### Move your project to a group - -Before you move your project to a group: - -- You must have the Owner role for the project. -- Remove any [container images](../user/packages/container_registry/index.md#move-or-rename-container-registry-repositories) -- Remove any npm packages. If you transfer a project to a different root namespace, the project must not contain any npm packages. When you update the path of a user or group, or transfer a subgroup or project, you must remove any npm packages first. You cannot update the root namespace of a project with npm packages. Make sure you update your .npmrc files to follow the naming convention and run npm publish if necessary. - -Now you're ready to move your project: - -1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Settings > General**. -1. Expand **Advanced**. -1. Under **Transfer project**, choose the group to transfer the project to. -1. Select **Transfer project**. -1. Enter the project's name and select **Confirm**. - -You are redirected to the project's new page. -If you have more than one personal project, you can repeat these steps for each -project. - -NOTE: -For more information about these migration steps, -see [Transferring your project into another namespace](../user/project/settings/index.md#transfer-a-project-to-another-namespace). -A migration might result in follow-up work to update the project path in -your related resources and tools, such as websites and package managers. - -### Work with your group - -You can now view your project in your group: - -1. On the top bar, select **Main menu > Groups** and find your group. -1. Look for your project under **Subgroups and projects**. - -Start enjoying the benefits of a group! For example, as the group Owner, you can -quickly view all unique users in your namespace: - -1. In your group, select **Settings > Usage Quotas**. -1. The **Seats** tab displays all users across all projects in your group. + + + + \ No newline at end of file diff --git a/doc/tutorials/move_personal_project_to_group/index.md b/doc/tutorials/move_personal_project_to_group/index.md new file mode 100644 index 00000000000..d3e695b78df --- /dev/null +++ b/doc/tutorials/move_personal_project_to_group/index.md @@ -0,0 +1,89 @@ +--- +stage: Data Stores +group: Tenant Scale +info: For assistance with this tutorial, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. +--- + +# Tutorial: Move your personal project to a group **(FREE SAAS)** + +If you created a project under a [personal namespace](../../user/namespace/index.md), +you can perform common tasks, like managing issue and merge requests, +and using source control and CI/CD. + +However, at some point you might outgrow your personal project and +want to move your project to a group namespace instead. With a group namespace, you can: + +- Give a group of users access to your project, rather than adding users one-by-one. +- View all issues and merge requests for all projects in the group. +- View all unique users in the group namespace, across all projects. +- Manage usage quotas. +- Start a trial or upgrade to a paid subscription tier. This option is important if you're + impacted by the [changes to user limits](https://about.gitlab.com/blog/2022/03/24/efficient-free-tier/), + and need more users. + +This tutorial shows you how to move your project from a personal namespace +to a group namespace. + +## Steps + +Here's an overview of the steps: + +1. [Create a group](#create-a-group). +1. [Move your project to a group](#move-your-project-to-a-group). +1. [Work with your group](#work-with-your-group). + +### Create a group + +To begin, make sure you have a suitable group to move your project to. +The group must allow the creation of projects, and you must have at least the +Maintainer role for the group. + +If you don't have a group, create one: + +1. On the top bar, select **Main menu > Groups > View all groups**. +1. On the right of the page, select **New group**. +1. In **Group name**, enter a name for the group. +1. In **Group URL**, enter a path for the group, which is used as the namespace. +1. Choose the [visibility level](../../user/public_access.md). +1. Optional. Fill in information to personalize your experience. +1. Select **Create group**. + +### Move your project to a group + +Before you move your project to a group: + +- You must have the Owner role for the project. +- Remove any [container images](../../user/packages/container_registry/index.md#move-or-rename-container-registry-repositories) +- Remove any npm packages. If you transfer a project to a different root namespace, the project must not contain any npm packages. When you update the path of a user or group, or transfer a subgroup or project, you must remove any npm packages first. You cannot update the root namespace of a project with npm packages. Make sure you update your .npmrc files to follow the naming convention and run npm publish if necessary. + +Now you're ready to move your project: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > General**. +1. Expand **Advanced**. +1. Under **Transfer project**, choose the group to transfer the project to. +1. Select **Transfer project**. +1. Enter the project's name and select **Confirm**. + +You are redirected to the project's new page. +If you have more than one personal project, you can repeat these steps for each +project. + +NOTE: +For more information about these migration steps, +see [Transferring your project into another namespace](../../user/project/settings/index.md#transfer-a-project-to-another-namespace). +A migration might result in follow-up work to update the project path in +your related resources and tools, such as websites and package managers. + +### Work with your group + +You can now view your project in your group: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. Look for your project under **Subgroups and projects**. + +Start enjoying the benefits of a group! For example, as the group Owner, you can +quickly view all unique users in your namespace: + +1. In your group, select **Settings > Usage Quotas**. +1. The **Seats** tab displays all users across all projects in your group. diff --git a/doc/tutorials/plan_and_track.md b/doc/tutorials/plan_and_track.md new file mode 100644 index 00000000000..35b552cdaa5 --- /dev/null +++ b/doc/tutorials/plan_and_track.md @@ -0,0 +1,18 @@ +--- +stage: none +group: Tutorials +info: For assistance with this tutorials page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. +--- + +# Plan and track your work + +Create a project to host your code, and plan your work using +issues, epics, and more. + +| Topic | Description | Good for beginners | +|-------|-------------|--------------------| +| [GitLab Agile Project Management](https://levelup.gitlab.com/courses/gitlab-agile-project-management) | Learn how to use planning features to manage your projects in this self-paced course. | **{star}** | +| [Create a project from a template](https://gitlab.com/projects/new#create_from_template) | Choose a project template and create a project with files to get you started. | | +| [Migrate to GitLab](../user/project/import/index.md) | If you are coming to GitLab from another platform, you can import or convert your projects. | | +| [Run an agile iteration](agile_sprint/index.md) | Use group, projects, and iterations to run an agile development iteration. | +| [Epics and Issue Boards](https://www.youtube.com/watch?v=I1bFIAQBHB8) | Find out how to use epics and issue boards for project management. | | diff --git a/doc/tutorials/scan_result_policy.md b/doc/tutorials/scan_result_policy.md new file mode 100644 index 00000000000..9aacd8eff7b --- /dev/null +++ b/doc/tutorials/scan_result_policy.md @@ -0,0 +1,11 @@ +--- +redirect_to: 'scan_result_policy/index.md' +remove_date: '2023-07-21' +--- + +This document was moved to [another location](scan_result_policy/index.md). + + + + + \ No newline at end of file diff --git a/doc/tutorials/scan_result_policy/index.md b/doc/tutorials/scan_result_policy/index.md new file mode 100644 index 00000000000..6f4feb9ec4f --- /dev/null +++ b/doc/tutorials/scan_result_policy/index.md @@ -0,0 +1,125 @@ +--- +stage: Govern +group: Security Policies +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Tutorial: Set up a scan result policy **(ULTIMATE)** + +This tutorial shows you how to create and configure a [scan result policy](../../user/application_security/policies/scan-result-policies.md). These policies can be set to take action based on scan results. +For example, in this tutorial, you'll set up a policy that requires approval from two specified users if a vulnerability is detected in a merge request. + +Prerequisites: + +The namespace used for this tutorial must: + +- Contain a minimum of three users, including your own. If you don't have two other users, you must first + create them. For details, see [Creating users](../../user/profile/account/create_accounts.md). + +To set up a scan result policy: + +1. [Create a test project](#create-a-test-project). +1. [Add a scan result policy](#add-a-scan-result-policy). +1. [Test the scan result policy](#test-the-scan-result-policy). + +## Create a test project + +1. On the top bar, select **Main menu > Projects**. +1. Select **New project**. +1. Select **Create blank project**. +1. Complete the fields. + - **Project name**: `sast-scan-result-policy`. + - Select the **Enable Static Application Security Testing (SAST)** checkbox. +1. Select **Create project**. + +## Add a scan result policy + +Next, you'll add a scan result policy to your test project: + +1. On the top bar, select **Main menu > Projects** and find the `sast-scan-result-policy` project. +1. On the left sidebar, go to **Security and Compliance > Policies**. +1. Select **New policy**. +1. In **Scan result policy**, select **Select policy**. +1. Complete the fields. + - **Name**: `sast-scan-result-policy` + - **Policy status**: **Enabled** +1. Add the following rule: + + ```plaintext + IF |Security Scan| from |SAST| find(s) more than |0| |All severity levels| |All vulnerability states| vulnerabilities in an open merge request targeting |All protected branches| + ``` + +1. Set **Actions** to the following: + + ```plaintext + THEN Require approval from | 2 | of the following approvers: + ``` + +1. Select two users. +1. Select **Configure with a merge request**. + + The application creates a new project to store the policies linked to it, and creates a merge request to define the policy. + +1. Select **Merge**. +1. On the top bar, select **Main menu > Projects** and select the `sast-scan-result-policy` project. +1. On the left sidebar, select **Security and Compliance > Policies**. + + You can see the list of policies added in the previous steps. + +## Test the scan result policy + +Nice work, you've created a scan result policy. To test it, create some vulnerabilities and check the result: + +1. On the top bar, select **Main menu > Projects** and select the `sast-scan-result-policy` project. +1. On the left sidebar, select **Repository > Files**. +1. From the **Add** (**{plus}**) dropdown list, select **New file**. +1. In the **Filename** field enter `main.ts`. +1. In the file's content, copy the following: + + ```typescript + // Non-literal require - tsr-detect-non-literal-require + var lib: String = 'fs' + require(lib) + + // Eval with variable - tsr-detect-eval-with-expression + var myeval: String = 'console.log("Hello.");'; + eval(myeval); + + // Unsafe Regexp - tsr-detect-unsafe-regexp + const regex: RegExp = /(x+x+)+y/; + + // Non-literal Regexp - tsr-detect-non-literal-regexp + var myregexpText: String = "/(x+x+)+y/"; + var myregexp: RegExp = new RegExp(myregexpText); + myregexp.test("(x+x+)+y"); + + // Markup escaping disabled - tsr-detect-disable-mustache-escape + var template: Object = new Object; + template.escapeMarkup = false; + + // Detects HTML injections - tsr-detect-html-injection + var element: Element = document.getElementById("mydiv"); + var content: String = "mycontent" + Element.innerHTML = content; + + // Timing attack - tsr-detect-possible-timing-attacks + var userInput: String = "Jane"; + var auth: String = "Jane"; + if (userInput == auth) { + console.log(userInput); + } + ``` + +1. In the **Commit message** field, enter `Add vulnerable file`. +1. In the **Target Branch** field, enter `test-branch`. +1. Select **Commit changes**. The **New merge request** form opens. +1. Select **Create merge request**. +1. In the new merge request, select `Create merge request`. + + Wait for the pipeline to complete. This could be a few minutes. + +The merge request security widget confirms that security scanning detected one potential +vulnerability. As defined in the scan result policy, the merge request is blocked and waiting for +approval. + +You now know how to set up and use scan result policies to catch vulnerabilities! diff --git a/doc/tutorials/secure_application.md b/doc/tutorials/secure_application.md new file mode 100644 index 00000000000..5b7d8733d63 --- /dev/null +++ b/doc/tutorials/secure_application.md @@ -0,0 +1,17 @@ +--- +stage: none +group: Tutorials +info: For assistance with this tutorials page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-other-projects-and-subjects. +--- + +# Secure your application and check compliance + +GitLab can check your application for security vulnerabilities and that it meets compliance requirements. + +| Topic | Description | Good for beginners | +|-------|-------------|--------------------| +| [Set up dependency scanning](https://about.gitlab.com/blog/2021/01/14/try-dependency-scanning/) | Try out dependency scanning, which checks for known vulnerabilities in dependencies. | **{star}** | +| [Create a compliance pipeline](compliance_pipeline/index.md) | Learn how to create compliance pipelines for your groups. | **{star}** | +| [Set up a scan result policy](scan_result_policy/index.md) | Learn how to configure a scan result policy that takes action based on scan results. | **{star}** | +| [Get started with GitLab application security](../user/application_security/get-started-security.md) | Follow recommended steps to set up security tools. | | +| [GitLab Security Essentials](https://levelup.gitlab.com/courses/security-essentials) | Learn about the essential security capabilities of GitLab in this self-paced course. | | diff --git a/doc/update/background_migrations.md b/doc/update/background_migrations.md index a55d2af8dd4..be6ad43fa8a 100644 --- a/doc/update/background_migrations.md +++ b/doc/update/background_migrations.md @@ -124,8 +124,10 @@ If you get this error, [check the batched background migration options](#databas ### Pause batched background migrations in GitLab 14.x -To pause an ongoing batched background migration, use the `disable` command above. -This command causes the migration to complete the current batch, and then wait to start the next batch. +To pause an ongoing batched background migration, +[disable the batched background migrations feature](#enable-or-disable-background-migrations). +Disabling the feature completes the current batch of migrations, then waits to start +the next batch until after the feature is enabled again. Use the following database queries to see the state of the current batched background migration: @@ -201,6 +203,47 @@ To disable it: Feature.disable(:optimize_batched_migrations) ``` +### Parallel execution + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104027) +> in GitLab 15.7, [behind a feature flag](../user/feature_flags.md), +> [enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/372316). +> - Enabled on GitLab.com. +> - Recommended for production use. +> - For GitLab self-managed instances, GitLab administrators can opt to +> [disable it](#enable-or-disable-parallel-execution-for-batched-background-migrations). + +There can be [risks when disabling released features](../administration/feature_flags.md#risks-when-disabling-released-features). +Refer to this feature's version history for more details. + +To speed up the execution of batched background migrations, two migrations are executed at the same time. + +[GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) can change +the number of batched background migrations executed in parallel: + +```ruby +ApplicationSetting.update_all(database_max_running_batched_background_migrations: 4) +``` + +#### Enable or disable parallel execution for batched background migrations + +Parallel execution for batched background migrations is under development but ready for production use. +It is deployed behind a feature flag that is **enabled by default**. +[GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) +can opt to disable it. + +To enable it: + +```ruby +Feature.enable(:batched_migrations_parallel_execution) +``` + +To disable it: + +```ruby +Feature.disable(:batched_migrations_parallel_execution) +``` + ## Troubleshooting ### Enable or disable background migrations diff --git a/doc/update/deprecations.md b/doc/update/deprecations.md index ff2a018cd23..e30782c70a3 100644 --- a/doc/update/deprecations.md +++ b/doc/update/deprecations.md @@ -2,6 +2,7 @@ stage: none group: none info: "See the Technical Writers assigned to Development Guidelines: https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments-to-development-guidelines" +toc: false --- # Deprecations by version @@ -18,188 +19,180 @@ For deprecation authors (usually Product Managers and Engineering Managers): - To add a deprecation, use the example.yml file in `/data/deprecations/templates` as a template. - For more information about authoring deprecations, check the the deprecation item guidance: - https://about.gitlab.com/handbook/marketing/blog/release-posts/#creating-a-deprecation-entry + https://about.gitlab.com/handbook/marketing/blog/release-posts/#update-the-deprecations-and-removals-docs For deprecation reviewers (Technical Writers only): - To update the deprecation doc, run: `bin/rake gitlab:docs:compile_deprecations` - To verify the deprecations doc is up to date, run: `bin/rake gitlab:docs:check_deprecations` - For more information about updating the deprecation doc, see the deprecation doc update guidance: - https://about.gitlab.com/handbook/marketing/blog/release-posts/#update-the-deprecations-doc + https://about.gitlab.com/handbook/marketing/blog/release-posts/#update-the-deprecations-and-removals-docs --> {::options parse_block_html="true" /} -In each release, GitLab announces features that are deprecated and no longer recommended for use. +These GitLab features are deprecated and no longer recommended for use. Each deprecated feature will be removed in a future release. Some features cause breaking changes when they are removed. +On GitLab.com, deprecated features can be removed at any time during the month leading up to the release. + **{rss}** **To be notified of upcoming breaking changes**, add this URL to your RSS feed reader: `https://about.gitlab.com/breaking-changes.xml` -DISCLAIMER: -This page contains information related to upcoming products, features, and functionality. -It is important to note that the information presented is for informational purposes only. -Please do not rely on this information for purchasing or planning purposes. -As with all projects, the items mentioned on this page are subject to change or delay. -The development, release, and timing of any products, features, or functionality remain at the -sole discretion of GitLab Inc. -
+You can also view [REST API](https://docs.gitlab.com/ee/api/rest/deprecations.html) +and [GraphQL](https://docs.gitlab.com/ee/api/graphql/removed_items.html) deprecations/removals. -
+
+
-## Announced in 15.9 +## GitLab 17.0 -
+
### Accessibility Testing is deprecated -Planned removal: GitLab 17.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/390424). +
Due to low customer usage, Accessibility Testing is deprecated and will be removed. There is no planned replacement and users should stop using Accessibility Testing before GitLab 17.0.
-
- -### Browser Performance Testing is deprecated - -Planned removal: GitLab 17.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-Due to limited customer usage, Browser Performance Testing is deprecated and will be removed. There is no planned replacement and users should stop using Browser Performance Testing before GitLab 17.0. +### Atlassian Crowd OmniAuth provider +
+- Announced in: GitLab 15.3 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/369117).
-
- -### CI/CD jobs will fail when no secret is returned from Hashicorp Vault +The `omniauth_crowd` gem that provides GitLab with the Atlassian Crowd OmniAuth provider will be removed in our +next major release, GitLab 16.0. This gem sees very little use and its +[lack of compatibility](https://github.com/robdimarco/omniauth_crowd/issues/37) with OmniAuth 2.0 is +[blocking our upgrade](https://gitlab.com/gitlab-org/gitlab/-/issues/30073). -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-When using the native HashiCorp Vault integration, CI/CD jobs will fail when no secret is returned from Vault. Make sure your configuration always return a secret, or update your pipeline to handle this change, before GitLab 16.0. +### Auto DevOps support for Herokuish is deprecated +
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/211643).
-
- -### Default CI/CD job token (`CI_JOB_TOKEN`) scope changed - -Planned removal: GitLab 16.0 +Auto DevOps support for Herokuish is deprecated in favor of [Cloud Native Buildpacks](https://docs.gitlab.com/ee/topics/autodevops/stages.html#auto-build-using-cloud-native-buildpacks). You should [migrate your builds from Herokuish to Cloud Native Buildpacks](https://docs.gitlab.com/ee/topics/autodevops/stages.html#moving-from-herokuish-to-cloud-native-buildpacks). From GitLab 14.0, Auto Build uses Cloud Native Buildpacks by default. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +Because Cloud Native Buildpacks do not support automatic testing, the Auto Test feature of Auto DevOps is also deprecated. -In GitLab 14.4 we introduced the ability to limit the "outbound" scope of the CI/CD job token (`CI_JOB_TOKEN`) to make it more secure. You can prevent job tokens from your project's pipelines from being used to access other projects. If needed, you can list specific projects that you want to access with your project's job tokens. +
-In 15.9 we extended this functionality with a better solution, an "inbound" scope limit. You can prevent the job tokens from _other_ projects from being used to access your project. With this feature, you can optionally list specific projects that you want to allow to access your project with _their_ job token. +
-In 16.0, this inbound scope limit will be the only option available for all projects, and the outbound limit setting will be removed. To prepare for this change, you can enable the ["inbound" CI/CD job token limit](https://docs.gitlab.com/ee/ci/jobs/ci_job_token.html#configure-the-job-token-scope-limit) feature now, and list any projects that need to access your project. +### Browser Performance Testing is deprecated +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/388719).
-
- -### Development dependencies reported for PHP and Python +Due to limited customer usage, Browser Performance Testing is deprecated and will be removed. There is no planned replacement and users should stop using Browser Performance Testing before GitLab 17.0. -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-In GitLab 16.0 the GitLab Dependency Scanning analyzer will begin reporting development dependencies for both Python/pipenv and PHP/composer projects. Users who do not wish to have these development dependencies reported should set `DS_INCLUDE_DEV_DEPENDENCIES: false` in their CI/CD file. +### CiRunner.projects default sort is changing to `id_desc` +
+- Announced in: GitLab 16.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/372117).
-
- -### Embedding Grafana panels in Markdown is deprecated +The `CiRunner.projects`'s field default sort order value will change from `id_asc` to `id_desc`. +If you rely on the order of the returned projects to be `id_asc`, change your scripts to make the choice explicit. -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The ability to add Grafana panels in GitLab Flavored Markdown is deprecated in 15.9 and will be removed in 16.0. -We intend to replace this feature with the ability to [embed charts](https://gitlab.com/groups/gitlab-org/opstrace/-/epics/33) with the [GitLab Observability UI](https://gitlab.com/gitlab-org/opstrace/opstrace-ui). +### CiRunnerUpgradeStatusType GraphQL type renamed to CiRunnerUpgradeStatus +
+- Announced in: GitLab 16.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/409332).
-
+The `CiRunnerUpgradeStatusType` GraphQL type has been renamed to `CiRunnerUpgradeStatus`. In GitLab 17.0, +the aliasing for the `CiRunnerUpgradeStatusType` type will be removed. -### Error Tracking UI in GitLab Rails is deprecated +
-Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +### DAST ZAP advanced configuration variables deprecation -The [Error Tracking UI](https://docs.gitlab.com/ee/operations/error_tracking.html) is deprecated in 15.9 and will be removed in 16.0. In future versions, you should use the [GitLab Observability UI](https://gitlab.com/gitlab-org/opstrace/opstrace-ui/), which will gradually be made available on GitLab.com over the next few releases. +
+- Announced in: GitLab 15.7 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/383467). +
-During the transition to the GitLab Observability UI, we will migrate the [GitLab Observability Backend](https://gitlab.com/gitlab-org/opstrace/opstrace) from a per-cluster deployment model to a per-tenant deployment model. Because [Integrated Error Tracking](https://docs.gitlab.com/ee/operations/error_tracking.html#integrated-error-tracking) is in Open Beta, we will not migrate any existing user data. For more details about the migration, see the direction pages for: +With the new browser-based DAST analyzer GA in GitLab 15.7, we are working towards making it the default DAST analyzer at some point in the future. In preparation for this, the following legacy DAST variables are being deprecated and scheduled for removal in GitLab 17.0: `DAST_ZAP_CLI_OPTIONS` and `DAST_ZAP_LOG_CONFIGURATION`. These variables allowed for advanced configuration of the legacy DAST analyzer, which was based on OWASP ZAP. The new browser-based analyzer will not include the same functionality, as these were specific to how ZAP worked. -- [Observability](https://about.gitlab.com/direction/monitor/observability/data-visualization/). -- The [Observability Backend](https://about.gitlab.com/direction/monitor/observability/data-management/). -- [Data visualization](https://about.gitlab.com/direction/monitor/observability/data-visualization/). +These three variables will be removed in GitLab 17.0.
-
+
-### External field in GraphQL ReleaseAssetLink type - -Planned removal: GitLab 16.0 +### Deprecate legacy shell escaping and quoting runner shell executor -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.11 +- End of Support: GitLab 17.9 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/406679). +
-In the [GraphQL API](https://docs.gitlab.com/ee/api/graphql), the `external` field of [`ReleaseAssetLink` type](https://docs.gitlab.com/ee/api/graphql/reference/index.html#releaseassetlink) was used to indicate whether a [release link](https://docs.gitlab.com/ee/user/project/releases/release_fields.html#links) is internal or external to your GitLab instance. -As of GitLab 15.9, we treat all release links as external, and therefore, this field is deprecated in GitLab 15.9, and will be removed in GitLab 16.0. -To avoid any disruptions to your workflow, please stop using the `external` field because it will be removed and will not be replaced. +The runner's legacy escape sequence mechanism to handle variable expansion implements a sub-optimal implementation of Ansi-C quoting. This method means that the runner would expand arguments included in double quotes. As of 15.11, we are deprecating the legacy escaping and quoting methods in the runner shell executor.
-
- -### External field in Releases and Release Links APIs +
-Planned removal: GitLab 16.0 +### DingTalk OmniAuth provider -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.10 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/390855). +
-In [Releases API](https://docs.gitlab.com/ee/api/releases) and [Release Links API](https://docs.gitlab.com/ee/api/releases/links.html), the `external` field was used to indicate whether a [release link](https://docs.gitlab.com/ee/user/project/releases/release_fields.html#links) is internal or external to your GitLab instance. -As of GitLab 15.9, we treat all release links as external, and therefore, this field is deprecated in GitLab 15.9, and will be removed in GitLab 16.0. -To avoid any disruptions to your workflow, please stop using the `external` field because it will be removed and will not be replaced. +The `omniauth-dingtalk` gem that provides GitLab with the DingTalk OmniAuth provider will be removed in our next +major release, GitLab 17.0. This gem sees very little use and is better suited for JiHu edition.
-
+
### Filepath field in Releases and Release Links APIs -Planned removal: GitLab 17.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/groups/gitlab-org/-/epics/9661). +
Support for specifying a `filepath` for a direct asset link in the [Releases API](https://docs.gitlab.com/ee/api/releases) and [Release Links API](https://docs.gitlab.com/ee/api/releases/links.html) is deprecated in GitLab 15.9 and will be @@ -217,359 +210,308 @@ To avoid any disruptions, you should replace `filepath` with `direct_asset_path`
-
+
-### GitLab Runner platforms and setup instructions in GraphQL API - -Planned removal: GitLab 17.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. - -The `runnerPlatforms` and `runnerSetup` queries to get GitLab Runner platforms and installation instructions -are deprecated and will be removed from the GraphQL API. For installation instructions, you should use the -[GitLab Runner documentation](https://docs.gitlab.com/runner/) +### GitLab Helm chart values `gitlab.kas.privateApi.*` are deprecated +
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/4097).
-
- -### HashiCorp Vault integration will no longer use CI_JOB_JWT by default - -Planned removal: GitLab 16.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. - -As part of our effort to improve the security of your CI workflows using JWT and OIDC, the native HashiCorp integration is also being updated in GitLab 16.0. Any projects that use the [`secrets:vault`](https://docs.gitlab.com/ee/ci/yaml/#secretsvault) keyword to retrieve secrets from Vault will need to be [configured to use ID tokens](https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html#configure-automatic-id-token-authentication). +We introduced the `global.kas.tls.*` Helm values to facilitate TLS communication between KAS and your Helm chart components. +The old values `gitlab.kas.privateApi.tls.enabled` and `gitlab.kas.privateApi.tls.secretName` are deprecated and scheduled for removal in GitLab 17.0. -To be prepared for this change, you should do the following before GitLab 16.0: +Because the new values provide a streamlined, comprehensive method to enable TLS for KAS, you should use `global.kas.tls.*` instead of `gitlab.kas.privateApi.tls.*`. The `gitlab.kas.privateApi.tls.*` For more information, see: -- [Disable the use of JSON web tokens](https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html#enable-automatic-id-token-authentication) in the pipeline. -- Ensure the bound audience is prefixed with `https://`. -- Use the new [`id_tokens`](https://docs.gitlab.com/ee/ci/yaml/#id_tokens) keyword - and configure the `aud` claim. +- The [merge request](https://gitlab.com/gitlab-org/charts/gitlab/-/merge_requests/2888) that introduces the `global.kas.tls.*` values. +- The [deprecated `gitlab.kas.privateApi.tls.*` documentation](https://docs.gitlab.com/charts/charts/gitlab/kas/index.html#enable-tls-communication-through-the-gitlabkasprivateapi-attributes-deprecated). +- The [new `global.kas.tls.*` documentation](https://docs.gitlab.com/charts/charts/globals.html#tls-settings-1).
-
- -### Legacy Praefect configuration method - -Planned removal: GitLab 16.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. - -Previously, Praefect configuration keys were scattered throughout the configuration file. Now, these are in a single configuration structure that matches -Praefect configuration so the previous configuration method is deprecated. - -The single configuration structure available from GitLab 15.9, though backwards compatibility is maintained. Once removed, Praefect must be configured using the single -configuration structure. You should update the configuration of Praefect at your earliest convenience. See -[GitLab 15.9 upgrade information](https://docs.gitlab.com/ee/update/#1590). +
-This change brings Praefect configuration in Omnibus GitLab in line with the configuration structure of Praefect. Previously, the hierarchies and configuration keys -didn't match. The change improves consistency between Omnibus GitLab and source installs and enables us to provide better documentation and tooling for both. +### GitLab Runner platforms and setup instructions in GraphQL API +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/387937).
-
+The `runnerPlatforms` and `runnerSetup` queries to get GitLab Runner platforms and installation instructions +are deprecated and will be removed from the GraphQL API. For installation instructions, you should use the +[GitLab Runner documentation](https://docs.gitlab.com/runner/) -### Legacy URLs replaced or removed +
-Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +### GitLab Runner registration token in Runner Operator -GitLab 16.0 removes legacy URLs from the GitLab application. +
+- Announced in: GitLab 15.6 +- End of Support: GitLab 17.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/382077). +
-When subgroups were introduced in GitLab 9.0, a `/-/` delimiter was added to URLs to signify the end of a group path. All GitLab URLs now use this delimiter for project, group, and instance level features. +The [`runner-registration-token`](https://docs.gitlab.com/runner/install/operator.html#install-the-kubernetes-operator) parameter that uses the OpenShift and Kubernetes Vanilla Operator to install a runner on Kubernetes is deprecated. Authentication tokens will be used to register runners instead. Registration tokens, and support for certain configuration arguments, +will be disabled behind a feature flag in GitLab 16.6 and removed in GitLab 17.0. The configuration arguments disabled for authentication tokens are: -URLs that do not use the `/-/` delimiter are planned for removal in GitLab 16.0. For the full list of these URLs, along with their replacements, see [issue 28848](https://gitlab.com/gitlab-org/gitlab/-/issues/28848#release-notes). +- `--locked` +- `--access-level` +- `--run-untagged` +- `--tag-list` -Update any scripts or bookmarks that reference the legacy URLs. GitLab APIs are not affected by this change. +This change is a breaking change. You should use an [authentication token](../ci/runners/register_runner.md) in the `gitlab-runner register` command instead.
-
- -### License Compliance CI Template - -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +### GraphQL type, `RunnerMembershipFilter` renamed to `CiRunnerMembershipFilter` -The GitLab [License Compliance](https://docs.gitlab.com/ee/user/compliance/license_compliance/) CI template is now deprecated and is scheduled for removal in the GitLab 16.0 release. Users who wish to continue using GitLab for License Compliance should remove the License Compliance template from their CI pipeline and add the [Dependency Scanning template](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuration). The Dependency Scanning template is now capable of gathering the required license information so it is no longer necessary to run a separate License Compliance job. The License Compliance CI template should not be removed prior to verifying that the `license_scanning_sbom_scanner` and `package_metadata_synchronization` flags are enabled for the instance and that the instance has been upgraded to a version that supports [the new method of license scanning](https://docs.gitlab.com/ee/user/compliance/license_scanning_of_cyclonedx_files/). +
+- Announced in: GitLab 16.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/409333). +
-| CI Pipeline Includes | GitLab <= 15.8 | 15.9 <= GitLab < 16.0 | GitLab >= 16.0 | -| ------------- | ------------- | ------------- | ------------- | -| Both DS and LS templates | License data from LS job is used | License data from LS job is used | License data from DS job is used | -| DS template is included but LS template is not | No license data | License data from DS job is used | License data from DS job is used | -| LS template is included but DS template is not | License data from LS job is used | License data from LS job is used | No license data | +The GraphQL type, `RunnerMembershipFilter`, has been renamed to `CiRunnerMembershipFilter`. In GitLab 17.0, +the aliasing for the `RunnerMembershipFilter` type will be removed.
-
- -### License-Check and the Policies tab on the License Compliance page +
-Planned removal: GitLab 16.0 +### GraphQL: The `DISABLED_WITH_OVERRIDE` value of the `SharedRunnersSetting` enum is deprecated. Use `DISABLED_AND_OVERRIDABLE` instead -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/385636). +
-The [License-Check feature](https://docs.gitlab.com/ee/user/compliance/license_check_rules.html) is now deprecated and is scheduled for removal in GitLab 16.0. Additionally, the Policies tab on the License Compliance page and all APIs related to the License-Check feature are deprecated and planned for removal in GitLab 16.0. Users who wish to continue to enforce approvals based on detected licenses are encouraged to create a new [License Approval policy](https://docs.gitlab.com/ee/user/compliance/license_approval_policies.html) instead. +In GitLab 17.0, the `DISABLED_WITH_OVERRIDE` value of the `SharedRunnersSetting` GraphQL enum type will be replaced with the value, `DISABLED_AND_OVERRIDABLE`.
-
+
### Load Performance Testing is deprecated -Planned removal: GitLab 17.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/388723). +
Due to low customer usage, Load Performance Testing is deprecated and will be removed. There is no planned replacement and users should stop using Load Performance Testing before GitLab 17.0.
-
- -### Old versions of JSON web tokens are deprecated - -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. - -Now that we have released [ID tokens](https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html) -with OIDC support, the old JSON web tokens are deprecated and will be removed. -Both the `CI_JOB_JWT` and `CI_JOB_JWT_V2` tokens, exposed to jobs as predefined variables, -will no longer be available in GitLab 16.0. - -To prepare for this change, you should: - -- Configure your pipelines to use the fully configurable and more secure - [`id_token`](https://docs.gitlab.com/ee/ci/yaml/index.html#id_tokens) keyword instead. -- [Enable the **Limit JSON Web Token (JWT) access**](https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html#enable-automatic-id-token-authentication) - setting, which prevents the old tokens from being exposed to any jobs. This setting - will be permanently enabled for all projects in GitLab 16.0. +### Maintainer role providing the ability to change Package settings using GraphQL API +
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/370471).
-
- -### Option to delete projects immediately is deprecated from deletion protection settings +The ability for users with the Maintainer role to change the **Packages and registries** settings for a group using +the GraphQL API is deprecated in GitLab 15.8 and will be removed in GitLab 17.0. These settings include: -Planned removal: GitLab 16.0 +- [Allowing or preventing duplicate package uploads](https://docs.gitlab.com/ee/user/packages/maven_repository/#do-not-allow-duplicate-maven-packages). +- [Package request forwarding](https://docs.gitlab.com/ee/user/packages/maven_repository/#request-forwarding-to-maven-central). +- [Enabling lifecycle rules for the Dependency Proxy](https://docs.gitlab.com/ee/user/packages/dependency_proxy/reduce_dependency_proxy_storage.html). -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +In GitLab 17.0 and later, you must have the Owner role for a group to change the **Packages and registries** +settings for the group using either the GitLab UI or GraphQL API. -The project deletion protection setting in the Admin Area had an option to delete projects immediately. Starting with 16.0, this option will no longer be available, and delayed project deletion will become the default behavior. +
-The option will no longer appear as a group setting. Self-managed users will still have the option to define the deletion delay period, and SaaS users have a non-adjustable default retention period of 7 days. Users can still delete the project immediately from the project settings. +
-The option to delete projects immediately by default was deprecated to prevent users from accidentally taking this action and permanently losing projects. +### Package pipelines in API payload is paginated +
+- Announced in: GitLab 14.5 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/289956).
-
+A request to the API for `/api/v4/projects/:id/packages` returns a paginated result of packages. Each package lists all of its pipelines in this response. This is a performance concern, as it's possible for a package to have hundreds or thousands of associated pipelines. -### Queue selector for running Sidekiq is deprecated +In milestone 17.0, we will remove the `pipelines` attribute from the API response. -End of Support: GitLab 16.0
-Planned removal: GitLab 17.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-Running Sidekiq with a [queue selector](https://docs.gitlab.com/ee/administration/sidekiq/processing_specific_job_classes.html#queue-selectors) (having multiple processes listening to a set of queues) and [negate settings](https://docs.gitlab.com/ee/administration/sidekiq/processing_specific_job_classes.html#negate-settings) is deprecated and will be fully removed in 17.0. +### PipelineSecurityReportFinding projectFingerprint GraphQL field -You can migrate away from queue selectors to [listening to all queues in all processes](https://docs.gitlab.com/ee/administration/sidekiq/extra_sidekiq_processes.html#start-multiple-processes). For example, if Sidekiq is currently running with 4 processes (denoted by 4 elements in `sidekiq['queue_groups']` in `/etc/gitlab/gitlab.rb`) with queue selector (`sidekiq['queue_selector'] = true`), you can change Sidekiq to listen to all queues in all 4 processes,for example `sidekiq['queue_groups'] = ['*'] * 4`. This approach is also recommended in our [Reference Architecture](https://docs.gitlab.com/ee/administration/reference_architectures/5k_users.html#configure-sidekiq). Note that Sidekiq can effectively run as many processes as the number of CPUs in the machine. +
+- Announced in: GitLab 15.1 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/343475). +
-While the above approach is recommended for most instances, Sidekiq can also be run using [routing rules](https://docs.gitlab.com/ee/administration/sidekiq/processing_specific_job_classes.html#routing-rules) which is also being used on GitLab.com. You can follow the [migration guide from queue selectors to routing rules](https://docs.gitlab.com/ee/administration/sidekiq/processing_specific_job_classes.html#migrating-from-queue-selectors-to-routing-rules). You need to take care with the migration to avoid losing jobs entirely. +The [`project_fingerprint`](https://gitlab.com/groups/gitlab-org/-/epics/2791) attribute of vulnerability findings is being deprecated in favor of a `uuid` attribute. By using UUIDv5 values to identify findings, we can easily associate any related entity with a finding. The `project_fingerprint` attribute is no longer being used to track findings, and will be removed in GitLab 17.0.
-
+
-### Remove offset pagination from Jobs API +### PostgreSQL 13 deprecated -Planned removal: GitLab 16.0 +
+- Announced in: GitLab 16.0 +- End of Support: GitLab 17.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/groups/gitlab-org/-/epics/9065). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +GitLab follows an [annual upgrade cadence for PostgreSQL](https://about.gitlab.com/handbook/engineering/development/enablement/data_stores/database/postgresql-upgrade-cadence.html). -A request to the API for `/api/v4/projects/:id/jobs` can return a paginated list of jobs. Projects can contain hundreds or thousands of jobs, so using an offset to paginate through them is slow. Users should instead use [`keyset-based pagination`](https://docs.gitlab.com/ee/api/rest/index.html#keyset-based-pagination) when requesting consecutive pages of results. +Support for PostgreSQL 13 is scheduled for removal in GitLab 17.0. +In GitLab 17.0, PostgreSQL 14 becomes the minimum required PostgreSQL version. -In milestone 16.0 we will remove offset-based pagination. +PostgreSQL 13 will be supported for the full GitLab 16 release cycle. +PostgreSQL 14 will also be supported for instances that want to upgrade prior to GitLab 17.0.
-
+
-### Required Pipeline Configuration is deprecated +### Queue selector for running Sidekiq is deprecated -Planned removal: GitLab 16.0 +
+- Announced in: GitLab 15.9 +- End of Support: GitLab 16.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/390787). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +Running Sidekiq with a [queue selector](https://docs.gitlab.com/ee/administration/sidekiq/processing_specific_job_classes.html#queue-selectors) (having multiple processes listening to a set of queues) and [negate settings](https://docs.gitlab.com/ee/administration/sidekiq/processing_specific_job_classes.html#negate-settings) is deprecated and will be fully removed in 17.0. -Required Pipeline Configuration will be removed in the 16.0 release. This impacts self-managed users on the Ultimate license. +You can migrate away from queue selectors to [listening to all queues in all processes](https://docs.gitlab.com/ee/administration/sidekiq/extra_sidekiq_processes.html#start-multiple-processes). For example, if Sidekiq is currently running with 4 processes (denoted by 4 elements in `sidekiq['queue_groups']` in `/etc/gitlab/gitlab.rb`) with queue selector (`sidekiq['queue_selector'] = true`), you can change Sidekiq to listen to all queues in all 4 processes,for example `sidekiq['queue_groups'] = ['*'] * 4`. This approach is also recommended in our [Reference Architecture](https://docs.gitlab.com/ee/administration/reference_architectures/5k_users.html#configure-sidekiq). Note that Sidekiq can effectively run as many processes as the number of CPUs in the machine. -We recommend replacing this with an alternative [compliance solution](https://docs.gitlab.com/ee/user/group/compliance_frameworks.html#compliance-pipelines) -that is available now. We recommend this alternative solution because it provides greater flexibility, allowing required pipelines to be assigned to specific compliance framework labels. +While the above approach is recommended for most instances, Sidekiq can also be run using [routing rules](https://docs.gitlab.com/ee/administration/sidekiq/processing_specific_job_classes.html#routing-rules) which is also being used on GitLab.com. You can follow the [migration guide from queue selectors to routing rules](https://docs.gitlab.com/ee/administration/sidekiq/processing_specific_job_classes.html#migrating-from-queue-selectors-to-routing-rules). You need to take care with the migration to avoid losing jobs entirely.
-
- -### SAST analyzer coverage changing in GitLab 16.0 +
-Planned removal: GitLab 16.0 +### Registration tokens and server-side runner arguments in `POST /api/v4/runners` endpoint -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.6 +- End of Support: GitLab 17.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/379743). +
-GitLab SAST uses various [analyzers](https://docs.gitlab.com/ee/user/application_security/sast/analyzers/) to scan code for vulnerabilities. +The support for registration tokens and certain runner configuration arguments in the `POST` method operation on the `/api/v4/runners` endpoint is deprecated. +This endpoint [registers](https://docs.gitlab.com/ee/api/runners.html#register-a-new-runner) a runner +with a GitLab instance at the instance, group, or project level through the API. Registration tokens, and support for certain configuration arguments, +will be disabled behind a feature flag in GitLab 16.6 and removed in GitLab 17.0. The configuration arguments disabled for authentication tokens are: -We're reducing the number of supported analyzers used by default in GitLab SAST. -This is part of our long-term strategy to deliver a faster, more consistent user experience across different programming languages. +- `--locked` +- `--access-level` +- `--run-untagged` +- `--maximum-timeout` +- `--paused` +- `--tag-list` +- `--maintenance-note` -Starting in GitLab 16.0, the GitLab SAST CI/CD template will no longer use the following analyzers, and they will enter End of Support status: +This change is a breaking change. You should [create a runner in the UI](../ci/runners/register_runner.md) to add configurations, and use the authentication token in the `gitlab-runner register` command instead. -- [Security Code Scan](https://gitlab.com/gitlab-org/security-products/analyzers/security-code-scan) (.NET) -- [PHPCS Security Audit](https://gitlab.com/gitlab-org/security-products/analyzers/phpcs-security-audit) (PHP) +
-We'll remove these analyzers from the [SAST CI/CD template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml) and replace them with GitLab-supported detection rules and the [Semgrep-based analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep). -Effective immediately, these analyzers will receive only security updates; other routine improvements or updates are not guaranteed. -After these analyzers reach End of Support, no further updates will be provided. -However, we won't delete container images previously published for these analyzers or remove the ability to run them by using a custom CI/CD pipeline job. +
-We will also remove Scala from the scope of the [SpotBugs-based analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) and replace it with the [Semgrep-based analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep). -This change will make it simpler to scan Scala code; compilation will no longer be required. -This change will be reflected in the automatic language detection portion of the [GitLab-managed SAST CI/CD template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml). -Note that the SpotBugs-based analyzer will continue to cover Groovy and Kotlin. +### Registration tokens and server-side runner arguments in `gitlab-runner register` command -If you've already dismissed a vulnerability finding from one of the deprecated analyzers, the replacement attempts to respect your previous dismissal. The system behavior depends on: +
+- Announced in: GitLab 15.6 +- End of Support: GitLab 17.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/380872). +
-- whether you've excluded the Semgrep-based analyzer from running in the past. -- which analyzer first discovered the vulnerabilities shown in the project's Vulnerability Report. +Registration tokens and certain configuration arguments in the command `gitlab-runner register` that [registers](https://docs.gitlab.com/runner/register/) a runner, are deprecated. +Authentication tokens will be used to register runners instead. Registration tokens, and support for certain configuration arguments, +will be disabled behind a feature flag in GitLab 16.6 and removed in GitLab 17.0. The configuration arguments disabled for authentication tokens are: -See [Vulnerability translation documentation](https://docs.gitlab.com/ee/user/application_security/sast/analyzers.html#vulnerability-translation) for further details. +- `--locked` +- `--access-level` +- `--run-untagged` +- `--maximum-timeout` +- `--paused` +- `--tag-list` +- `--maintenance-note` -If you applied customizations to any of the affected analyzers or if you currently disable the Semgrep analyzer in your pipelines, you must take action as detailed in the [deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/390416#breaking-change). +This change is a breaking change. You should [create a runner in the UI](../ci/runners/register_runner.md) to add configurations, and use the authentication token in the `gitlab-runner register` command instead.
-
- -### Secure analyzers major version update - -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. - -The Secure stage will be bumping the major versions of its analyzers in tandem with the GitLab 16.0 release. This bump will enable a clear delineation for analyzers, between: +### Required Pipeline Configuration is deprecated -- Those released prior to May 22, 2023 -- Those released after May 22, 2023 +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/389467). +
-If you are not using the default included templates, or have pinned your analyzer versions you will need to update your CI/CD job definition to either remove the pinned version or to update the latest major version. -Users of GitLab 13.0-15.10 will continue to experience analyzer updates as normal until the release of GitLab 16.0, following which all newly fixed bugs and released features will be released only in the new major version of the analyzers. We do not backport bugs and features to deprecated versions as per our [maintenance policy](https://docs.gitlab.com/ee/policy/maintenance.html). As required, security patches will be backported within the latest 3 minor releases. -Specifically, the following are being deprecated and will no longer be updated after 16.0 GitLab release: +Required Pipeline Configuration will be removed in the 17.0 release. This impacts self-managed users on the Ultimate license. -- API Fuzzing: version 2 -- Container Scanning: version 5 -- Coverage-guided fuzz testing: version 3 -- Dependency Scanning: version 3 -- Dynamic Application Security Testing (DAST): version 3 -- DAST API: version 2 -- IaC Scanning: version 3 -- License Scanning: version 4 -- Secret Detection: version 4 -- Static Application Security Testing (SAST): version 3 of [all analyzers](https://docs.gitlab.com/ee/user/application_security/sast/#supported-languages-and-frameworks) - - `brakeman`: version 3 - - `flawfinder`: version 3 - - `kubesec`: version 3 - - `mobsf`: version 3 - - `nodejs-scan`: version 3 - - `phpcs-security-audit`: version 3 - - `pmd-apex`: version 3 - - `security-code-scan`: version 3 - - `semgrep`: version 3 - - `sobelow`: version 3 - - `spotbugs`: version 3 +We recommend replacing this with an alternative [compliance solution](https://docs.gitlab.com/ee/user/group/compliance_frameworks.html#compliance-pipelines) +that is available now. We recommend this alternative solution because it provides greater flexibility, allowing required pipelines to be assigned to specific compliance framework labels.
-
- -### Secure scanning CI/CD templates will use new job `rules` - -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +### Self-managed certificate-based integration with Kubernetes -GitLab-managed CI/CD templates for security scanning will be updated in the GitLab 16.0 release. -The updates will include improvements already released in the Latest versions of the CI/CD templates. -We released these changes in the Latest template versions because they have the potential to disrupt customized CI/CD pipeline configurations. +
+- Announced in: GitLab 14.5 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/groups/gitlab-org/configure/-/epics/8). +
-In all updated templates, we're: +The certificate-based integration with Kubernetes [will be deprecated and removed](https://about.gitlab.com/blog/2021/11/15/deprecating-the-cert-based-kubernetes-integration/). -- Adding support for running scans in merge request (MR) pipelines. -- Updating the definition of variables like `SAST_DISABLED` and `DEPENDENCY_SCANNING_DISABLED` to disable scanning only if the value is `"true"`. Previously, even if the value were `"false"`, scanning would be disabled. +As a self-managed customer, we are introducing the [feature flag](../administration/feature_flags.md#enable-or-disable-the-feature) `certificate_based_clusters` in GitLab 15.0 so you can keep your certificate-based integration enabled. However, the feature flag will be disabled by default, so this change is a **breaking change**. -The following templates will be updated: +In GitLab 17.0 we will remove both the feature and its related code. Until the final removal in 17.0, features built on this integration will continue to work, if you enable the feature flag. Until the feature is removed, GitLab will continue to fix security and critical issues as they arise. -- API Fuzzing: [`API-Fuzzing.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml) -- Container Scanning: [`Container-Scanning.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml) -- Coverage-Guided Fuzzing: [`Coverage-Fuzzing.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml) -- DAST: [`DAST.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml) -- DAST API: [`DAST-API.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml) -- Dependency Scanning: [`Dependency-Scanning.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml) -- IaC Scanning: [`SAST-IaC.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml) -- SAST: [`SAST.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml) -- Secret Detection: [`Secret-Detection.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Secret-Detction.gitlab-ci.yml) +For a more robust, secure, forthcoming, and reliable integration with Kubernetes, we recommend you use the +[agent for Kubernetes](https://docs.gitlab.com/ee/user/clusters/agent/) to connect Kubernetes clusters with GitLab. [How do I migrate?](https://docs.gitlab.com/ee/user/infrastructure/clusters/migrate_to_gitlab_agent.html) -We recommend that you test your pipelines before the 16.0 release if you use one of the templates listed above and you do any of the following: +Although an explicit removal date is set, we don't plan to remove this feature until the new solution has feature parity. +For more information about the blockers to removal, see [this issue](https://gitlab.com/gitlab-org/configure/general/-/issues/199). - 1. You override `rules` for your security scanning jobs. - 1. You use the `_DISABLED` variables but set a value other than `"true"`. +For updates and details about this deprecation, follow [this epic](https://gitlab.com/groups/gitlab-org/configure/-/epics/8).
-
+
### Single database connection is deprecated -Planned removal: GitLab 17.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/387898). +
Previously, [GitLab's database](https://docs.gitlab.com/omnibus/settings/database.html) configuration had a single `main:` section. This is being deprecated. The new @@ -582,16 +524,16 @@ automatically from GitLab 16.0 onwards.
-
+
### Slack notifications integration -End of Support: GitLab 17.0
-Planned removal: GitLab 17.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.9 +- End of Support: GitLab 17.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/372411). +
As we're consolidating all Slack capabilities into the GitLab for Slack app, we're [deprecating the Slack notifications @@ -602,582 +544,590 @@ we'll be introducing support in [this epic](https://gitlab.com/groups/gitlab-org
-
- -### Support for Praefect custom metrics endpoint configuration +
-Planned removal: GitLab 16.0 +### Support for REST API endpoints that reset runner registration tokens -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.7 +- End of Support: GitLab 17.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/383341). +
-Support for using the `prometheus_exclude_database_from_default_metrics` configuration value is deprecated in GitLab -15.9 and will be removed in GitLab 16.0. We are removing this configuration value because using it is non-performant. -This change means the following metrics will become unavailable on `/metrics`: +The support for runner registration tokens is deprecated. As a consequence, the REST API endpoints to reset a registration token are also deprecated and will +be removed in GitLab 17.0. +The deprecated endpoints are: -- `gitaly_praefect_unavailable_repositories`. -- `gitaly_praefect_verification_queue_depth`. -- `gitaly_praefect_replication_queue_depth`. +- `POST /runners/reset_registration_token` +- `POST /projects/:id/runners/reset_registration_token` +- `POST /groups/:id/runners/reset_registration_token` -This may require updating your metrics collection targets to also scrape `/db_metrics`. +We plan to implement a new method to bind runners to a GitLab instance +as part of the new [GitLab Runner token architecture](https://docs.gitlab.com/ee/architecture/blueprints/runner_tokens/). +The work is planned in [this epic](https://gitlab.com/groups/gitlab-org/-/epics/7633). +This new architecture introduces a new method for registering runners and will eliminate the legacy +[runner registration token](https://docs.gitlab.com/ee/security/token_overview.html#runner-registration-tokens). +From GitLab 17.0 and later, the runner registration methods implemented by the new GitLab Runner token architecture will be the only supported methods.
-
+
### The GitLab legacy requirement IID is deprecated in favor of work item IID -Planned removal: GitLab 17.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/390263). +
We will be transitioning to a new IID as a result of moving requirements to a [work item type](https://docs.gitlab.com/ee/development/work_items.html#work-items-and-work-item-types). Users should begin using the new IID as support for the legacy IID and existing formatting will end in GitLab 17.0. The legacy requirement IID remains available until its removal in GitLab 17.0.
-
- -### Trigger jobs can mirror downstream pipeline status exactly - -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. - -In some cases, like when a downstream pipeline had the `passed with warnings` status, trigger jobs that were using [`strategy: depend`](https://docs.gitlab.com/ee/ci/yaml/index.html#strategydepend) did not mirror the status of the downstream pipeline exactly. In GitLab 16.0 trigger jobs will show the exact same status as the the downstream pipeline. If your pipeline relied on this behavior, you should update your pipeline to handle the more accurate status. +### The Visual Reviews tool is deprecated +
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/387751).
-
- -
-## Announced in 15.8 - -
- -### Approvers and Approver Group fields in Merge Request Approval API - -Planned removal: GitLab 16.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. - -The endpoint to get the configuration of approvals for a project returns empty arrays for `approvers` and `approval_groups`. These fields were deprecated in favor of the endpoint to [get project-level rules](https://docs.gitlab.com/ee/api/merge_request_approvals.html#get-project-level-rules) for a merge request. API users are encouraged to switch to this endpoint instead. These fields will be removed from the `get configuration` endpoint in v5 of the GitLab REST API. +Due to limited customer usage and capabilities, the Visual Reviews feature for Review Apps is deprecated and will be removed. There is no planned replacement and users should stop using Visual Reviews before GitLab 17.0.
-
- -### Auto DevOps no longer provisions a PostgreSQL database by default - -Planned removal: GitLab 16.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. - -Currently, Auto DevOps provisions an in-cluster PostgreSQL database by default. -In GitLab 16.0, databases will be provisioned only for users who opt in. This -change supports production deployments that require more robust database management. +
-If you want Auto DevOps to provision an in-cluster database, -set the `POSTGRES_ENABLED` CI/CD variable to `true`. +### The `gitlab-runner exec` command is deprecated +
+- Announced in: GitLab 15.7 +- End of Support: GitLab 17.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/385235).
-
+The [`gitlab-runner exec`](https://docs.gitlab.com/runner/commands/#gitlab-runner-exec) command is deprecated and will be fully removed from GitLab Runner in 16.0. The `gitlab-runner exec` feature was initially developed to provide the ability to validate a GitLab CI pipeline on a local system without needing to commit the updates to a GitLab instance. However, with the continued evolution of GitLab CI, replicating all GitLab CI features into `gitlab-runner exec` was no longer viable. Pipeline syntax and validation [simulation](https://docs.gitlab.com/ee/ci/pipeline_editor/#simulate-a-cicd-pipeline) are available in the GitLab pipeline editor. -### Auto DevOps support for Herokuish is deprecated +
-Planned removal: GitLab 17.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +### Trigger jobs can mirror downstream pipeline status exactly -Auto DevOps support for Herokuish is deprecated in favor of [Cloud Native Buildpacks](https://docs.gitlab.com/ee/topics/autodevops/stages.html#auto-build-using-cloud-native-buildpacks). You should [migrate your builds from Herokuish to Cloud Native Buildpacks](https://docs.gitlab.com/ee/topics/autodevops/stages.html#moving-from-herokuish-to-cloud-native-buildpacks). From GitLab 14.0, Auto Build uses Cloud Native Buildpacks by default. +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/285493). +
-Because Cloud Native Buildpacks do not support automatic testing, the Auto Test feature of Auto DevOps is also deprecated. +In some cases, like when a downstream pipeline had the `passed with warnings` status, trigger jobs that were using [`strategy: depend`](https://docs.gitlab.com/ee/ci/yaml/index.html#strategydepend) did not mirror the status of the downstream pipeline exactly. In GitLab 17.0 trigger jobs will show the exact same status as the the downstream pipeline. If your pipeline relied on this behavior, you should update your pipeline to handle the more accurate status.
-
+
-### Automatic backup upload using Openstack Swift and Rackspace APIs +### `runnerRegistrationToken` parameter for GitLab Runner Helm Chart -End of Support: GitLab 15.10
-Planned removal: GitLab 15.10 +
+- Announced in: GitLab 15.6 +- End of Support: GitLab 17.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/381111). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +The [`runnerRegistrationToken`](https://docs.gitlab.com/runner/install/kubernetes.html#required-configuration) parameter to use the GitLab Helm Chart to install a runner on Kubernetes is deprecated. -We are deprecating support for [uploading backups to remote storage](https://docs.gitlab.com/ee/raketasks/backup_gitlab.html#upload-backups-to-a-remote-cloud-storage) using Openstack Swift and Rackspace APIs. The support for these APIs depends on third-party libraries that are no longer actively maintained and have not been updated for Ruby 3. GitLab is switching over to Ruby 3 prior to EOL of Ruby 2 in order to stay up to date on security patches. +We plan to implement a new method to bind runners to a GitLab instance leveraging `runnerToken` +as part of the new [GitLab Runner token architecture](https://docs.gitlab.com/ee/architecture/blueprints/runner_tokens/). +The work is planned in [this epic](https://gitlab.com/groups/gitlab-org/-/epics/7633). -- If you're using OpenStack, you need to change you configuration to use the S3 API instead of Swift. -- If you're using Rackspace storage, you need to switch to a different provider or manually upload the backup file after the backup task is complete. +From GitLab 17.0 and later, the methods to register runners introduced by the new GitLab Runner token architecture will be the only supported methods.
-
- -### Azure Storage Driver defaults to the correct root prefix - -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +### `sidekiq` delivery method for `incoming_email` and `service_desk_email` is deprecated -The Azure Storage Driver writes to `//` as the default root directory. This default root directory appears in some places within the Azure UI as `//`. We have maintained this legacy behavior to support older deployments using this storage driver. However, when moving to Azure from another storage driver, this behavior hides all your data until you configure the storage driver to build root paths without an extra leading slash by setting `trimlegacyrootprefix: true`. +
+- Announced in: GitLab 16.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/398132). +
-The new default configuration for the storage driver will set `trimlegacyrootprefix: true`, and `/` will be the default root directory. You can add `trimlegacyrootprefix: false` to your current configuration to avoid any disruptions. +The `sidekiq` delivery method for `incoming_email` and `service_desk_email` is deprecated and is +scheduled for removal in GitLab 17.0. -This breaking change will happen in GitLab 16.0. +GitLab uses a separate process called `mail_room` to ingest emails. Currently, GitLab administrators +can configure their GitLab instances to use `sidekiq` or `webhook` delivery methods to deliver ingested +emails from `mail_room` to GitLab. -
+Using the deprecated `sidekiq` delivery method, `mail_room` writes the job data directly to the GitLab +Redis queue. This means that there is a hard coupling between the delivery method and the Redis +configuration. Another disadvantage is that framework optimizations such as job payload compression are missed. -
+Using the `webhook` delivery method, `mail_room` pushes the ingested email body to the GitLab +API. That way `mail_room` does not need to know your Redis configuration and the GitLab application +adds the processing job. `mail_room` authenticates with a shared secret key. -### Conan project-level search endpoint returns project-specific results +Reconfiguring an Omnibus installation generates this secret key file automatically, +so no secret file configuration setting is needed. -Planned removal: GitLab 16.0 +You can configure a custom secret key file (32 characters base 64 encoded) by running a command +like below and referencing the secret file in `incoming_email_secret_file` and +`service_desk_email_secret_file` (always specify the absolute path): -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +```shell +echo $( ruby -rsecurerandom -e "puts SecureRandom.base64(32)" ) > ~/.gitlab-mailroom-secret +``` -You can use the GitLab Conan repository with [project-level](https://docs.gitlab.com/ee/user/packages/conan_repository/#add-a-remote-for-your-project) or [instance-level](https://docs.gitlab.com/ee/user/packages/conan_repository/#add-a-remote-for-your-instance) endpoints. Each level supports the conan search command. However, the search endpoint for the project level is also returning packages from outside the target project. +If you run GitLab on more than one machine, you need to provide the secret key file for each machine. -This unintended functionality is deprecated in GitLab 15.8 and will be removed in GitLab 16.0. The search endpoint for the project level will only return packages from the target project. +We highly encourage GitLab administrators to start using the `webhook` delivery method for +`incoming_email_delivery_method` and `service_desk_email_delivery_method` instead of `sidekiq`.
-
- -### Configuring Redis config file paths using environment variables is deprecated +
-Planned removal: GitLab 16.0 +### project.pipeline.securityReportFindings GraphQL query -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.1 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/343475). +
-You can no longer specify Redis configuration file locations -using the environment variables like `GITLAB_REDIS_CACHE_CONFIG_FILE` or -`GITLAB_REDIS_QUEUES_CONFIG_FILE`. Use the default -config file locations instead, for example `config/redis.cache.yml` or -`config/redis.queues.yml`. +Previous work helped [align the vulnerabilities calls for pipeline security tabs](https://gitlab.com/gitlab-org/gitlab/-/issues/343469) to match the vulnerabilities calls for project-level and group-level vulnerability reports. This helped the frontend have a more consistent interface. The old `project.pipeline.securityReportFindings` query was formatted differently than other vulnerability data calls. Now that it has been replaced with the new `project.pipeline.vulnerabilities` field, the old `project.pipeline.securityReportFindings` is being deprecated and will be removed in GitLab 17.0. +
-
- -### Container Registry pull-through cache +
-Planned removal: GitLab 16.0 +## GitLab 16.6 -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The Container Registry pull-through cache is deprecated in GitLab 15.8 and will be removed in GitLab 16.0. While the Container Registry pull-through cache functionality is useful, we have not made significant changes to this feature. You can use the upstream version of the container registry to achieve the same functionality. Removing the pull-through cache allows us also to remove the upstream client code without sacrificing functionality. +### Error Tracking UI in GitLab Rails is deprecated +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/389991).
-
- -### Cookie authorization in the GitLab for Jira Cloud app - -Planned removal: GitLab 16.0 +The [Error Tracking UI](https://docs.gitlab.com/ee/operations/error_tracking.html) is deprecated in 15.9 and will be removed in 16.6 (milestone might change) once GitLab Observability UI is made available. In future versions, you should use the [GitLab Observability UI](https://gitlab.com/gitlab-org/opstrace/opstrace-ui/), which will gradually be made available on GitLab.com over the next few releases. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +During the transition to the GitLab Observability UI, we will migrate the [GitLab Observability Backend](https://gitlab.com/gitlab-org/opstrace/opstrace) from a per-cluster deployment model to a per-tenant deployment model. Because [Integrated Error Tracking](https://docs.gitlab.com/ee/operations/error_tracking.html#integrated-error-tracking) is in Open Beta, we will not migrate any existing user data. For more details about the migration, see the direction pages for: -Cookie authentication in the GitLab for Jira Cloud app is now deprecated in favor of OAuth authentication. -You must [set up OAuth authentication](https://docs.gitlab.com/ee/integration/jira/connect-app.html#set-up-oauth-authentication) -to continue to use the GitLab for Jira Cloud app. Without OAuth, you will not be able to manage linked namespaces. +- [Observability](https://about.gitlab.com/direction/monitor/observability/data-visualization/). +- The [Observability Backend](https://about.gitlab.com/direction/monitor/observability/data-management/). +- [Data visualization](https://about.gitlab.com/direction/monitor/observability/data-visualization/). +
-
- -### Dependency Scanning support for Java 13, 14, 15, and 16 +
-Planned removal: GitLab 16.0 +## GitLab 16.5 -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-GitLab has deprecated Dependency Scanning support for Java versions 13, 14, 15, and 16 and plans to remove that support in the upcoming GitLab 16.0 release. This is consistent with [Oracle's support policy](https://www.oracle.com/java/technologies/java-se-support-roadmap.html) as Oracle Premier and Extended Support for these versions has ended. This also allows GitLab to focus Dependency Scanning Java support on LTS versions moving forward. +### HashiCorp Vault integration will no longer use CI_JOB_JWT by default +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/366798).
-
+As part of our effort to improve the security of your CI workflows using JWT and OIDC, the native HashiCorp integration is also being updated in GitLab 16.0. Any projects that use the [`secrets:vault`](https://docs.gitlab.com/ee/ci/yaml/#secretsvault) keyword to retrieve secrets from Vault will need to be [configured to use the ID tokens](https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html#configure-automatic-id-token-authentication). ID tokens were introduced in 15.7. -### Deployment API returns error when `updated_at` and `updated_at` are not used together +To prepare for this change, use the new [`id_tokens`](https://docs.gitlab.com/ee/ci/yaml/#id_tokens) +keyword and configure the `aud` claim. Ensure the bound audience is prefixed with `https://`. -Planned removal: GitLab 16.0 +In GitLab 15.9 to 15.11, you can [enable the **Limit JSON Web Token (JWT) access**](https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html#enable-automatic-id-token-authentication) +setting, which prevents the old tokens from being exposed to any jobs and enables +[ID token authentication for the `secrets:vault` keyword](https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html#configure-automatic-id-token-authentication). -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +In GitLab 16.0 and later: -The Deployment API will now return an error when `updated_at` filtering and `updated_at` sorting are not used together. Some users were using filtering by `updated_at` to fetch "latest" deployment without using `updated_at` sorting, which may produce wrong results. You should instead use them together, or migrate to filtering by `finished_at` and sorting by `finished_at` which will give you "latest deployments" in a consistent way. +- This setting will be removed. +- CI/CD jobs that use the `id_tokens` keyword can use ID tokens with `secrets:vault`, + and will not have any `CI_JOB_JWT*` tokens available. +- Jobs that do not use the `id_tokens` keyword will continue to have the `CI_JOB_JWT*` + tokens available until GitLab 16.5.
-
- -### Developer role providing the ability to import projects to a group - -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. - -The ability for users with the Developer role for a group to import projects to that group is deprecated in GitLab -15.8 and will be removed in GitLab 16.0. From GitLab 16.0, only users with at least the Maintainer role for a group -will be able to import projects to that group. +### Old versions of JSON web tokens are deprecated +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/366798).
-
+[ID tokens](https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html) with OIDC support +were introduced in GitLab 15.7. These tokens are more configurable than the old JSON web tokens (JWTs), are OIDC compliant, +and only available in CI/CD jobs that explictly have ID tokens configured. +ID tokens are more secure than the old `CI_JOB_JWT*` JSON web tokens which are exposed in every job, +and as a result these old JSON web tokens are deprecated: -### GitLab Helm chart values `gitlab.kas.privateApi.*` are deprecated +- `CI_JOB_JWT` +- `CI_JOB_JWT_V1` +- `CI_JOB_JWT_V2` -Planned removal: GitLab 17.0 +To prepare for this change, configure your pipelines to use [ID tokens](https://docs.gitlab.com/ee/ci/yaml/index.html#id_tokens) +instead of the deprecated tokens. For OIDC compliance, the `iss` claim now uses +the fully qualified domain name, for example `https://example.com`, previously +introduced with the `CI_JOB_JWT_V2` token. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +In GitLab 15.9 to 15.11, you can [enable the **Limit JSON Web Token (JWT) access**](https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html#enable-automatic-id-token-authentication) +setting, which prevents the old tokens from being exposed to any jobs and enables +[ID token authentication for the `secrets:vault` keyword](https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html#configure-automatic-id-token-authentication). -We introduced the `global.kas.tls.*` Helm values to facilitate TLS communication between KAS and your Helm chart components. -The old values `gitlab.kas.privateApi.tls.enabled` and `gitlab.kas.privateApi.tls.secretName` are deprecated and scheduled for removal in GitLab 17.0. +In GitLab 16.0 and later: -Because the new values provide a streamlined, comprehensive method to enable TLS for KAS, you should use `global.kas.tls.*` instead of `gitlab.kas.privateApi.tls.*`. The `gitlab.kas.privateApi.tls.*` For more information, see: +- This setting will be removed. +- CI/CD jobs that use the `id_tokens` keyword can use ID tokens with `secrets:vault`, + and will not have any `CI_JOB_JWT*` tokens available. +- Jobs that do not use the `id_tokens` keyword will continue to have the `CI_JOB_JWT*` + tokens available until GitLab 16.5. -- The [merge request](https://gitlab.com/gitlab-org/charts/gitlab/-/merge_requests/2888) that introduces the `global.kas.tls.*` values. -- The [deprecated `gitlab.kas.privateApi.tls.*` documentation](https://docs.gitlab.com/charts/charts/gitlab/kas/index.html#enable-tls-communication-through-the-gitlabkasprivateapi-attributes-deprecated). -- The [new `global.kas.tls.*` documentation](https://docs.gitlab.com/charts/charts/globals.html#tls-settings-1). +In GitLab 16.5, the deprecated tokens will be completely removed and will no longer +be available in CI/CD jobs. +
-
- -### GitLab.com importer - -Planned removal: GitLab 16.0 +
-The [GitLab.com importer](https://docs.gitlab.com/ee/user/project/import/gitlab_com.html) is deprecated in GitLab 15.8 and will be removed in GitLab 16.0. +## GitLab 16.3 -The GitLab.com importer was introduced in 2015 for importing a project from GitLab.com to a self-managed GitLab instance through the UI. -This feature is available on self-managed instances only. [Migrating GitLab groups and projects by direct transfer](https://docs.gitlab.com/ee/user/group/import/#migrate-groups-by-direct-transfer-recommended) -supersedes the GitLab.com importer and provides a more cohesive importing functionality. +
-See [migrated group items](https://docs.gitlab.com/ee/user/group/import/#migrated-group-items) and [migrated project items](https://docs.gitlab.com/ee/user/group/import/#migrated-project-items) for an overview. +### Bundled Grafana deprecated and disabled +
+- Announced in: GitLab 16.0 +- End of Support: GitLab 16.3 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/7772).
-
- -### GraphQL: The `DISABLED_WITH_OVERRIDE` value of the `SharedRunnersSetting` enum is deprecated. Use `DISABLED_AND_OVERRIDABLE` instead +The version of [Grafana bundled with Omnibus GitLab](https://docs.gitlab.com/omnibus/settings/grafana.html) is +disabled in 16.0 and will be removed in 16.3. +If you are using the bundled Grafana, you must migrate to either: -Planned removal: GitLab 16.0 +- Another implementation of Grafana. For more information, see + [Switch to new Grafana instance](https://docs.gitlab.com/ee/administration/monitoring/performance/grafana_configuration.html#switch-to-new-grafana-instance). +- Another observability platform of your choice. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +The version of Grafana that is currently provided is no longer a supported version. -In GitLab 16.0, the `DISABLED_WITH_OVERRIDE` value of the `SharedRunnersSetting` GraphQL enum type will be replaced with the value, `DISABLED_AND_OVERRIDABLE`. +In GitLab versions 16.0 to 16.2, you can still [re-enable the bundled Grafana](https://docs.gitlab.com/ee/administration/monitoring/performance/grafana_configuration.html#temporary-workaround). +However, enabling the bundled Grafana will no longer work from GitLab 16.3. +
-
- -### Limit personal access token and deploy token's access with external authorization +
-Planned removal: GitLab 16.0 +## GitLab 16.1 -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-With external authorization enabled, personal access tokens (PATs) and deploy tokens must no longer be able to access container or package registries. This defense-in-depth security measure will be deployed in 16.0. For users that use PATs and deploy tokens to access these registries, this measure breaks this use of these tokens. Disable external authorization to use tokens with container or package registries. +### GitLab Runner images based on Alpine 3.12, 3.13, 3.14 +
+- Announced in: GitLab 15.11 +- End of Support: GitLab 16.1 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/29639).
-
- -### Live Preview no longer available in the Web IDE - -Planned removal: GitLab 15.9 +We will stop publishing runner images based on the following, end-of-life Alpine versions: -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. - -The Live Preview feature of the Web IDE was intended to provide a client-side preview of static web applications. However, complex configuration steps and a narrow set of supported project types have limited its utility. With the introduction of the Web IDE Beta in GitLab 15.7, you can now connect to a full server-side runtime environment. With upcoming support for installing extensions in the Web IDE, we'll also support more advanced workflows than those available with Live Preview. As of GitLab 15.9, Live Preview is no longer available in the Web IDE. +- Alpine 3.12 +- Alpine 3.13 +- Alpine 3.14 (end-of-life on 2023-05-23)
-
+
-### Maintainer role providing the ability to change Package settings using GraphQL API - -Planned removal: GitLab 16.0 +### License Compliance CI Template -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/387561). +
-The ability for users with the Maintainer role to change the **Packages and registries** settings for a group using -the GraphQL API is deprecated in GitLab 15.8 and will be removed in GitLab 16.0. These settings include: +**Update:** We previously announced we would remove the existing License Compliance CI template in GitLab 16.0. However, due to performance issues with the [license scanning of CycloneDX files](https://docs.gitlab.com/ee/user/compliance/license_scanning_of_cyclonedx_files/) we will do this change in 16.1 instead. -- [Allowing or preventing duplicate package uploads](https://docs.gitlab.com/ee/user/packages/maven_repository/#do-not-allow-duplicate-maven-packages). -- [Package request forwarding](https://docs.gitlab.com/ee/user/packages/maven_repository/#request-forwarding-to-maven-central). -- [Enabling lifecycle rules for the Dependency Proxy](https://docs.gitlab.com/ee/user/packages/dependency_proxy/reduce_dependency_proxy_storage.html). +The GitLab [License Compliance](https://docs.gitlab.com/ee/user/compliance/license_compliance/) CI template is now deprecated and is scheduled for removal in the GitLab 16.1 release. Users who wish to continue using GitLab for License Compliance should remove the License Compliance template from their CI pipeline and add the [Dependency Scanning template](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuration). The Dependency Scanning template is now capable of gathering the required license information so it is no longer necessary to run a separate License Compliance job. The License Compliance CI template should not be removed prior to verifying that the `license_scanning_sbom_scanner` and `package_metadata_synchronization` flags are enabled for the instance and that the instance has been upgraded to a version that supports [the new method of license scanning](https://docs.gitlab.com/ee/user/compliance/license_scanning_of_cyclonedx_files/). -In GitLab 16.0 and later, you must have Owner role for a group to change the **Packages and registries** -settings for the group using either the GitLab UI or GraphQL API. +| CI Pipeline Includes | GitLab <= 15.8 | 15.9 <= GitLab < 16.1 | GitLab >= 16.1 | +| ------------- | ------------- | ------------- | ------------- | +| Both DS and LS templates | License data from LS job is used | License data from LS job is used | License data from DS job is used | +| DS template is included but LS template is not | No license data | License data from DS job is used | License data from DS job is used | +| LS template is included but DS template is not | License data from LS job is used | License data from LS job is used | No license data | +
-
- -### Non-standard default Redis ports are deprecated - -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +## GitLab 16.0 -If GitLab starts without any Redis configuration file present, -GitLab assumes it can connect to three Redis servers at `localhost:6380`, -`localhost:6381` and `localhost:6382`. We are changing this behavior -so GitLab assumes there is one Redis server at `localhost:6379`. +
-Administrators who want to keep the three servers must configure -the Redis URLs by editing the `config/redis.cache.yml`,`config/redis.queues.yml` -and `config/redis.shared_state.yml` files. +### Auto DevOps no longer provisions a PostgreSQL database by default +
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/343988).
-
+Currently, Auto DevOps provisions an in-cluster PostgreSQL database by default. +In GitLab 16.0, databases will be provisioned only for users who opt in. This +change supports production deployments that require more robust database management. -### Null value for `private_profile` attribute in User API is deprecated +If you want Auto DevOps to provision an in-cluster database, +set the `POSTGRES_ENABLED` CI/CD variable to `true`. -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-When creating and updating users through the API, `null` was a valid value for the `private_profile` attribute, which would internally be converted to the default value. Starting with 16.0, `null` will no longer be a valid value for this parameter, and the response will be a 400 if used. Now the only valid values are `true` and `false`. +### Azure Storage Driver defaults to the correct root prefix +
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/container-registry/-/issues/854).
-
+The Azure Storage Driver writes to `//` as the default root directory. This default root directory appears in some places within the Azure UI as `//`. We have maintained this legacy behavior to support older deployments using this storage driver. However, when moving to Azure from another storage driver, this behavior hides all your data until you configure the storage driver to build root paths without an extra leading slash by setting `trimlegacyrootprefix: true`. -### Projects API field `operations_access_level` is deprecated +The new default configuration for the storage driver will set `trimlegacyrootprefix: true`, and `/` will be the default root directory. You can add `trimlegacyrootprefix: false` to your current configuration to avoid any disruptions. + +This breaking change will happen in GitLab 16.0. -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-We are deprecating the `operations_access_level` field in the Projects API. This field has been replaced by fields to control specific features: `releases_access_level`, `environments_access_level`, `feature_flags_access_level`, `infrastructure_access_level`, and `monitor_access_level`. +### Bundled Grafana Helm Chart is deprecated +
+- Announced in: GitLab 15.10 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/4353).
-
+The Grafana Helm chart that is bundled with the GitLab Helm Chart is deprecated and will be removed in the GitLab Helm Chart 7.0 release (releasing along with GitLab 16.0). -### Rake task for importing bare repositories +The bundled Grafana Helm chart is an optional service that can be turned on to provide the Grafana UI connected to the GitLab Helm Chart's Prometheus metrics. -Planned removal: GitLab 16.0 +The version of Grafana that the GitLab Helm Chart is currently providing is no longer a supported Grafana version. +If you're using the bundled Grafana, you should switch to the [newer chart version from Grafana Labs](https://artifacthub.io/packages/helm/grafana/grafana) +or a Grafana Operator from a trusted provider. -The [Rake task for importing bare repositories](https://docs.gitlab.com/ee/raketasks/import.html) `gitlab:import:repos` is deprecated in GitLab 15.8 and will be removed in GitLab 16.0. +In your new Grafana instance, you can [configure the GitLab provided Prometheus as a data source](https://docs.gitlab.com/ee/administration/monitoring/performance/grafana_configuration.html#integration-with-gitlab-ui) +and [connect Grafana to the GitLab UI](https://docs.gitlab.com/ee/administration/monitoring/performance/grafana_configuration.html#integration-with-gitlab-ui). -This Rake task imports a directory tree of repositories into a GitLab instance. These repositories must have been -managed by GitLab previously, because the Rake task relies on the specific directory structure or a specific custom Git setting in order to work (`gitlab.fullpath`). +
-Importing repositories using this Rake task has limitations. The Rake task: +
-- Only knows about project and project wiki repositories and doesn't support repositories for designs, group wikis, or snippets. -- Permits you to import non-hashed storage projects even though these aren't supported. -- Relies on having Git config `gitlab.fullpath` set. [Epic 8953](https://gitlab.com/groups/gitlab-org/-/epics/8953) proposes removing support for this setting. +### CAS OmniAuth provider -Alternatives to using the `gitlab:import:repos` Rake task include: +
+- Announced in: GitLab 15.3 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/369127). +
-- Migrating projects using either [an export file](https://docs.gitlab.com/ee/user/project/settings/import_export.html) or - [direct transfer](https://docs.gitlab.com/ee/user/group/import/#migrate-groups-by-direct-transfer-recommended) migrate repositories as well. -- Importing a [repository by URL](https://docs.gitlab.com/ee/user/project/import/repo_by_url.html). -- Importing a [repositories from a non-GitLab source](https://docs.gitlab.com/ee/user/project/import/). +The `omniauth-cas3` gem that provides GitLab with the CAS OmniAuth provider will be removed in our next major +release, GitLab 16.0. This gem sees very little use and its lack of upstream maintenance is preventing GitLab's +[upgrade to OmniAuth 2.0](https://gitlab.com/gitlab-org/gitlab/-/issues/30073).
-
+
-### Support for third party registries +### CI/CD jobs will fail when no secret is returned from Hashicorp Vault -Planned removal: GitLab 16.0 +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/353080). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +When using the native HashiCorp Vault integration, CI/CD jobs will fail when no secret is returned from Vault. Make sure your configuration always return a secret, or update your pipeline to handle this change, before GitLab 16.0. -Support for third-party container registries is deprecated in GitLab 15.8 and will be [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/376217) in GitLab 16.0. Supporting both GitLab's Container Registry and third-party container registries is challenging for maintenance, code quality, and backward compatibility. This hinders our ability to stay [efficient](https://about.gitlab.com/handbook/values/#efficiency). +
-Since we released the new [GitLab Container Registry](https://gitlab.com/groups/gitlab-org/-/epics/5523) version for GitLab.com, we've started to implement additional features that are not available in third-party container registries. These new features have allowed us to achieve significant performance improvements, such as [cleanup policies](https://gitlab.com/groups/gitlab-org/-/epics/8379). We are focusing on delivering [new features](https://gitlab.com/groups/gitlab-org/-/epics/5136), most of which will require functionalities only available on the GitLab Container Registry. This deprecation allows us to reduce fragmentation and user frustration in the long term by focusing on delivering a more robust integrated registry experience and feature set. +
-Moving forward, we'll continue to invest in developing and releasing new features that will only be available in the GitLab Container Registry. +### Changing MobSF-based SAST analyzer behavior in multi-module Android projects +
+- Announced in: GitLab 16.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/408396).
-
+**Update:** We previously announced a change to how the MobSF-based GitLab SAST analyzer would scan multi-module Android projects. +We've cancelled that change, and no action is required. -### Test system hook endpoint +Instead of changing which single module would be scanned, we [improved multi-module support](https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/merge_requests/73). -End of Support: GitLab 16.0
-Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The [test system hook](https://docs.gitlab.com/ee/api/system_hooks.html#test-system-hook) endpoint returns dummy data. -This endpoint is now deprecated and will be removed from the GitLab codebase. +### Changing merge request approvals with the `/approvals` API endpoint +
+- Announced in: GitLab 14.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/353097).
-
+To change the approvals required for a merge request, you should no longer use the `/approvals` API endpoint, which was deprecated in GitLab 14.0. -### The API no longer returns revoked tokens for the agent for Kubernetes +Instead, use the [`/approval_rules` endpoint](https://docs.gitlab.com/ee/api/merge_request_approvals.html#merge-request-level-mr-approvals) to [create](https://docs.gitlab.com/ee/api/merge_request_approvals.html#create-merge-request-level-rule) or [update](https://docs.gitlab.com/ee/api/merge_request_approvals.html#update-merge-request-level-rule) the approval rules for a merge request. -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-Currently, GET requests to the [Cluster Agents API](https://docs.gitlab.com/ee/api/cluster_agents.html#list-tokens-for-an-agent) -endpoints can return revoked tokens. In GitLab 16.0, GET requests will not return revoked tokens. +### Conan project-level search endpoint returns project-specific results -You should review your calls to these endpoints and ensure you do not use revoked tokens. +
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/384455). +
-This change affects the following REST and GraphQL API endpoints: +You can use the GitLab Conan repository with [project-level](https://docs.gitlab.com/ee/user/packages/conan_repository/#add-a-remote-for-your-project) or [instance-level](https://docs.gitlab.com/ee/user/packages/conan_repository/#add-a-remote-for-your-instance) endpoints. Each level supports the conan search command. However, the search endpoint for the project level is also returning packages from outside the target project. -- REST API: - - [List tokens](https://docs.gitlab.com/ee/api/cluster_agents.html#list-tokens-for-an-agent) - - [Get a single token](https://docs.gitlab.com/ee/api/cluster_agents.html#get-a-single-agent-token) -- GraphQL: - - [`ClusterAgent.tokens`](https://docs.gitlab.com/ee/api/graphql/reference/#clusteragenttokens) +This unintended functionality is deprecated in GitLab 15.8 and will be removed in GitLab 16.0. The search endpoint for the project level will only return packages from the target project.
-
- -### The Visual Reviews tool is deprecated +
-Planned removal: GitLab 17.0 +### Configuration fields in GitLab Runner Helm Chart -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.6 +- End of Support: GitLab 16.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/379064). +
-Due to limited customer usage and capabilities, the Visual Reviews feature for Review Apps is deprecated and will be removed. There is no planned replacement and users should stop using Visual Reviews before GitLab 17.0. +From GitLab 13.6, users can [specify any runner configuration in the GitLab Runner Helm chart](https://docs.gitlab.com/runner/install/kubernetes.html). When we implemented this feature, we deprecated values in the GitLab Helm Chart configuration that were specific to GitLab Runner. These fields are deprecated and we plan to remove them in v1.0 of the GitLab Runner Helm chart.
-
+
-### The latest Terraform templates will overwrite current stable templates +### Configuring Redis config file paths using environment variables is deprecated -Planned removal: GitLab 16.0 +
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/388255). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +You can no longer specify Redis configuration file locations +using the environment variables like `GITLAB_REDIS_CACHE_CONFIG_FILE` or +`GITLAB_REDIS_QUEUES_CONFIG_FILE`. Use the default +config file locations instead, for example `config/redis.cache.yml` or +`config/redis.queues.yml`. -With every major GitLab version, we update the stable Terraform templates with the current latest templates. -This change affects the [quickstart](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml) -and the [base](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml) templates. +
-Because the new templates ship with default rules, the update might break your Terraform pipelines. -For example, if your Terraform jobs are triggered as a downstream pipeline, the rules won't trigger your jobs -in GitLab 16.0. +
-To accommodate the changes, you might need to adjust the [`rules`](https://docs.gitlab.com/ee/ci/yaml/#rules) in your -`.gitlab-ci.yml` file. +### Container Registry pull-through cache +
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/container-registry/-/issues/842).
-
- -### `environment_tier` parameter for DORA API +The Container Registry [pull-through cache](https://docs.docker.com/registry/recipes/mirror/) is deprecated in GitLab 15.8 and will be removed in GitLab 16.0. The pull-through cache is part of the upstream [Docker Distribution project](https://github.com/distribution/distribution). However, we are removing the pull-through cache in favor of the GitLab Dependency Proxy, which allows you to proxy and cache container images from Docker Hub. Removing the pull-through cache allows us also to remove the upstream client code without sacrificing functionality. -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-To avoid confusion and duplication, the `environment_tier` parameter is deprecated in favor of the `environment_tiers` parameter. The new `environment_tiers` parameter allows DORA APIs to return aggregated data for multiple tiers at the same time. The `environment_tier` parameter will be removed in GitLab 16.0. +### Container Scanning variables that reference Docker +
+- Announced in: GitLab 15.4 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/371840).
-
- -### openSUSE Leap 15.3 packages - -Planned removal: GitLab 15.11 +All Container Scanning variables that are prefixed by `DOCKER_` in variable name are deprecated. This includes the `DOCKER_IMAGE`, `DOCKER_PASSWORD`, `DOCKER_USER`, and `DOCKERFILE_PATH` variables. Support for these variables will be removed in the GitLab 16.0 release. Use the [new variable names](https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-cicd-variables) `CS_IMAGE`, `CS_REGISTRY_PASSWORD`, `CS_REGISTRY_USER`, and `CS_DOCKERFILE_PATH` in place of the deprecated names. -Distribution support and security updates for openSUSE Leap 15.3 [ended December 2022](https://en.opensuse.org/Lifetime#Discontinued_distributions). +
-Starting in GitLab 15.7 we started providing packages for openSUSE Leap 15.4, and will stop providing packages for openSUSE Leap 15.3 in the 15.11 milestone. +
-- Switch from the openSUSE Leap 15.3 packages to the provided 15.4 packages. +### Cookie authorization in the GitLab for Jira Cloud app -
+
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/387299).
-
+Cookie authentication in the GitLab for Jira Cloud app is now deprecated in favor of OAuth authentication. +On self-managed, you must [set up OAuth authentication](https://docs.gitlab.com/ee/integration/jira/connect-app.html#set-up-oauth-authentication-for-self-managed-instances) +to continue to use the GitLab for Jira Cloud app. Without OAuth, you can't manage linked namespaces. -## Announced in 15.7 +
-
+
### DAST API scans using DAST template is deprecated -Planned removal: GitLab 16.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.7 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/384198). +
With the move to the new DAST API analyzer and the `DAST-API.gitlab-ci.yml` template for DAST API scans, we will be removing the ability to scan APIs with the DAST analyzer. Use of the `DAST.gitlab-ci.yml` or `DAST-latest.gitlab-ci.yml` templates for API scans is deprecated as of GitLab 15.7 and will no longer work in GitLab 16.0. Please use `DAST-API.gitlab-ci.yml` template and refer to the [DAST API analyzer](https://docs.gitlab.com/ee/user/application_security/dast_api/#configure-dast-api-with-an-openapi-specification) documentation for configuration details.
-
+
### DAST API variables -Planned removal: GitLab 16.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.7 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/383467). +
With the switch to the new DAST API analyzer in GitLab 15.6, two legacy DAST API variables are being deprecated. The variables `DAST_API_HOST_OVERRIDE` and `DAST_API_SPECIFICATION` will no longer be used for DAST API scans. @@ -1189,508 +1139,508 @@ These two variables will be removed in GitLab 16.0.
-
+
-### DAST ZAP advanced configuration variables deprecation - -Planned removal: GitLab 16.0 +### DAST report variables deprecation -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.7 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/384340). +
-With the new browser-based DAST analyzer GA in GitLab 15.7, we are working towards making it the default DAST analyzer at some point in the future. In preparation for this, the following legacy DAST variables are being deprecated and scheduled for removal in GitLab 16.0: `DAST_ZAP_CLI_OPTIONS` and `DAST_ZAP_LOG_CONFIGURATION`. These variables allowed for advanced configuration of the legacy DAST analyzer, which was based on OWASP ZAP. The new browser-based analyzer will not include the same functionality, as these were specific to how ZAP worked. +With the new browser-based DAST analyzer GA in GitLab 15.7, we are working towards making it the default DAST analyzer at some point in the future. In preparation for this, the following legacy DAST variables are being deprecated and scheduled for removal in GitLab 16.0: `DAST_HTML_REPORT`, `DAST_XML_REPORT`, and `DAST_MARKDOWN_REPORT`. These reports relied on the legacy DAST analyzer and we do not plan to implement them in the new browser-based analyzer. As of GitLab 16.0, these report artifacts will no longer be generated. These three variables will be removed in GitLab 16.0.
-
+
-### DAST report variables deprecation +### Default CI/CD job token (`CI_JOB_TOKEN`) scope changed + +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/395708). +
-Planned removal: GitLab 16.0 +In GitLab 14.4 we introduced the ability to [limit your project's CI/CD job token](https://docs.gitlab.com/ee/ci/jobs/ci_job_token.html#limit-your-projects-job-token-access) (`CI_JOB_TOKEN`) access to make it more secure. You can prevent job tokens **from your project's** pipelines from being used to **access other projects**. When enabled with no other configuration, your pipelines cannot access other projects. To use the job token to access other projects from your pipeline, you must list those projects explicitly in the **Limit CI_JOB_TOKEN access** setting's allowlist, and you must be a maintainer in all the projects. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +The job token functionality was updated in 15.9 with a better security setting to [allow access to your project with a job token](https://docs.gitlab.com/ee/ci/jobs/ci_job_token.html#allow-access-to-your-project-with-a-job-token). When enabled with no other configuration, job tokens **from other projects** cannot **access your project**. Similar to the older setting, you can optionally allow other projects to access your project with a job token if you list those projects explicitly in the **Allow access to this project with a CI_JOB_TOKEN** setting's allowlist. With this new setting, you must be a maintainer in your own project, but only need to have the Guest role in the other projects. -With the new browser-based DAST analyzer GA in GitLab 15.7, we are working towards making it the default DAST analyzer at some point in the future. In preparation for this, the following legacy DAST variables are being deprecated and scheduled for removal in GitLab 16.0: `DAST_HTML_REPORT`, `DAST_XML_REPORT`, and `DAST_MARKDOWN_REPORT`. These reports relied on the legacy DAST analyzer and we do not plan to implement them in the new browser-based analyzer. As of GitLab 16.0, these report artifacts will no longer be generated. +As a result, the **Limit** setting is deprecated in preference of the better **Allow access** setting. In GitLab 16.0 the **Limit** setting will be disabled by default for all new projects. In projects with this setting currently enabled, it will continue to function as expected, but you will not be able to add any more projects to the allowlist. If the setting is disabled in any project, it will not be possible to re-enable this setting in 16.0 or later. -These three variables will be removed in GitLab 16.0. +In 17.0, we plan to remove the **Limit** setting completely, and set the **Allow access** setting to enabled for all projects. This change ensures a higher level of security between projects. If you currently use the **Limit** setting, you should update your projects to use the **Allow access** setting instead. If other projects access your project with a job token, you must add them to the **Allow access** allowlist. + +To prepare for this change, users on GitLab.com or self-managed GitLab 15.9 or later can enable the **Allow access** setting now and add the other projects. It will not be possible to disable the setting in 17.0 or later.
-
+
-### KAS Metrics Port in GitLab Helm Chart - -End of Support: GitLab 16.0
-Planned removal: GitLab 16.0 +### Dependency Scanning support for Java 13, 14, 15, and 16 -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/387560). +
-The `gitlab.kas.metrics.port` has been deprecated in favor of the new `gitlab.kas.observability.port` configuration field for the [GitLab Helm Chart](https://gitlab.com/gitlab-org/charts/gitlab/-/merge_requests/2839). -This port is used for much more than just metrics, which warranted this change to avoid confusion in configuration. +GitLab has deprecated Dependency Scanning support for Java versions 13, 14, 15, and 16 and plans to remove that support in the upcoming GitLab 16.0 release. This is consistent with [Oracle's support policy](https://www.oracle.com/java/technologies/java-se-support-roadmap.html) as Oracle Premier and Extended Support for these versions has ended. This also allows GitLab to focus Dependency Scanning Java support on LTS versions moving forward.
-
- -### Shimo integration +
-End of Support: GitLab 16.0
-Planned removal: GitLab 16.0 +### Deployment API returns error when `updated_at` and `updated_at` are not used together -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/328500). +
-The [Shimo Workspace integration](https://docs.gitlab.com/ee/user/project/integrations/shimo.html) has been deprecated -and will be moved to the JiHu GitLab codebase. +The Deployment API will now return an error when `updated_at` filtering and `updated_at` sorting are not used together. Some users were using filtering by `updated_at` to fetch "latest" deployment without using `updated_at` sorting, which may produce wrong results. You should instead use them together, or migrate to filtering by `finished_at` and sorting by `finished_at` which will give you "latest deployments" in a consistent way.
-
+
-### Single merge request changes API endpoint +### Deprecate legacy Gitaly configuration methods -Planned removal: GitLab 16.0 +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352609). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +Using environment variables `GIT_CONFIG_SYSTEM` and `GIT_CONFIG_GLOBAL` to configure Gitaly is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/352609). +These variables are being replaced with standard [`config.toml` Gitaly configuration](https://docs.gitlab.com/ee/administration/gitaly/reference.html). -The endpoint to get [changes from a single merge request](https://docs.gitlab.com/ee/api/merge_requests.html#get-single-merge-request-changes) has been deprecated in favor the [list merge request diffs](https://docs.gitlab.com/ee/api/merge_requests.html#list-merge-request-diffs) endpoint. API users are encouraged to switch to the new diffs endpoint instead. The `changes from a single merge request` endpoint will be removed in v5 of the GitLab REST API. +GitLab instances that use `GIT_CONFIG_SYSTEM` and `GIT_CONFIG_GLOBAL` to configure Gitaly should switch to configuring using +`config.toml`.
-
+
-### Support for REST API endpoints that reset runner registration tokens - -End of Support: GitLab 16.6
-Planned removal: GitLab 17.0 +### Deprecated Consul http metrics -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.10 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/7278). +
-The support for runner registration tokens is deprecated. As a consequence, the REST API endpoints to reset a registration token are also deprecated and will -be removed in GitLab 17.0. -The deprecated endpoints are: +The Consul provided in the GitLab Omnibus package will no longer provide older deprecated Consul metrics starting in GitLab 16.0. -- `POST /runners/reset_registration_token` -- `POST /projects/:id/runners/reset_registration_token` -- `POST /groups/:id/runners/reset_registration_token` +In GitLab 14.0, [Consul was updated to 1.9.6](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/5344), +which deprecated some telemetry metrics from being at the `consul.http` path. In GitLab 16.0, the `consul.http` path will be removed. -In GitLab 15.8, we plan to implement a new method to bind runners to a GitLab instance, -as part of the new [GitLab Runner token architecture](https://docs.gitlab.com/ee/architecture/blueprints/runner_tokens/). -This new architecture introduces a new method for registering runners and will eliminate the legacy -[runner registration token](https://docs.gitlab.com/ee/security/token_overview.html#runner-registration-tokens). -From GitLab 17.0 and later, the runner registration methods implemented by the new GitLab Runner token architecture will be the only supported methods. +If you have monitoring that consumes Consul metrics, update them to use `consul.api.http` instead of `consul.http`. +For more information, see [the deprecation notes for Consul 1.9.0](https://github.com/hashicorp/consul/releases/tag/v1.9.0).
-
- -### Support for periods (`.`) in Terraform state names might break existing states - -End of Support: GitLab 16.0
-Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +### Deprecation and planned removal for `CI_PRE_CLONE_SCRIPT` variable on GitLab SaaS -Previously, Terraform state names containing periods were not supported. However, you could still use state names with periods via a workaround. +
+- Announced in: GitLab 15.9 +- End of Support: GitLab 16.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/391896). +
-GitLab 15.7 [adds full support](https://docs.gitlab.com/ee/user/infrastructure/iac/troubleshooting.html#state-not-found-if-the-state-name-contains-a-period) for state names that contain periods. If you used a workaround to handle these state names, your jobs might fail, or it might look like you've run Terraform for the first time. +The [`CI_PRE_CLONE_SCRIPT` variable](https://docs.gitlab.com/ee/ci/runners/saas/linux_saas_runner.html#pre-clone-script) supported by GitLab SaaS Runners is deprecated as of GitLab 15.9 and will be removed in 16.0. The `CI_PRE_CLONE_SCRIPT` variable enables you to run commands in your CI/CD job prior to the runner executing Git init and get fetch. For more information about how this feature works, see [Pre-clone script](https://docs.gitlab.com/ee/ci/runners/saas/linux_saas_runner.html#pre-clone-script). As an alternative, you can use the [`pre_get_sources_script`](https://docs.gitlab.com/ee/ci/yaml/#hookspre_get_sources_script). -To resolve the issue: +
- 1. Change any references to the state file by excluding the period and any characters that follow. - - For example, if your state name is `state.name`, change all references to `state`. - 1. Run your Terraform commands. +
-To use the full state name, including the period, [migrate to the full state file](https://docs.gitlab.com/ee/user/infrastructure/iac/terraform_state.html#migrate-to-a-gitlab-managed-terraform-state). +### Developer role providing the ability to import projects to a group +
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/387891).
-
- -### The Phabricator task importer is deprecated +The ability for users with the Developer role for a group to import projects to that group is deprecated in GitLab +15.8 and will be removed in GitLab 16.0. From GitLab 16.0, only users with at least the Maintainer role for a group +will be able to import projects to that group. -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The [Phabricator task importer](https://docs.gitlab.com/ee/user/project/import/phabricator.html) is being deprecated. Phabricator itself as a project is no longer actively maintained since June 1, 2021. We haven't observed imports using this tool. There has been no activity on the open related issues on GitLab. +### Development dependencies reported for PHP and Python +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/375505).
-
- -### The `gitlab-runner exec` command is deprecated +In GitLab 16.0 the GitLab Dependency Scanning analyzer will begin reporting development dependencies for both Python/pipenv and PHP/composer projects. Users who do not wish to have these development dependencies reported should set `DS_INCLUDE_DEV_DEPENDENCIES: false` in their CI/CD file. -End of Support: GitLab 17.0
-Planned removal: GitLab 17.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The [`gitlab-runner exec`](https://docs.gitlab.com/runner/commands/#gitlab-runner-exec) command is deprecated and will be fully removed from GitLab Runner in 16.0. The `gitlab-runner exec` feature was initially developed to provide the ability to validate a GitLab CI pipeline on a local system without needing to commit the updates to a GitLab instance. However, with the continued evolution of GitLab CI, replicating all GitLab CI features into `gitlab-runner exec` was no longer viable. Pipeline syntax and validation [simulation](https://docs.gitlab.com/ee/ci/pipeline_editor/#simulate-a-cicd-pipeline) are available in the GitLab pipeline editor. +### Embedding Grafana panels in Markdown is deprecated +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/389477).
-
- -### ZenTao integration +The ability to add Grafana panels in GitLab Flavored Markdown is deprecated in 15.9 and will be removed in 16.0. +We intend to replace this feature with the ability to [embed charts](https://gitlab.com/groups/gitlab-org/opstrace/-/epics/33) with the [GitLab Observability UI](https://gitlab.com/gitlab-org/opstrace/opstrace-ui). -End of Support: GitLab 16.0
-Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The [ZenTao product integration](https://docs.gitlab.com/ee/user/project/integrations/zentao.html) has been deprecated -and will be moved to the JiHu GitLab codebase. +### Enforced validation of CI/CD parameter character lengths +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/372770).
-
- -### `POST ci/lint` API endpoint deprecated +While CI/CD [job names](https://docs.gitlab.com/ee/ci/jobs/index.html#job-name-limitations) have a strict 255 character limit, other CI/CD parameters do not yet have validations ensuring they also stay under the limit. -Planned removal: GitLab 16.0 +In GitLab 16.0, validation will be added to strictly limit the following to 255 characters as well: -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +- The `stage` keyword. +- The `ref`, which is the Git branch or tag name for the pipeline. +- The `description` and `target_url` parameter, used by external CI/CD integrations. -The `POST ci/lint` API endpoint is deprecated in 15.7, and will be removed in 16.0. This endpoint does not validate the full range of CI/CD configuration options. Instead, use [`POST /projects/:id/ci/lint`](https://docs.gitlab.com/ee/api/lint.html#validate-a-ci-yaml-configuration-with-a-namespace), which properly validates CI/CD configuration. +Users on self-managed instances should update their pipelines to ensure they do not use parameters that exceed 255 characters. Users on GitLab.com do not need to make any changes, as these are already limited in that database. -
-
+
-## Announced in 15.6 +### Environment search query requires at least three characters -
+
+- Announced in: GitLab 15.10 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/382532). +
-### Configuration fields in GitLab Runner Helm Chart +From GitLab 16.0, when you search for environments with the API, you must use at least three characters. This change helps us ensure the scalability of the search operation. -End of Support: GitLab 16.0
-Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-From GitLab 13.6, users can [specify any runner configuration in the GitLab Runner Helm chart](https://docs.gitlab.com/runner/install/kubernetes.html). When we implemented this feature, we deprecated values in the GitLab Helm Chart configuration that were specific to GitLab Runner. These fields are deprecated and we plan to remove them in v1.0 of the GitLab Runner Helm chart. +### External field in GraphQL ReleaseAssetLink type +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/).
-
- -### GitLab Runner registration token in Runner Operator +In the [GraphQL API](https://docs.gitlab.com/ee/api/graphql/), the `external` field of [`ReleaseAssetLink` type](https://docs.gitlab.com/ee/api/graphql/reference/index.html#releaseassetlink) was used to indicate whether a [release link](https://docs.gitlab.com/ee/user/project/releases/release_fields.html#links) is internal or external to your GitLab instance. +As of GitLab 15.9, we treat all release links as external, and therefore, this field is deprecated in GitLab 15.9, and will be removed in GitLab 16.0. +To avoid any disruptions to your workflow, please stop using the `external` field because it will be removed and will not be replaced. -End of Support: GitLab 16.6
-Planned removal: GitLab 17.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The [`runner-registration-token`](https://docs.gitlab.com/runner/install/operator.html#install-the-kubernetes-operator) parameter that uses the OpenShift and k8s Vanilla Operator to install a runner on Kubernetes is deprecated. GitLab plans to introduce a new [GitLab Runner token architecture](https://docs.gitlab.com/ee/architecture/blueprints/runner_tokens/) in GitLab 15.8, which introduces a new method for registering runners and eliminates the legacy runner registration token. +### External field in Releases and Release Links APIs +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/).
-
- -### Registration tokens and server-side runner arguments in `POST /api/v4/runners` endpoint - -End of Support: GitLab 16.6
-Planned removal: GitLab 17.0 +In [Releases API](https://docs.gitlab.com/ee/api/releases/) and [Release Links API](https://docs.gitlab.com/ee/api/releases/links.html), the `external` field was used to indicate whether a [release link](https://docs.gitlab.com/ee/user/project/releases/release_fields.html#links) is internal or external to your GitLab instance. +As of GitLab 15.9, we treat all release links as external, and therefore, this field is deprecated in GitLab 15.9, and will be removed in GitLab 16.0. +To avoid any disruptions to your workflow, please stop using the `external` field because it will be removed and will not be replaced. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The support for registration tokens and certain runner configuration arguments in the `POST` method operation on the `/api/v4/runners` endpoint is deprecated. -This endpoint [registers](https://docs.gitlab.com/ee/api/runners.html#register-a-new-runner) a runner -with a GitLab instance at the instance, group, or project level through the API. We plan to remove the support for -registration tokens and certain configuration arguments in this endpoint in GitLab 17.0. +
-In GitLab 15.10, we plan to implement a new method to bind runners to a GitLab instance, -as part of the new [GitLab Runner token architecture](https://docs.gitlab.com/ee/architecture/blueprints/runner_tokens/). -This new architecture introduces a new method for registering runners and will eliminate the legacy -[runner registration token](https://docs.gitlab.com/ee/security/token_overview.html#runner-registration-tokens). -From GitLab 17.0 and later, the runner registration methods implemented by the new GitLab Runner token architecture will be the only supported methods. +### Geo: Project repository redownload is deprecated +
+- Announced in: GitLab 15.11 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/388868).
-
- -### Registration tokens and server-side runner arguments in `gitlab-runner register` command +In secondary Geo sites, the button to "Redownload" a project repository is +deprecated. The redownload logic has inherent data consistency issues which +are difficult to resolve when encountered. The button will be removed in +GitLab 16.0. -End of Support: GitLab 16.6
-Planned removal: GitLab 17.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The support for registration tokens and certain configuration arguments in the command to [register](https://docs.gitlab.com/runner/register/) a runner, `gitlab-runner register` is deprecated. -GitLab plans to introduce a new [GitLab Runner token architecture](https://docs.gitlab.com/ee/architecture/blueprints/runner_tokens/) in GitLab 15.10, -which introduces a new method for registering runners and eliminates the legacy -[runner registration token](https://docs.gitlab.com/ee/security/token_overview.html#runner-registration-tokens). -The new method will involve creating the runner in the GitLab UI and passing the -[runner authentication token](https://docs.gitlab.com/ee/security/token_overview.html#runner-authentication-tokens-also-called-runner-tokens) -to the `gitlab-runner register` command. +### GitLab self-monitoring project +
+- Announced in: GitLab 14.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/348909).
-
+GitLab self-monitoring gives administrators of self-hosted GitLab instances the tools to monitor the health of their instances. This feature is deprecated in GitLab 14.9, and is scheduled for removal in 16.0. -### `runnerRegistrationToken` parameter for GitLab Runner Helm Chart +
-End of Support: GitLab 16.6
-Planned removal: GitLab 17.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +### GitLab.com importer -The [`runnerRegistrationToken`](https://docs.gitlab.com/runner/install/kubernetes.html#required-configuration) parameter to use the GitLab Helm Chart to install a runner on Kubernetes is deprecated. +
+- Announced in: GitLab 15.8 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-com/Product/-/issues/4895). +
-As part of the new [GitLab Runner token architecture](https://docs.gitlab.com/ee/architecture/blueprints/runner_tokens/), in GitLab 15.8 we plan to introduce: +The [GitLab.com importer](https://docs.gitlab.com/ee/user/project/import/gitlab_com.html) is deprecated in GitLab 15.8 and will be removed in GitLab 16.0. -- A new method to bind runners to a GitLab instance leveraging `runnerToken`. -- A unique system ID saved to the `config.toml`, which will ensure traceability between jobs and runners. +The GitLab.com importer was introduced in 2015 for importing a project from GitLab.com to a self-managed GitLab instance through the UI. +This feature is available on self-managed instances only. [Migrating GitLab groups and projects by direct transfer](https://docs.gitlab.com/ee/user/group/import/#migrate-groups-by-direct-transfer-recommended) +supersedes the GitLab.com importer and provides a more cohesive importing functionality. -From GitLab 17.0 and later, the methods to register runners introduced by the new GitLab Runner token architecture will be the only supported methods. +See [migrated group items](https://docs.gitlab.com/ee/user/group/import/#migrated-group-items) and [migrated project items](https://docs.gitlab.com/ee/user/group/import/#migrated-project-items) for an overview.
-
- -### merge_status API field +
-Planned removal: GitLab 16.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. - -The `merge_status` field in the [merge request API](https://docs.gitlab.com/ee/api/merge_requests.html#merge-status) has been deprecated in favor of the `detailed_merge_status` field which more correctly identifies all of the potential statuses that a merge request can be in. API users are encouraged to use the new `detailed_merge_status` field instead. The `merge_status` field will be removed in v5 of the GitLab REST API. +### GraphQL API Runner status will not return `paused` -
+
+- Announced in: GitLab 14.5 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/344648).
-
+The GitLab Runner GraphQL API endpoints will not return `paused` or `active` as a status in GitLab 16.0. +In a future v5 of the REST API, the endpoints for GitLab Runner will also not return `paused` or `active`. -## Announced in 15.5 +A runner's status will only relate to runner contact status, such as: +`online`, `offline`, or `not_connected`. Status `paused` or `active` will no longer appear. -
+When checking if a runner is `paused`, API users are advised to check the boolean attribute +`paused` to be `true` instead. When checking if a runner is `active`, check if `paused` is `false`. -### File Type variable expansion in `.gitlab-ci.yml` +
-Planned removal: GitLab 15.7 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +### GraphQL API legacyMode argument for Runner status -Previously, variables that referenced or applied alias file variables expanded the value of the `File` type variable. For example, the file contents. This behavior was incorrect because it did not comply with typical shell variable expansion rules. To leak secrets or sensitive information stored in `File` type variables, a user could run an $echo command with the variable as an input parameter. +
+- Announced in: GitLab 15.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/360545). +
-This breaking change fixes this issue but could disrupt user workflows that work around the behavior. With this change, job variable expansions that reference or apply alias file variables, expand to the file name or path of the `File` type variable, instead of its value, such as the file contents. +The `legacyMode` argument to the `status` field in `RunnerType` will be rendered non-functional in the 16.0 release +as part of the deprecations details in the [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/351109). + +In GitLab 16.0 and later, the `status` field will act as if `legacyMode` is null. The `legacyMode` argument will +be present during the 16.x cycle to avoid breaking the API signature, and will be removed altogether in the +17.0 release.
-
+
### GraphQL field `confidential` changed to `internal` on notes -Planned removal: GitLab 16.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.5 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/371485). +
The `confidential` field for a `Note` will be deprecated and renamed to `internal`.
-
+
-### vulnerabilityFindingDismiss GraphQL mutation +### Jira DVCS connector for Jira Cloud -Planned removal: GitLab 16.0 +
+- Announced in: GitLab 15.1 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/groups/gitlab-org/-/epics/7508). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +The [Jira DVCS connector](https://docs.gitlab.com/ee/integration/jira/dvcs/) for Jira Cloud has been deprecated and will be removed in GitLab 16.0. If you're using the Jira DVCS connector with Jira Cloud, migrate to the [GitLab for Jira Cloud app](https://docs.gitlab.com/ee/integration/jira/connect-app.html). -The `VulnerabilityFindingDismiss` GraphQL mutation is being deprecated and will be removed in GitLab 16.0. This mutation was not used often as the Vulnerability Finding ID was not available to users (this field was [deprecated in 15.3](https://docs.gitlab.com/ee/update/deprecations.html#use-of-id-field-in-vulnerabilityfindingdismiss-mutation)). Users should instead use `VulnerabilityDismiss` to dismiss vulnerabilities in the Vulnerability Report or `SecurityFindingDismiss` for security findings in the CI Pipeline Security tab. +The Jira DVCS connector is also deprecated for Jira 8.13 and earlier. You can only use the Jira DVCS connector with Jira Server or Jira Data Center in Jira 8.14 and later. -
-
+
-## Announced in 15.4 +### KAS Metrics Port in GitLab Helm Chart -
+
+- Announced in: GitLab 15.7 +- End of Support: GitLab 16.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/383039). +
-### Container Scanning variables that reference Docker +The `gitlab.kas.metrics.port` has been deprecated in favor of the new `gitlab.kas.observability.port` configuration field for the [GitLab Helm Chart](https://gitlab.com/gitlab-org/charts/gitlab/-/merge_requests/2839). +This port is used for much more than just metrics, which warranted this change to avoid confusion in configuration. -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-All Container Scanning variables that are prefixed by `DOCKER_` in variable name are deprecated. This includes the `DOCKER_IMAGE`, `DOCKER_PASSWORD`, `DOCKER_USER`, and `DOCKERFILE_PATH` variables. Support for these variables will be removed in the GitLab 16.0 release. Use the [new variable names](https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-cicd-variables) `CS_IMAGE`, `CS_REGISTRY_PASSWORD`, `CS_REGISTRY_USER`, and `CS_DOCKERFILE_PATH` in place of the deprecated names. +### Legacy Gitaly configuration method +
+- Announced in: GitLab 15.10 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/393574).
-
- -### Non-expiring access tokens +Gitaly configuration within Omnibus GitLab has been updated such that all Gitaly related configuration keys are in a single +configuration structure that matches the standard Gitaly configuration. As such, the previous configuration structure is deprecated. -Planned removal: GitLab 16.0 +The single configuration structure is available from GitLab 15.10, though backwards compatibility is maintained. Once removed, Gitaly must be configured using the single +configuration structure. You should update the configuration of Gitaly at your earliest convenience. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +The change improves consistency between Omnibus GitLab and source installs and enables us to provide better documentation and tooling for both. -Access tokens that have no expiration date are valid indefinitely, which presents a security risk if the access token -is divulged. Because access tokens that have an exipiration date are better, from GitLab 15.3 we -[populate a default expiration date](https://gitlab.com/gitlab-org/gitlab/-/issues/348660). +You should update to the new configuration structure as soon as possible using +[the upgrade instructions](https://docs.gitlab.com/ee/update/#gitaly-omnibus-gitlab-configuration-structure-change). -In GitLab 16.0, any [personal](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html), -[project](https://docs.gitlab.com/ee/user/project/settings/project_access_tokens.html), or -[group](https://docs.gitlab.com/ee/user/group/settings/group_access_tokens.html) access token that does not have an -expiration date will automatically have an expiration date set at one year. +
-We recommend giving your access tokens an expiration date in line with your company's security policies before the -default is applied: +
-- On GitLab.com during the 16.0 milestone. -- On GitLab self-managed instances when they are upgraded to 16.0. +### Legacy Praefect configuration method +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/390291).
-
- -### Starboard directive in the config for the GitLab Agent for Kubernetes - -Planned removal: GitLab 16.0 +Previously, Praefect configuration keys were scattered throughout the configuration file. Now, these are in a single configuration structure that matches +Praefect configuration so the previous configuration method is deprecated. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +The single configuration structure available from GitLab 15.9, though backwards compatibility is maintained. Once removed, Praefect must be configured using the single +configuration structure. You should update your Praefect configuration as soon as possible using +[the upgrade instructions](https://docs.gitlab.com/ee/update/#praefect-omnibus-gitlab-configuration-structure-change). -GitLab's operational container scanning capabilities no longer require starboard to be installed. Consequently, use of the `starboard:` directive in the configuration file for the GitLab Agent for Kubernetes is now deprecated and is scheduled for removal in GitLab 16.0. Update your configuration file to use the `container_scanning:` directive. +This change brings Praefect configuration in Omnibus GitLab in line with the configuration structure of Praefect. Previously, the hierarchies and configuration keys +didn't match. The change improves consistency between Omnibus GitLab and source installs and enables us to provide better documentation and tooling for both.
-
- -### Toggle behavior of `/draft` quick action in merge requests +
-Planned removal: GitLab 16.0 +### Legacy URLs replaced or removed -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/214217). +
-In order to make the behavior of toggling the draft status of a merge request more clear via a quick action, we're deprecating and removing the toggle behavior of the `/draft` quick action. Beginning with the 16.0 release of GitLab, `/draft` will only set a merge request to Draft and a new `/ready` quick action will be used to remove the draft status. +GitLab 16.0 removes legacy URLs from the GitLab application. -
+When subgroups were introduced in GitLab 9.0, a `/-/` delimiter was added to URLs to signify the end of a group path. All GitLab URLs now use this delimiter for project, group, and instance level features. -
+URLs that do not use the `/-/` delimiter are planned for removal in GitLab 16.0. For the full list of these URLs, along with their replacements, see [issue 28848](https://gitlab.com/gitlab-org/gitlab/-/issues/28848#release-notes). -### Vulnerability confidence field +Update any scripts or bookmarks that reference the legacy URLs. GitLab APIs are not affected by this change. -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-In GitLab 15.3, [security report schemas below version 15 were deprecated](https://docs.gitlab.com/ee/update/deprecations.html#security-report-schemas-version-14xx). -The `confidence` attribute on vulnerability findings exists only in schema versions before `15-0-0`, and therefore is effectively deprecated since GitLab 15.4 supports schema version `15-0-0`. To maintain consistency -between the reports and our public APIs, the `confidence` attribute on any vulnerability-related components of our GraphQL API is now deprecated and will be -removed in 16.0. +### License-Check and the Policies tab on the License Compliance page +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/390417).
-
- -
-## Announced in 15.3 +The [License-Check feature](https://docs.gitlab.com/ee/user/compliance/license_check_rules.html) is now deprecated and is scheduled for removal in GitLab 16.0. Additionally, the Policies tab on the License Compliance page and all APIs related to the License-Check feature are deprecated and planned for removal in GitLab 16.0. Users who wish to continue to enforce approvals based on detected licenses are encouraged to create a new [License Approval policy](https://docs.gitlab.com/ee/user/compliance/license_approval_policies.html) instead. -
+
-### Atlassian Crowd OmniAuth provider +
-Planned removal: GitLab 16.0 +### Limit personal access token and deploy token's access with external authorization -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/387721). +
-The `omniauth_crowd` gem that provides GitLab with the Atlassian Crowd OmniAuth provider will be removed in our -next major release, GitLab 16.0. This gem sees very little use and its -[lack of compatibility](https://github.com/robdimarco/omniauth_crowd/issues/37) with OmniAuth 2.0 is -[blocking our upgrade](https://gitlab.com/gitlab-org/gitlab/-/issues/30073). +With external authorization enabled, personal access tokens (PATs) and deploy tokens must no longer be able to access container or package registries. This defense-in-depth security measure will be deployed in 16.0. For users that use PATs and deploy tokens to access these registries, this measure breaks this use of these tokens. Disable external authorization to use tokens with container or package registries.
-
+
-### Bundled Grafana deprecated +### Major bundled Helm Chart updates for the GitLab Helm Chart -Planned removal: GitLab 15.4 +
+- Announced in: GitLab 15.10 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/3442). +
-In GitLab 15.4, we will be swapping the bundled Grafana to a fork of Grafana maintained by GitLab. +To coincide with GitLab 16.0, the GitLab Helm Chart will release the 7.0 major version. The following major bundled chart updates will be included: -There was an [identified CVE for Grafana](https://nvd.nist.gov/vuln/detail/CVE-2022-31107), and to mitigate this security vulnerability, we must swap to our own fork because the older version of Grafana we were bundling is no longer receiving long-term support. +- In GitLab 16.0, [PostgreSQL 12 support is being removed, and PostgreSQL 13 is becoming the new minimum](#postgresql-12-deprecated). + - Installs using production-ready external databases will need to complete their migration to a newer PostgreSQL version before upgrading. + - Installs using the [non-production bundled PostgreSQL 12 chart](https://docs.gitlab.com/charts/installation/tools.html#postgresql) will have the chart upgraded to the new version. For more information, [see issue 4118](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/4118) +- Installs using the [non-production bundled Redis chart](https://docs.gitlab.com/charts/installation/tools.html#redis) will have the chart upgraded to a newer version. For more information, [see issue 3375](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/3375) +- Installs using the [bundled cert-manager chart](https://docs.gitlab.com/charts/installation/tls.html#option-1-cert-manager-and-lets-encrypt) will have the chart upgraded to a newer version. For more information, [see issue 4313](https://gitlab.com/gitlab-org/charts/gitlab/-/issues/4313) -This is not expected to cause any incompatibilities with the previous version of Grafana. Neither when using our bundled version, nor when using an external instance of Grafana. +The full GitLab Helm Chart 7.0 upgrade steps will be available in the [upgrade docs](https://docs.gitlab.com/charts/installation/upgrade.html).
-
- -### CAS OmniAuth provider +
-Planned removal: GitLab 16.0 +### Managed Licenses API -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/390417). +
-The `omniauth-cas3` gem that provides GitLab with the CAS OmniAuth provider will be removed in our next major -release, GitLab 16.0. This gem sees very little use and its lack of upstream maintenance is preventing GitLab's -[upgrade to OmniAuth 2.0](https://gitlab.com/gitlab-org/gitlab/-/issues/30073). +The [Managed Licenses API](https://docs.gitlab.com/ee/api/managed_licenses.html) is now deprecated and is scheduled for removal in GitLab 16.0.
-
+
### Maximum number of active pipelines per project limit (`ci_active_pipelines`) -Planned removal: GitLab 16.0 +
+- Announced in: GitLab 15.3 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/368195). +
The [**Maximum number of active pipelines per project** limit](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#set-cicd-limits) was never enabled by default and will be removed in GitLab 16.0. This limit can also be configured in the Rails console under [`ci_active_pipelines`](https://docs.gitlab.com/ee/administration/instance_limits.html#number-of-pipelines-running-concurrently). Instead, use the other recommended rate limits that offer similar protection: @@ -1699,1341 +1649,1638 @@ The [**Maximum number of active pipelines per project** limit](https://docs.gitl
-
+
-### Redis 5 deprecated - -End of Support: GitLab 15.6
-Planned removal: GitLab 16.0 +### Monitor performance metrics through Prometheus -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 14.7 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/346541). +
-With GitLab 13.9, in the Omnibus GitLab package and GitLab Helm chart 4.9, the Redis version [was updated to Redis 6](https://about.gitlab.com/releases/2021/02/22/gitlab-13-9-released/#omnibus-improvements). -Redis 5 has reached the end of life in April 2022 and will no longer be supported as of GitLab 15.6. -If you are using your own Redis 5.0 instance, you should upgrade it to Redis 6.0 or higher before upgrading to GitLab 16.0 or higher. +By displaying data stored in a Prometheus instance, GitLab allows users to view performance metrics. GitLab also displays visualizations of these metrics in dashboards. The user can connect to a previously-configured external Prometheus instance, or set up Prometheus as a GitLab Managed App. +However, since certificate-based integration with Kubernetes clusters is deprecated in GitLab, the metrics functionality in GitLab that relies on Prometheus is also deprecated. This includes the metrics visualizations in dashboards. GitLab is working to develop a single user experience based on [Opstrace](https://about.gitlab.com/press/releases/2021-12-14-gitlab-acquires-opstrace-to-expand-its-devops-platform-with-open-source-observability-solution.html). An [issue exists](https://gitlab.com/groups/gitlab-org/-/epics/6976) for you to follow work on the Opstrace integration.
-
- -### Security report schemas version 14.x.x +
-Planned removal: GitLab 16.0 +### Non-expiring access tokens -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.4 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/369122). +
-Version 14.x.x [security report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas) are deprecated. +Access tokens that have no expiration date are valid indefinitely, which presents a security risk if the access token +is divulged. Because access tokens that have an exipiration date are better, from GitLab 15.3 we +[populate a default expiration date](https://gitlab.com/gitlab-org/gitlab/-/issues/348660). -In GitLab 15.8 and later, [security report scanner integrations](https://docs.gitlab.com/ee/development/integrations/secure.html) that use schema version 14.x.x will display a deprecation warning in the pipeline's **Security** tab. +In GitLab 16.0, any [personal](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html), +[project](https://docs.gitlab.com/ee/user/project/settings/project_access_tokens.html), or +[group](https://docs.gitlab.com/ee/user/group/settings/group_access_tokens.html) access token that does not have an +expiration date will automatically have an expiration date set at one year. -In GitLab 16.0 and later, the feature will be removed. Security reports that use schema version 14.x.x will cause an error in the pipeline's **Security** tab. +We recommend giving your access tokens an expiration date in line with your company's security policies before the +default is applied: -For more information, refer to [security report validation](https://docs.gitlab.com/ee/user/application_security/#security-report-validation). +- On GitLab.com during the 16.0 milestone. +- On GitLab self-managed instances when they are upgraded to 16.0.
-
+
-### Use of `id` field in vulnerabilityFindingDismiss mutation +### Non-standard default Redis ports are deprecated -Planned removal: GitLab 16.0 +
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/388269). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +If GitLab starts without any Redis configuration file present, +GitLab assumes it can connect to three Redis servers at `localhost:6380`, +`localhost:6381` and `localhost:6382`. We are changing this behavior +so GitLab assumes there is one Redis server at `localhost:6379`. -You can use the vulnerabilityFindingDismiss GraphQL mutation to set the status of a vulnerability finding to `Dismissed`. Previously, this mutation used the `id` field to identify findings uniquely. However, this did not work for dismissing findings from the pipeline security tab. Therefore, using the `id` field as an identifier has been dropped in favor of the `uuid` field. Using the 'uuid' field as an identifier allows you to dismiss the finding from the pipeline security tab. +Administrators who want to keep the three servers must configure +the Redis URLs by editing the `config/redis.cache.yml`,`config/redis.queues.yml` +and `config/redis.shared_state.yml` files. -
-
+
-## Announced in 15.2 +### Option to delete projects immediately is deprecated from deletion protection settings -
+
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/389557). +
-### Remove `job_age` parameter from `POST /jobs/request` Runner endpoint +The group and project deletion protection setting in the Admin Area had an option to delete groups and projects immediately. Starting with 16.0, this option will no longer be available, and delayed group and project deletion will become the default behavior. -Planned removal: GitLab 16.0 +The option will no longer appear as a group setting. Self-managed users will still have the option to define the deletion delay period, and SaaS users have a non-adjustable default retention period of 7 days. Users can still immediately delete the project from the project settings, and the group from the group settings. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +The option to delete groups and projects immediately by default was deprecated to prevent users from accidentally taking this action and permanently losing groups and projects. -The `job_age` parameter, returned from the `POST /jobs/request` API endpoint used in communication with GitLab Runner, was never used by any GitLab or Runner feature. This parameter will be removed in GitLab 16.0. - -This could be a breaking change for anyone that developed their own runner that relies on this parameter being returned by the endpoint. This is not a breaking change for anyone using an officially released version of GitLab Runner, including public shared runners on GitLab.com. - -
-
+
-## Announced in 15.1 +### PipelineSecurityReportFinding name GraphQL field -
+
+- Announced in: GitLab 15.1 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/346335). +
-### Jira GitHub Enterprise DVCS integration +Previously, the [PipelineSecurityReportFinding GraphQL type was updated](https://gitlab.com/gitlab-org/gitlab/-/issues/335372) to include a new `title` field. This field is an alias for the current `name` field, making the less specific `name` field redundant. The `name` field will be removed from the PipelineSecurityReportFinding type in GitLab 16.0. -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The [Jira DVCS Connector](https://docs.gitlab.com/ee/integration/jira/dvcs/) (which enables the [Jira Development Panel](https://support.atlassian.com/jira-software-cloud/docs/view-development-information-for-an-issue/)), will no longer support Jira Cloud users starting with GitLab 16.0. The [GitLab for Jira App](https://docs.gitlab.com/ee/integration/jira/connect-app.html) has always been recommended for Jira Cloud users, and it will be required instead of the DVCS connector. If you are a Jira Cloud user, we recommended you begin migrating to the GitLab for Jira App. -Any Jira Server and Jira Data Center users will need to confirm they are not using the GitHub Enterprise Connector to enable the GitLab DVCS integration, but they may continue to use the [native GitLab DVCS integration](https://docs.gitlab.com/ee/integration/jira/dvcs/) (supported in Jira 8.14 and later). +### PostgreSQL 12 deprecated +
+- Announced in: GitLab 15.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/349185).
-
+Support for PostgreSQL 12 is scheduled for removal in GitLab 16.0. +In GitLab 16.0, PostgreSQL 13 becomes the minimum required PostgreSQL version. -### PipelineSecurityReportFinding name GraphQL field +PostgreSQL 12 will be supported for the full GitLab 15 release cycle. +PostgreSQL 13 will also be supported for instances that want to upgrade prior to GitLab 16.0. -Planned removal: GitLab 16.0 +Support for PostgreSQL 13 was added to Geo in GitLab 15.2. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-Previously, the [PipelineSecurityReportFinding GraphQL type was updated](https://gitlab.com/gitlab-org/gitlab/-/issues/335372) to include a new `title` field. This field is an alias for the current `name` field, making the less specific `name` field redundant. The `name` field will be removed from the PipelineSecurityReportFinding type in GitLab 16.0. +
-
+### Projects API field `operations_access_level` is deprecated -
+
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/385798). +
-### PipelineSecurityReportFinding projectFingerprint GraphQL field +We are deprecating the `operations_access_level` field in the Projects API. This field has been replaced by fields to control specific features: `releases_access_level`, `environments_access_level`, `feature_flags_access_level`, `infrastructure_access_level`, and `monitor_access_level`. -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The [`project_fingerprint`](https://gitlab.com/groups/gitlab-org/-/epics/2791) attribute of vulnerability findings is being deprecated in favor of a `uuid` attribute. By using UUIDv5 values to identify findings, we can easily associate any related entity with a finding. The `project_fingerprint` attribute is no longer being used to track findings, and will be removed in GitLab 16.0. +### Rake task for importing bare repositories +
+- Announced in: GitLab 15.8 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-com/Product/-/issues/5255).
-
+The [Rake task for importing bare repositories](https://docs.gitlab.com/ee/raketasks/import.html) `gitlab:import:repos` is deprecated in GitLab 15.8 and will be removed in GitLab 16.0. + +This Rake task imports a directory tree of repositories into a GitLab instance. These repositories must have been +managed by GitLab previously, because the Rake task relies on the specific directory structure or a specific custom Git setting in order to work (`gitlab.fullpath`). -### REST API Runner maintainer_note +Importing repositories using this Rake task has limitations. The Rake task: -Planned removal: GitLab 16.0 +- Only knows about project and project wiki repositories and doesn't support repositories for designs, group wikis, or snippets. +- Permits you to import non-hashed storage projects even though these aren't supported. +- Relies on having Git config `gitlab.fullpath` set. [Epic 8953](https://gitlab.com/groups/gitlab-org/-/epics/8953) proposes removing support for this setting. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +Alternatives to using the `gitlab:import:repos` Rake task include: -The `maintainer_note` argument in the `POST /runners` REST endpoint was deprecated in GitLab 14.8 and replaced with the `maintenance_note` argument. -The `maintainer_note` argument will be removed in GitLab 16.0. +- Migrating projects using either [an export file](https://docs.gitlab.com/ee/user/project/settings/import_export.html) or + [direct transfer](https://docs.gitlab.com/ee/user/group/import/#migrate-groups-by-direct-transfer-recommended) migrate repositories as well. +- Importing a [repository by URL](https://docs.gitlab.com/ee/user/project/import/repo_by_url.html). +- Importing a [repositories from a non-GitLab source](https://docs.gitlab.com/ee/user/project/import/).
-
+
-### Vulnerability Report sort by Tool +### Redis 5 deprecated -Planned removal: GitLab 15.3 +
+- Announced in: GitLab 15.3 +- End of Support: GitLab 15.6 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/331468). +
-The ability to sort the Vulnerability Report by the `Tool` column (scan type) was disabled and put behind a feature flag in GitLab 14.10 due to a refactor -of the underlying data model. The feature flag has remained off by default as further refactoring will be required to ensure sorting -by this value remains performant. Due to very low usage of the `Tool` column for sorting, the feature flag will instead be removed in -GitLab 15.3 to simplify the codebase and prevent any unwanted performance degradation. +With GitLab 13.9, in the Omnibus GitLab package and GitLab Helm chart 4.9, the Redis version [was updated to Redis 6](https://about.gitlab.com/releases/2021/02/22/gitlab-13-9-released/#omnibus-improvements). +Redis 5 has reached the end of life in April 2022 and will no longer be supported as of GitLab 15.6. +If you are using your own Redis 5.0 instance, you should upgrade it to Redis 6.0 or higher before upgrading to GitLab 16.0 or higher.
-
+
-### project.pipeline.securityReportFindings GraphQL query +### Remove `job_age` parameter from `POST /jobs/request` Runner endpoint -Planned removal: GitLab 16.0 +
+- Announced in: GitLab 15.2 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/334253). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +The `job_age` parameter, returned from the `POST /jobs/request` API endpoint used in communication with GitLab Runner, was never used by any GitLab or Runner feature. This parameter will be removed in GitLab 16.0. -Previous work helped [align the vulnerabilities calls for pipeline security tabs](https://gitlab.com/gitlab-org/gitlab/-/issues/343469) to match the vulnerabilities calls for project-level and group-level vulnerability reports. This helped the frontend have a more consistent interface. The old `project.pipeline.securityReportFindings` query was formatted differently than other vulnerability data calls. Now that it has been replaced with the new `project.pipeline.vulnerabilities` field, the old `project.pipeline.securityReportFindings` is being deprecated and will be removed in GitLab 16.0. +This could be a breaking change for anyone that developed their own runner that relies on this parameter being returned by the endpoint. This is not a breaking change for anyone using an officially released version of GitLab Runner, including public shared runners on GitLab.com.
+ +
+ +### SAST analyzer coverage changing in GitLab 16.0 + +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/390416).
-
+GitLab SAST uses various [analyzers](https://docs.gitlab.com/ee/user/application_security/sast/analyzers/) to scan code for vulnerabilities. -## Announced in 15.0 +We're reducing the number of supported analyzers used by default in GitLab SAST. +This is part of our long-term strategy to deliver a faster, more consistent user experience across different programming languages. -
+Starting in GitLab 16.0, the GitLab SAST CI/CD template will no longer use the [Security Code Scan](https://gitlab.com/gitlab-org/security-products/analyzers/security-code-scan)-based analyzer for .NET, and it will enter End of Support status. +We'll remove this analyzer from the [SAST CI/CD template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml) and replace it with GitLab-supported detection rules for C# in the [Semgrep-based analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep). -### CiCdSettingsUpdate mutation renamed to ProjectCiCdSettingsUpdate +Effective immediately, this analyzer will receive only security updates; other routine improvements or updates are not guaranteed. +After this analyzer reaches End of Support in GitLab 16.0, no further updates will be provided. +However, we won't delete container images previously published for this analyzer or remove the ability to run it by using a custom CI/CD pipeline job. -Planned removal: GitLab 16.0 +If you've already dismissed a vulnerability finding from the deprecated analyzer, the replacement attempts to respect your previous dismissal. The system behavior depends on: -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +- whether you've excluded the Semgrep-based analyzer from running in the past. +- which analyzer first discovered the vulnerabilities shown in the project's Vulnerability Report. -The `CiCdSettingsUpdate` mutation was renamed to `ProjectCiCdSettingsUpdate` in GitLab 15.0. -The `CiCdSettingsUpdate` mutation will be removed in GitLab 16.0. -Any user scripts that use the `CiCdSettingsUpdate` mutation must be updated to use `ProjectCiCdSettingsUpdate` -instead. +See [Vulnerability translation documentation](https://docs.gitlab.com/ee/user/application_security/sast/analyzers.html#vulnerability-translation) for further details. -
+If you applied customizations to the affected analyzer, or if you currently disable the Semgrep-based analyzer in your pipelines, you must take action as detailed in the [deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/390416#breaking-change). -
+**Update:** We've reduced the scope of this change. We will no longer make the following changes in GitLab 16.0: -### GraphQL API legacyMode argument for Runner status +1. Remove support for the analyzer based on [PHPCS Security Audit](https://gitlab.com/gitlab-org/security-products/analyzers/phpcs-security-audit) and replace it with GitLab-managed detection rules in the [Semgrep-based analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep). +1. Remove Scala from the scope of the [SpotBugs-based analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) and replace it with GitLab-managed detection rules in the [Semgrep-based analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep). -Planned removal: GitLab 16.0 +Work to replace the PHPCS Security Audit-based analyzer is tracked in [issue 364060](https://gitlab.com/gitlab-org/gitlab/-/issues/364060) and work to migrate Scala scanning to the Semgrep-based analyzer is tracked in [issue 362958](https://gitlab.com/gitlab-org/gitlab/-/issues/362958). -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The `legacyMode` argument to the `status` field in `RunnerType` will be rendered non-functional in the 16.0 release -as part of the deprecations details in the [issue](https://gitlab.com/gitlab-org/gitlab/-/issues/351109). +
-In GitLab 16.0 and later, the `status` field will act as if `legacyMode` is null. The `legacyMode` argument will -be present during the 16.x cycle to avoid breaking the API signature, and will be removed altogether in the -17.0 release. +### Secure analyzers major version update +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/390912).
-
+The Secure stage will be bumping the major versions of its analyzers in tandem with the GitLab 16.0 release. This bump will enable a clear delineation for analyzers, between: -### PostgreSQL 12 deprecated +- Those released prior to May 22, 2023 +- Those released after May 22, 2023 -Planned removal: GitLab 16.0 +If you are not using the default included templates, or have pinned your analyzer versions you will need to update your CI/CD job definition to either remove the pinned version or to update the latest major version. +Users of GitLab 13.0-15.10 will continue to experience analyzer updates as normal until the release of GitLab 16.0, following which all newly fixed bugs and released features will be released only in the new major version of the analyzers. We do not backport bugs and features to deprecated versions as per our [maintenance policy](https://docs.gitlab.com/ee/policy/maintenance.html). As required, security patches will be backported within the latest 3 minor releases. +Specifically, the following are being deprecated and will no longer be updated after 16.0 GitLab release: -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +- API Fuzzing: version 2 +- Container Scanning: version 5 +- Coverage-guided fuzz testing: version 3 +- Dependency Scanning: version 3 +- Dynamic Application Security Testing (DAST): version 3 +- DAST API: version 2 +- IaC Scanning: version 3 +- License Scanning: version 4 +- Secret Detection: version 4 +- Static Application Security Testing (SAST): version 3 of [all analyzers](https://docs.gitlab.com/ee/user/application_security/sast/#supported-languages-and-frameworks) + - `brakeman`: version 3 + - `flawfinder`: version 3 + - `kubesec`: version 3 + - `mobsf`: version 3 + - `nodejs-scan`: version 3 + - `phpcs-security-audit`: version 3 + - `pmd-apex`: version 3 + - `security-code-scan`: version 3 + - `semgrep`: version 3 + - `sobelow`: version 3 + - `spotbugs`: version 3 -Support for PostgreSQL 12 is scheduled for removal in GitLab 16.0. -In GitLab 16.0, PostgreSQL 13 becomes the minimum required PostgreSQL version. +
-PostgreSQL 12 will be supported for the full GitLab 15 release cycle. -PostgreSQL 13 will also be supported for instances that want to upgrade prior to GitLab 16.0. +
-Upgrading to PostgreSQL 13 is not yet supported for GitLab instances with Geo enabled. Geo support for PostgreSQL 13 will be announced in a minor release version of GitLab 15, after the process is fully supported and validated. For more information, read the Geo related verifications on the [support epic for PostgreSQL 13](https://gitlab.com/groups/gitlab-org/-/epics/3832). +### Secure scanning CI/CD templates will use new job `rules` +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/391822).
-
+GitLab-managed CI/CD templates for security scanning will be updated in the GitLab 16.0 release. +The updates will include improvements already released in the Latest versions of the CI/CD templates. +We released these changes in the Latest template versions because they have the potential to disrupt customized CI/CD pipeline configurations. -### Vulnerability Report sort by State +In all updated templates, we're updating the definition of variables like `SAST_DISABLED` and `DEPENDENCY_SCANNING_DISABLED` to disable scanning only if the value is `"true"`. Previously, even if the value were `"false"`, scanning would be disabled. -Planned removal: GitLab 15.3 +The following templates will be updated: -The ability to sort the Vulnerability Report by the `State` column was disabled and put behind a feature flag in GitLab 14.10 due to a refactor -of the underlying data model. The feature flag has remained off by default as further refactoring will be required to ensure sorting -by this value remains performant. Due to very low usage of the `State` column for sorting, the feature flag will instead be removed to simplify the codebase and prevent any unwanted performance degradation. +- API Fuzzing: [`API-Fuzzing.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml) +- Container Scanning: [`Container-Scanning.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml) +- Coverage-Guided Fuzzing: [`Coverage-Fuzzing.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml) +- DAST: [`DAST.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml) +- DAST API: [`DAST-API.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml) +- Dependency Scanning: [`Dependency-Scanning.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml) +- IaC Scanning: [`SAST-IaC.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml) +- SAST: [`SAST.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml) +- Secret Detection: [`Secret-Detection.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Secret-Detction.gitlab-ci.yml) + +We recommend that you test your pipelines before the 16.0 release if you use one of the templates listed above and you use the `_DISABLED` variables but set a value other than `"true"`. + +**Update:** We previously announced that we would update the `rules` on the affected templates to run in [merge request pipelines](https://docs.gitlab.com/ee/ci/pipelines/merge_request_pipelines.html) by default. +However, due to compatibility issues [discussed in the deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/388988#note_1372629948), we will no longer make this change in GitLab 16.0. We will still release the changes to the `_DISABLED` variables as described above.
+ +
+ +### Security report schemas version 14.x.x + +
+- Announced in: GitLab 15.3 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/366477).
-
+Version 14.x.x [security report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas) are deprecated. -## Announced in 14.10 +In GitLab 15.8 and later, [security report scanner integrations](https://docs.gitlab.com/ee/development/integrations/secure.html) that use schema version 14.x.x will display a deprecation warning in the pipeline's **Security** tab. -
+In GitLab 16.0 and later, the feature will be removed. Security reports that use schema version 14.x.x will cause an error in the pipeline's **Security** tab. -### Dependency Scanning default Java version changed to 17 +For more information, refer to [security report validation](https://docs.gitlab.com/ee/user/application_security/#security-report-validation). -Planned removal: GitLab 15.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-In GitLab 15.0, for Dependency Scanning, the default version of Java that the scanner expects will be updated from 11 to 17. Java 17 is [the most up-to-date Long Term Support (LTS) version](https://en.wikipedia.org/wiki/Java_version_history). Dependency scanning continues to support the same [range of versions (8, 11, 13, 14, 15, 16, 17)](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#supported-languages-and-package-managers), only the default version is changing. If your project uses the previous default of Java 11, be sure to [set the `DS_Java_Version` variable to match](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning). +### Shimo integration +
+- Announced in: GitLab 15.7 +- End of Support: GitLab 16.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/377824).
-
+The [Shimo Workspace integration](https://docs.gitlab.com/ee/user/project/integrations/shimo.html) has been deprecated +and will be moved to the JiHu GitLab codebase. + +
-### Outdated indices of Advanced Search migrations +
-Planned removal: GitLab 15.0 +### Starboard directive in the config for the GitLab Agent for Kubernetes -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.4 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/368828). +
-As Advanced Search migrations usually require support multiple code paths for a long period of time, it’s important to clean those up when we safely can. We use GitLab major version upgrades as a safe time to remove backward compatibility for indices that have not been fully migrated. See the [upgrade documentation](https://docs.gitlab.com/ee/update/index.html#upgrading-to-a-new-major-version) for details. +GitLab's operational container scanning capabilities no longer require starboard to be installed. Consequently, use of the `starboard:` directive in the configuration file for the GitLab Agent for Kubernetes is now deprecated and is scheduled for removal in GitLab 16.0. Update your configuration file to use the `container_scanning:` directive.
-
+
-### Toggle notes confidentiality on APIs +### Support for Praefect custom metrics endpoint configuration -Planned removal: GitLab 16.0 +
+- Announced in: GitLab 15.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/390266). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +Support for using the `prometheus_exclude_database_from_default_metrics` configuration value is deprecated in GitLab +15.9 and will be removed in GitLab 16.0. We are removing this configuration value because using it is non-performant. +This change means the following metrics will become unavailable on `/metrics`: -Toggling notes confidentiality with REST and GraphQL APIs is being deprecated. Updating notes confidential attribute is no longer supported by any means. We are changing this to simplify the experience and prevent private information from being unintentionally exposed. +- `gitaly_praefect_unavailable_repositories`. +- `gitaly_praefect_verification_queue_depth`. +- `gitaly_praefect_replication_queue_depth`. + +This may require updating your metrics collection targets to also scrape `/db_metrics`. -
-
+
-## Announced in 14.9 +### Support for periods (`.`) in Terraform state names might break existing states -
+
+- Announced in: GitLab 15.7 +- End of Support: GitLab 16.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/385564). +
-### Background upload for object storage +Previously, Terraform state names containing periods were not supported. However, you could still use state names with periods via a workaround. -Planned removal: GitLab 15.0 +GitLab 15.7 [adds full support](https://docs.gitlab.com/ee/user/infrastructure/iac/troubleshooting.html#state-not-found-if-the-state-name-contains-a-period) for state names that contain periods. If you used a workaround to handle these state names, your jobs might fail, or it might look like you've run Terraform for the first time. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +To resolve the issue: -To reduce the overall complexity and maintenance burden of GitLab's [object storage feature](https://docs.gitlab.com/ee/administration/object_storage.html), support for using `background_upload` to upload files is deprecated and will be fully removed in GitLab 15.0. Review the [15.0 specific changes](https://docs.gitlab.com/omnibus/update/gitlab_15_changes.html) for the [removed background uploads settings for object storage](https://docs.gitlab.com/omnibus/update/gitlab_15_changes.html#removed-background-uploads-settings-for-object-storage). + 1. Change any references to the state file by excluding the period and any characters that follow. + - For example, if your state name is `state.name`, change all references to `state`. + 1. Run your Terraform commands. -This impacts a small subset of object storage providers: +To use the full state name, including the period, [migrate to the full state file](https://docs.gitlab.com/ee/user/infrastructure/iac/terraform_state.html#migrate-to-a-gitlab-managed-terraform-state). -- **OpenStack** Customers using OpenStack need to change their configuration to use the S3 API instead of Swift. -- **RackSpace** Customers using RackSpace-based object storage need to migrate data to a different provider. +
-GitLab will publish additional guidance to assist affected customers in migrating. +
+ +### The API no longer returns revoked tokens for the agent for Kubernetes +
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/382129).
-
+Currently, GET requests to the [Cluster Agents API](https://docs.gitlab.com/ee/api/cluster_agents.html#list-tokens-for-an-agent) +endpoints can return revoked tokens. In GitLab 16.0, GET requests will not return revoked tokens. -### Deprecate support for Debian 9 +You should review your calls to these endpoints and ensure you do not use revoked tokens. -Planned removal: GitLab 15.1 +This change affects the following REST and GraphQL API endpoints: -Long term service and support (LTSS) for [Debian 9 Stretch ends in July 2022](https://wiki.debian.org/LTS). Therefore, we will no longer support the Debian 9 distribution for the GitLab package. Users can upgrade to Debian 10 or Debian 11. +- REST API: + - [List tokens](https://docs.gitlab.com/ee/api/cluster_agents.html#list-tokens-for-an-agent) + - [Get a single token](https://docs.gitlab.com/ee/api/cluster_agents.html#get-a-single-agent-token) +- GraphQL: + - [`ClusterAgent.tokens`](https://docs.gitlab.com/ee/api/graphql/reference/#clusteragenttokens)
-
+
-### GitLab Pages running as daemon +### The Phabricator task importer is deprecated -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 15.7 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-com/Product/-/issues/4894). +
-In 15.0, support for daemon mode for GitLab Pages will be removed. +The [Phabricator task importer](https://docs.gitlab.com/ee/user/project/import/phabricator.html) is being deprecated. Phabricator itself as a project is no longer actively maintained since June 1, 2021. We haven't observed imports using this tool. There has been no activity on the open related issues on GitLab.
-
+
-### GitLab self-monitoring project +### The latest Terraform templates will overwrite current stable templates -Planned removal: GitLab 16.0 +
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/386001). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +With every major GitLab version, we update the stable Terraform templates with the current latest templates. +This change affects the [quickstart](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml) +and the [base](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml) templates. -GitLab self-monitoring gives administrators of self-hosted GitLab instances the tools to monitor the health of their instances. This feature is deprecated in GitLab 14.9, and is scheduled for removal in 16.0. +Because the new templates ship with default rules, the update might break your Terraform pipelines. +For example, if your Terraform jobs are triggered as a downstream pipeline, the rules won't trigger your jobs +in GitLab 16.0. + +To accommodate the changes, you might need to adjust the [`rules`](https://docs.gitlab.com/ee/ci/yaml/#rules) in your +`.gitlab-ci.yml` file.
-
+
-### GraphQL permissions change for Package settings +### Toggle behavior of `/draft` quick action in merge requests -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 15.4 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/365365). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +In order to make the behavior of toggling the draft status of a merge request more clear via a quick action, we're deprecating and removing the toggle behavior of the `/draft` quick action. Beginning with the 16.0 release of GitLab, `/draft` will only set a merge request to Draft and a new `/ready` quick action will be used to remove the draft status. -The GitLab Package stage offers a Package Registry, Container Registry, and Dependency Proxy to help you manage all of your dependencies using GitLab. Each of these product categories has a variety of settings that can be adjusted using the API. +
-The permissions model for GraphQL is being updated. After 15.0, users with the Guest, Reporter, and Developer role can no longer update these settings: +
-- [Package Registry settings](https://docs.gitlab.com/ee/api/graphql/reference/#packagesettings) -- [Container Registry cleanup policy](https://docs.gitlab.com/ee/api/graphql/reference/#containerexpirationpolicy) -- [Dependency Proxy time-to-live policy](https://docs.gitlab.com/ee/api/graphql/reference/#dependencyproxyimagettlgrouppolicy) -- [Enabling the Dependency Proxy for your group](https://docs.gitlab.com/ee/api/graphql/reference/#dependencyproxysetting) +### Toggle notes confidentiality on APIs +
+- Announced in: GitLab 14.10 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/350670).
-
+Toggling notes confidentiality with REST and GraphQL APIs is being deprecated. Updating notes confidential attribute is no longer supported by any means. We are changing this to simplify the experience and prevent private information from being unintentionally exposed. -### Move `custom_hooks_dir` setting from GitLab Shell to Gitaly +
-Planned removal: GitLab 15.0 +
-The [`custom_hooks_dir`](https://docs.gitlab.com/ee/administration/server_hooks.html#create-a-global-server-hook-for-all-repositories) setting is now configured in Gitaly, and will be removed from GitLab Shell in GitLab 15.0. +### Use of `id` field in vulnerabilityFindingDismiss mutation +
+- Announced in: GitLab 15.3 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/367166).
-
- -### Permissions change for downloading Composer dependencies - -Planned removal: GitLab 14.10 +You can use the vulnerabilityFindingDismiss GraphQL mutation to set the status of a vulnerability finding to `Dismissed`. Previously, this mutation used the `id` field to identify findings uniquely. However, this did not work for dismissing findings from the pipeline security tab. Therefore, using the `id` field as an identifier has been dropped in favor of the `uuid` field. Using the 'uuid' field as an identifier allows you to dismiss the finding from the pipeline security tab. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The GitLab Composer repository can be used to push, search, fetch metadata about, and download PHP dependencies. All these actions require authentication, except for downloading dependencies. +
-Downloading Composer dependencies without authentication is deprecated in GitLab 14.9, and will be removed in GitLab 15.0. Starting with GitLab 15.0, you must authenticate to download Composer dependencies. +### Use of third party container registries is deprecated +
+- Announced in: GitLab 15.8 +- End of Support: GitLab 16.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/376216).
-
+Using third-party container registries with GitLab as an auth endpoint is deprecated in GitLab 15.8 and the [end of support](https://docs.gitlab.com/ee/development/deprecation_guidelines/#terminology) is scheduled for GitLab 16.0. This impacts self-managed customers that have connected their external registry to the GitLab user interface to find, view, and delete container images. -### htpasswd Authentication for the Container Registry - -Planned removal: GitLab 15.0 +Supporting both GitLab's Container Registry as well as third-party container registries is challenging for maintenance, code quality, and backward compatibility. This hinders our ability to stay [efficient](https://about.gitlab.com/handbook/values/#efficiency). As a result we will not support this functionality moving forward. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +This change will not impact your ability to pull and push container images to external registries using pipelines. -The Container Registry supports [authentication](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/configuration.md#auth) with `htpasswd`. It relies on an [Apache `htpasswd` file](https://httpd.apache.org/docs/2.4/programs/htpasswd.html), with passwords hashed using `bcrypt`. +Since we released the new [GitLab Container Registry](https://gitlab.com/groups/gitlab-org/-/epics/5523) version for GitLab.com, we've started to implement additional features that are not available in third-party container registries. These new features have allowed us to achieve significant performance improvements, such as [cleanup policies](https://gitlab.com/groups/gitlab-org/-/epics/8379). We are focusing on delivering [new features](https://gitlab.com/groups/gitlab-org/-/epics/5136), most of which will require functionalities only available on the GitLab Container Registry. This deprecation allows us to reduce fragmentation and user frustration in the long term by focusing on delivering a more robust integrated registry experience and feature set. -Since it isn't used in the context of GitLab (the product), `htpasswd` authentication will be deprecated in GitLab 14.9 and removed in GitLab 15.0. +Moving forward, we'll continue to invest in developing and releasing new features that will only be available in the GitLab Container Registry.
-
+
-### user_email_lookup_limit API field +### Vulnerability confidence field -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 15.4 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/372332). +
+ +In GitLab 15.3, [security report schemas below version 15 were deprecated](https://docs.gitlab.com/ee/update/deprecations.html#security-report-schemas-version-14xx). +The `confidence` attribute on vulnerability findings exists only in schema versions before `15-0-0`, and therefore is effectively deprecated since GitLab 15.4 supports schema version `15-0-0`. To maintain consistency +between the reports and our public APIs, the `confidence` attribute on any vulnerability-related components of our GraphQL API is now deprecated and will be +removed in 16.0. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The `user_email_lookup_limit` [API field](https://docs.gitlab.com/ee/api/settings.html) is deprecated and will be removed in GitLab 15.0. Until GitLab 15.0, `user_email_lookup_limit` is aliased to `search_rate_limit` and existing workflows will continue to work. +
-Any API calls attempting to change the rate limits for `user_email_lookup_limit` should use `search_rate_limit` instead. +### Work items path with global ID at the end of the path is deprecated -
+
+- Announced in: GitLab 15.10 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/393836).
-
+Usage of global IDs in work item URLs is deprecated. In the future, only internal IDs (IID) will be supported. -## Announced in 14.8 +Because GitLab supports multiple work item types, a path such as `https://gitlab.com/gitlab-org/gitlab/-/work_items/` can display, for example, a [task](https://docs.gitlab.com/ee/user/tasks.html) or an [OKR](https://docs.gitlab.com/ee/user/okrs.html). -
+In GitLab 15.10 we added support for using internal IDs (IID) in that path by appending a query param at +the end (`iid_path`) in the following format: `https://gitlab.com/gitlab-org/gitlab/-/work_items/?iid_path=true`. -### Configurable Gitaly `per_repository` election strategy +In GitLab 16.0 we will remove the ability to use a global ID in the work items path. The number at the end of the path will be considered an internal ID (IID) without the need of adding a query param at the end. Only the following format will be supported: `https://gitlab.com/gitlab-org/gitlab/-/work_items/`. -Planned removal: GitLab 14.9 +
-Configuring the `per_repository` Gitaly election strategy is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/352612). -`per_repository` has been the only option since GitLab 14.0. +
-This change is part of regular maintenance to keep our codebase clean. +### ZenTao integration +
+- Announced in: GitLab 15.7 +- End of Support: GitLab 16.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/377825).
-
+The [ZenTao product integration](https://docs.gitlab.com/ee/user/project/integrations/zentao.html) has been deprecated +and will be moved to the JiHu GitLab codebase. -### Container Network and Host Security +
-Planned removal: GitLab 15.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. - -All functionality related to GitLab's Container Network Security and Container Host Security categories is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to evaluate the following open source projects as potential solutions that can be installed and managed outside of GitLab: [AppArmor](https://gitlab.com/apparmor/apparmor), [Cilium](https://github.com/cilium/cilium), [Falco](https://github.com/falcosecurity/falco), [FluentD](https://github.com/fluent/fluentd), [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/). To integrate these technologies into GitLab, add the desired Helm charts into your copy of the [Cluster Management Project Template](https://docs.gitlab.com/ee/user/clusters/management_project_template.html). Deploy these Helm charts in production by calling commands through GitLab [CI/CD](https://docs.gitlab.com/ee/user/clusters/agent/ci_cd_workflow.html). - -As part of this change, the following specific capabilities within GitLab are now deprecated, and are scheduled for removal in GitLab 15.0: - -- The **Security & Compliance > Threat Monitoring** page. -- The `Network Policy` security policy type, as found on the **Security & Compliance > Policies** page. -- The ability to manage integrations with the following technologies through GitLab: AppArmor, Cilium, Falco, FluentD, and Pod Security Policies. -- All APIs related to the above functionality. - -For additional context, or to provide feedback regarding this change, please reference our open [deprecation issue](https://gitlab.com/groups/gitlab-org/-/epics/7476). +### `CI_BUILD_*` predefined variables +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352957).
-
+The predefined CI/CD variables that start with `CI_BUILD_*` were deprecated in GitLab 9.0, and will be removed in GitLab 16.0. If you still use these variables, be sure to change to the replacement [predefined variables](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html) which are functionally identical: -### Dependency Scanning Python 3.9 and 3.6 image deprecation +| Removed variable | Replacement variable | +| --------------------- |------------------------ | +| `CI_BUILD_BEFORE_SHA` | `CI_COMMIT_BEFORE_SHA` | +| `CI_BUILD_ID` | `CI_JOB_ID` | +| `CI_BUILD_MANUAL` | `CI_JOB_MANUAL` | +| `CI_BUILD_NAME` | `CI_JOB_NAME` | +| `CI_BUILD_REF` | `CI_COMMIT_SHA` | +| `CI_BUILD_REF_NAME` | `CI_COMMIT_REF_NAME` | +| `CI_BUILD_REF_SLUG` | `CI_COMMIT_REF_SLUG` | +| `CI_BUILD_REPO` | `CI_REPOSITORY_URL` | +| `CI_BUILD_STAGE` | `CI_JOB_STAGE` | +| `CI_BUILD_TAG` | `CI_COMMIT_TAG` | +| `CI_BUILD_TOKEN` | `CI_JOB_TOKEN` | +| `CI_BUILD_TRIGGERED` | `CI_PIPELINE_TRIGGERED` | -Planned removal: GitLab 15.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-For those using Dependency Scanning for Python projects, we are deprecating the default `gemnasium-python:2` image which uses Python 3.6 as well as the custom `gemnasium-python:2-python-3.9` image which uses Python 3.9. The new default image as of GitLab 15.0 will be for Python 3.9 as it is a [supported version](https://endoflife.date/python) and 3.6 [is no longer supported](https://endoflife.date/python). +### `POST ci/lint` API endpoint deprecated -For users using Python 3.9 or 3.9-compatible projects, you should not need to take action and dependency scanning should begin to work in GitLab 15.0. If you wish to test the new container now please run a test pipeline in your project with this container (which will be removed in 15.0). Use the Python 3.9 image: +
+- Announced in: GitLab 15.7 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/381669). +
-```yaml -gemnasium-python-dependency_scanning: - image: - name: registry.gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python:2-python-3.9 -``` +The `POST ci/lint` API endpoint is deprecated in 15.7, and will be removed in 16.0. This endpoint does not validate the full range of CI/CD configuration options. Instead, use [`POST /projects/:id/ci/lint`](https://docs.gitlab.com/ee/api/lint.html#validate-a-ci-yaml-configuration-with-a-namespace), which properly validates CI/CD configuration. -For users using Python 3.6, as of GitLab 15.0 you will no longer be able to use the default template for dependency scanning. You will need to switch to use the deprecated `gemnasium-python:2` analyzer image. If you are impacted by this please comment in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/351503) so we can extend the removal if needed. +
-For users using the 3.9 special exception image, you must instead use the default value and no longer override your container. To verify if you are using the 3.9 special exception image, check your `.gitlab-ci.yml` file for the following reference: +
-```yaml -gemnasium-python-dependency_scanning: - image: - name: registry.gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python:2-python-3.9 -``` +### `environment_tier` parameter for DORA API +
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/365939).
-
+To avoid confusion and duplication, the `environment_tier` parameter is deprecated in favor of the `environment_tiers` parameter. The new `environment_tiers` parameter allows DORA APIs to return aggregated data for multiple tiers at the same time. The `environment_tier` parameter will be removed in GitLab 16.0. -### Deprecate Geo Admin UI Routes +
-Planned removal: GitLab 15.0 +
-In GitLab 13.0, we introduced new project and design replication details routes in the Geo Admin UI. These routes are `/admin/geo/replication/projects` and `/admin/geo/replication/designs`. We kept the legacy routes and redirected them to the new routes. In GitLab 15.0, we will remove support for the legacy routes `/admin/geo/projects` and `/admin/geo/designs`. Please update any bookmarks or scripts that may use the legacy routes. +### `started` iteration state +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/334018).
-
+The `started` iteration state in the [iterations GraphQL API](https://https://docs.gitlab.com/ee/api/graphql/reference/index.html#iterationstate) +and [iterations REST API](https://docs.gitlab.com/ee/api/iterations.html#list-project-iterations) is deprecated. -### Deprecate custom Geo:db:* Rake tasks - -Planned removal: GitLab 15.0 - -In GitLab 14.8, we are [replacing the `geo:db:*` Rake tasks with built-in tasks](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77269/diffs) that are now possible after [switching the Geo tracking database to use Rails' 6 support of multiple databases](https://gitlab.com/groups/gitlab-org/-/epics/6458). -The following `geo:db:*` tasks will be replaced with their corresponding `db:*:geo` tasks: +The GraphQL API version will be removed in GitLab 16.0. This state is being replaced with the `current` state (already available) +which aligns with the naming for other time-based entities, such as milestones. -- `geo:db:drop` -> `db:drop:geo` -- `geo:db:create` -> `db:create:geo` -- `geo:db:setup` -> `db:setup:geo` -- `geo:db:migrate` -> `db:migrate:geo` -- `geo:db:rollback` -> `db:rollback:geo` -- `geo:db:version` -> `db:version:geo` -- `geo:db:reset` -> `db:reset:geo` -- `geo:db:seed` -> `db:seed:geo` -- `geo:schema:load:geo` -> `db:schema:load:geo` -- `geo:db:schema:dump` -> `db:schema:dump:geo` -- `geo:db:migrate:up` -> `db:migrate:up:geo` -- `geo:db:migrate:down` -> `db:migrate:down:geo` -- `geo:db:migrate:redo` -> `db:migrate:redo:geo` -- `geo:db:migrate:status` -> `db:migrate:status:geo` -- `geo:db:test:prepare` -> `db:test:prepare:geo` -- `geo:db:test:load` -> `db:test:load:geo` -- `geo:db:test:purge` -> `db:test:purge:geo` +We plan to continue to support the `started` state in REST API version until the next v5 REST API version.
-
- -### Deprecate feature flag PUSH_RULES_SUPERSEDE_CODE_OWNERS +
-Planned removal: GitLab 15.0 +### vulnerabilityFindingDismiss GraphQL mutation -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 15.5 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/375645). +
-The feature flag `PUSH_RULES_SUPERSEDE_CODE_OWNERS` is being removed in GitLab 15.0. Upon its removal, push rules will supersede Code Owners. Even if Code Owner approval is required, a push rule that explicitly allows a specific user to push code supersedes the Code Owners setting. +The `VulnerabilityFindingDismiss` GraphQL mutation is being deprecated and will be removed in GitLab 16.0. This mutation was not used often as the Vulnerability Finding ID was not available to users (this field was [deprecated in 15.3](https://docs.gitlab.com/ee/update/deprecations.html#use-of-id-field-in-vulnerabilityfindingdismiss-mutation)). Users should instead use `VulnerabilityDismiss` to dismiss vulnerabilities in the Vulnerability Report or `SecurityFindingDismiss` for security findings in the CI Pipeline Security tab. +
-
+
-### Deprecate legacy Gitaly configuration methods +## GitLab 15.11 -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +### openSUSE Leap 15.3 packages -Using environment variables `GIT_CONFIG_SYSTEM` and `GIT_CONFIG_GLOBAL` to configure Gitaly is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/352609). -These variables are being replaced with standard [`config.toml` Gitaly configuration](https://docs.gitlab.com/ee/administration/gitaly/reference.html). +
+- Announced in: GitLab 15.8 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/7371). +
-GitLab instances that use `GIT_CONFIG_SYSTEM` and `GIT_CONFIG_GLOBAL` to configure Gitaly should switch to configuring using -`config.toml`. +Distribution support and security updates for openSUSE Leap 15.3 [ended December 2022](https://en.opensuse.org/Lifetime#Discontinued_distributions). -
+Starting in GitLab 15.7 we started providing packages for openSUSE Leap 15.4, and will stop providing packages for openSUSE Leap 15.3 in the 15.11 milestone. -
+- Switch from the openSUSE Leap 15.3 packages to the provided 15.4 packages. -### Elasticsearch 6.8 +
+
-Planned removal: GitLab 15.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +## GitLab 15.10 -Elasticsearch 6.8 is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. -Customers using Elasticsearch 6.8 need to upgrade their Elasticsearch version to 7.x prior to upgrading to GitLab 15.0. -We recommend using the latest version of Elasticsearch 7 to benefit from all Elasticsearch improvements. +
-Elasticsearch 6.8 is also incompatible with Amazon OpenSearch, which we [plan to support in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/327560). +### Automatic backup upload using Openstack Swift and Rackspace APIs +
+- Announced in: GitLab 15.8 +- End of Support: GitLab 15.10 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/387976).
-
+We are deprecating support for [uploading backups to remote storage](https://docs.gitlab.com/ee/raketasks/backup_gitlab.html#upload-backups-to-a-remote-cloud-storage) using Openstack Swift and Rackspace APIs. The support for these APIs depends on third-party libraries that are no longer actively maintained and have not been updated for Ruby 3. GitLab is switching over to Ruby 3 prior to EOL of Ruby 2 in order to stay up to date on security patches. -### External status check API breaking changes +- If you're using OpenStack, you need to change you configuration to use the S3 API instead of Swift. +- If you're using Rackspace storage, you need to switch to a different provider or manually upload the backup file after the backup task is complete. -Planned removal: GitLab 15.0 +
+
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The [external status check API](https://docs.gitlab.com/ee/api/status_checks.html) was originally implemented to -support pass-by-default requests to mark a status check as passing. Pass-by-default requests are now deprecated. -Specifically, the following are deprecated: +## GitLab 15.9 -- Requests that do not contain the `status` field. -- Requests that have the `status` field set to `approved`. +
-Beginning in GitLab 15.0, status checks will only be updated to a passing state if the `status` field is both present -and set to `passed`. Requests that: +### Live Preview no longer available in the Web IDE -- Do not contain the `status` field will be rejected with a `422` error. For more information, see [the relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/338827). -- Contain any value other than `passed` will cause the status check to fail. For more information, see [the relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/339039). +
+- Announced in: GitLab 15.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/383889). +
-To align with this change, API calls to list external status checks will also return the value of `passed` rather than -`approved` for status checks that have passed. +The Live Preview feature of the Web IDE was intended to provide a client-side preview of static web applications. However, complex configuration steps and a narrow set of supported project types have limited its utility. With the introduction of the Web IDE Beta in GitLab 15.7, you can now connect to a full server-side runtime environment. With upcoming support for installing extensions in the Web IDE, we'll also support more advanced workflows than those available with Live Preview. As of GitLab 15.9, Live Preview is no longer available in the Web IDE.
-
+
-### GraphQL API Runner will not accept `status` filter values of `active` or `paused` +### SaaS certificate-based integration with Kubernetes -Planned removal: GitLab 16.0 +
+- Announced in: GitLab 14.5 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/groups/gitlab-org/configure/-/epics/8). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +The certificate-based integration with Kubernetes will be [deprecated and removed](https://about.gitlab.com/blog/2021/11/15/deprecating-the-cert-based-kubernetes-integration/). As a GitLab SaaS customer, on new namespaces, you will no longer be able to integrate GitLab and your cluster using the certificate-based approach as of GitLab 15.0. The integration for current users will be enabled per namespace. -The GitLab Runner GraphQL endpoints will stop accepting `paused` or `active` as a status value in GitLab 16.0. +For a more robust, secure, forthcoming, and reliable integration with Kubernetes, we recommend you use the +[agent for Kubernetes](https://docs.gitlab.com/ee/user/clusters/agent/) to connect Kubernetes clusters with GitLab. [How do I migrate?](https://docs.gitlab.com/ee/user/infrastructure/clusters/migrate_to_gitlab_agent.html) -A runner's status will only relate to runner contact status, such as: `online`, `offline`. -Status values `paused` or `active` will no longer be accepted and will be replaced by the `paused` query parameter. +Although an explicit removal date is set, we don't plan to remove this feature until the new solution has feature parity. +For more information about the blockers to removal, see [this issue](https://gitlab.com/gitlab-org/configure/general/-/issues/199). -When checking for paused runners, API users are advised to specify `paused: true` as the query parameter. -When checking for active runners, specify `paused: false`. +For updates and details about this deprecation, follow [this epic](https://gitlab.com/groups/gitlab-org/configure/-/epics/8). -The REST API endpoints will follow in the same direction in a future REST v5 API, however the new `paused` -status value can be used in place of `active` since GitLab 14.8. +GitLab self-managed customers can still use the feature [with a feature flag](https://docs.gitlab.com/ee/update/deprecations.html#self-managed-certificate-based-integration-with-kubernetes). +
-
- -### GraphQL ID and GlobalID compatibility - -Planned removal: GitLab 15.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-We are removing a non-standard extension to our GraphQL processor, which we added for backwards compatibility. This extension modifies the validation of GraphQL queries, allowing the use of the `ID` type for arguments where it would normally be rejected. -Some arguments originally had the type `ID`. These were changed to specific -kinds of `ID`. This change may be a breaking change if you: +## GitLab 15.7 -- Use GraphQL. -- Use the `ID` type for any argument in your query signatures. +
-Some field arguments still have the `ID` type. These are typically for -IID values, or namespace paths. An example is `Query.project(fullPath: ID!)`. +### File Type variable expansion in `.gitlab-ci.yml` -For a list of affected and unaffected field arguments, -see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352832). +
+- Announced in: GitLab 15.5 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/29407). +
-You can test if this change affects you by validating -your queries locally, using schema data fetched from a GitLab server. -You can do this by using the GraphQL explorer tool for the relevant GitLab -instance. For example: `https://gitlab.com/-/graphql-explorer`. +Previously, variables that referenced or applied alias file variables expanded the value of the `File` type variable. For example, the file contents. This behavior was incorrect because it did not comply with typical shell variable expansion rules. To leak secrets or sensitive information stored in `File` type variables, a user could run an $echo command with the variable as an input parameter. -For example, the following query illustrates the breaking change: +This breaking change fixes this issue but could disrupt user workflows that work around the behavior. With this change, job variable expansions that reference or apply alias file variables, expand to the file name or path of the `File` type variable, instead of its value, such as the file contents. -```graphql -# a query using the deprecated type of Query.issue(id:) -# WARNING: This will not work after GitLab 15.0 -query($id: ID!) { - deprecated: issue(id: $id) { - title, description - } -} -``` +
+
-The query above will not work after GitLab 15.0 is released, because the type -of `Query.issue(id:)` is actually `IssueID!`. +
-Instead, you should use one of the following two forms: +## GitLab 15.6 -```graphql -# This will continue to work -query($id: IssueID!) { - a: issue(id: $id) { - title, description - } - b: issue(id: "gid://gitlab/Issue/12345") { - title, description - } -} -``` +
-This query works now, and will continue to work after GitLab 15.0. -You should convert any queries in the first form (using `ID` as a named type in the signature) -to one of the other two forms (using the correct appropriate type in the signature, or using -an inline argument expression). +### NFS for Git repository storage +
+- Announced in: GitLab 14.0
-
+With the general availability of Gitaly Cluster ([introduced in GitLab 13.0](https://about.gitlab.com/releases/2020/05/22/gitlab-13-0-released/)), we have deprecated development (bugfixes, performance improvements, etc) for NFS for Git repository storage in GitLab 14.0. We will continue to provide technical support for NFS for Git repositories throughout 14.x, but we will remove all support for NFS on November 22, 2022. This was originally planned for May 22, 2022, but in an effort to allow continued maturity of Gitaly Cluster, we have chosen to extend our deprecation of support date. Please see our official [Statement of Support](https://about.gitlab.com/support/statement-of-support/#gitaly-and-nfs) for further information. -### OAuth tokens without expiration +Gitaly Cluster offers tremendous benefits for our customers such as: -Planned removal: GitLab 15.0 +- [Variable replication factors](https://docs.gitlab.com/ee/administration/gitaly/index.html#replication-factor). +- [Strong consistency](https://docs.gitlab.com/ee/administration/gitaly/index.html#strong-consistency). +- [Distributed read capabilities](https://docs.gitlab.com/ee/administration/gitaly/index.html#distributed-reads). -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +We encourage customers currently using NFS for Git repositories to plan their migration by reviewing our documentation on [migrating to Gitaly Cluster](https://docs.gitlab.com/ee/administration/gitaly/index.html#migrate-to-gitaly-cluster). -By default, all new applications expire access tokens after 2 hours. In GitLab 14.2 and earlier, OAuth access tokens -had no expiration. In GitLab 15.0, an expiry will be automatically generated for any existing token that does not -already have one. +
+
-You should [opt in](https://docs.gitlab.com/ee/integration/oauth_provider.html#expiring-access-tokens) to expiring -tokens before GitLab 15.0 is released: +
-1. Edit the application. -1. Select **Expire access tokens** to enable them. Tokens must be revoked or they don’t expire. +## GitLab 15.4 -
+
-
+### Bundled Grafana deprecated -### Optional enforcement of PAT expiration +
+- Announced in: GitLab 15.3 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6972). +
-Planned removal: GitLab 15.0 +In GitLab 15.4, we will be swapping the bundled Grafana to a fork of Grafana maintained by GitLab. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +There was an [identified CVE for Grafana](https://nvd.nist.gov/vuln/detail/CVE-2022-31107), and to mitigate this security vulnerability, we must swap to our own fork because the older version of Grafana we were bundling is no longer receiving long-term support. -The feature to disable enforcement of PAT expiration is unusual from a security perspective. -We have become concerned that this unusual feature could create unexpected behavior for users. -Unexpected behavior in a security feature is inherently dangerous, so we have decided to remove this feature. +This is not expected to cause any incompatibilities with the previous version of Grafana. Neither when using our bundled version, nor when using an external instance of Grafana.
-
- -### Optional enforcement of SSH expiration +
-Planned removal: GitLab 15.0 +### SAST analyzer consolidation and CI/CD template changes -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352554). +
-The feature to disable enforcement of SSH expiration is unusual from a security perspective. -We have become concerned that this unusual feature could create unexpected behavior for users. -Unexpected behavior in a security feature is inherently dangerous, so we have decided to remove this feature. +GitLab SAST uses various [analyzers](https://docs.gitlab.com/ee/user/application_security/sast/analyzers/) to scan code for vulnerabilities. -
+We are reducing the number of analyzers used in GitLab SAST as part of our long-term strategy to deliver a better and more consistent user experience. +Streamlining the set of analyzers will also enable faster [iteration](https://about.gitlab.com/handbook/values/#iteration), better [results](https://about.gitlab.com/handbook/values/#results), and greater [efficiency](https://about.gitlab.com/handbook/values/#efficiency) (including a reduction in CI runner usage in most cases). -
+In GitLab 15.4, GitLab SAST will no longer use the following analyzers: -### Out-of-the-box SAST support for Java 8 +- [ESLint](https://gitlab.com/gitlab-org/security-products/analyzers/eslint) (JavaScript, TypeScript, React) +- [Gosec](https://gitlab.com/gitlab-org/security-products/analyzers/gosec) (Go) +- [Bandit](https://gitlab.com/gitlab-org/security-products/analyzers/bandit) (Python) -Planned removal: GitLab 15.0 +NOTE: +This change was originally planned for GitLab 15.0 and was postponed to GitLab 15.4. +See [the removal notice](./removals.md#sast-analyzer-consolidation-and-cicd-template-changes) for further details. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +These analyzers will be removed from the [GitLab-managed SAST CI/CD template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml) and replaced with the [Semgrep-based analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep). +Effective immediately, they will receive only security updates; other routine improvements or updates are not guaranteed. +After these analyzers reach End of Support, no further updates will be provided. +We will not delete container images previously published for these analyzers; any such change would be announced as a [deprecation, removal, or breaking change announcement](https://about.gitlab.com/handbook/marketing/blog/release-posts/#deprecations-removals-and-breaking-changes). -The [GitLab SAST SpotBugs analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) scans [Java, Scala, Groovy, and Kotlin code](https://docs.gitlab.com/ee/user/application_security/sast/#supported-languages-and-frameworks) for security vulnerabilities. -For technical reasons, the analyzer must first compile the code before scanning. -Unless you use the [pre-compilation strategy](https://docs.gitlab.com/ee/user/application_security/sast/#pre-compilation), the analyzer attempts to automatically compile your project's code. +We will also remove Java from the scope of the [SpotBugs](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) analyzer and replace it with the [Semgrep-based analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep). +This change will make it simpler to scan Java code; compilation will no longer be required. +This change will be reflected in the automatic language detection portion of the [GitLab-managed SAST CI/CD template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml). Note that the SpotBugs-based analyzer will continue to cover Groovy, Kotlin, and Scala. -In GitLab versions prior to 15.0, the analyzer image includes Java 8 and Java 11 runtimes to facilitate compilation. +If you've already dismissed a vulnerability finding from one of the deprecated analyzers, the replacement attempts to respect your previous dismissal. The system behavior depends on: -In GitLab 15.0, we will: +- whether you’ve excluded the Semgrep-based analyzer from running in the past. +- which analyzer first discovered the vulnerabilities shown in the project’s Vulnerability Report. -- Remove Java 8 from the analyzer image to reduce the size of the image. -- Add Java 17 to the analyzer image to make it easier to compile with Java 17. +See [Vulnerability translation documentation](https://docs.gitlab.com/ee/user/application_security/sast/analyzers.html#vulnerability-translation) for further details. -If you rely on Java 8 being present in the analyzer environment, you must take action as detailed in the [deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352549#breaking-change). +If you applied customizations to any of the affected analyzers or if you currently disable the Semgrep analyzer in your pipelines, you must take action as detailed in the [deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352554#breaking-change). +
-
- -### Querying Usage Trends via the `instanceStatisticsMeasurements` GraphQL node +
-Planned removal: GitLab 15.0 +## GitLab 15.3 -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The `instanceStatisticsMeasurements` GraphQL node has been renamed to `usageTrendsMeasurements` in 13.10 and the old field name has been marked as deprecated. To fix the existing GraphQL queries, replace `instanceStatisticsMeasurements` with `usageTrendsMeasurements`. +### Vulnerability Report sort by State +
+- Announced in: GitLab 15.0 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/360516).
-
+The ability to sort the Vulnerability Report by the `State` column was disabled and put behind a feature flag in GitLab 14.10 due to a refactor +of the underlying data model. The feature flag has remained off by default as further refactoring will be required to ensure sorting +by this value remains performant. Due to very low usage of the `State` column for sorting, the feature flag will instead be removed to simplify the codebase and prevent any unwanted performance degradation. -### REST and GraphQL API Runner usage of `active` replaced by `paused` +
-Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +### Vulnerability Report sort by Tool -Occurrences of the `active` identifier in the GitLab Runner GraphQL API endpoints will be -renamed to `paused` in GitLab 16.0. +
+- Announced in: GitLab 15.1 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/362962). +
-- For the GraphQL API, this change affects: - - the `CiRunner` property - - the `RunnerUpdateInput` input type for the `runnerUpdate` mutation - - the `runners` and `Group.runners` queries -- In v4 of the REST API, starting in GitLab 14.8, you can use the `paused` property in place of `active` -- In v5 of the REST API, this change will affect: - - endpoints taking or returning `active` property, such as: - - `GET /runners` - - `GET /runners/all` - - `GET /runners/:id` / `PUT /runners/:id` - - `PUT --form "active=false" /runners/:runner_id` - - `GET /projects/:id/runners` / `POST /projects/:id/runners` - - `GET /groups/:id/runners` +The ability to sort the Vulnerability Report by the `Tool` column (scan type) was disabled and put behind a feature flag in GitLab 14.10 due to a refactor +of the underlying data model. The feature flag has remained off by default as further refactoring will be required to ensure sorting +by this value remains performant. Due to very low usage of the `Tool` column for sorting, the feature flag will instead be removed in +GitLab 15.3 to simplify the codebase and prevent any unwanted performance degradation. + +
+
+ +
-The 16.0 release of GitLab Runner will start using the `paused` property when registering runners. +## GitLab 15.1 +
+ +### Deprecate support for Debian 9 + +
+- Announced in: GitLab 14.9
-
+Long term service and support (LTSS) for [Debian 9 Stretch ends in July 2022](https://wiki.debian.org/LTS). Therefore, we will no longer support the Debian 9 distribution for the GitLab package. Users can upgrade to Debian 10 or Debian 11. -### Request profiling +
+
-Planned removal: GitLab 15.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +## GitLab 15.0 -[Request profiling](https://docs.gitlab.com/ee/administration/monitoring/performance/index.html) is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. +
-We're working on [consolidating our profiling tools](https://gitlab.com/groups/gitlab-org/-/epics/7327) and making them more easily accessible. -We [evaluated](https://gitlab.com/gitlab-org/gitlab/-/issues/350152) the use of this feature and we found that it is not widely used. -It also depends on a few third-party gems that are not actively maintained anymore, have not been updated for the latest version of Ruby, or crash frequently when profiling heavy page loads. +### Audit events for repository push events -For more information, check the [summary section of the deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352488#deprecation-summary). +
+- Announced in: GitLab 14.3 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/337993). +
+ +Audit events for [repository events](https://docs.gitlab.com/ee/administration/audit_events.html#removed-events) are now deprecated and will be removed in GitLab 15.0. + +These events have always been disabled by default and had to be manually enabled with a +feature flag. Enabling them can cause too many events to be generated which can +dramatically slow down GitLab instances. For this reason, they are being removed.
-
+
-### Required pipeline configurations in Premium tier +### Background upload for object storage -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 14.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/26600). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +To reduce the overall complexity and maintenance burden of GitLab's [object storage feature](https://docs.gitlab.com/ee/administration/object_storage.html), support for using `background_upload` to upload files is deprecated and will be fully removed in GitLab 15.0. Review the [15.0 specific changes](https://docs.gitlab.com/omnibus/update/gitlab_15_changes.html) for the [removed background uploads settings for object storage](https://docs.gitlab.com/omnibus/update/gitlab_15_changes.html#removed-background-uploads-settings-for-object-storage). -The [required pipeline configuration](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#required-pipeline-configuration) feature is deprecated in GitLab 14.8 for Premium customers and is scheduled for removal in GitLab 15.0. This feature is not deprecated for GitLab Ultimate customers. +This impacts a small subset of object storage providers: -This change to move the feature to GitLab's Ultimate tier is intended to help our features better align with our [pricing philosophy](https://about.gitlab.com/company/pricing/#three-tiers) as we see demand for this feature originating primarily from executives. +- **OpenStack** Customers using OpenStack need to change their configuration to use the S3 API instead of Swift. +- **RackSpace** Customers using RackSpace-based object storage need to migrate data to a different provider. -This change will also help GitLab remain consistent in its tiering strategy with the other related Ultimate-tier features of: -[Security policies](https://docs.gitlab.com/ee/user/application_security/policies/) and [compliance framework pipelines](https://docs.gitlab.com/ee/user/project/settings/index.html#compliance-pipeline-configuration). +GitLab will publish additional guidance to assist affected customers in migrating.
-
+
-### Retire-JS Dependency Scanning tool +### CI/CD job name length limit + +
+- Announced in: GitLab 14.6 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/342800). +
-Planned removal: GitLab 15.0 +In GitLab 15.0 we are going to limit the number of characters in CI/CD job names to 255. Any pipeline with job names that exceed the 255 character limit will stop working after the 15.0 release. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-As of 14.8 the retire.js job is being deprecated from Dependency Scanning. It will continue to be included in our CI/CD template while deprecated. We are removing retire.js from Dependency Scanning on May 22, 2022 in GitLab 15.0. JavaScript scanning functionality will not be affected as it is still being covered by Gemnasium. +
-If you have explicitly excluded retire.js using DS_EXCLUDED_ANALYZERS you will need to clean up (remove the reference) in 15.0. If you have customized your pipeline's Dependency Scanning configuration related to the `retire-js-dependency_scanning` job you will want to switch to gemnasium-dependency_scanning before the removal in 15.0, to prevent your pipeline from failing. If you have not used the DS_EXCLUDED_ANALYZERS to reference retire.js, or customized your template specifically for retire.js, you will not need to take action. +### Changing an instance (shared) runner to a project (specific) runner +
+- Announced in: GitLab 14.5 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/345347).
-
+In GitLab 15.0, you can no longer change an instance (shared) runner to a project (specific) runner. -### SAST analyzer consolidation and CI/CD template changes +Users often accidentally change instance runners to project runners, and they're unable to change them back. GitLab does not allow you to change a project runner to a shared runner because of the security implications. A runner meant for one project could be set to run jobs for an entire instance. -Planned removal: GitLab 15.4 +Administrators who need to add runners for multiple projects can register a runner for one project, then go to the Admin view and choose additional projects. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-GitLab SAST uses various [analyzers](https://docs.gitlab.com/ee/user/application_security/sast/analyzers/) to scan code for vulnerabilities. +
-We are reducing the number of analyzers used in GitLab SAST as part of our long-term strategy to deliver a better and more consistent user experience. -Streamlining the set of analyzers will also enable faster [iteration](https://about.gitlab.com/handbook/values/#iteration), better [results](https://about.gitlab.com/handbook/values/#results), and greater [efficiency](https://about.gitlab.com/handbook/values/#efficiency) (including a reduction in CI runner usage in most cases). +### Container Network and Host Security -In GitLab 15.4, GitLab SAST will no longer use the following analyzers: +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +
-- [ESLint](https://gitlab.com/gitlab-org/security-products/analyzers/eslint) (JavaScript, TypeScript, React) -- [Gosec](https://gitlab.com/gitlab-org/security-products/analyzers/gosec) (Go) -- [Bandit](https://gitlab.com/gitlab-org/security-products/analyzers/bandit) (Python) +All functionality related to GitLab's Container Network Security and Container Host Security categories is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to evaluate the following open source projects as potential solutions that can be installed and managed outside of GitLab: [AppArmor](https://gitlab.com/apparmor/apparmor), [Cilium](https://github.com/cilium/cilium), [Falco](https://github.com/falcosecurity/falco), [FluentD](https://github.com/fluent/fluentd), [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/). To integrate these technologies into GitLab, add the desired Helm charts into your copy of the [Cluster Management Project Template](https://docs.gitlab.com/ee/user/clusters/management_project_template.html). Deploy these Helm charts in production by calling commands through GitLab [CI/CD](https://docs.gitlab.com/ee/user/clusters/agent/ci_cd_workflow.html). -NOTE: -This change was originally planned for GitLab 15.0 and was postponed to GitLab 15.4. -See [the removal notice](./removals.md#sast-analyzer-consolidation-and-cicd-template-changes) for further details. +As part of this change, the following specific capabilities within GitLab are now deprecated, and are scheduled for removal in GitLab 15.0: -These analyzers will be removed from the [GitLab-managed SAST CI/CD template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml) and replaced with the [Semgrep-based analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep). -Effective immediately, they will receive only security updates; other routine improvements or updates are not guaranteed. -After these analyzers reach End of Support, no further updates will be provided. -We will not delete container images previously published for these analyzers; any such change would be announced as a [deprecation, removal, or breaking change announcement](https://about.gitlab.com/handbook/marketing/blog/release-posts/#deprecations-removals-and-breaking-changes). +- The **Security & Compliance > Threat Monitoring** page. +- The `Network Policy` security policy type, as found on the **Security & Compliance > Policies** page. +- The ability to manage integrations with the following technologies through GitLab: AppArmor, Cilium, Falco, FluentD, and Pod Security Policies. +- All APIs related to the above functionality. -We will also remove Java from the scope of the [SpotBugs](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) analyzer and replace it with the [Semgrep-based analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep). -This change will make it simpler to scan Java code; compilation will no longer be required. -This change will be reflected in the automatic language detection portion of the [GitLab-managed SAST CI/CD template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml). Note that the SpotBugs-based analyzer will continue to cover Groovy, Kotlin, and Scala. +For additional context, or to provide feedback regarding this change, please reference our open [deprecation issue](https://gitlab.com/groups/gitlab-org/-/epics/7476). -If you've already dismissed a vulnerability finding from one of the deprecated analyzers, the replacement attempts to respect your previous dismissal. The system behavior depends on: +
-- whether you’ve excluded the Semgrep-based analyzer from running in the past. -- which analyzer first discovered the vulnerabilities shown in the project’s Vulnerability Report. +
-See [Vulnerability translation documentation](https://docs.gitlab.com/ee/user/application_security/sast/analyzers.html#vulnerability-translation) for further details. +### Container scanning schemas below 14.0.0 -If you applied customizations to any of the affected analyzers or if you currently disable the Semgrep analyzer in your pipelines, you must take action as detailed in the [deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352554#breaking-change). +
+- Announced in: GitLab 14.7 +
+ +[Container scanning report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) +versions earlier than 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation +against the schema version declared in the report will also no longer be supported in GitLab 15.0. + +Third-party tools that [integrate with GitLab by outputting a container scanning security report](https://docs.gitlab.com/ee/development/integrations/secure.html#report) +as a pipeline job artifact are affected. You must ensure that all output reports adhere to the correct schema with a minimum version of 14.0.0. Reports with a lower version or that fail to validate against the declared schema version will not be processed, and vulnerability findings will not display in MRs, pipelines, or Vulnerability Reports. + +To help with the transition, from GitLab 14.10, non-compliant reports will display a +[warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) +in the Vulnerability Report.
-
+
-### SAST support for .NET 2.1 +### Coverage guided fuzzing schemas below 14.0.0 -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 14.7 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +[Coverage guided fuzzing report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) +below version 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation +against the schema version declared in the report will also no longer be supported in GitLab 15.0. -The GitLab SAST Security Code Scan analyzer scans .NET code for security vulnerabilities. -For technical reasons, the analyzer must first build the code to scan it. +Third-party tools that [integrate with GitLab by outputting a coverage guided fuzzing security report](https://docs.gitlab.com/ee/development/integrations/secure.html#report) +as a pipeline job artifact are affected. You must ensure that all output reports adhere to the correct +schema with a minimum version of 14.0.0. Any reports with a lower version or that fail to validate +against the declared schema version will not be processed, and vulnerability +findings will not display in MRs, pipelines, or Vulnerability Reports. -In GitLab versions prior to 15.0, the default analyzer image (version 2) includes support for: +To help with the transition, from GitLab 14.10, non-compliant reports will display a +[warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) +in the Vulnerability Report. -- .NET 2.1 -- .NET 3.0 and .NET Core 3.0 -- .NET Core 3.1 -- .NET 5.0 +
-In GitLab 15.0, we will change the default major version for this analyzer from version 2 to version 3. This change: +
-- Adds [severity values for vulnerabilities](https://gitlab.com/gitlab-org/gitlab/-/issues/350408) along with [other new features and improvements](https://gitlab.com/gitlab-org/security-products/analyzers/security-code-scan/-/blob/master/CHANGELOG.md). -- Removes .NET 2.1 support. -- Adds support for .NET 6.0, Visual Studio 2019, and Visual Studio 2022. +### DAST schemas below 14.0.0 -Version 3 was [announced in GitLab 14.6](https://about.gitlab.com/releases/2021/12/22/gitlab-14-6-released/#sast-support-for-net-6) and made available as an optional upgrade. +
+- Announced in: GitLab 14.7 +
-If you rely on .NET 2.1 support being present in the analyzer image by default, you must take action as detailed in the [deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352553#breaking-change). +[DAST report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) +versions earlier than 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation +against the schema version declared in the report will also no longer be supported as of GitLab 15.0. + +Third-party tools that [integrate with GitLab by outputting a DAST security report](https://docs.gitlab.com/ee/development/integrations/secure.html#report) +as a pipeline job artifact are affected. You must ensure that all output reports adhere to the correct +schema with a minimum version of 14.0.0. Reports with a lower version or that fail to validate +against the declared schema version will not be processed, and vulnerability +findings will not display in MRs, pipelines, or Vulnerability Reports. + +To help with the transition, from GitLab 14.10, non-compliant reports will cause a +[warning to be displayed](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) +in the Vulnerability Report.
-
+
-### Secret Detection configuration variables deprecated +### Dependency Scanning Python 3.9 and 3.6 image deprecation -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/334060). +
-To make it simpler and more reliable to [customize GitLab Secret Detection](https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings), we're deprecating some of the variables that you could previously set in your CI/CD configuration. +For those using Dependency Scanning for Python projects, we are deprecating the default `gemnasium-python:2` image which uses Python 3.6 as well as the custom `gemnasium-python:2-python-3.9` image which uses Python 3.9. The new default image as of GitLab 15.0 will be for Python 3.9 as it is a [supported version](https://endoflife.date/python) and 3.6 [is no longer supported](https://endoflife.date/python). -The following variables currently allow you to customize the options for historical scanning, but interact poorly with the [GitLab-managed CI/CD template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml) and are now deprecated: +For users using Python 3.9 or 3.9-compatible projects, you should not need to take action and dependency scanning should begin to work in GitLab 15.0. If you wish to test the new container now please run a test pipeline in your project with this container (which will be removed in 15.0). Use the Python 3.9 image: -- `SECRET_DETECTION_COMMIT_FROM` -- `SECRET_DETECTION_COMMIT_TO` -- `SECRET_DETECTION_COMMITS` -- `SECRET_DETECTION_COMMITS_FILE` +```yaml +gemnasium-python-dependency_scanning: + image: + name: registry.gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python:2-python-3.9 +``` -The `SECRET_DETECTION_ENTROPY_LEVEL` previously allowed you to configure rules that only considered the entropy level of strings in your codebase, and is now deprecated. -This type of entropy-only rule created an unacceptable number of incorrect results (false positives) and is no longer supported. +For users using Python 3.6, as of GitLab 15.0 you will no longer be able to use the default template for dependency scanning. You will need to switch to use the deprecated `gemnasium-python:2` analyzer image. If you are impacted by this please comment in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/351503) so we can extend the removal if needed. -In GitLab 15.0, we'll update the Secret Detection [analyzer](https://docs.gitlab.com/ee/user/application_security/terminology/#analyzer) to ignore these deprecated options. -You'll still be able to configure historical scanning of your commit history by setting the [`SECRET_DETECTION_HISTORIC_SCAN` CI/CD variable](https://docs.gitlab.com/ee/user/application_security/secret_detection/#available-cicd-variables). +For users using the 3.9 special exception image, you must instead use the default value and no longer override your container. To verify if you are using the 3.9 special exception image, check your `.gitlab-ci.yml` file for the following reference: -For further details, see [the deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352565). +```yaml +gemnasium-python-dependency_scanning: + image: + name: registry.gitlab.com/gitlab-org/security-products/analyzers/gemnasium-python:2-python-3.9 +```
-
+
-### Secure and Protect analyzer images published in new location +### Dependency Scanning default Java version changed to 17 -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 14.10 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +In GitLab 15.0, for Dependency Scanning, the default version of Java that the scanner expects will be updated from 11 to 17. Java 17 is [the most up-to-date Long Term Support (LTS) version](https://en.wikipedia.org/wiki/Java_version_history). Dependency scanning continues to support the same [range of versions (8, 11, 13, 14, 15, 16, 17)](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#supported-languages-and-package-managers), only the default version is changing. If your project uses the previous default of Java 11, be sure to [set the `DS_Java_Version` variable to match](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning). -GitLab uses various [analyzers](https://docs.gitlab.com/ee/user/application_security/terminology/#analyzer) to [scan for security vulnerabilities](https://docs.gitlab.com/ee/user/application_security/). -Each analyzer is distributed as a container image. +
-Starting in GitLab 14.8, new versions of GitLab Secure and Protect analyzers are published to a new registry location under `registry.gitlab.com/security-products`. +
-We will update the default value of [GitLab-managed CI/CD templates](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/Security) to reflect this change: +### Dependency scanning schemas below 14.0.0 -- For all analyzers except Container Scanning, we will update the variable `SECURE_ANALYZERS_PREFIX` to the new image registry location. -- For Container Scanning, the default image address is already updated. There is no `SECURE_ANALYZERS_PREFIX` variable for Container Scanning. +
+- Announced in: GitLab 14.7 +
-In a future release, we will stop publishing images to `registry.gitlab.com/gitlab-org/security-products/analyzers`. -Once this happens, you must take action if you manually pull images and push them into a separate registry. This is commonly the case for [offline deployments](https://docs.gitlab.com/ee/user/application_security/offline_deployments/index.html). -Otherwise, you won't receive further updates. +[Dependency scanning report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) +versions earlier than 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation +against the schema version declared in the report will also no longer be supported as of GitLab 15.0. -See the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352564) for more details. +Third-party tools that [integrate with GitLab by outputting a Dependency scanning security report](https://docs.gitlab.com/ee/development/integrations/secure.html#report) +as a pipeline job artifact are affected. You must ensure that all output reports adhere to the correct +schema with a minimum version of 14.0.0. Reports with a lower version or that fail to validate +against the declared schema version will not be processed, and vulnerability +findings will not display in MRs, pipelines, or Vulnerability Reports. + +To help with the transition, from GitLab 14.10, non-compliant reports will cause a +[warning to be displayed](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) +in the Vulnerability Report.
-
+
-### Secure and Protect analyzer major version update +### Deprecate Geo Admin UI Routes -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 14.8 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/351345). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +In GitLab 13.0, we introduced new project and design replication details routes in the Geo Admin UI. These routes are `/admin/geo/replication/projects` and `/admin/geo/replication/designs`. We kept the legacy routes and redirected them to the new routes. In GitLab 15.0, we will remove support for the legacy routes `/admin/geo/projects` and `/admin/geo/designs`. Please update any bookmarks or scripts that may use the legacy routes. -The Secure and Protect stages will be bumping the major versions of their analyzers in tandem with the GitLab 15.0 release. This major bump will enable a clear delineation for analyzers, between: +
-- Those released prior to May 22, 2022, which generate reports that _are not_ subject to stringent schema validation. -- Those released after May 22, 2022, which generate reports that _are_ subject to stringent schema validation. +
-If you are not using the default inclusion templates, or have pinned your analyzer versions you will need to update your CI/CD job definition to either remove the pinned version or to update the latest major version. -Users of GitLab 12.0-14.10 will continue to experience analyzer updates as normal until the release of GitLab 15.0, following which all newly fixed bugs and newly released features in the new major versions of the analyzers will not be available in the deprecated versions because we do not backport bugs and new features as per our [maintenance policy](https://docs.gitlab.com/ee/policy/maintenance.html). As required security patches will be backported within the latest 3 minor releases. -Specifically, the following are being deprecated and will no longer be updated after 15.0 GitLab release: +### Deprecate custom Geo:db:* Rake tasks -- API Security: version 1 -- Container Scanning: version 4 -- Coverage-guided fuzz testing: version 2 -- Dependency Scanning: version 2 -- Dynamic Application Security Testing (DAST): version 2 -- Infrastructure as Code (IaC) Scanning: version 1 -- License Scanning: version 3 -- Secret Detection: version 3 -- Static Application Security Testing (SAST): version 2 of [all analyzers](https://docs.gitlab.com/ee/user/application_security/sast/#supported-languages-and-frameworks), except `gosec` which is currently at version 3 - - `bandit`: version 2 - - `brakeman`: version 2 - - `eslint`: version 2 - - `flawfinder`: version 2 - - `gosec`: version 3 - - `kubesec`: version 2 - - `mobsf`: version 2 - - `nodejs-scan`: version 2 - - `phpcs-security-audit`: version 2 - - `pmd-apex`: version 2 - - `security-code-scan`: version 2 - - `semgrep`: version 2 - - `sobelow`: version 2 - - `spotbugs`: version 2 +
+- Announced in: GitLab 14.8 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/351945). +
+ +In GitLab 14.8, we are [replacing the `geo:db:*` Rake tasks with built-in tasks](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/77269/diffs) that are now possible after [switching the Geo tracking database to use Rails' 6 support of multiple databases](https://gitlab.com/groups/gitlab-org/-/epics/6458). +The following `geo:db:*` tasks will be replaced with their corresponding `db:*:geo` tasks: + +- `geo:db:drop` -> `db:drop:geo` +- `geo:db:create` -> `db:create:geo` +- `geo:db:setup` -> `db:setup:geo` +- `geo:db:migrate` -> `db:migrate:geo` +- `geo:db:rollback` -> `db:rollback:geo` +- `geo:db:version` -> `db:version:geo` +- `geo:db:reset` -> `db:reset:geo` +- `geo:db:seed` -> `db:seed:geo` +- `geo:schema:load:geo` -> `db:schema:load:geo` +- `geo:db:schema:dump` -> `db:schema:dump:geo` +- `geo:db:migrate:up` -> `db:migrate:up:geo` +- `geo:db:migrate:down` -> `db:migrate:down:geo` +- `geo:db:migrate:redo` -> `db:migrate:redo:geo` +- `geo:db:migrate:status` -> `db:migrate:status:geo` +- `geo:db:test:prepare` -> `db:test:prepare:geo` +- `geo:db:test:load` -> `db:test:load:geo` +- `geo:db:test:purge` -> `db:test:purge:geo`
-
+
-### Support for gRPC-aware proxy deployed between Gitaly and rest of GitLab +### Deprecate feature flag PUSH_RULES_SUPERSEDE_CODE_OWNERS -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/262019). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +The feature flag `PUSH_RULES_SUPERSEDE_CODE_OWNERS` is being removed in GitLab 15.0. Upon its removal, push rules will supersede Code Owners. Even if Code Owner approval is required, a push rule that explicitly allows a specific user to push code supersedes the Code Owners setting. -Although not recommended or documented, it was possible to deploy a gRPC-aware proxy between Gitaly and -the rest of GitLab. For example, NGINX and Envoy. The ability to deploy a gRPC-aware proxy is -[deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/352517). If you currently use a gRPC-aware proxy for -Gitaly connections, you should change your proxy configuration to use TCP or TLS proxying (OSI layer 4) instead. +
-Gitaly Cluster became incompatible with gRPC-aware proxies in GitLab 13.12. Now all GitLab installations will be incompatible with -gRPC-aware proxies, even without Gitaly Cluster. +
-By sending some of our internal RPC traffic through a custom protocol (instead of gRPC) we -increase throughput and reduce Go garbage collection latency. For more information, see -the [relevant epic](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/463). +### Elasticsearch 6.8 +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/350275).
-
+Elasticsearch 6.8 is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. +Customers using Elasticsearch 6.8 need to upgrade their Elasticsearch version to 7.x prior to upgrading to GitLab 15.0. +We recommend using the latest version of Elasticsearch 7 to benefit from all Elasticsearch improvements. -### Test coverage project CI/CD setting +Elasticsearch 6.8 is also incompatible with Amazon OpenSearch, which we [plan to support in GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/327560). -Planned removal: GitLab 15.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-To simplify setting a test coverage pattern, in GitLab 15.0 the -[project setting for test coverage parsing](https://docs.gitlab.com/ee/ci/pipelines/settings.html#add-test-coverage-results-using-project-settings-removed) -is being removed. +### Enforced validation of security report schemas -Instead, using the project’s `.gitlab-ci.yml`, provide a regular expression with the `coverage` keyword to set -testing coverage results in merge requests. +
+- Announced in: GitLab 14.7 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/groups/gitlab-org/-/epics/6968). +
+ +[Security report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) +versions earlier than 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation +against the schema version declared in the report will also no longer be supported in GitLab 15.0. + +Security tools that [integrate with GitLab by outputting security reports](https://docs.gitlab.com/ee/development/integrations/secure.html#report) +as pipeline job artifacts are affected. You must ensure that all output reports adhere to the correct +schema with a minimum version of 14.0.0. Reports with a lower version or that fail to validate +against the declared schema version will not be processed, and vulnerability +findings will not display in MRs, pipelines, or Vulnerability Reports. + +To help with the transition, from GitLab 14.10, non-compliant reports will display a +[warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) +in the Vulnerability Report.
-
+
-### Vulnerability Check +### External status check API breaking changes -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +The [external status check API](https://docs.gitlab.com/ee/api/status_checks.html) was originally implemented to +support pass-by-default requests to mark a status check as passing. Pass-by-default requests are now deprecated. +Specifically, the following are deprecated: -The vulnerability check feature is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. We encourage you to migrate to the new security approvals feature instead. You can do so by navigating to **Security & Compliance > Policies** and creating a new Scan Result Policy. +- Requests that do not contain the `status` field. +- Requests that have the `status` field set to `approved`. -The new security approvals feature is similar to vulnerability check. For example, both can require approvals for MRs that contain security vulnerabilities. However, security approvals improve the previous experience in several ways: +Beginning in GitLab 15.0, status checks will only be updated to a passing state if the `status` field is both present +and set to `passed`. Requests that: -- Users can choose who is allowed to edit security approval rules. An independent security or compliance team can therefore manage rules in a way that prevents development project maintainers from modifying the rules. -- Multiple rules can be created and chained together to allow for filtering on different severity thresholds for each scanner type. -- A two-step approval process can be enforced for any desired changes to security approval rules. -- A single set of security policies can be applied to multiple development projects to allow for ease in maintaining a single, centralized ruleset. +- Do not contain the `status` field will be rejected with a `422` error. For more information, see [the relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/338827). +- Contain any value other than `passed` will cause the status check to fail. For more information, see [the relevant issue](https://gitlab.com/gitlab-org/gitlab/-/issues/339039). + +To align with this change, API calls to list external status checks will also return the value of `passed` rather than +`approved` for status checks that have passed.
-
+
-### `CI_BUILD_*` predefined variables +### GitLab Pages running as daemon -Planned removal: GitLab 16.0 +
+- Announced in: GitLab 14.9 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +In 15.0, support for daemon mode for GitLab Pages will be removed. -The predefined CI/CD variables that start with `CI_BUILD_*` were deprecated in GitLab 9.0, and will be removed in GitLab 16.0. If you still use these variables, be sure to change to the replacement [predefined variables](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html) which are functionally identical: +
-| Removed variable | Replacement variable | -| --------------------- |------------------------ | -| `CI_BUILD_BEFORE_SHA` | `CI_COMMIT_BEFORE_SHA` | -| `CI_BUILD_ID` | `CI_JOB_ID` | -| `CI_BUILD_MANUAL` | `CI_JOB_MANUAL` | -| `CI_BUILD_NAME` | `CI_JOB_NAME` | -| `CI_BUILD_REF` | `CI_COMMIT_SHA` | -| `CI_BUILD_REF_NAME` | `CI_COMMIT_REF_NAME` | -| `CI_BUILD_REF_SLUG` | `CI_COMMIT_REF_SLUG` | -| `CI_BUILD_REPO` | `CI_REPOSITORY_URL` | -| `CI_BUILD_STAGE` | `CI_JOB_STAGE` | -| `CI_BUILD_TAG` | `CI_COMMIT_TAG` | -| `CI_BUILD_TOKEN` | `CI_JOB_TOKEN` | -| `CI_BUILD_TRIGGERED` | `CI_PIPELINE_TRIGGERED` | +
+ +### GitLab Serverless +
+- Announced in: GitLab 14.3 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/groups/gitlab-org/configure/-/epics/6).
-
+GitLab Serverless is a feature set to support Knative-based serverless development with automatic deployments and monitoring. -### `projectFingerprint` in `PipelineSecurityReportFinding` GraphQL +We decided to remove the GitLab Serverless features as they never really resonated with our users. Besides, given the continuous development of Kubernetes and Knative, our current implementations do not even work with recent versions. -Planned removal: GitLab 15.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The `projectFingerprint` field in the [PipelineSecurityReportFinding](https://docs.gitlab.com/ee/api/graphql/reference/index.html#pipelinesecurityreportfinding) -GraphQL object is being deprecated. This field contains a "fingerprint" of security findings used to determine uniqueness. -The method for calculating fingerprints has changed, resulting in different values. Going forward, the new values will be -exposed in the UUID field. Data previously available in the projectFingerprint field will eventually be removed entirely. +### Godep support in License Compliance + +
+- Announced in: GitLab 14.7 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/327057). +
+ +The Godep dependency manager for Golang was deprecated in 2020 by Go and +has been replaced with Go modules. +To reduce our maintenance cost we are deprecating License Compliance for Godep projects as of 14.7 +and will remove it in GitLab 15.0 + +
+ +
+ +### GraphQL ID and GlobalID compatibility + +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/257883). +
+ +We are removing a non-standard extension to our GraphQL processor, which we added for backwards compatibility. This extension modifies the validation of GraphQL queries, allowing the use of the `ID` type for arguments where it would normally be rejected. +Some arguments originally had the type `ID`. These were changed to specific +kinds of `ID`. This change may be a breaking change if you: + +- Use GraphQL. +- Use the `ID` type for any argument in your query signatures. + +Some field arguments still have the `ID` type. These are typically for +IID values, or namespace paths. An example is `Query.project(fullPath: ID!)`. + +For a list of affected and unaffected field arguments, +see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352832). + +You can test if this change affects you by validating +your queries locally, using schema data fetched from a GitLab server. +You can do this by using the GraphQL explorer tool for the relevant GitLab +instance. For example: `https://gitlab.com/-/graphql-explorer`. + +For example, the following query illustrates the breaking change: + +```graphql +# a query using the deprecated type of Query.issue(id:) +# WARNING: This will not work after GitLab 15.0 +query($id: ID!) { + deprecated: issue(id: $id) { + title, description + } +} +``` + +The query above will not work after GitLab 15.0 is released, because the type +of `Query.issue(id:)` is actually `IssueID!`. + +Instead, you should use one of the following two forms: + +```graphql +# This will continue to work +query($id: IssueID!) { + a: issue(id: $id) { + title, description + } + b: issue(id: "gid://gitlab/Issue/12345") { + title, description + } +} +``` + +This query works now, and will continue to work after GitLab 15.0. +You should convert any queries in the first form (using `ID` as a named type in the signature) +to one of the other two forms (using the correct appropriate type in the signature, or using +an inline argument expression).
-
+
+ +### GraphQL permissions change for Package settings -### `started` iterations API field +
+- Announced in: GitLab 14.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +
-Planned removal: GitLab 15.0 +The GitLab Package stage offers a Package Registry, Container Registry, and Dependency Proxy to help you manage all of your dependencies using GitLab. Each of these product categories has a variety of settings that can be adjusted using the API. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +The permissions model for GraphQL is being updated. After 15.0, users with the Guest, Reporter, and Developer role can no longer update these settings: -The `started` field in the [iterations API](https://docs.gitlab.com/ee/api/iterations.html#list-project-iterations) is being deprecated and will be removed in GitLab 15.0. This field is being replaced with the `current` field (already available) which aligns with the naming for other time-based entities, such as milestones. +- [Package Registry settings](https://docs.gitlab.com/ee/api/graphql/reference/#packagesettings) +- [Container Registry cleanup policy](https://docs.gitlab.com/ee/api/graphql/reference/#containerexpirationpolicy) +- [Dependency Proxy time-to-live policy](https://docs.gitlab.com/ee/api/graphql/reference/#dependencyproxyimagettlgrouppolicy) +- [Enabling the Dependency Proxy for your group](https://docs.gitlab.com/ee/api/graphql/reference/#dependencyproxysetting)
+ +
+ +### Known host required for GitLab Runner SSH executor + +
+- Announced in: GitLab 14.5 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/28192).
-
+In [GitLab 14.3](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3074), we added a configuration setting in the GitLab Runner `config.toml` file. This setting, [`[runners.ssh.disable_strict_host_key_checking]`](https://docs.gitlab.com/runner/executors/ssh.html#security), controls whether or not to use strict host key checking with the SSH executor. -## Announced in 14.7 +In GitLab 15.0 and later, the default value for this configuration option will change from `true` to `false`. This means that strict host key checking will be enforced when using the GitLab Runner SSH executor. -
+
-### Container scanning schemas below 14.0.0 +
-Planned removal: GitLab 15.0 +### Legacy approval status names from License Compliance API -[Container scanning report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) -versions earlier than 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation -against the schema version declared in the report will also no longer be supported in GitLab 15.0. +
+- Announced in: GitLab 14.6 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/335707). +
-Third-party tools that [integrate with GitLab by outputting a container scanning security report](https://docs.gitlab.com/ee/development/integrations/secure.html#report) -as a pipeline job artifact are affected. You must ensure that all output reports adhere to the correct schema with a minimum version of 14.0.0. Reports with a lower version or that fail to validate against the declared schema version will not be processed, and vulnerability findings will not display in MRs, pipelines, or Vulnerability Reports. +We deprecated legacy names for approval status of license policy (blacklisted, approved) in the `managed_licenses` API but they are still used in our API queries and responses. They will be removed in 15.0. -To help with the transition, from GitLab 14.10, non-compliant reports will display a -[warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) -in the Vulnerability Report. +If you are using our License Compliance API you should stop using the `approved` and `blacklisted` query parameters, they are now `allowed` and `denied`. In 15.0 the responses will also stop using `approved` and `blacklisted` so you need to adjust any of your custom tools to use the old and new values so they do not break with the 15.0 release.
-
+
-### Coverage guided fuzzing schemas below 14.0.0 +### Legacy database configuration -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 14.3 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/338182). +
-[Coverage guided fuzzing report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) -below version 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation -against the schema version declared in the report will also no longer be supported in GitLab 15.0. +The syntax of [GitLabs database](https://docs.gitlab.com/omnibus/settings/database.html) +configuration located in `database.yml` is changing and the legacy format is deprecated. The legacy format +supported using a single PostgreSQL adapter, whereas the new format is changing to support multiple databases. The `main:` database needs to be defined as a first configuration item. -Third-party tools that [integrate with GitLab by outputting a coverage guided fuzzing security report](https://docs.gitlab.com/ee/development/integrations/secure.html#report) -as a pipeline job artifact are affected. You must ensure that all output reports adhere to the correct -schema with a minimum version of 14.0.0. Any reports with a lower version or that fail to validate -against the declared schema version will not be processed, and vulnerability -findings will not display in MRs, pipelines, or Vulnerability Reports. +This deprecation mainly impacts users compiling GitLab from source because Omnibus will handle this configuration automatically. -To help with the transition, from GitLab 14.10, non-compliant reports will display a -[warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) -in the Vulnerability Report. +
+ +
+ +### Logging in GitLab +
+- Announced in: GitLab 14.7 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/346485).
-
+The logging features in GitLab allow users to install the ELK stack (Elasticsearch, Logstash, and Kibana) to aggregate and manage application logs. Users can search for relevant logs in GitLab. However, since deprecating certificate-based integration with Kubernetes clusters and GitLab Managed Apps, we don't have a recommended solution for logging within GitLab. For more information, you can follow the issue for [integrating Opstrace with GitLab](https://gitlab.com/groups/gitlab-org/-/epics/6976). -### DAST schemas below 14.0.0 +
-Planned removal: GitLab 15.0 +
-[DAST report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) -versions earlier than 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation -against the schema version declared in the report will also no longer be supported as of GitLab 15.0. +### Move `custom_hooks_dir` setting from GitLab Shell to Gitaly -Third-party tools that [integrate with GitLab by outputting a DAST security report](https://docs.gitlab.com/ee/development/integrations/secure.html#report) -as a pipeline job artifact are affected. You must ensure that all output reports adhere to the correct -schema with a minimum version of 14.0.0. Reports with a lower version or that fail to validate -against the declared schema version will not be processed, and vulnerability -findings will not display in MRs, pipelines, or Vulnerability Reports. +
+- Announced in: GitLab 14.9 +
+ +The [`custom_hooks_dir`](https://docs.gitlab.com/ee/administration/server_hooks.html#create-a-global-server-hook-for-all-repositories) setting is now configured in Gitaly, and will be removed from GitLab Shell in GitLab 15.0. + +
+ +
+ +### OAuth implicit grant + +
+- Announced in: GitLab 14.0 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +
+ +The OAuth implicit grant authorization flow will be removed in our next major release, GitLab 15.0. Any applications that use OAuth implicit grant should switch to alternative [supported OAuth flows](https://docs.gitlab.com/ee/api/oauth2.html). + +
+ +
+ +### OAuth tokens without expiration + +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +
+ +By default, all new applications expire access tokens after 2 hours. In GitLab 14.2 and earlier, OAuth access tokens +had no expiration. In GitLab 15.0, an expiry will be automatically generated for any existing token that does not +already have one. + +You should [opt in](https://docs.gitlab.com/ee/integration/oauth_provider.html#expiring-access-tokens) to expiring +tokens before GitLab 15.0 is released: -To help with the transition, from GitLab 14.10, non-compliant reports will cause a -[warning to be displayed](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) -in the Vulnerability Report. +1. Edit the application. +1. Select **Expire access tokens** to enable them. Tokens must be revoked or they don’t expire.
-
+
-### Dependency scanning schemas below 14.0.0 +### OmniAuth Kerberos gem -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 14.3 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/337384). +
-[Dependency scanning report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) -versions earlier than 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation -against the schema version declared in the report will also no longer be supported as of GitLab 15.0. +The `omniauth-kerberos` gem will be removed in our next major release, GitLab 15.0. -Third-party tools that [integrate with GitLab by outputting a Dependency scanning security report](https://docs.gitlab.com/ee/development/integrations/secure.html#report) -as a pipeline job artifact are affected. You must ensure that all output reports adhere to the correct -schema with a minimum version of 14.0.0. Reports with a lower version or that fail to validate -against the declared schema version will not be processed, and vulnerability -findings will not display in MRs, pipelines, or Vulnerability Reports. +This gem has not been maintained and has very little usage. We therefore plan to remove support for this authentication method and recommend using the Kerberos [SPNEGO](https://en.wikipedia.org/wiki/SPNEGO) integration instead. You can follow the [upgrade instructions](https://docs.gitlab.com/ee/integration/kerberos.html#upgrading-from-password-based-to-ticket-based-kerberos-sign-ins) to upgrade from the `omniauth-kerberos` integration to the supported one. -To help with the transition, from GitLab 14.10, non-compliant reports will cause a -[warning to be displayed](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) -in the Vulnerability Report. +Note that we are not deprecating the Kerberos SPNEGO integration, only the old password-based Kerberos integration.
-
+
-### Enforced validation of security report schemas +### Optional enforcement of PAT expiration -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/351962). +
-[Security report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) -versions earlier than 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation -against the schema version declared in the report will also no longer be supported in GitLab 15.0. +The feature to disable enforcement of PAT expiration is unusual from a security perspective. +We have become concerned that this unusual feature could create unexpected behavior for users. +Unexpected behavior in a security feature is inherently dangerous, so we have decided to remove this feature. -Security tools that [integrate with GitLab by outputting security reports](https://docs.gitlab.com/ee/development/integrations/secure.html#report) -as pipeline job artifacts are affected. You must ensure that all output reports adhere to the correct -schema with a minimum version of 14.0.0. Reports with a lower version or that fail to validate -against the declared schema version will not be processed, and vulnerability -findings will not display in MRs, pipelines, or Vulnerability Reports. +
-To help with the transition, from GitLab 14.10, non-compliant reports will display a -[warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) -in the Vulnerability Report. +
+### Optional enforcement of SSH expiration + +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/351963).
-
+The feature to disable enforcement of SSH expiration is unusual from a security perspective. +We have become concerned that this unusual feature could create unexpected behavior for users. +Unexpected behavior in a security feature is inherently dangerous, so we have decided to remove this feature. -### Godep support in License Compliance +
-Planned removal: GitLab 15.0 +
-The Godep dependency manager for Golang was deprecated in 2020 by Go and -has been replaced with Go modules. -To reduce our maintenance cost we are deprecating License Compliance for Godep projects as of 14.7 -and will remove it in GitLab 15.0 +### Out-of-the-box SAST support for Java 8 +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352549).
-
+The [GitLab SAST SpotBugs analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) scans [Java, Scala, Groovy, and Kotlin code](https://docs.gitlab.com/ee/user/application_security/sast/#supported-languages-and-frameworks) for security vulnerabilities. +For technical reasons, the analyzer must first compile the code before scanning. +Unless you use the [pre-compilation strategy](https://docs.gitlab.com/ee/user/application_security/sast/#pre-compilation), the analyzer attempts to automatically compile your project's code. -### Logging in GitLab +In GitLab versions prior to 15.0, the analyzer image includes Java 8 and Java 11 runtimes to facilitate compilation. -Planned removal: GitLab 15.0 +In GitLab 15.0, we will: -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +- Remove Java 8 from the analyzer image to reduce the size of the image. +- Add Java 17 to the analyzer image to make it easier to compile with Java 17. -The logging features in GitLab allow users to install the ELK stack (Elasticsearch, Logstash, and Kibana) to aggregate and manage application logs. Users can search for relevant logs in GitLab. However, since deprecating certificate-based integration with Kubernetes clusters and GitLab Managed Apps, we don't have a recommended solution for logging within GitLab. For more information, you can follow the issue for [integrating Opstrace with GitLab](https://gitlab.com/groups/gitlab-org/-/epics/6976). +If you rely on Java 8 being present in the analyzer environment, you must take action as detailed in the [deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352549#breaking-change).
-
- -### Monitor performance metrics through Prometheus +
-Planned removal: GitLab 16.0 +### Outdated indices of Advanced Search migrations -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 14.10 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/359133). +
-By displaying data stored in a Prometheus instance, GitLab allows users to view performance metrics. GitLab also displays visualizations of these metrics in dashboards. The user can connect to a previously-configured external Prometheus instance, or set up Prometheus as a GitLab Managed App. -However, since certificate-based integration with Kubernetes clusters is deprecated in GitLab, the metrics functionality in GitLab that relies on Prometheus is also deprecated. This includes the metrics visualizations in dashboards. GitLab is working to develop a single user experience based on [Opstrace](https://about.gitlab.com/press/releases/2021-12-14-gitlab-acquires-opstrace-to-expand-its-devops-platform-with-open-source-observability-solution.html). An [issue exists](https://gitlab.com/groups/gitlab-org/-/epics/6976) for you to follow work on the Opstrace integration. +As Advanced Search migrations usually require support multiple code paths for a long period of time, it’s important to clean those up when we safely can. We use GitLab major version upgrades as a safe time to remove backward compatibility for indices that have not been fully migrated. See the [upgrade documentation](https://docs.gitlab.com/ee/update/index.html#upgrading-to-a-new-major-version) for details.
-
+
### Pseudonymizer -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 14.7 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/219952). +
The Pseudonymizer feature is generally unused, can cause production issues with large databases, @@ -3042,425 +3289,478 @@ It is now considered deprecated, and will be removed in GitLab 15.0.
-
- -### SAST schemas below 14.0.0 - -Planned removal: GitLab 15.0 +
-[SAST report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) -versions earlier than 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation -against the schema version declared in the report will also no longer be supported as of GitLab 15.0. +### Querying Usage Trends via the `instanceStatisticsMeasurements` GraphQL node -Third-party tools that [integrate with GitLab by outputting a SAST security report](https://docs.gitlab.com/ee/development/integrations/secure.html#report) -as a pipeline job artifact are affected. You must ensure that all output reports adhere to the correct -schema with a minimum version of 14.0.0. Reports with a lower version or that fail to validate -against the declared schema version will not be processed, and vulnerability -findings will not display in MRs, pipelines, or Vulnerability Reports. +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/332323). +
-To help with the transition, from GitLab 14.10, non-compliant reports will display a -[warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) -in the Vulnerability Report. +The `instanceStatisticsMeasurements` GraphQL node has been renamed to `usageTrendsMeasurements` in 13.10 and the old field name has been marked as deprecated. To fix the existing GraphQL queries, replace `instanceStatisticsMeasurements` with `usageTrendsMeasurements`.
-
+
-### Secret detection schemas below 14.0.0 +### Request profiling -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352488). +
-[Secret detection report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) -versions earlier than 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation -against the schema version declared in the report will also no longer be supported as of GitLab 15.0. +[Request profiling](https://docs.gitlab.com/ee/administration/monitoring/performance/index.html) is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. -Third-party tools that [integrate with GitLab by outputting a Secret detection security report](https://docs.gitlab.com/ee/development/integrations/secure.html#report) -as a pipeline job artifact are affected. You must ensure that all output reports adhere to the correct -schema with a minimum version of 14.0.0. Reports with a lower version or that fail to validate -against the declared schema version will not be processed, and vulnerability -findings will not display in MRs, pipelines, or Vulnerability Reports. +We're working on [consolidating our profiling tools](https://gitlab.com/groups/gitlab-org/-/epics/7327) and making them more easily accessible. +We [evaluated](https://gitlab.com/gitlab-org/gitlab/-/issues/350152) the use of this feature and we found that it is not widely used. +It also depends on a few third-party gems that are not actively maintained anymore, have not been updated for the latest version of Ruby, or crash frequently when profiling heavy page loads. -To help with the transition, from GitLab 14.10, non-compliant reports will display a -[warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) -in the Vulnerability Report. +For more information, check the [summary section of the deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352488#deprecation-summary).
-
+
-### Sidekiq metrics and health checks configuration +### Required pipeline configurations in Premium tier -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +The [required pipeline configuration](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#required-pipeline-configuration) feature is deprecated in GitLab 14.8 for Premium customers and is scheduled for removal in GitLab 15.0. This feature is not deprecated for GitLab Ultimate customers. -Exporting Sidekiq metrics and health checks using a single process and port is deprecated. -Support will be removed in 15.0. +This change to move the feature to GitLab's Ultimate tier is intended to help our features better align with our [pricing philosophy](https://about.gitlab.com/company/pricing/#three-tiers) as we see demand for this feature originating primarily from executives. -We have updated Sidekiq to export [metrics and health checks from two separate processes](https://gitlab.com/groups/gitlab-org/-/epics/6409) -to improve stability and availability and prevent data loss in edge cases. -As those are two separate servers, a configuration change will be required in 15.0 -to explicitly set separate ports for metrics and health-checks. -The newly introduced settings for `sidekiq['health_checks_*']` -should always be set in `gitlab.rb`. -For more information, check the documentation for [configuring Sidekiq](https://docs.gitlab.com/ee/administration/sidekiq.html). +This change will also help GitLab remain consistent in its tiering strategy with the other related Ultimate-tier features of: +[Security policies](https://docs.gitlab.com/ee/user/application_security/policies/) and [compliance framework pipelines](https://docs.gitlab.com/ee/user/project/settings/index.html#compliance-pipeline-configuration). -These changes also require updates in either Prometheus to scrape the new endpoint or k8s health-checks to target the new -health-check port to work properly, otherwise either metrics or health-checks will disappear. +
-For the deprecation period those settings are optional -and GitLab will default the Sidekiq health-checks port to the same port as `sidekiq_exporter` -and only run one server (not changing the current behaviour). -Only if they are both set and a different port is provided, a separate metrics server will spin up -to serve the Sidekiq metrics, similar to the way Sidekiq will behave in 15.0. +
+ +### Retire-JS Dependency Scanning tool +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/350510).
-
+As of 14.8 the retire.js job is being deprecated from Dependency Scanning. It will continue to be included in our CI/CD template while deprecated. We are removing retire.js from Dependency Scanning on May 22, 2022 in GitLab 15.0. JavaScript scanning functionality will not be affected as it is still being covered by Gemnasium. -### Static Site Editor +If you have explicitly excluded retire.js using DS_EXCLUDED_ANALYZERS you will need to clean up (remove the reference) in 15.0. If you have customized your pipeline's Dependency Scanning configuration related to the `retire-js-dependency_scanning` job you will want to switch to gemnasium-dependency_scanning before the removal in 15.0, to prevent your pipeline from failing. If you have not used the DS_EXCLUDED_ANALYZERS to reference retire.js, or customized your template specifically for retire.js, you will not need to take action. -Planned removal: GitLab 15.0 +
-The Static Site Editor will no longer be available starting in GitLab 15.0. Improvements to the Markdown editing experience across GitLab will deliver smiliar benefit but with a wider reach. Incoming requests to the Static Site Editor will be redirected to the [Web IDE](https://docs.gitlab.com/ee/user/project/web_ide/index.html). +
-Current users of the Static Site Editor can view the [documentation](https://docs.gitlab.com/ee/user/project/web_ide/index.html) for more information, including how to remove the configuration files from existing projects. +### SAST schemas below 14.0.0 +
+- Announced in: GitLab 14.7
-
+[SAST report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) +versions earlier than 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation +against the schema version declared in the report will also no longer be supported as of GitLab 15.0. -### Tracing in GitLab +Third-party tools that [integrate with GitLab by outputting a SAST security report](https://docs.gitlab.com/ee/development/integrations/secure.html#report) +as a pipeline job artifact are affected. You must ensure that all output reports adhere to the correct +schema with a minimum version of 14.0.0. Reports with a lower version or that fail to validate +against the declared schema version will not be processed, and vulnerability +findings will not display in MRs, pipelines, or Vulnerability Reports. -Planned removal: GitLab 15.0 +To help with the transition, from GitLab 14.10, non-compliant reports will display a +[warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) +in the Vulnerability Report. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-Tracing in GitLab is an integration with Jaeger, an open-source end-to-end distributed tracing system. GitLab users can navigate to their Jaeger instance to gain insight into the performance of a deployed application, tracking each function or microservice that handles a given request. Tracing in GitLab is deprecated in GitLab 14.7, and scheduled for removal in 15.0. To track work on a possible replacement, see the issue for [Opstrace integration with GitLab](https://gitlab.com/groups/gitlab-org/-/epics/6976). +
-
+### SAST support for .NET 2.1 -
+
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352553). +
-### `artifacts:reports:cobertura` keyword +The GitLab SAST Security Code Scan analyzer scans .NET code for security vulnerabilities. +For technical reasons, the analyzer must first build the code to scan it. -Planned removal: GitLab 15.0 +In GitLab versions prior to 15.0, the default analyzer image (version 2) includes support for: -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +- .NET 2.1 +- .NET 3.0 and .NET Core 3.0 +- .NET Core 3.1 +- .NET 5.0 -Currently, test coverage visualizations in GitLab only support Cobertura reports. Starting 15.0, the -`artifacts:reports:cobertura` keyword will be replaced by -[`artifacts:reports:coverage_report`](https://gitlab.com/gitlab-org/gitlab/-/issues/344533). Cobertura will be the -only supported report file in 15.0, but this is the first step towards GitLab supporting other report types. +In GitLab 15.0, we will change the default major version for this analyzer from version 2 to version 3. This change: -
+- Adds [severity values for vulnerabilities](https://gitlab.com/gitlab-org/gitlab/-/issues/350408) along with [other new features and improvements](https://gitlab.com/gitlab-org/security-products/analyzers/security-code-scan/-/blob/master/CHANGELOG.md). +- Removes .NET 2.1 support. +- Adds support for .NET 6.0, Visual Studio 2019, and Visual Studio 2022. -
+Version 3 was [announced in GitLab 14.6](https://about.gitlab.com/releases/2021/12/22/gitlab-14-6-released/#sast-support-for-net-6) and made available as an optional upgrade. -### merged_by API field +If you rely on .NET 2.1 support being present in the analyzer image by default, you must take action as detailed in the [deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352553#breaking-change). -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The `merged_by` field in the [merge request API](https://docs.gitlab.com/ee/api/merge_requests.html#list-merge-requests) has been deprecated in favor of the `merge_user` field which more correctly identifies who merged a merge request when performing actions (merge when pipeline succeeds, add to merge train) other than a simple merge. API users are encouraged to use the new `merge_user` field instead. The `merged_by` field will be removed in v5 of the GitLab REST API. +### Secret Detection configuration variables deprecated +
+- Announced in: GitLab 14.8 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352565).
-
- -
-## Announced in 14.6 +To make it simpler and more reliable to [customize GitLab Secret Detection](https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings), we're deprecating some of the variables that you could previously set in your CI/CD configuration. -
+The following variables currently allow you to customize the options for historical scanning, but interact poorly with the [GitLab-managed CI/CD template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml) and are now deprecated: -### CI/CD job name length limit +- `SECRET_DETECTION_COMMIT_FROM` +- `SECRET_DETECTION_COMMIT_TO` +- `SECRET_DETECTION_COMMITS` +- `SECRET_DETECTION_COMMITS_FILE` -Planned removal: GitLab 15.0 +The `SECRET_DETECTION_ENTROPY_LEVEL` previously allowed you to configure rules that only considered the entropy level of strings in your codebase, and is now deprecated. +This type of entropy-only rule created an unacceptable number of incorrect results (false positives) and is no longer supported. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +In GitLab 15.0, we'll update the Secret Detection [analyzer](https://docs.gitlab.com/ee/user/application_security/terminology/#analyzer) to ignore these deprecated options. +You'll still be able to configure historical scanning of your commit history by setting the [`SECRET_DETECTION_HISTORIC_SCAN` CI/CD variable](https://docs.gitlab.com/ee/user/application_security/secret_detection/#available-cicd-variables). -In GitLab 15.0 we are going to limit the number of characters in CI/CD job names to 255. Any pipeline with job names that exceed the 255 character limit will stop working after the 15.0 release. +For further details, see [the deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/352565).
-
+
-### Legacy approval status names from License Compliance API +### Secret detection schemas below 14.0.0 -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 14.7 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +[Secret detection report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/releases) +versions earlier than 14.0.0 will no longer be supported in GitLab 15.0. Reports that do not pass validation +against the schema version declared in the report will also no longer be supported as of GitLab 15.0. -We deprecated legacy names for approval status of license policy (blacklisted, approved) in the `managed_licenses` API but they are still used in our API queries and responses. They will be removed in 15.0. +Third-party tools that [integrate with GitLab by outputting a Secret detection security report](https://docs.gitlab.com/ee/development/integrations/secure.html#report) +as a pipeline job artifact are affected. You must ensure that all output reports adhere to the correct +schema with a minimum version of 14.0.0. Reports with a lower version or that fail to validate +against the declared schema version will not be processed, and vulnerability +findings will not display in MRs, pipelines, or Vulnerability Reports. -If you are using our License Compliance API you should stop using the `approved` and `blacklisted` query parameters, they are now `allowed` and `denied`. In 15.0 the responses will also stop using `approved` and `blacklisted` so you need to adjust any of your custom tools to use the old and new values so they do not break with the 15.0 release. +To help with the transition, from GitLab 14.10, non-compliant reports will display a +[warning](https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_672853791) +in the Vulnerability Report.
-
- -### `type` and `types` keyword in CI/CD configuration - -Planned removal: GitLab 15.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. - -The `type` and `types` CI/CD keywords will be removed in GitLab 15.0. Pipelines that use these keywords will stop working, so you must switch to `stage` and `stages`, which have the same behavior. +### Secure and Protect analyzer images published in new location +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352564).
-
+GitLab uses various [analyzers](https://docs.gitlab.com/ee/user/application_security/terminology/#analyzer) to [scan for security vulnerabilities](https://docs.gitlab.com/ee/user/application_security/). +Each analyzer is distributed as a container image. -### apiFuzzingCiConfigurationCreate GraphQL mutation +Starting in GitLab 14.8, new versions of GitLab Secure and Protect analyzers are published to a new registry location under `registry.gitlab.com/security-products`. -Planned removal: GitLab 15.0 +We will update the default value of [GitLab-managed CI/CD templates](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates/Security) to reflect this change: -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +- For all analyzers except Container Scanning, we will update the variable `SECURE_ANALYZERS_PREFIX` to the new image registry location. +- For Container Scanning, the default image address is already updated. There is no `SECURE_ANALYZERS_PREFIX` variable for Container Scanning. -The API Fuzzing configuration snippet is now being generated client-side and does not require an -API request anymore. We are therefore deprecating the `apiFuzzingCiConfigurationCreate` mutation -which isn't being used in GitLab anymore. +In a future release, we will stop publishing images to `registry.gitlab.com/gitlab-org/security-products/analyzers`. +Once this happens, you must take action if you manually pull images and push them into a separate registry. This is commonly the case for [offline deployments](https://docs.gitlab.com/ee/user/application_security/offline_deployments/index.html). +Otherwise, you won't receive further updates. + +See the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352564) for more details.
-
+
-### bundler-audit Dependency Scanning tool +### Secure and Protect analyzer major version update + +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/350936). +
-Planned removal: GitLab 15.0 +The Secure and Protect stages will be bumping the major versions of their analyzers in tandem with the GitLab 15.0 release. This major bump will enable a clear delineation for analyzers, between: -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +- Those released prior to May 22, 2022, which generate reports that _are not_ subject to stringent schema validation. +- Those released after May 22, 2022, which generate reports that _are_ subject to stringent schema validation. -As of 14.6 bundler-audit is being deprecated from Dependency Scanning. It will continue to be in our CI/CD template while deprecated. We are removing bundler-audit from Dependency Scanning on May 22, 2022 in 15.0. After this removal Ruby scanning functionality will not be affected as it is still being covered by Gemnasium. +If you are not using the default inclusion templates, or have pinned your analyzer versions you will need to update your CI/CD job definition to either remove the pinned version or to update the latest major version. +Users of GitLab 12.0-14.10 will continue to experience analyzer updates as normal until the release of GitLab 15.0, following which all newly fixed bugs and newly released features in the new major versions of the analyzers will not be available in the deprecated versions because we do not backport bugs and new features as per our [maintenance policy](https://docs.gitlab.com/ee/policy/maintenance.html). As required security patches will be backported within the latest 3 minor releases. +Specifically, the following are being deprecated and will no longer be updated after 15.0 GitLab release: -If you have explicitly excluded bundler-audit using DS_EXCLUDED_ANALYZERS you will need to clean up (remove the reference) in 15.0. If you have customized your pipeline's Dependency Scanning configuration, for example to edit the `bundler-audit-dependency_scanning` job, you will want to switch to gemnasium-dependency_scanning before removal in 15.0, to prevent your pipeline from failing. If you have not used the DS_EXCLUDED_ANALYZERS to reference bundler-audit, or customized your template specifically for bundler-audit, you will not need to take action. +- API Security: version 1 +- Container Scanning: version 4 +- Coverage-guided fuzz testing: version 2 +- Dependency Scanning: version 2 +- Dynamic Application Security Testing (DAST): version 2 +- Infrastructure as Code (IaC) Scanning: version 1 +- License Scanning: version 3 +- Secret Detection: version 3 +- Static Application Security Testing (SAST): version 2 of [all analyzers](https://docs.gitlab.com/ee/user/application_security/sast/#supported-languages-and-frameworks), except `gosec` which is currently at version 3 + - `bandit`: version 2 + - `brakeman`: version 2 + - `eslint`: version 2 + - `flawfinder`: version 2 + - `gosec`: version 3 + - `kubesec`: version 2 + - `mobsf`: version 2 + - `nodejs-scan`: version 2 + - `phpcs-security-audit`: version 2 + - `pmd-apex`: version 2 + - `security-code-scan`: version 2 + - `semgrep`: version 2 + - `sobelow`: version 2 + - `spotbugs`: version 2 -
-
+
-## Announced in 14.5 - -
- -### Changing an instance (shared) runner to a project (specific) runner +### Sidekiq metrics and health checks configuration -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 14.7 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/347509). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +Exporting Sidekiq metrics and health checks using a single process and port is deprecated. +Support will be removed in 15.0. -In GitLab 15.0, you can no longer change an instance (shared) runner to a project (specific) runner. +We have updated Sidekiq to export [metrics and health checks from two separate processes](https://gitlab.com/groups/gitlab-org/-/epics/6409) +to improve stability and availability and prevent data loss in edge cases. +As those are two separate servers, a configuration change will be required in 15.0 +to explicitly set separate ports for metrics and health-checks. +The newly introduced settings for `sidekiq['health_checks_*']` +should always be set in `gitlab.rb`. +For more information, check the documentation for [configuring Sidekiq](https://docs.gitlab.com/ee/administration/sidekiq/index.html). -Users often accidentally change instance runners to project runners, and they're unable to change them back. GitLab does not allow you to change a project runner to a shared runner because of the security implications. A runner meant for one project could be set to run jobs for an entire instance. +These changes also require updates in either Prometheus to scrape the new endpoint or k8s health-checks to target the new +health-check port to work properly, otherwise either metrics or health-checks will disappear. -Administrators who need to add runners for multiple projects can register a runner for one project, then go to the Admin view and choose additional projects. +For the deprecation period those settings are optional +and GitLab will default the Sidekiq health-checks port to the same port as `sidekiq_exporter` +and only run one server (not changing the current behaviour). +Only if they are both set and a different port is provided, a separate metrics server will spin up +to serve the Sidekiq metrics, similar to the way Sidekiq will behave in 15.0.
-
- -### GraphQL API Runner status will not return `paused` - -Planned removal: GitLab 16.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +### Static Site Editor -The GitLab Runner GraphQL API endpoints will not return `paused` or `active` as a status in GitLab 16.0. -In a future v5 of the REST API, the endpoints for GitLab Runner will also not return `paused` or `active`. +
+- Announced in: GitLab 14.7 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/347137). +
-A runner's status will only relate to runner contact status, such as: -`online`, `offline`, or `not_connected`. Status `paused` or `active` will no longer appear. +The Static Site Editor will no longer be available starting in GitLab 15.0. Improvements to the Markdown editing experience across GitLab will deliver smiliar benefit but with a wider reach. Incoming requests to the Static Site Editor will be redirected to the [Web IDE](https://docs.gitlab.com/ee/user/project/web_ide/index.html). -When checking if a runner is `paused`, API users are advised to check the boolean attribute -`paused` to be `true` instead. When checking if a runner is `active`, check if `paused` is `false`. +Current users of the Static Site Editor can view the [documentation](https://docs.gitlab.com/ee/user/project/web_ide/index.html) for more information, including how to remove the configuration files from existing projects.
-
+
-### Known host required for GitLab Runner SSH executor - -Planned removal: GitLab 15.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +### Support for SLES 12 SP2 -In [GitLab 14.3](https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/3074), we added a configuration setting in the GitLab Runner `config.toml` file. This setting, [`[runners.ssh.disable_strict_host_key_checking]`](https://docs.gitlab.com/runner/executors/ssh.html#security), controls whether or not to use strict host key checking with the SSH executor. +
+- Announced in: GitLab 14.5 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +
-In GitLab 15.0 and later, the default value for this configuration option will change from `true` to `false`. This means that strict host key checking will be enforced when using the GitLab Runner SSH executor. +Long term service and support (LTSS) for SUSE Linux Enterprise Server (SLES) 12 SP2 [ended on March 31, 2021](https://www.suse.com/lifecycle/). The CA certificates on SP2 include the expired DST root certificate, and it's not getting new CA certificate package updates. We have implemented some [workarounds](https://gitlab.com/gitlab-org/gitlab-omnibus-builder/-/merge_requests/191), but we will not be able to continue to keep the build running properly.
-
+
-### Package pipelines in API payload is paginated +### Support for gRPC-aware proxy deployed between Gitaly and rest of GitLab -Planned removal: GitLab 16.0 +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +Although not recommended or documented, it was possible to deploy a gRPC-aware proxy between Gitaly and +the rest of GitLab. For example, NGINX and Envoy. The ability to deploy a gRPC-aware proxy is +[deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/352517). If you currently use a gRPC-aware proxy for +Gitaly connections, you should change your proxy configuration to use TCP or TLS proxying (OSI layer 4) instead. -A request to the API for `/api/v4/projects/:id/packages` returns a paginated result of packages. Each package lists all of its pipelines in this response. This is a performance concern, as it's possible for a package to have hundreds or thousands of associated pipelines. +Gitaly Cluster became incompatible with gRPC-aware proxies in GitLab 13.12. Now all GitLab installations will be incompatible with +gRPC-aware proxies, even without Gitaly Cluster. -In milestone 16.0, we will remove the `pipelines` attribute from the API response. +By sending some of our internal RPC traffic through a custom protocol (instead of gRPC) we +increase throughput and reduce Go garbage collection latency. For more information, see +the [relevant epic](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/463).
-
+
-### SaaS certificate-based integration with Kubernetes +### Test coverage project CI/CD setting -Planned removal: GitLab 15.9 +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +To simplify setting a test coverage pattern, in GitLab 15.0 the +[project setting for test coverage parsing](https://docs.gitlab.com/ee/ci/pipelines/settings.html#add-test-coverage-results-using-project-settings-removed) +is being removed. -The certificate-based integration with Kubernetes will be [deprecated and removed](https://about.gitlab.com/blog/2021/11/15/deprecating-the-cert-based-kubernetes-integration/). As a GitLab SaaS customer, on new namespaces, you will no longer be able to integrate GitLab and your cluster using the certificate-based approach as of GitLab 15.0. The integration for current users will be enabled per namespace. +Instead, using the project’s `.gitlab-ci.yml`, provide a regular expression with the `coverage` keyword to set +testing coverage results in merge requests. -For a more robust, secure, forthcoming, and reliable integration with Kubernetes, we recommend you use the -[agent for Kubernetes](https://docs.gitlab.com/ee/user/clusters/agent/) to connect Kubernetes clusters with GitLab. [How do I migrate?](https://docs.gitlab.com/ee/user/infrastructure/clusters/migrate_to_gitlab_agent.html) +
-For updates and details about this deprecation, follow [this epic](https://gitlab.com/groups/gitlab-org/configure/-/epics/8). +
-GitLab self-managed customers can still use the feature [with a feature flag](https://docs.gitlab.com/ee/update/deprecations.html#self-managed-certificate-based-integration-with-kubernetes). +### Tracing in GitLab +
+- Announced in: GitLab 14.7 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/346540).
-
- -### Self-managed certificate-based integration with Kubernetes - -Planned removal: GitLab 17.0 +Tracing in GitLab is an integration with Jaeger, an open-source end-to-end distributed tracing system. GitLab users can navigate to their Jaeger instance to gain insight into the performance of a deployed application, tracking each function or microservice that handles a given request. Tracing in GitLab is deprecated in GitLab 14.7, and scheduled for removal in 15.0. To track work on a possible replacement, see the issue for [Opstrace integration with GitLab](https://gitlab.com/groups/gitlab-org/-/epics/6976). -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The certificate-based integration with Kubernetes [will be deprecated and removed](https://about.gitlab.com/blog/2021/11/15/deprecating-the-cert-based-kubernetes-integration/). +
-As a self-managed customer, we are introducing the [feature flag](../administration/feature_flags.md#enable-or-disable-the-feature) `certificate_based_clusters` in GitLab 15.0 so you can keep your certificate-based integration enabled. However, the feature flag will be disabled by default, so this change is a **breaking change**. +### Update to the Container Registry group-level API -In GitLab 17.0 we will remove both the feature and its related code. Until the final removal in 17.0, features built on this integration will continue to work, if you enable the feature flag. Until the feature is removed, GitLab will continue to fix security and critical issues as they arise. +
+- Announced in: GitLab 14.5 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/336912). +
-For a more robust, secure, forthcoming, and reliable integration with Kubernetes, we recommend you use the -[agent for Kubernetes](https://docs.gitlab.com/ee/user/clusters/agent/) to connect Kubernetes clusters with GitLab. [How do I migrate?](https://docs.gitlab.com/ee/user/infrastructure/clusters/migrate_to_gitlab_agent.html) +In milestone 15.0, support for the `tags` and `tags_count` parameters will be removed from the Container Registry API that [gets registry repositories from a group](../api/container_registry.md#within-a-group). -For updates and details about this deprecation, follow [this epic](https://gitlab.com/groups/gitlab-org/configure/-/epics/8). +The `GET /groups/:id/registry/repositories` endpoint will remain, but won't return any info about tags. To get the info about tags, you can use the existing `GET /registry/repositories/:id` endpoint, which will continue to support the `tags` and `tag_count` options as it does today. The latter must be called once per image repository.
-
+
-### Support for SLES 12 SP2 +### Value Stream Analytics filtering calculation change -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 14.5 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/343210). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +We are changing how the date filter works in Value Stream Analytics. Instead of filtering by the time that the issue or merge request was created, the date filter will filter by the end event time of the given stage. This will result in completely different figures after this change has rolled out. -Long term service and support (LTSS) for SUSE Linux Enterprise Server (SLES) 12 SP2 [ended on March 31, 2021](https://www.suse.com/lifecycle/). The CA certificates on SP2 include the expired DST root certificate, and it's not getting new CA certificate package updates. We have implemented some [workarounds](https://gitlab.com/gitlab-org/gitlab-omnibus-builder/-/merge_requests/191), but we will not be able to continue to keep the build running properly. +If you monitor Value Stream Analytics metrics and rely on the date filter, to avoid losing data, you must save the data prior to this change.
-
+
-### Update to the Container Registry group-level API +### Vulnerability Check -Planned removal: GitLab 15.0 +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +The vulnerability check feature is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. We encourage you to migrate to the new security approvals feature instead. You can do so by navigating to **Security & Compliance > Policies** and creating a new Scan Result Policy. -In milestone 15.0, support for the `tags` and `tags_count` parameters will be removed from the Container Registry API that [gets registry repositories from a group](../api/container_registry.md#within-a-group). +The new security approvals feature is similar to vulnerability check. For example, both can require approvals for MRs that contain security vulnerabilities. However, security approvals improve the previous experience in several ways: -The `GET /groups/:id/registry/repositories` endpoint will remain, but won't return any info about tags. To get the info about tags, you can use the existing `GET /registry/repositories/:id` endpoint, which will continue to support the `tags` and `tag_count` options as it does today. The latter must be called once per image repository. +- Users can choose who is allowed to edit security approval rules. An independent security or compliance team can therefore manage rules in a way that prevents development project maintainers from modifying the rules. +- Multiple rules can be created and chained together to allow for filtering on different severity thresholds for each scanner type. +- A two-step approval process can be enforced for any desired changes to security approval rules. +- A single set of security policies can be applied to multiple development projects to allow for ease in maintaining a single, centralized ruleset.
-
+
-### Value Stream Analytics filtering calculation change - -Planned removal: GitLab 15.0 +### `Versions` on base `PackageType` -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 14.5 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/327453). +
-We are changing how the date filter works in Value Stream Analytics. Instead of filtering by the time that the issue or merge request was created, the date filter will filter by the end event time of the given stage. This will result in completely different figures after this change has rolled out. +As part of the work to create a [Package Registry GraphQL API](https://gitlab.com/groups/gitlab-org/-/epics/6318), the Package group deprecated the `Version` type for the basic `PackageType` type and moved it to [`PackageDetailsType`](https://docs.gitlab.com/ee/api/graphql/reference/index.html#packagedetailstype). -If you monitor Value Stream Analytics metrics and rely on the date filter, to avoid losing data, you must save the data prior to this change. +In milestone 15.0, we will completely remove `Version` from `PackageType`.
-
+
-### `Versions` on base `PackageType` - -Planned removal: GitLab 15.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +### `artifacts:reports:cobertura` keyword -As part of the work to create a [Package Registry GraphQL API](https://gitlab.com/groups/gitlab-org/-/epics/6318), the Package group deprecated the `Version` type for the basic `PackageType` type and moved it to [`PackageDetailsType`](https://docs.gitlab.com/ee/api/graphql/reference/index.html#packagedetailstype). +
+- Announced in: GitLab 14.7 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/348980). +
-In milestone 15.0, we will completely remove `Version` from `PackageType`. +Currently, test coverage visualizations in GitLab only support Cobertura reports. Starting 15.0, the +`artifacts:reports:cobertura` keyword will be replaced by +[`artifacts:reports:coverage_report`](https://gitlab.com/gitlab-org/gitlab/-/issues/344533). Cobertura will be the +only supported report file in 15.0, but this is the first step towards GitLab supporting other report types.
-
+
### `defaultMergeCommitMessageWithDescription` GraphQL API field -Planned removal: GitLab 15.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 14.5 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/345451). +
The GraphQL API field `defaultMergeCommitMessageWithDescription` has been deprecated and will be removed in GitLab 15.0. For projects with a commit message template set, it will ignore the template.
-
+
### `dependency_proxy_for_private_groups` feature flag -Planned removal: GitLab 15.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 14.5 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/276777). +
We added a feature flag because [GitLab-#11582](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) changed how public groups use the Dependency Proxy. Prior to this change, you could use the Dependency Proxy without authentication. The change requires authentication to use the Dependency Proxy. @@ -3468,15 +3768,15 @@ In milestone 15.0, we will remove the feature flag entirely. Moving forward, you
-
+
### `pipelines` field from the `version` field -Planned removal: GitLab 15.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 14.5 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/342882). +
In GraphQL, there are two `pipelines` fields that you can use in a [`PackageDetailsType`](https://docs.gitlab.com/ee/api/graphql/reference/#packagedetailstype) to get the pipelines for package versions: @@ -3487,198 +3787,227 @@ To mitigate possible performance problems, we will remove the `versions` field's
-
+
-### `promote-db` command from `gitlab-ctl` +### `projectFingerprint` in `PipelineSecurityReportFinding` GraphQL + +
+- Announced in: GitLab 14.8 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +
+ +The `projectFingerprint` field in the [PipelineSecurityReportFinding](https://docs.gitlab.com/ee/api/graphql/reference/index.html#pipelinesecurityreportfinding) +GraphQL object is being deprecated. This field contains a "fingerprint" of security findings used to determine uniqueness. +The method for calculating fingerprints has changed, resulting in different values. Going forward, the new values will be +exposed in the UUID field. Data previously available in the projectFingerprint field will eventually be removed entirely. + +
-Planned removal: GitLab 15.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +### `promote-db` command from `gitlab-ctl` + +
+- Announced in: GitLab 14.5 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/345207). +
In GitLab 14.5, we introduced the command `gitlab-ctl promote` to promote any Geo secondary node to a primary during a failover. This command replaces `gitlab-ctl promote-db` which is used to promote database nodes in multi-node Geo secondary sites. `gitlab-ctl promote-db` will continue to function as-is and be available until GitLab 15.0. We recommend that Geo customers begin testing the new `gitlab-ctl promote` command in their staging environments and incorporating the new command in their failover procedures.
-
+
### `promote-to-primary-node` command from `gitlab-ctl` -Planned removal: GitLab 15.0 - -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 14.5 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/345207). +
In GitLab 14.5, we introduced the command `gitlab-ctl promote` to promote any Geo secondary node to a primary during a failover. This command replaces `gitlab-ctl promote-to-primary-node` which was only usable for single-node Geo sites. `gitlab-ctl promote-to-primary-node` will continue to function as-is and be available until GitLab 15.0. We recommend that Geo customers begin testing the new `gitlab-ctl promote` command in their staging environments and incorporating the new command in their failover procedures.
-
- -### openSUSE Leap 15.2 packages +
-Planned removal: GitLab 14.8 +### `type` and `types` keyword in CI/CD configuration -Distribution support and security updates for openSUSE Leap 15.2 are [ending December 2021](https://en.opensuse.org/Lifetime#openSUSE_Leap). +
+- Announced in: GitLab 14.6 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +
-Starting in 14.5 we are providing packages for openSUSE Leap 15.3, and will stop providing packages for openSUSE Leap 15.2 in the 14.8 milestone. +The `type` and `types` CI/CD keywords will be removed in GitLab 15.0. Pipelines that use these keywords will stop working, so you must switch to `stage` and `stages`, which have the same behavior.
-
- -
-## Announced in 14.3 +
-
+### apiFuzzingCiConfigurationCreate GraphQL mutation -### Audit events for repository push events +
+- Announced in: GitLab 14.6 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/333233). +
-Planned removal: GitLab 15.0 +The API Fuzzing configuration snippet is now being generated client-side and does not require an +API request anymore. We are therefore deprecating the `apiFuzzingCiConfigurationCreate` mutation +which isn't being used in GitLab anymore. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-Audit events for [repository events](https://docs.gitlab.com/ee/administration/audit_events.html#removed-events) are now deprecated and will be removed in GitLab 15.0. +
-These events have always been disabled by default and had to be manually enabled with a -feature flag. Enabling them can cause too many events to be generated which can -dramatically slow down GitLab instances. For this reason, they are being removed. +### bundler-audit Dependency Scanning tool +
+- Announced in: GitLab 14.6 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/289832).
-
- -### GitLab Serverless +As of 14.6 bundler-audit is being deprecated from Dependency Scanning. It will continue to be in our CI/CD template while deprecated. We are removing bundler-audit from Dependency Scanning on May 22, 2022 in 15.0. After this removal Ruby scanning functionality will not be affected as it is still being covered by Gemnasium. -Planned removal: GitLab 15.0 +If you have explicitly excluded bundler-audit using DS_EXCLUDED_ANALYZERS you will need to clean up (remove the reference) in 15.0. If you have customized your pipeline's Dependency Scanning configuration, for example to edit the `bundler-audit-dependency_scanning` job, you will want to switch to gemnasium-dependency_scanning before removal in 15.0, to prevent your pipeline from failing. If you have not used the DS_EXCLUDED_ANALYZERS to reference bundler-audit, or customized your template specifically for bundler-audit, you will not need to take action. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-GitLab Serverless is a feature set to support Knative-based serverless development with automatic deployments and monitoring. +
-We decided to remove the GitLab Serverless features as they never really resonated with our users. Besides, given the continuous development of Kubernetes and Knative, our current implementations do not even work with recent versions. +### htpasswd Authentication for the Container Registry +
+- Announced in: GitLab 14.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/).
-
- -### Legacy database configuration +The Container Registry supports [authentication](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs/configuration.md#auth) with `htpasswd`. It relies on an [Apache `htpasswd` file](https://httpd.apache.org/docs/2.4/programs/htpasswd.html), with passwords hashed using `bcrypt`. -Planned removal: GitLab 15.0 +Since it isn't used in the context of GitLab (the product), `htpasswd` authentication will be deprecated in GitLab 14.9 and removed in GitLab 15.0. -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The syntax of [GitLabs database](https://docs.gitlab.com/omnibus/settings/database.html) -configuration located in `database.yml` is changing and the legacy format is deprecated. The legacy format -supported using a single PostgreSQL adapter, whereas the new format is changing to support multiple databases. The `main:` database needs to be defined as a first configuration item. +
-This deprecation mainly impacts users compiling GitLab from source because Omnibus will handle this configuration automatically. +### user_email_lookup_limit API field +
+- Announced in: GitLab 14.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/).
-
+The `user_email_lookup_limit` [API field](https://docs.gitlab.com/ee/api/settings.html) is deprecated and will be removed in GitLab 15.0. Until GitLab 15.0, `user_email_lookup_limit` is aliased to `search_rate_limit` and existing workflows will continue to work. -### OmniAuth Kerberos gem +Any API calls attempting to change the rate limits for `user_email_lookup_limit` should use `search_rate_limit` instead. -Planned removal: GitLab 15.0 +
+
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
-The `omniauth-kerberos` gem will be removed in our next major release, GitLab 15.0. +## GitLab 14.10 -This gem has not been maintained and has very little usage. We therefore plan to remove support for this authentication method and recommend using the Kerberos [SPNEGO](https://en.wikipedia.org/wiki/SPNEGO) integration instead. You can follow the [upgrade instructions](https://docs.gitlab.com/ee/integration/kerberos.html#upgrading-from-password-based-to-ticket-based-kerberos-sign-ins) to upgrade from the `omniauth-kerberos` integration to the supported one. +
-Note that we are not deprecating the Kerberos SPNEGO integration, only the old password-based Kerberos integration. +### Permissions change for downloading Composer dependencies +
+- Announced in: GitLab 14.9 +- This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/).
-
- -
- -## Announced in 14.2 -
+The GitLab Composer repository can be used to push, search, fetch metadata about, and download PHP dependencies. All these actions require authentication, except for downloading dependencies. -### Release CLI distributed as a generic package +Downloading Composer dependencies without authentication is deprecated in GitLab 14.9, and will be removed in GitLab 15.0. Starting with GitLab 15.0, you must authenticate to download Composer dependencies. -Planned removal: GitLab 14.6 +
+
-The [release-cli](https://gitlab.com/gitlab-org/release-cli) will be released as a [generic package](https://gitlab.com/gitlab-org/release-cli/-/packages) starting in GitLab 14.2. We will continue to deploy it as a binary to S3 until GitLab 14.5 and stop distributing it in S3 in GitLab 14.6. +
-
+## GitLab 14.9 -
+
-### Rename Task Runner pod to Toolbox +### Configurable Gitaly `per_repository` election strategy -Planned removal: GitLab 14.5 +
+- Announced in: GitLab 14.8 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/gitlab/-/issues/352612). +
-The Task Runner pod is used to execute periodic housekeeping tasks within the GitLab application and is often confused with the GitLab Runner. Thus, [Task Runner will be renamed to Toolbox](https://gitlab.com/groups/gitlab-org/charts/-/epics/25). +Configuring the `per_repository` Gitaly election strategy is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/352612). +`per_repository` has been the only option since GitLab 14.0. -This will result in the rename of the sub-chart: `gitlab/task-runner` to `gitlab/toolbox`. Resulting pods will be named along the lines of `{{ .Release.Name }}-toolbox`, which will often be `gitlab-toolbox`. They will be locatable with the label `app=toolbox`. +This change is part of regular maintenance to keep our codebase clean.
-
- -## Announced in 14.0 +
-
+## GitLab 14.8 -### Changing merge request approvals with the `/approvals` API endpoint +
-Planned removal: GitLab 16.0 +### openSUSE Leap 15.2 packages -WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +
+- Announced in: GitLab 14.5 +- To discuss this change or learn more, see the [deprecation issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6427). +
-To change the approvals required for a merge request, you should no longer use the `/approvals` API endpoint, which was deprecated in GitLab 14.0. +Distribution support and security updates for openSUSE Leap 15.2 are [ending December 2021](https://en.opensuse.org/Lifetime#openSUSE_Leap). -Instead, use the [`/approval_rules` endpoint](https://docs.gitlab.com/ee/api/merge_request_approvals.html#merge-request-level-mr-approvals) to [create](https://docs.gitlab.com/ee/api/merge_request_approvals.html#create-merge-request-level-rule) or [update](https://docs.gitlab.com/ee/api/merge_request_approvals.html#update-merge-request-level-rule) the approval rules for a merge request. +Starting in 14.5 we are providing packages for openSUSE Leap 15.3, and will stop providing packages for openSUSE Leap 15.2 in the 14.8 milestone. +
-
- -### NFS for Git repository storage +
-Planned removal: GitLab 15.6 +## GitLab 14.6 -With the general availability of Gitaly Cluster ([introduced in GitLab 13.0](https://about.gitlab.com/releases/2020/05/22/gitlab-13-0-released/)), we have deprecated development (bugfixes, performance improvements, etc) for NFS for Git repository storage in GitLab 14.0. We will continue to provide technical support for NFS for Git repositories throughout 14.x, but we will remove all support for NFS on November 22, 2022. This was originally planned for May 22, 2022, but in an effort to allow continued maturity of Gitaly Cluster, we have chosen to extend our deprecation of support date. Please see our official [Statement of Support](https://about.gitlab.com/support/statement-of-support/#gitaly-and-nfs) for further information. +
-Gitaly Cluster offers tremendous benefits for our customers such as: +### Release CLI distributed as a generic package -- [Variable replication factors](https://docs.gitlab.com/ee/administration/gitaly/index.html#replication-factor). -- [Strong consistency](https://docs.gitlab.com/ee/administration/gitaly/index.html#strong-consistency). -- [Distributed read capabilities](https://docs.gitlab.com/ee/administration/gitaly/index.html#distributed-reads). +
+- Announced in: GitLab 14.2 +
-We encourage customers currently using NFS for Git repositories to plan their migration by reviewing our documentation on [migrating to Gitaly Cluster](https://docs.gitlab.com/ee/administration/gitaly/index.html#migrate-to-gitaly-cluster). +The [release-cli](https://gitlab.com/gitlab-org/release-cli) will be released as a [generic package](https://gitlab.com/gitlab-org/release-cli/-/packages) starting in GitLab 14.2. We will continue to deploy it as a binary to S3 until GitLab 14.5 and stop distributing it in S3 in GitLab 14.6. +
-
+
-### OAuth implicit grant +## GitLab 14.5 -Planned removal: GitLab 15.0 +
-WARNING: -This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). -Review the details carefully before upgrading. +### Rename Task Runner pod to Toolbox -The OAuth implicit grant authorization flow will be removed in our next major release, GitLab 15.0. Any applications that use OAuth implicit grant should switch to alternative [supported OAuth flows](https://docs.gitlab.com/ee/api/oauth2.html). +
+- Announced in: GitLab 14.2 +
+ +The Task Runner pod is used to execute periodic housekeeping tasks within the GitLab application and is often confused with the GitLab Runner. Thus, [Task Runner will be renamed to Toolbox](https://gitlab.com/groups/gitlab-org/charts/-/epics/25). + +This will result in the rename of the sub-chart: `gitlab/task-runner` to `gitlab/toolbox`. Resulting pods will be named along the lines of `{{ .Release.Name }}-toolbox`, which will often be `gitlab-toolbox`. They will be locatable with the label `app=toolbox`.
+ +DISCLAIMER: +This page contains information related to upcoming products, features, and functionality. +It is important to note that the information presented is for informational purposes only. +Please do not rely on this information for purchasing or planning purposes. +As with all projects, the items mentioned on this page are subject to change or delay. +The development, release, and timing of any products, features, or functionality remain at the +sole discretion of GitLab Inc. diff --git a/doc/update/index.md b/doc/update/index.md index d08368663da..f1b8bb17f14 100644 --- a/doc/update/index.md +++ b/doc/update/index.md @@ -23,7 +23,7 @@ has additional information about upgrading, including: ## Upgrade based on installation method Depending on the installation method and your GitLab version, there are multiple -official ways to update GitLab: +official ways to upgrade GitLab: - [Linux packages (Omnibus GitLab)](#linux-packages-omnibus-gitlab) - [Source installations](#installation-from-source) @@ -33,11 +33,11 @@ official ways to update GitLab: ### Linux packages (Omnibus GitLab) The [package upgrade guide](package/index.md) -contains the steps needed to update a package installed by official GitLab +contains the steps needed to upgrade a package installed by official GitLab repositories. There are also instructions when you want to -[update to a specific version](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). +[upgrade to a specific version](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories). ### Installation from source @@ -63,7 +63,7 @@ editions, and they are based on the Omnibus package. See how to ### Installation using Helm GitLab can be deployed into a Kubernetes cluster using Helm. -Instructions on how to update a cloud-native deployment are in +Instructions on how to upgrade a cloud-native deployment are in [a separate document](https://docs.gitlab.com/charts/installation/upgrade.html). Use the [version mapping](https://docs.gitlab.com/charts/installation/version_mappings.html) @@ -76,7 +76,7 @@ See the guide to [plan your GitLab upgrade](plan_your_upgrade.md). ## Check for background migrations before upgrading Certain releases may require different migrations to be -finished before you update to the newer version. +finished before you upgrade to the newer version. For more information, see [background migrations](background_migrations.md). @@ -103,15 +103,15 @@ To address the above two scenarios, it is advised to do the following prior to u 1. Wait until all jobs are finished. 1. Upgrade GitLab. -1. [Update GitLab Runner](https://docs.gitlab.com/runner/install/index.html) to the same version +1. [Upgrade GitLab Runner](https://docs.gitlab.com/runner/install/index.html) to the same version as your GitLab version. Both versions [should be the same](https://docs.gitlab.com/runner/#gitlab-runner-versions). 1. Unpause your runners and unblock new jobs from starting by reverting the previous `/etc/gitlab/gitlab.rb` change. -## Checking for pending Advanced Search migrations **(PREMIUM SELF)** +## Checking for pending advanced search migrations **(PREMIUM SELF)** This section is only applicable if you have enabled the [Elasticsearch integration](../integration/advanced_search/elasticsearch.md) **(PREMIUM SELF)**. -Major releases require all [Advanced Search migrations](../integration/advanced_search/elasticsearch.md#advanced-search-migrations) +Major releases require all [advanced search migrations](../integration/advanced_search/elasticsearch.md#advanced-search-migrations) to be finished from the most recent minor release in your current version before the major version upgrade. You can find pending migrations by running the following command: @@ -129,16 +129,16 @@ cd /home/git/gitlab sudo -u git -H bundle exec rake gitlab:elastic:list_pending_migrations ``` -### What do you do if your Advanced Search migrations are stuck? +### What do you do if your advanced search migrations are stuck? -In GitLab 15.0, an Advanced Search migration named `DeleteOrphanedCommit` can be permanently stuck +In GitLab 15.0, an advanced search migration named `DeleteOrphanedCommit` can be permanently stuck in a pending state across upgrades. This issue [is corrected in GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89539). -If you are a self-managed customer who uses GitLab 15.0 with Advanced Search, you will experience performance degradation. +If you are a self-managed customer who uses GitLab 15.0 with advanced search, you will experience performance degradation. To clean up the migration, upgrade to 15.1 or later. -For other Advanced Search migrations stuck in pending, see [how to retry a halted migration](../integration/advanced_search/elasticsearch.md#retry-a-halted-migration). +For other advanced search migrations stuck in pending, see [how to retry a halted migration](../integration/advanced_search/elasticsearch.md#retry-a-halted-migration). ### What do you do for the error `Elasticsearch version not compatible` @@ -151,7 +151,7 @@ Read how to [upgrade without downtime](zero_downtime.md). ## Upgrading to a new major version Upgrading the *major* version requires more attention. -Backward-incompatible changes and migrations are reserved for major versions. +Backward-incompatible changes are reserved for major versions. Follow the directions carefully as we cannot guarantee that upgrading between major versions is seamless. @@ -166,7 +166,7 @@ It's also important to ensure that any [background migrations have been fully co before upgrading to a new major version. If you have enabled the [Elasticsearch integration](../integration/advanced_search/elasticsearch.md) **(PREMIUM SELF)**, then -[ensure all Advanced Search migrations are completed](#checking-for-pending-advanced-search-migrations) in the last minor version in +[ensure all advanced search migrations are completed](#checking-for-pending-advanced-search-migrations) in the last minor version in your current version before proceeding with the major version upgrade. @@ -196,11 +196,11 @@ accordingly, while also consulting the NOTE: When not explicitly specified, upgrade GitLab to the latest available patch -release rather than the first patch release, for example `13.8.8` instead of `13.8.0`. -This includes versions you must stop at on the upgrade path as there may +release of the `major`.`minor` release rather than the first patch release, for example `13.8.8` instead of `13.8.0`. +This includes `major`.`minor` versions you must stop at on the upgrade path as there may be fixes for issues relating to the upgrade process. Specifically around a [major version](#upgrading-to-a-new-major-version), -crucial database schema and migration patches are included in the latest patch releases. +crucial database schema and migration patches may be included in the latest patch releases. ## Upgrading between editions @@ -218,16 +218,16 @@ The following guides are for subscribers of the Enterprise Edition only. If you wish to upgrade your GitLab installation from Community to Enterprise Edition, follow the guides below based on the installation method: -- [Source CE to EE update guides](upgrading_from_ce_to_ee.md) - The steps are very similar +- [Source CE to EE upgrade guides](upgrading_from_ce_to_ee.md) - The steps are very similar to a version upgrade: stop the server, get the code, update configuration files for the new functionality, install libraries and do migrations, update the init script, start the application and check its status. -- [Omnibus CE to EE](package/convert_to_ee.md) - Follow this guide to update your Omnibus +- [Omnibus CE to EE](package/convert_to_ee.md) - Follow this guide to upgrade your Omnibus GitLab Community Edition to the Enterprise Edition. - [Docker CE to EE](../install/docker.md#convert-community-edition-to-enterprise-edition) - - Follow this guide to update your GitLab Community Edition container to an Enterprise Edition container. + Follow this guide to upgrade your GitLab Community Edition container to an Enterprise Edition container. - [Helm chart (Kubernetes) CE to EE](https://docs.gitlab.com/charts/installation/deployment.html#convert-community-edition-to-enterprise-edition) - - Follow this guide to update your GitLab Community Edition Helm deployment to Enterprise Edition. + Follow this guide to upgrade your GitLab Community Edition Helm deployment to Enterprise Edition. ### Enterprise to Community Edition @@ -237,7 +237,7 @@ possible. ## Version-specific upgrading instructions -Each month, major, minor, or patch releases of GitLab are published along with a +Each month, major or minor as well as possibly patch releases of GitLab are published along with a [release post](https://about.gitlab.com/releases/categories/releases/). You should read the release posts for all versions you're passing over. At the end of major and minor release posts, there are three sections to look for specifically: @@ -264,10 +264,60 @@ NOTE: Specific information that follow related to Ruby and Git versions do not apply to [Omnibus installations](https://docs.gitlab.com/omnibus/) and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with appropriate Ruby and Git versions and are not using system binaries for Ruby and Git. There is no need to install Ruby or Git when utilizing these two approaches. +### 16.0.0 + +- Sidekiq jobs are only routed to `default` and `mailers` queues by default, and as a result, + every Sidekiq process also listens to those queues to ensure all jobs are processed across + all queues. This behavior does not apply if you have configured the [routing rules](../administration/sidekiq/processing_specific_job_classes.md#routing-rules). + +### 15.11.1 + +- Many [project importers](../user/project/import/index.md) and [group importers](../user/group/import/index.md) now + require the Maintainer role instead of only requiring the Developer role. For more information, see the documentation + for any importers you use. + +### 15.11.0 + +- Upgrades to GitLab 15.11 directly from GitLab versions 15.5.0 and earlier on self-managed installs will fail due to a missing migration until the fix for [issue 408304](https://gitlab.com/gitlab-org/gitlab/-/issues/408304) is released in version 15.11.3. Affected users wanting to upgrade to 15.11 can either: + - Perform an intermediate upgrade to any version between 15.5 and 15.10 before upgrading to 15.11, or + - Target version 15.11.3 or later. + +### 15.10.5 + +- Many [project importers](../user/project/import/index.md) and [group importers](../user/group/import/index.md) now + require the Maintainer role instead of only requiring the Developer role. For more information, see the documentation + for any importers you use. + +### 15.10.0 + +- Gitaly configuration changes significantly in Omnibus GitLab 16.0. You can begin migrating to the new structure in Omnibus GitLab 15.10 while backwards compatibility is + maintained in the lead up to Omnibus GitLab 16.0. [Read more about this change](#gitaly-omnibus-gitlab-configuration-structure-change). + ### 15.9.0 -- This version removes `SanitizeConfidentialTodos` background migration [added](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87908/diffs) in 15.6, which removed any user inaccessible to-do items. Make sure that this migration is finished before upgrading to 15.9. -- As part of the [CI Partitioning effort](../architecture/blueprints/ci_data_decay/pipeline_partitioning.md), a [new Foreign Key](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/107547) was added to `ci_builds_needs`. On GitLab instances with large CI tables, adding this constraint can take longer than usual. Make sure that this migration is finished before upgrading to 15.9. +- **Upgrade to patch release 15.9.3 or later**. This provides fixes for two database migration bugs: + - Patch releases 15.9.0, 15.9.1, 15.9.2 have [a bug that can cause data loss](#user-profile-data-loss-bug-in-159x) from the user profile fields. + - The second [bug fix](https://gitlab.com/gitlab-org/gitlab/-/issues/394760) ensures it is possible to upgrade directly from 15.4.x. +- As part of the [CI Partitioning effort](../architecture/blueprints/ci_data_decay/pipeline_partitioning.md), a [new Foreign Key](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/107547) was added to `ci_builds_needs`. On GitLab instances with large CI tables, adding this constraint can take longer than usual. +- Praefect's metadata verifier's [invalid metadata deletion behavior](../administration/gitaly/praefect.md#enable-deletions) is now enabled by default. + + The metadata verifier processes replica records in the Praefect database and verifies the replicas actually exist on the Gitaly nodes. If the replica doesn't exist, its + metadata record is deleted. This enables Praefect to fix situations where a replica has a metadata record indicating it's fine but, in reality, it doesn't exist on disk. + After the metadata record is deleted, Praefect's reconciler schedules a replication job to recreate the replica. + + Because of past issues with the state management logic, there may be invalid metadata records in the database. These could exist, for example, because of incomplete + deletions of repositories or partially completed renames. The verifier deletes these stale replica records of affected repositories. These repositories may show up as + unavailable repositories in the metrics and `praefect dataloss` sub-command because of the replica records being removed. If you encounter such repositories, remove + the repository using `praefect remove-repository` to remove the repository's remaining records. + + You can find repositories with invalid metadata records prior in GitLab 15.0 and later by searching for the log records outputted by the verifier. [Read more about repository verification, and to see an example log entry](../administration/gitaly/praefect.md#repository-verification). +- Praefect configuration changes significantly in Omnibus GitLab 16.0. You can begin migrating to the new structure in Omnibus GitLab 15.9 while backwards compatibility is + maintained in the lead up to Omnibus GitLab 16.0. [Read more about this change](#praefect-omnibus-gitlab-configuration-structure-change). +- For **self-compiled (source) installations**, with the addition of `gitlab-sshd` the Kerberos headers are needed to build GitLab Shell. + + ```shell + sudo apt install libkrb5-dev + ``` ### 15.8.2 @@ -320,7 +370,7 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap ### 15.7.2 -- Geo: [Container registry push events are rejected](https://gitlab.com/gitlab-org/gitlab/-/issues/386389) by the `/api/v4/container_registry_event/events` endpoint resulting in Geo secondary sites not being aware of updates to container registry images and subsequently not replicating the updates. Secondary sites may contain out of date container images after a failover as a consequence. This impacts versions 15.6.0 - 15.6.6 and 15.7.0 - 15.7.2. If you're using Geo with container repositories, you are advised to upgrade to GitLab 15.6.7, 15.7.3, or 15.8.0 which contain a fix for this issue and avoid potential data loss after a failover. +- Geo: [Container registry push events are rejected](https://gitlab.com/gitlab-org/gitlab/-/issues/386389) by the `/api/v4/container_registry_event/events` endpoint resulting in Geo secondary sites not being aware of updates to container registry images and subsequently not replicating the upgrades. Secondary sites may contain out of date container images after a failover as a consequence. This impacts versions 15.6.0 - 15.6.6 and 15.7.0 - 15.7.2. If you're using Geo with container repositories, you are advised to upgrade to GitLab 15.6.7, 15.7.3, or 15.8.0 which contain a fix for this issue and avoid potential data loss after a failover. - Geo: We discovered an issue where [replication and verification of projects and wikis was not keeping up](https://gitlab.com/gitlab-org/gitlab/-/issues/387980) on small number of Geo installations. Your installation may be affected if you see some projects and/or wikis persistently in the "Queued" state for verification. This can lead to data loss after a failover. - Impacted versions: GitLab versions 15.6.x, 15.7.x, and 15.8.0 - 15.8.2. - Versions containing fix: GitLab 15.8.3 and later. @@ -364,6 +414,7 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap consistent in our documentation and product defaults. For example, previously: + - Omnibus GitLab default (`sidekiq['max_concurrency']`): 50 - From source installation default: 50 - Helm chart default (`gitlab.sidekiq.concurrency`): 25 @@ -476,7 +527,7 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap ### 15.5.3 -- GitLab 15.4.0 introduced a default [Sidekiq routing rule](../administration/sidekiq/extra_sidekiq_routing.md) that routes all jobs to the `default` queue. For instances using [queue selectors](../administration/sidekiq/processing_specific_job_classes.md#queue-selectors), this will cause [performance problems](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1991) as some Sidekiq processes will be idle. +- GitLab 15.4.0 introduced a default [Sidekiq routing rule](../administration/sidekiq/processing_specific_job_classes.md#routing-rules) that routes all jobs to the `default` queue. For instances using [queue selectors](../administration/sidekiq/processing_specific_job_classes.md#queue-selectors-deprecated), this causes [performance problems](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1991) as some Sidekiq processes will be idle. - The default routing rule has been reverted in 15.5.4, so upgrading to that version or later will return to the previous behavior. - If a GitLab instance now listens only to the `default` queue (which is not currently recommended), it will be required to add this routing rule back in `/etc/gitlab/gitlab.rb`: @@ -488,7 +539,7 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap ### 15.5.2 -- GitLab 15.4.0 introduced a default [Sidekiq routing rule](../administration/sidekiq/extra_sidekiq_routing.md) that routes all jobs to the `default` queue. For instances using [queue selectors](../administration/sidekiq/processing_specific_job_classes.md#queue-selectors), this will cause [performance problems](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1991) as some Sidekiq processes will be idle. +- GitLab 15.4.0 introduced a default [Sidekiq routing rule](../administration/sidekiq/processing_specific_job_classes.md#routing-rules) that routes all jobs to the `default` queue. For instances using [queue selectors](../administration/sidekiq/processing_specific_job_classes.md#queue-selectors-deprecated), this causes [performance problems](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1991) as some Sidekiq processes will be idle. - The default routing rule has been reverted in 15.5.4, so upgrading to that version or later will return to the previous behavior. - If a GitLab instance now listens only to the `default` queue (which is not currently recommended), it will be required to add this routing rule back in `/etc/gitlab/gitlab.rb`: @@ -500,7 +551,7 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap ### 15.5.1 -- GitLab 15.4.0 introduced a default [Sidekiq routing rule](../administration/sidekiq/extra_sidekiq_routing.md) that routes all jobs to the `default` queue. For instances using [queue selectors](../administration/sidekiq/processing_specific_job_classes.md#queue-selectors), this will cause [performance problems](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1991) as some Sidekiq processes will be idle. +- GitLab 15.4.0 introduced a default [Sidekiq routing rule](../administration/sidekiq/processing_specific_job_classes.md#routing-rules) that routes all jobs to the `default` queue. For instances using [queue selectors](../administration/sidekiq/processing_specific_job_classes.md#queue-selectors-deprecated), this causes [performance problems](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1991) as some Sidekiq processes will be idle. - The default routing rule has been reverted in 15.5.4, so upgrading to that version or later will return to the previous behavior. - If a GitLab instance now listens only to the `default` queue (which is not currently recommended), it will be required to add this routing rule back in `/etc/gitlab/gitlab.rb`: @@ -512,7 +563,7 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap ### 15.5.0 -- GitLab 15.4.0 introduced a default [Sidekiq routing rule](../administration/sidekiq/extra_sidekiq_routing.md) that routes all jobs to the `default` queue. For instances using [queue selectors](../administration/sidekiq/processing_specific_job_classes.md#queue-selectors), this will cause [performance problems](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1991) as some Sidekiq processes will be idle. +- GitLab 15.4.0 introduced a default [Sidekiq routing rule](../administration/sidekiq/processing_specific_job_classes.md#routing-rules) that routes all jobs to the `default` queue. For instances using [queue selectors](../administration/sidekiq/processing_specific_job_classes.md#queue-selectors-deprecated), this causes [performance problems](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1991) as some Sidekiq processes will be idle. - The default routing rule has been reverted in 15.5.4, so upgrading to that version or later will return to the previous behavior. - If a GitLab instance now listens only to the `default` queue (which is not currently recommended), it will be required to add this routing rule back in `/etc/gitlab/gitlab.rb`: @@ -564,7 +615,7 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap - GitLab 15.4.0 includes a [batched background migration](background_migrations.md#batched-background-migrations) to [remove incorrect values from `expire_at` in `ci_job_artifacts` table](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89318). This migration might take hours or days to complete on larger GitLab instances. - By default, Gitaly and Praefect nodes use the time server at `pool.ntp.org`. If your instance can not connect to `pool.ntp.org`, [configure the `NTP_HOST` variable](../administration/gitaly/praefect.md#customize-time-server-setting). -- GitLab 15.4.0 introduced a default [Sidekiq routing rule](../administration/sidekiq/extra_sidekiq_routing.md) that routes all jobs to the `default` queue. For instances using [queue selectors](../administration/sidekiq/processing_specific_job_classes.md#queue-selectors), this will cause [performance problems](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1991) as some Sidekiq processes will be idle. +- GitLab 15.4.0 introduced a default [Sidekiq routing rule](../administration/sidekiq/processing_specific_job_classes.md#routing-rules) that routes all jobs to the `default` queue. For instances using [queue selectors](../administration/sidekiq/processing_specific_job_classes.md#queue-selectors-deprecated), this causes [performance problems](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1991) as some Sidekiq processes will be idle. - The default routing rule has been reverted in 15.4.5, so upgrading to that version or later will return to the previous behavior. - If a GitLab instance now listens only to the `default` queue (which is not currently recommended), it will be required to add this routing rule back in `/etc/gitlab/gitlab.rb`: @@ -584,6 +635,12 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap migration might take multiple hours or days to complete on larger GitLab instances. Make sure the migration has completed successfully before upgrading to 15.7.0 or later. - Due to [a bug introduced in GitLab 15.4](https://gitlab.com/gitlab-org/gitlab/-/issues/390155), if one or more Git repositories in Gitaly Cluster is [unavailable](../administration/gitaly/recovery.md#unavailable-repositories), then [Repository checks](../administration/repository_checks.md#repository-checks) and [Geo replication and verification](../administration/geo/index.md) stop running for all project or project wiki repositories in the affected Gitaly Cluster. The bug was fixed by [reverting the change in GitLab 15.9.0](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/110823). Before upgrading to this version, check if you have any "unavailable" repositories. See [the bug issue](https://gitlab.com/gitlab-org/gitlab/-/issues/390155) for more information. +- A redesigned sign-in page is enabled by default in GitLab 15.4 and later, with improvements shipping in later releases. For more information, see [epic 8557](https://gitlab.com/groups/gitlab-org/-/epics/8557). + It can be disabled with a feature flag. Start [a Rails console](../administration/operations/rails_console.md) and run: + + ```ruby + Feature.disable(:restyle_login_page) + ``` ### 15.3.4 @@ -705,10 +762,42 @@ A [license caching issue](https://gitlab.com/gitlab-org/gitlab/-/issues/376706) Gitaly. The previous implementation in GitLab Shell was removed in GitLab 15.0. With this change, global server hooks are stored only inside a subdirectory named after the hook type. Global server hooks can no longer be a single hook file in the root of the custom hooks directory. For example, you must use `/.d/*` rather than `/`. + - Use `gitaly['custom_hooks_dir']` in `gitlab.rb` ([introduced in 14.3](https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/4208)) + for Omnibus GitLab. This replaces `gitlab_shell['custom_hooks_dir']`. - [Incorrect deletion of object storage files on Geo secondary sites](https://gitlab.com/gitlab-org/gitlab/-/issues/371397) can occur in certain situations. See [Geo: Incorrect object storage LFS file deletion on secondary site issue in GitLab 15.0.0 to 15.3.2](#geo-incorrect-object-storage-lfs-file-deletion-on-secondary-sites-in-gitlab-1500-to-1532). - The `FF_GITLAB_REGISTRY_HELPER_IMAGE` [feature flag](../administration/feature_flags.md#enable-or-disable-the-feature) is removed and helper images are always pulled from GitLab Registry. - The `AES256-GCM-SHA384` SSL cipher is no longer allowed by NGINX. See how you can [add the cipher back](https://docs.gitlab.com/omnibus/update/gitlab_15_changes.html#aes256-gcm-sha384-ssl-cipher-no-longer-allowed-by-default-by-nginx) to the allow list. +- Support for more than one database has been added to GitLab. For **self-compiled (source) installations**, + `config/database.yml` must include a database name in the database configuration. + The `main: database` must be first. If an invalid or deprecated syntax is used, an error is generated + during application start: + + ```plaintext + ERROR: This installation of GitLab uses unsupported 'config/database.yml'. + The main: database needs to be defined as a first configuration item instead of primary. (RuntimeError) + ``` + + Previously, the `config/database.yml` file looked like the following: + + ```yaml + production: + adapter: postgresql + encoding: unicode + database: gitlabhq_production + ... + ``` + + Starting with GitLab 15.0, it must define a `main` database first: + + ```yaml + production: + main: + adapter: postgresql + encoding: unicode + database: gitlabhq_production + ... + ``` ### 14.10.0 @@ -739,11 +828,11 @@ A [license caching issue](https://gitlab.com/gitlab-org/gitlab/-/issues/376706) These [batched background migrations](background_migrations.md#batched-background-migrations) update whole database tables to ensure corresponding records in `namespaces` table for each record in `projects` table. - After you update to 14.9.0 or a later 14.9 patch version, + After you upgrade to 14.9.0 or a later 14.9 patch version, [batched background migrations must finish](background_migrations.md#batched-background-migrations) - before you update to a later version. + before you upgrade to a later version. - If the migrations are not finished and you try to update to a later version, + If the migrations are not finished and you try to upgrade to a later version, you see errors like: ```plaintext @@ -804,13 +893,13 @@ A [license caching issue](https://gitlab.com/gitlab-org/gitlab/-/issues/376706) [background migration `PopulateTopicsNonPrivateProjectsCount`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79140) that may remain stuck permanently in a **pending** state. - To clean up this stuck job, run the following in the [GitLab Rails Console](../administration/operations/rails_console.md): + To clean up this stuck job, run the following in the [GitLab Rails Console](../administration/operations/rails_console.md): - ```ruby - Gitlab::Database::BackgroundMigrationJob.pending.where(class_name: "PopulateTopicsNonPrivateProjectsCount").find_each do |job| - puts Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded("PopulateTopicsNonPrivateProjectsCount", job.arguments) - end - ``` + ```ruby + Gitlab::Database::BackgroundMigrationJob.pending.where(class_name: "PopulateTopicsNonPrivateProjectsCount").find_each do |job| + puts Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded("PopulateTopicsNonPrivateProjectsCount", job.arguments) + end + ``` - If upgrading from a version earlier than 14.3.0, to avoid [an issue with job retries](https://gitlab.com/gitlab-org/gitlab/-/issues/357822), first upgrade @@ -883,11 +972,11 @@ or [init scripts](upgrading_from_source.md#configure-sysv-init-script) by [follo To clean up this stuck job, run the following in the [GitLab Rails Console](../administration/operations/rails_console.md): - ```ruby - Gitlab::Database::BackgroundMigrationJob.pending.where(class_name: "UpdateVulnerabilityOccurrencesLocation").find_each do |job| - puts Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded("UpdateVulnerabilityOccurrencesLocation", job.arguments) - end - ``` + ```ruby + Gitlab::Database::BackgroundMigrationJob.pending.where(class_name: "UpdateVulnerabilityOccurrencesLocation").find_each do |job| + puts Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded("UpdateVulnerabilityOccurrencesLocation", job.arguments) + end + ``` - Upgrading to 14.5 (or later) [might encounter a one hour timeout](https://gitlab.com/gitlab-org/gitlab/-/issues/354211) owing to a long running database data change. @@ -901,6 +990,19 @@ or [init scripts](upgrading_from_source.md#configure-sysv-init-script) by [follo [There is a workaround to complete the data change and the upgrade manually](package/index.md#mixlibshelloutcommandtimeout-rails_migrationgitlab-rails--command-timed-out-after-3600s) +- As part of [enabling real-time issue assignees](https://gitlab.com/gitlab-org/gitlab/-/issues/330117), Action Cable is now enabled by default. + For **self-compiled (source) installations**, `config/cable.yml` is required to be present. + + Configure this by running: + + ```shell + cd /home/git/gitlab + sudo -u git -H cp config/cable.yml.example config/cable.yml + + # Change the Redis socket path if you are not using the default Debian / Ubuntu configuration + sudo -u git -H editor config/cable.yml + ``` + ### 14.4.4 - For [zero-downtime upgrades](zero_downtime.md) on a GitLab cluster with separate Web and API nodes, you must enable the `paginated_tree_graphql_query` [feature flag](../administration/feature_flags.md#enable-or-disable-the-feature) _before_ upgrading GitLab Web nodes to 14.4. @@ -928,13 +1030,13 @@ or [init scripts](upgrading_from_source.md#configure-sysv-init-script) by [follo [background migration `PopulateTopicsTotalProjectsCountCache`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71033) that may remain stuck permanently in a **pending** state when the instance lacks records that match the migration's target. - To clean up this stuck job, run the following in the [GitLab Rails Console](../administration/operations/rails_console.md): + To clean up this stuck job, run the following in the [GitLab Rails Console](../administration/operations/rails_console.md): - ```ruby - Gitlab::Database::BackgroundMigrationJob.pending.where(class_name: "PopulateTopicsTotalProjectsCountCache").find_each do |job| - puts Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded("PopulateTopicsTotalProjectsCountCache", job.arguments) - end - ``` + ```ruby + Gitlab::Database::BackgroundMigrationJob.pending.where(class_name: "PopulateTopicsTotalProjectsCountCache").find_each do |job| + puts Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded("PopulateTopicsTotalProjectsCountCache", job.arguments) + end + ``` ### 14.3.0 @@ -1148,9 +1250,9 @@ Long running batched background database migrations: After you update to 14.0.5 or a later 14.0 patch version, [batched background migrations must finish](background_migrations.md#batched-background-migrations) - before you update to a later version. + before you upgrade to a later version. - If the migrations are not finished and you try to update to a later version, + If the migrations are not finished and you try to upgrade to a later version, you see an error like: ```plaintext @@ -1177,7 +1279,7 @@ Other issues: - 14.0.5 or a later 14.0.Z patch release. - 14.1.0 or a later 14.1.Z patch release. 1. [Batched background migrations must finish](background_migrations.md#batched-background-migrations) - before you update to a later version [and may take longer than usual](#1400). + before you upgrade to a later version [and may take longer than usual](#1400). ### 13.12.0 @@ -1247,7 +1349,7 @@ See [Maintenance mode issue in GitLab 13.9 to 14.4](#maintenance-mode-issue-in-g DETAIL: trigger trigger_0d588df444c8 on table application_settings depends on column asset_proxy_whitelist of table application_settings ``` - To work around this bug, follow the previous steps to complete the update. + To work around this bug, follow the previous steps to complete the upgrade. More details are available [in this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/324160). - See [Maintenance mode issue in GitLab 13.9 to 14.4](#maintenance-mode-issue-in-gitlab-139-to-144). @@ -1366,12 +1468,23 @@ all servers must first be upgraded to 13.1.Z before upgrading to 13.2.0 or later #### Custom Rack Attack initializers -From GitLab 13.0.1, custom Rack Attack initializers (`config/initializers/rack_attack.rb`) are replaced with initializers -supplied with GitLab during upgrades. We recommend you use these GitLab-supplied initializers. +From GitLab 13.1, custom Rack Attack initializers (`config/initializers/rack_attack.rb`) are replaced with initializers +supplied with GitLab during upgrades. You should use these GitLab-supplied initializers. If you persist your own Rack Attack initializers between upgrades, you might [get `500` errors](https://gitlab.com/gitlab-org/gitlab/-/issues/334681) when [upgrading to GitLab 14.0 and later](#1400). +For **self-compiled (source) installations**, the Rack Attack initializer on GitLab +was renamed from [`config/initializers/rack_attack_new.rb` to `config/initializers/rack_attack.rb`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/33072). +The rename was part of [deprecating Rack Attack throttles on Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4750). + +If `rack_attack.rb` has been created on your installation, consider creating a backup before updating: + +```shell +cd /home/git/gitlab +cp config/initializers/rack_attack.rb config/initializers/rack_attack_backup.rb +``` + ### 12.10.0 - The final patch release (12.10.14) @@ -1425,6 +1538,335 @@ After upgraded to 11.11.8 you can safely upgrade to 12.0.Z. See our [documentation on upgrade paths](../policy/maintenance.md#upgrade-recommendations) for more information. +### User profile data loss bug in 15.9.x + +There is a database migration bug in 15.9.0, 15.9.1, and 15.9.2 that can cause data loss from the user profile fields `linkedin`, `twitter`, `skype`, `website_url`, `location`, and `organization`. + +This bug is fixed in patch releases 15.9.3 and later. + +The following upgrade path also works around the bug: + +1. Upgrade to GitLab 15.6.x, 15.7.x, or 15.8.x. +1. [Ensure batched background migrations](background_migrations.md#batched-background-migrations) are complete. +1. Upgrade to an earlier GitLab 15.9 patch release that doesn't have the bug fix. + +It is not then required to upgrade to 15.9.3 or later for this issue. + +[Read the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/393216) for more information. + +### Gitaly: Omnibus GitLab configuration structure change + +Gitaly configuration structure in Omnibus GitLab [changes](https://gitlab.com/gitlab-org/gitaly/-/issues/4467) in GitLab 16.0 to be consistent with the Gitaly configuration +structure used in source installs. + +As a result of this change, a single hash under `gitaly['configuration']` holds most Gitaly +configuration. Some `gitaly['..']` configuration options will continue to be used by Omnibus GitLab 16.0 and later: + +- `enable` +- `dir` +- `bin_path` +- `env_directory` +- `env` +- `open_files_ulimit` +- `consul_service_name` +- `consul_service_meta` + +Migrate by moving your existing configuration under the new structure. The new structure is supported from Omnibus GitLab 15.10. + +The new structure is documented below with the old keys described in a comment above the new keys. When applying the new structure to your configuration: + +1. Replace the `...` with the value from the old key. +1. Skip any keys you haven't configured a value for previously. +1. Remove the old keys from the configuration once migrated. +1. Optional but recommended. Include a trailing comma for all hash keys so the hash remains valid when keys are re-ordered or additional keys are added. + + ```ruby +gitaly['configuration'] = { + # gitaly['socket_path'] + socket_path: ..., + # gitaly['runtime_dir'] + runtime_dir: ..., + # gitaly['listen_addr'] + listen_addr: ..., + # gitaly['prometheus_listen_addr'] + prometheus_listen_addr: ..., + # gitaly['tls_listen_addr'] + tls_listen_addr: ..., + tls: { + # gitaly['certificate_path'] + certificate_path: ..., + # gitaly['key_path'] + key_path: ..., + }, + # gitaly['graceful_restart_timeout'] + graceful_restart_timeout: ..., + logging: { + # gitaly['logging_level'] + level: ..., + # gitaly['logging_format'] + format: ..., + # gitaly['logging_sentry_dsn'] + sentry_dsn: ..., + # gitaly['logging_ruby_sentry_dsn'] + ruby_sentry_dsn: ..., + # gitaly['logging_sentry_environment'] + sentry_environment: ..., + # gitaly['log_directory'] + dir: ..., + }, + prometheus: { + # gitaly['prometheus_grpc_latency_buckets']. The old value was configured as a string + # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2]. + grpc_latency_buckets: ..., + }, + auth: { + # gitaly['auth_token'] + token: ..., + # gitaly['auth_transitioning'] + transitioning: ..., + }, + git: { + # gitaly['git_catfile_cache_size'] + catfile_cache_size: ..., + # gitaly['git_bin_path'] + bin_path: ..., + # gitaly['use_bundled_git'] + use_bundled_binaries: ..., + # gitaly['gpg_signing_key_path'] + signing_key: ..., + # gitaly['gitconfig']. This is still an array but the type of the elements have changed. + config: [ + { + # Previously the elements contained 'section', and 'subsection' in addition to 'key'. Now + # these all should be concatenated into just 'key', separated by dots. For example, + # {section: 'first', subsection: 'middle', key: 'last', value: 'value'}, should become + # {key: 'first.middle.last', value: 'value'}. + key: ..., + value: ..., + }, + ], + }, + # Storage could previously be configured through either gitaly['storage'] or 'git_data_dirs'. Migrate + # the relevant configuration according to the instructions below. + # For 'git_data_dirs', migrate only the 'path' to the gitaly['configuration'] and leave the rest of it untouched. + storage: [ + { + # gitaly['storage'][]['name'] + # + # git_data_dirs[]. The storage name was configured as a key in the map. + name: ..., + # gitaly['storage'][]['path'] + # + # git_data_dirs[]['path']. Use the value from git_data_dirs[]['path'] and append '/repositories' to it. + # + # For example, if the path in 'git_data_dirs' was '/var/opt/gitlab/git-data', use + # '/var/opt/gitlab/git-data/repositories'. The '/repositories' extension was automatically + # appended to the path configured in `git_data_dirs`. + path: ..., + }, + ], + hooks: { + # gitaly['custom_hooks_dir'] + custom_hooks_dir: ..., + }, + daily_maintenance: { + # gitaly['daily_maintenance_disabled'] + disabled: ..., + # gitaly['daily_maintenance_start_hour'] + start_hour: ..., + # gitaly['daily_maintenance_start_minute'] + start_minute: ..., + # gitaly['daily_maintenance_duration'] + duration: ..., + # gitaly['daily_maintenance_storages'] + storages: ..., + }, + cgroups: { + # gitaly['cgroups_mountpoint'] + mountpoint: ..., + # gitaly['cgroups_hierarchy_root'] + hierarchy_root: ..., + # gitaly['cgroups_memory_bytes'] + memory_bytes: ..., + # gitaly['cgroups_cpu_shares'] + cpu_shares: ..., + repositories: { + # gitaly['cgroups_repositories_count'] + count: ..., + # gitaly['cgroups_repositories_memory_bytes'] + memory_bytes: ..., + # gitaly['cgroups_repositories_cpu_shares'] + cpu_shares: ..., + } + }, + # gitaly['concurrency']. While the structure is the same, the string keys in the array elements + # should be replaced by symbols as elsewhere. {'key' => 'value'}, should become {key: 'value'}. + concurrency: ..., + # gitaly['rate_limiting']. While the structure is the same, the string keys in the array elements + # should be replaced by symbols as elsewhere. {'key' => 'value'}, should become {key: 'value'}. + rate_limiting: ..., + pack_objects_cache: { + # gitaly['pack_objects_cache_enabled'] + enabled: ..., + # gitaly['pack_objects_cache_dir'] + dir: ..., + # gitaly['pack_objects_cache_max_age'] + max_age: ..., + } +} +``` + +### Praefect: Omnibus GitLab configuration structure change + +Praefect configuration structure in Omnibus GitLab [changes](https://gitlab.com/gitlab-org/gitaly/-/issues/4467) in GitLab 16.0 to be consistent with the Praefect configuration +structure used in source installs. + +As a result of this change, a single hash under `praefect['configuration']` holds most Praefect +configuration. Some `praefect['..']` configuration options will continue to be used by Omnibus GitLab 16.0 and later: + +- `enable` +- `dir` +- `log_directory` +- `env_directory` +- `env` +- `wrapper_path` +- `auto_migrate` +- `consul_service_name` + +Migrate by moving your existing configuration under the new structure. The new structure is supported from Omnibus GitLab 15.9. + +The new structure is documented below with the old keys described in a comment above the new keys. When applying the new structure to your configuration: + +1. Replace the `...` with the value from the old key. +1. Skip any keys you haven't configured a value for previously. +1. Remove the old keys from the configuration once migrated. +1. Optional but recommended. Include a trailing comma for all hash keys so the hash remains valid when keys are re-ordered or additional keys are added. + +```ruby +praefect['configuration'] = { + # praefect['listen_addr'] + listen_addr: ..., + # praefect['socket_path'] + socket_path: ..., + # praefect['prometheus_listen_addr'] + prometheus_listen_addr: ..., + # praefect['tls_listen_addr'] + tls_listen_addr: ..., + # praefect['separate_database_metrics'] + prometheus_exclude_database_from_default_metrics: ..., + auth: { + # praefect['auth_token'] + token: ..., + # praefect['auth_transitioning'] + transitioning: ..., + }, + logging: { + # praefect['logging_format'] + format: ..., + # praefect['logging_level'] + level: ..., + }, + failover: { + # praefect['failover_enabled'] + enabled: ..., + }, + background_verification: { + # praefect['background_verification_delete_invalid_records'] + delete_invalid_records: ..., + # praefect['background_verification_verification_interval'] + verification_interval: ..., + }, + reconciliation: { + # praefect['reconciliation_scheduling_interval'] + scheduling_interval: ..., + # praefect['reconciliation_histogram_buckets']. The old value was configured as a string + # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2]. + histogram_buckets: ..., + }, + tls: { + # praefect['certificate_path'] + certificate_path: ..., + # praefect['key_path'] + key_path: ..., + }, + database: { + # praefect['database_host'] + host: ..., + # praefect['database_port'] + port: ..., + # praefect['database_user'] + user: ..., + # praefect['database_password'] + password: ..., + # praefect['database_dbname'] + dbname: ..., + # praefect['database_sslmode'] + sslmode: ..., + # praefect['database_sslcert'] + sslcert: ..., + # praefect['database_sslkey'] + sslkey: ..., + # praefect['database_sslrootcert'] + sslrootcert: ..., + session_pooled: { + # praefect['database_direct_host'] + host: ..., + # praefect['database_direct_port'] + port: ..., + # praefect['database_direct_user'] + user: ..., + # praefect['database_direct_password'] + password: ..., + # praefect['database_direct_dbname'] + dbname: ..., + # praefect['database_direct_sslmode'] + sslmode: ..., + # praefect['database_direct_sslcert'] + sslcert: ..., + # praefect['database_direct_sslkey'] + sslkey: ..., + # praefect['database_direct_sslrootcert'] + sslrootcert: ..., + } + }, + sentry: { + # praefect['sentry_dsn'] + sentry_dsn: ..., + # praefect['sentry_environment'] + sentry_environment: ..., + }, + prometheus: { + # praefect['prometheus_grpc_latency_buckets']. The old value was configured as a string + # such as '[0, 1, 2]'. The new value must be an array like [0, 1, 2]. + grpc_latency_buckets: ..., + }, + # praefect['graceful_stop_timeout'] + graceful_stop_timeout: ..., + + # praefect['virtual_storages']. The old value was a hash map but the new value is an array. + virtual_storage: [ + { + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]. The name was previously the key in + # the 'virtual_storages' hash. + name: ..., + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]. The old value was a hash map + # but the new value is an array. + node: [ + { + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]. Use NODE_NAME key as the + # storage. + storage: ..., + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]['address']. + address: ..., + # praefect['virtual_storages'][VIRTUAL_STORAGE_NAME]['nodes'][NODE_NAME]['token']. + token: ..., + }, + ], + } + ] +} +``` + ### Change to Praefect-generated replica paths in GitLab 15.3 New Git repositories created in Gitaly cluster no longer use the `@hashed` storage path. @@ -1438,19 +1880,26 @@ and pass the `@hashed` storage path to `-relative-path`. With this information, you can correctly install [server hooks](../administration/server_hooks.md). -### Maintenance mode issue in GitLab 13.9 to 14.4 +### Geo: LFS transfers redirect to primary from secondary site mid-session in GitLab 15.1.0 to 15.3.2 -When [Maintenance mode](../administration/maintenance_mode/index.md) is enabled, users cannot sign in with SSO, SAML, or LDAP. +LFS transfers can [redirect to the primary from secondary site mid-session](https://gitlab.com/gitlab-org/gitlab/-/issues/371571) causing failed pull and clone requests in GitLab 15.1.0 to 15.3.2 when [Geo proxying](../administration/geo/secondary_proxy/index.md) is enabled. Geo proxying is enabled by default in GitLab 15.1 and later. -Users who were signed in before Maintenance mode was enabled, continue to be signed in. If the administrator who enabled Maintenance mode loses their session, then they can't disable Maintenance mode via the UI. In that case, you can [disable Maintenance mode via the API or Rails console](../administration/maintenance_mode/index.md#disable-maintenance-mode). +This issue is resolved in GitLab 15.3.3, so customers with the following configuration should upgrade to 15.3.3 or later: -[This bug](https://gitlab.com/gitlab-org/gitlab/-/issues/329261) was fixed in GitLab 14.5.0 and backported into 14.4.3 and 14.3.5. +- LFS is enabled. +- LFS objects are being replicated across Geo sites. +- Repositories are being pulled by using a Geo secondary site. -### LFS objects import and mirror issue in GitLab 14.6.0 to 14.7.2 +### Geo: Incorrect object storage LFS file deletion on secondary sites in GitLab 15.0.0 to 15.3.2 -When Geo is enabled, LFS objects fail to be saved for imported or mirrored projects. +[Incorrect deletion of object storage files on Geo secondary sites](https://gitlab.com/gitlab-org/gitlab/-/issues/371397) +can occur in GitLab 15.0.0 to 15.3.2 in the following situations: -[This bug](https://gitlab.com/gitlab-org/gitlab/-/issues/352368) was fixed in GitLab 14.8.0 and backported into 14.7.3. +- GitLab-managed object storage replication is disabled, and LFS objects are created while importing a project with object storage enabled. +- GitLab-managed replication to sync object storage is enabled and subsequently disabled. + +This issue is resolved in 15.3.3. Customers who have both LFS enabled and LFS objects being replicated across Geo sites +should upgrade directly to 15.3.3 to reduce the risk of data loss on secondary sites. ### PostgreSQL segmentation fault issue @@ -1466,26 +1915,19 @@ by a database engine bug that causes a segmentation fault. Read more [in the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/364763). -### Geo: Incorrect object storage LFS file deletion on secondary sites in GitLab 15.0.0 to 15.3.2 - -[Incorrect deletion of object storage files on Geo secondary sites](https://gitlab.com/gitlab-org/gitlab/-/issues/371397) -can occur in GitLab 15.0.0 to 15.3.2 in the following situations: +### LFS objects import and mirror issue in GitLab 14.6.0 to 14.7.2 -- GitLab-managed object storage replication is disabled, and LFS objects are created while importing a project with object storage enabled. -- GitLab-managed replication to sync object storage is enabled and subsequently disabled. +When Geo is enabled, LFS objects fail to be saved for imported or mirrored projects. -This issue is resolved in 15.3.3. Customers who have both LFS enabled and LFS objects being replicated across Geo sites -should upgrade directly to 15.3.3 to reduce the risk of data loss on secondary sites. +[This bug](https://gitlab.com/gitlab-org/gitlab/-/issues/352368) was fixed in GitLab 14.8.0 and backported into 14.7.3. -### Geo: LFS transfers redirect to primary from secondary site mid-session in GitLab 15.1.0 to 15.3.2 +### Maintenance mode issue in GitLab 13.9 to 14.4 -LFS transfers can [redirect to the primary from secondary site mid-session](https://gitlab.com/gitlab-org/gitlab/-/issues/371571) causing failed pull and clone requests in GitLab 15.1.0 to 15.3.2 when [Geo proxying](../administration/geo/secondary_proxy/index.md) is enabled. Geo proxying is enabled by default in GitLab 15.1 and later. +When [Maintenance mode](../administration/maintenance_mode/index.md) is enabled, users cannot sign in with SSO, SAML, or LDAP. -This issue is resolved in GitLab 15.3.3, so customers with the following configuration should upgrade to 15.3.3 or later: +Users who were signed in before Maintenance mode was enabled, continue to be signed in. If the administrator who enabled Maintenance mode loses their session, then they can't disable Maintenance mode via the UI. In that case, you can [disable Maintenance mode via the API or Rails console](../administration/maintenance_mode/index.md#disable-maintenance-mode). -- LFS is enabled. -- LFS objects are being replicated across Geo sites. -- Repositories are being pulled by using a Geo secondary site. +[This bug](https://gitlab.com/gitlab-org/gitlab/-/issues/329261) was fixed in GitLab 14.5.0 and backported into 14.4.3 and 14.3.5. ## Miscellaneous diff --git a/doc/update/mysql_to_postgresql.md b/doc/update/mysql_to_postgresql.md deleted file mode 100644 index ad36a9ff534..00000000000 --- a/doc/update/mysql_to_postgresql.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -remove_date: '2023-02-28' -redirect_to: 'index.md' ---- - -# Migrating from MySQL to PostgreSQL (removed) **(FREE SELF)** - -Support for MySQL was removed in GitLab 12.1. diff --git a/doc/update/package/convert_to_ee.md b/doc/update/package/convert_to_ee.md index 7ebb860746b..0e8e0b1e569 100644 --- a/doc/update/package/convert_to_ee.md +++ b/doc/update/package/convert_to_ee.md @@ -68,7 +68,7 @@ The steps can be summed up to: NOTE: If you want to use `dpkg`/`rpm` instead of `apt-get`/`yum`, go through the first step to find the current GitLab version, then follow - [Update using a manually-downloaded package](index.md#upgrade-using-a-manually-downloaded-package), + [Upgrade using a manually-downloaded package](index.md#upgrade-using-a-manually-downloaded-package), and then [add your license](../../user/admin_area/license.md). 1. Install the `gitlab-ee` package. The install automatically @@ -116,7 +116,7 @@ The steps can be summed up to: sudo rm /etc/yum.repos.d/gitlab_gitlab-ce.repo ``` -1. Optional. [Set up the Elasticsearch integration](../../integration/advanced_search/elasticsearch.md) to enable [Advanced Search](../../user/search/advanced_search.md). +1. Optional. [Set up the Elasticsearch integration](../../integration/advanced_search/elasticsearch.md) to enable [advanced search](../../user/search/advanced_search.md). -That's it! You can now use GitLab Enterprise Edition! To update to a newer -version, follow [Update using the official repositories](index.md#upgrade-using-the-official-repositories). +That's it! You can now use GitLab Enterprise Edition! To upgrade to a newer +version, follow [Upgrade using the official repositories](index.md#upgrade-using-the-official-repositories). diff --git a/doc/update/package/index.md b/doc/update/package/index.md index 34c7c096a8d..3e0d09eb36e 100644 --- a/doc/update/package/index.md +++ b/doc/update/package/index.md @@ -159,7 +159,7 @@ a manual installation. If for some reason you don't use the official repositories, you can download the package and install it manually. This method can be used to either -install GitLab for the first time or update it. +install GitLab for the first time or upgrade it. To download and install GitLab: @@ -169,7 +169,7 @@ To download and install GitLab: and architecture. Next to the filename is a label indicating the distribution, as the filenames may be the same. 1. Find the package version you wish to install, and select the filename from the list. -1. Select **Download** in the upper right corner to download the package. +1. In the upper-right corner, select **Download**. 1. After the package is downloaded, install it by using one of the following commands and replacing `` with the package name you downloaded: @@ -294,7 +294,7 @@ To fix this issue: ### Error `Failed to connect to the internal GitLab API` on a separate GitLab Pages server -See [GitLab Pages troubleshooting](../../administration/pages/index.md#failed-to-connect-to-the-internal-gitlab-api). +See [GitLab Pages administration troubleshooting](../../administration/pages/troubleshooting.md#failed-to-connect-to-the-internal-gitlab-api). ### Error `An error occurred during the signature verification` when running `apt-get update` diff --git a/doc/update/patch_versions.md b/doc/update/patch_versions.md index 8e62718125b..b6530418f97 100644 --- a/doc/update/patch_versions.md +++ b/doc/update/patch_versions.md @@ -2,7 +2,6 @@ stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false --- # Universal update guide for patch versions of source installations **(FREE SELF)** diff --git a/doc/update/plan_your_upgrade.md b/doc/update/plan_your_upgrade.md index 5b4ecb96747..e1354ea3665 100644 --- a/doc/update/plan_your_upgrade.md +++ b/doc/update/plan_your_upgrade.md @@ -60,16 +60,6 @@ to ensure the major components of GitLab are working: 1. If using Elasticsearch, verify that searches are successful. -1. If you are using [Reply by Email](../administration/reply_by_email.md) or [Service Desk](../user/project/service_desk.md), - manually install the latest version of `gitlab-mail_room`: - - ```shell - gem install gitlab-mail_room - ``` - - NOTE: This step is necessary to avoid thread deadlocks and to support the latest MailRoom features. See - [this explanation](../development/emails.md#mailroom-gem-updates) for more details. - If in any case something goes wrong, see [how to troubleshoot](#troubleshooting). ## Rollback plan @@ -172,8 +162,8 @@ If you have Kubernetes clusters connected with GitLab, [upgrade your GitLab agen #### Elasticsearch -Before updating GitLab, confirm Advanced Search migrations are complete by -[checking for pending advanced search migrations](background_migrations.md). +Before updating GitLab, confirm advanced search migrations are complete by +[checking for pending advanced search migrations](index.md#checking-for-pending-advanced-search-migrations). After updating GitLab, you may have to upgrade [Elasticsearch if the new version breaks compatibility](../integration/advanced_search/elasticsearch.md#version-requirements). diff --git a/doc/update/removals.md b/doc/update/removals.md index ae42ba53fd5..7359d74c6f5 100644 --- a/doc/update/removals.md +++ b/doc/update/removals.md @@ -34,6 +34,678 @@ For removal reviewers (Technical Writers only): https://about.gitlab.com/handbook/marketing/blog/release-posts/#update-the-removals-doc --> +## Removed in 16.0 + +### Auto DevOps no longer provisions a database by default + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +Currently, Auto DevOps provisions an in-cluster PostgreSQL database by default. +In GitLab 16.0, databases will be provisioned only for users who opt in. This +change supports production deployments that require more robust database management. + +If you want Auto DevOps to provision an in-cluster database, +set the `POSTGRES_ENABLED` CI/CD variable to `true`. + +### Azure Storage Driver defaults to the correct root prefix + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The Azure Storage Driver used to write to `//` as the default root directory. This default root directory appeared in some places in the Azure UI as `//`. We maintained this legacy behavior to support older deployments using this storage driver. However, when moving to Azure from another storage driver, this behavior hides all your data until you configure the storage driver with `trimlegacyrootprefix: true` to build root paths without an extra leading slash. + +In GitLab 16.0, the new default configuration for the storage driver uses `trimlegacyrootprefix: true`, and `/` is the default root directory. You can set your configuration to `trimlegacyrootprefix: false` if needed, to revert to the previous behavior. + +### Bundled Grafana Helm Chart + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The Grafana Helm chart that was bundled with the GitLab Helm Chart is removed in the GitLab Helm Chart 7.0 release (releasing along with GitLab 16.0). + +The `global.grafana.enabled` setting for the GitLab Helm Chart has also been removed alongside the Grafana Helm chart. + +If you're using the bundled Grafana, you should switch to the [newer chart version from Grafana Labs](https://artifacthub.io/packages/helm/grafana/grafana) +or a Grafana Operator from a trusted provider. + +In your new Grafana instance, you can [configure the GitLab provided Prometheus as a data source](https://docs.gitlab.com/ee/administration/monitoring/performance/grafana_configuration.html#integration-with-gitlab-ui) +and [connect Grafana to the GitLab UI](https://docs.gitlab.com/ee/administration/monitoring/performance/grafana_configuration.html#integration-with-gitlab-ui). + +### CAS OmniAuth provider is removed + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The `omniauth-cas3` gem that provides GitLab with the CAS OmniAuth provider is being removed. You can no longer authenticate into a GitLab instance through CAS. This gem sees very little use. [The gem](https://rubygems.org/gems/omniauth-cas3/) has not had a new release in almost 5 years, which means that its dependencies are out of date and required manual patching during GitLab's [upgrade to OmniAuth 2.0](https://gitlab.com/gitlab-org/gitlab/-/issues/30073). + +### CiCdSettingsUpdate mutation renamed to ProjectCiCdSettingsUpdate + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The `CiCdSettingsUpdate` mutation was renamed to `ProjectCiCdSettingsUpdate` in GitLab 15.0. +The `CiCdSettingsUpdate` mutation will be removed in GitLab 16.0. +Any user scripts that use the `CiCdSettingsUpdate` mutation must be updated to use `ProjectCiCdSettingsUpdate` +instead. + +### Conan project-level search returns only project-specific results" + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The [GitLab Conan repository](https://docs.gitlab.com/ee/user/packages/conan_repository/) supports the `conan search` command, but when searching a project-level endpoint, instance-level Conan packages could have been returned. This unintended functionality is removed in GitLab 16.0. The search endpoint for the project level now only returns packages from the target project. + +### Configuring Redis config file paths using environment variables is no longer supported + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +You can no longer specify Redis configuration file locations +using the environment variables like `GITLAB_REDIS_CACHE_CONFIG_FILE` or +`GITLAB_REDIS_QUEUES_CONFIG_FILE`. Use the default +configuration file locations instead, for example `config/redis.cache.yml` or +`config/redis.queues.yml`. + +### Container Registry pull-through cache is removed + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The Container Registry [pull-through cache](https://docs.docker.com/registry/recipes/mirror/) was deprecated in GitLab 15.8 and removed in GitLab 16.0. This feature is part of the upstream [Docker Distribution project](https://github.com/distribution/distribution) but we are removing that code in favor of the GitLab Dependency Proxy. Use the GitLab Dependency Proxy to proxy and cache container images from Docker Hub. + +### Container Scanning variables that reference Docker removed + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +All Container Scanning variables with a name prefixed by `DOCKER_` have been removed. This includes: + +- `DOCKER_IMAGE` +- `DOCKER_PASSWORD` +- `DOCKER_USER` +- `DOCKERFILE_PATH` + +Instead, use the [new variable names](https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-cicd-variables): + +- `CS_IMAGE` +- `CS_REGISTRY_PASSWORD` +- `CS_REGISTRY_USER` +- `CS_DOCKERFILE_PATH` + +### Dependency Scanning ends support for Java 13, 14, 15, and 16 + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +Dependency Scanning no longer supports projects that use Java versions 13, 14, 15, and 16. + +### Developer role providing the ability to import projects to a group + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The ability for users with the Developer role for a group to import projects to that group was deprecated in GitLab +15.8 and is removed in GitLab 16.0. + +From GitLab 16.0, only users with at least the Maintainer role for a group can import projects to that group. + +### Embedding Grafana panels in Markdown is removed + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The ability to add Grafana panels in GitLab Flavored Markdown is removed. +We intend to replace this feature with the ability to [embed charts](https://gitlab.com/groups/gitlab-org/opstrace/-/epics/33) +with the [GitLab Observability UI](https://gitlab.com/gitlab-org/opstrace/opstrace-ui). + +### Enforced validation of CI/CD parameter character lengths + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +Previously, only CI/CD [job names](https://docs.gitlab.com/ee/ci/jobs/index.html#job-name-limitations) had a strict 255-character limit. Now, more CI/CD keywords are validated to ensure they stay under the limit. + +The following to 255 characters are now strictly limited to 255 characters: + +- The `stage` keyword. +- The `ref` parameter, which is the Git branch or tag name for the pipeline. +- The `description` and `target_url` parameters, used by external CI/CD integrations. + +Users on self-managed instances should update their pipelines to ensure they do not use parameters that exceed 255 characters. Users on GitLab.com do not need to make any changes, as these parameters are already limited in that database. + +### GitLab administrators must have permission to modify protected branches or tags + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +GitLab administrators can no longer perform actions on protected branches or tags unless they have been explicitly granted that permission. These actions include pushing and merging into a [protected branch](https://docs.gitlab.com/ee/user/project/protected_branches.html), unprotecting a branch, and creating [protected tags](https://docs.gitlab.com/ee/user/project/protected_tags.html). + +### GitLab.com importer + +The [GitLab.com importer](https://docs.gitlab.com/ee/user/project/import/gitlab_com.html) was deprecated in GitLab 15.8 and is removed in GitLab 16.0. + +The GitLab.com importer was introduced in 2015 for importing a project from GitLab.com to a self-managed GitLab instance through the UI. + +This feature was available on self-managed instances only. [Migrating GitLab groups and projects by direct transfer](https://docs.gitlab.com/ee/user/group/import/#migrate-groups-by-direct-transfer-recommended) +supersedes the GitLab.com importer and provides a more cohesive importing functionality. + +See [migrated group items](https://docs.gitlab.com/ee/user/group/import/#migrated-group-items) and [migrated project items](https://docs.gitlab.com/ee/user/group/import/#migrated-project-items) for an overview. + +### GraphQL API: Runner status no longer returns `PAUSED` and `ACTIVE` values + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +In GitLab 16.0 and later, the GraphQL query for runners will no longer return the statuses `PAUSED` and `ACTIVE`. + +- `PAUSED` has been replaced with the field, `paused: true`. +- `ACTIVE` has been replaced with the field, `paused: false`. + +### Jira DVCS connector for Jira Cloud and Jira 8.13 and earlier + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The [Jira DVCS connector](https://docs.gitlab.com/ee/integration/jira/dvcs/) for Jira Cloud was deprecated in GitLab 15.1 and has been removed in 16.0. Use the [GitLab for Jira Cloud app](https://docs.gitlab.com/ee/integration/jira/connect-app.html) instead. The Jira DVCS connector was also deprecated for Jira 8.13 and earlier. You can only use the Jira DVCS connector with Jira Data Center or Jira Server in Jira 8.14 and later. Upgrade your Jira instance to Jira 8.14 or later, and reconfigure the Jira integration in your GitLab instance. +If you cannot upgrade your Jira instance in time and are on GitLab self-managed version, we offer a workaround until GitLab 16.6. This breaking change is deployed in GitLab 16.0 behind a feature flag named `jira_dvcs_end_of_life_amnesty`. The flag is disabled by default, but you can ask an administrator to enable the flag at any time. For questions related to this announcement, see the [feedback issue](https://gitlab.com/gitlab-org/gitlab/-/issues/408185). + +### Legacy Gitaly configuration method + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +Previously, Gitaly configuration keys for Omnibus GitLab were scattered throughout the configuration file. In GitLab +15.10, we added support for a single configuration structure that matches Gitaly internal configuration. Both methods +of configuring Gitaly were supported in parallel. + +In GitLab 16.0, we removed support for the former configuration method and now only support the new configuration +method. + +Before upgrading to GitLab 16.0, administrators must migrate to the new single configuration structure. For +instructions, see [Gitaly - Omnibus GitLab configuration structure change](https://docs.gitlab.com/ee/update/#gitaly-omnibus-gitlab-configuration-structure-change). + +### Legacy Gitaly configuration methods with variables + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The environment variables `GIT_CONFIG_SYSTEM` and `GIT_CONFIG_GLOBAL` were deprecated in GitLab 14.8 and are removed +in GitLab 16.0. These variables are replaced with standard +[`config.toml` Gitaly configuration](https://docs.gitlab.com/ee/administration/gitaly/reference.html). + +GitLab instances that use `GIT_CONFIG_SYSTEM` and `GIT_CONFIG_GLOBAL` to configure Gitaly must switch to configuring +using `config.toml`. + +### Legacy Praefect configuration method + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +Previously, Praefect configuration keys for Omnibus GitLab were scattered throughout the configuration file. In GitLab +15.9, we added support for a single configuration structure that matches Praefect internal configuration. Both methods +of configuring Praefect were supported in parallel. + +In GitLab 16.0, we removed support for the former configuration method and now only support the new configuration +method. + +Before upgrading to GitLab 16.0, administrators must migrate to the new single configuration structure. For +instructions, see [Praefect - Omnibus GitLab configuration structure change](https://docs.gitlab.com/ee/update/#praefect-omnibus-gitlab-configuration-structure-change). + +### License-Check and the Policies tab on the License Compliance page + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The License Check Policies feature has been removed. Additionally, the Policies tab on the License Compliance page and all APIs related to the License Check feature have been removed. To enforce approvals based on detected licenses, use the [License Approval policy](https://docs.gitlab.com/ee/user/compliance/license_approval_policies.html) feature instead. + +### Limit CI_JOB_TOKEN scope is disabled + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +In GitLab 14.4 we introduced the ability to [limit your project's CI/CD job token](https://docs.gitlab.com/ee/ci/jobs/ci_job_token.html#limit-your-projects-job-token-access) (`CI_JOB_TOKEN`) access to make it more secure. You could use the **Limit CI_JOB_TOKEN access** setting to prevent job tokens from your project's pipelines from being used to **access other projects**. When enabled with no other configuration, your pipelines could not access any other projects. To use job tokens to access other projects from your project's pipelines, you needed to list those other projects explicitly in the setting's allowlist, and you needed to be a maintainer in _all_ the projects. You might have seen this mentioned as the "outbound scope" of the job token. + +The job token functionality was updated in 15.9 with a [better security setting](https://docs.gitlab.com/ee/ci/jobs/ci_job_token.html#allow-access-to-your-project-with-a-job-token). Instead of securing your own project's job tokens from accessing other projects, the new workflow is to secure your own project from being accessed by other projects' job tokens without authorization. You can see this as an "inbound scope" for job tokens. When this new **Allow access to this project with a CI_JOB_TOKEN** setting is enabled with no other configuration, job tokens from other projects cannot **access your project**. If you want a project to have access to your own project, you must list it in the new setting's allowlist. You must be a maintainer in your own project to control the new allowlist, but you only need to have the Guest role in the other projects. This new setting is enabled by default for all new projects. + +In GitLab 16.0, the old **Limit CI_JOB_TOKEN access** setting is disabled by default for all **new** projects. In existing projects with this setting currently enabled, it will continue to function as expected, but you are unable to add any more projects to the old allowlist. If the setting is disabled in any project, it is not possible to re-enable this setting in 16.0 or later. To control access between your projects, use the new **Allow access** setting instead. + +In 17.0, we plan to remove the **Limit** setting completely, and set the **Allow access** setting to enabled for all projects. This change ensures a higher level of security between projects. If you currently use the **Limit** setting, you should update your projects to use the **Allow access** setting instead. If other projects access your project with a job token, you must add them to the **Allow access** setting's allowlist. + +To prepare for this change, users on GitLab.com or self-managed GitLab 15.9 or later can enable the **Allow access** setting now and add the other projects. It will not be possible to disable the setting in 17.0 or later. + +### Managed Licenses API + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The [Managed Licenses API](https://archives.docs.gitlab.com/15.8/ee/api/managed_licenses.html) has been removed. To enforce approvals in merge requests when non-compliant licenses are detected, use the [License Approval policy](https://docs.gitlab.com/ee/user/compliance/license_approval_policies.html) feature instead. + +Our [GraphQL APIs](https://docs.gitlab.com/ee/api/graphql/reference/) can be used to create a Security Policy Project, [update the policy.yml](https://docs.gitlab.com/ee/api/graphql/reference/#mutationscanexecutionpolicycommit) in the Security Policy Project, and enforce those policies. + +To query a list of dependencies and components, use our [Dependencies REST API](https://docs.gitlab.com/ee/api/dependencies.html) or [export from the Dependency List](https://docs.gitlab.com/ee/user/application_security/dependency_list/). + +### Maximum number of active pipelines per project limit (`ci_active_pipelines`) + +The [**Maximum number of active pipelines per project** limit](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#set-cicd-limits) has been removed. Instead, use the other recommended rate limits that offer similar protection: + +- [**Pipelines rate limits**](https://docs.gitlab.com/ee/user/admin_area/settings/rate_limit_on_pipelines_creation.html). +- [**Total number of jobs in currently active pipelines**](https://docs.gitlab.com/ee/user/admin_area/settings/continuous_integration.html#set-cicd-limits). + +### Monitoring performance metrics through Prometheus is removed + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +We previously launched a solution that allows you to view performance metrics by displaying data stored in a Prometheus instance. +The Prometheus instance can be set up as a GitLab-managed app or you can connect a previously configured Prometheus instance. +The latter is known as an "external Prometheus" in GitLab. The value we provided was to enable you to easily set up monitoring +(using GitLab Managed Apps) and have the visualization of the metrics all in the same tool you used to build the application. + +However, as we are removing certificate-based integrations, the full monitoring experience is also deprecated as you will not +have the option to easily set up Prometheus from GitLab. Furthermore, we plan to consolidate on +a focused observability dashboard experience instead of having multiple paths to view metrics. Because of this, we are also removing the external +Prometheus experience, together with the metrics visualization capability. + +This removal only refers to the GitLab Metrics capabilities, and **does not** include: + +- Deprecating [alerts for Prometheus](https://gitlab.com/gitlab-org/gitlab/-/issues/338834). +- [Capabilities that GitLab comes with that allow operators of GitLab to retrieve metrics from those instances](https://docs.gitlab.com/ee/administration/monitoring/prometheus/gitlab_metrics.html). + +### Non-expiring access tokens no longer supported + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +Currently, you can create access tokens that have no expiration date. These access tokens are valid indefinitely, which presents a security risk if the access token is +divulged. Because expiring access tokens are better, from GitLab 15.4 we [populate a default expiration date](https://gitlab.com/gitlab-org/gitlab/-/issues/348660). + +In GitLab 16.0, any personal, project, or group access token that does not have an expiration date will automatically have an expiration date set at 365 days later than the current date. + +### Non-standard default Redis ports are no longer supported + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +If GitLab starts without any Redis configuration file present, +GitLab assumes it can connect to three Redis servers at `localhost:6380`, +`localhost:6381` and `localhost:6382`. We are changing this behavior +so GitLab assumes there is one Redis server at `localhost:6379`. + +If you want to keep using the three servers, you must configure +the Redis URLs by editing the `config/redis.cache.yml`,`config/redis.queues.yml`, +and `config/redis.shared_state.yml` files. + +### PipelineSecurityReportFinding name GraphQL field + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +Previously, the [PipelineSecurityReportFinding GraphQL type was updated](https://gitlab.com/gitlab-org/gitlab/-/issues/335372) to include a new `title` field. This field is an alias for the current `name` field, making the less specific `name` field redundant. The `name` field is removed from the PipelineSecurityReportFinding type in GitLab 16.0. + +### PostgreSQL 12 compatibility + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +In GitLab 16.0, PostgreSQL 13 is the minimum supported PostgreSQL version. PostgreSQL 12 is no longer shipped with the GitLab Omnibus package. +Before upgrading to GitLab 16.0, if you are: + +- Still using GitLab-packaged PostgreSQL 12, you must [perform a database upgrade](https://docs.gitlab.com/omnibus/settings/database.html#upgrade-packaged-postgresql-server) + to PostgreSQL 13. +- Using an externally-provided PostgreSQL 12, you must upgrade to PostgreSQL 13 or later to meet the + [minimum version requirements](https://docs.gitlab.com/ee/install/requirements.html#postgresql-requirements). + +### Praefect custom metrics endpoint configuration + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +Support for using the `prometheus_exclude_database_from_default_metrics` configuration value was deprecated in +GitLab 15.9 and is removed in GitLab 16.0. We made this change to improve the performance of Praefect. +All metrics that scrape the Praefect database are now exported to the `/db_metrics` endpoint. + +You must update your metrics collection targets to use the `/db_metrics` endpoint. + +### Project REST API field `operations_access_level` removed + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +In project REST API endpoints, the `operations_access_level` field +is removed in favor of more specialized fields like: + +- `releases_access_level` +- `environments_access_level` +- `feature_flags_access_level` +- `infrastructure_access_level` +- `monitor_access_level` + +### Rake task for importing bare repositories + +The [Rake task for importing bare repositories](https://docs.gitlab.com/ee/raketasks/import.html) `gitlab:import:repos` was deprecated in GitLab 15.8 and is removed in GitLab 16.0. + +This Rake task imported a directory tree of repositories into a GitLab instance. These repositories must have been +managed by GitLab previously, because the Rake task relied on the specific directory structure or a specific custom Git setting in order to work (`gitlab.fullpath`). + +Importing repositories using this Rake task had limitations. The Rake task: + +- Only knew about project and project wiki repositories and didn't support repositories for designs, group wikis, or snippets. +- Permitted you to import non-hashed storage projects even though these aren't supported. +- Relied on having Git config `gitlab.fullpath` set. [Epic 8953](https://gitlab.com/groups/gitlab-org/-/epics/8953) proposes removing support for this setting. + +Alternatives to using the `gitlab:import:repos` Rake task include: + +- Migrating projects using either [an export file](https://docs.gitlab.com/ee/user/project/settings/import_export.html) or + [direct transfer](https://docs.gitlab.com/ee/user/group/import/#migrate-groups-by-direct-transfer-recommended) migrate repositories as well. +- Importing a [repository by URL](https://docs.gitlab.com/ee/user/project/import/repo_by_url.html). +- Importing a [repositories from a non-GitLab source](https://docs.gitlab.com/ee/user/project/import/). + +### Redis 5 compatibility + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +In GitLab 13.9, we updated the Omnibus GitLab package and GitLab Helm chart 4.9 to Redis 6. Redis 5 reached end of life in April 2022 and is not supported. + +GitLab 16.0, we have removed support for Redis 5. If you are using your own Redis 5.0 instance, you must upgrade it to Redis 6.0 or later before upgrading to GitLab 16.0 +or later. + +### Removal of job_age parameter in `POST /jobs/request` Runner endpoint + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The `job_age` parameter, returned from the `POST /jobs/request` API endpoint used in communication with GitLab Runner, has been removed in GitLab 16.0. + +This could be a breaking change for anyone that developed their own runner that relies on this parameter being returned by the endpoint. This is not a breaking change for anyone using an officially released version of GitLab Runner, including public shared runners on GitLab.com. + +### Remove legacy configuration fields in GitLab Runner Helm Chart + +In GitLab 13.6 and later, users can [specify any runner configuration in the GitLab Runner Helm chart](https://docs.gitlab.com/runner/install/kubernetes.html). When this features was released, we deprecated the fields in the GitLab Helm Chart configuration specific to the runner. As of v1.0 of the GitLab Runner Helm chart (GitLab 16.0), the following fields have been removed and are no longer supported: + +- `image` +- `rbac.resources` +- `rbac.verbs` +- `runners.image` +- `runners.imagePullSecrets` +- `runners.imagePullPolicy` +- `runners.requestConcurrency` +- `runners.privileged` +- `runners.namespace` +- `runners.pollTimeout` +- `runners.outputLimit` +- `runners.cache.cacheType` +- `runners.cache.cachePath` +- `runners.cache.cacheShared` +- `runners.cache.s3ServerAddress` +- `runners.cache.s3BucketLocation` +- `runners.cache.s3CacheInsecure` +- `runners.cache.gcsBucketName` +- `runners.builds` +- `runners.services` +- `runners.helpers` +- `runners.pod_security_context` +- `runners.serviceAccountName` +- `runners.cloneUrl` +- `runners.nodeSelector` +- `runners.nodeTolerations` +- `runners.podLabels` +- `runners.podAnnotations` +- `runners.env` + +### Remove the deprecated `environment_tier` parameter from the DORA API + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The `environment_tier` parameter has been superseded by the `environment_tiers` parameter. + +If you use the `environment_tier` parameter in your integration (REST or GraphQL) then you need to replace it with the `environment_tiers` parameter which accepts an array of strings. + +### Removed `external` field from GraphQL `ReleaseAssetLink` type + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +From GitLab 15.9, all Release links are external. The `external` field of the `ReleaseAssetLink` type was deprecated in 15.9, and removed in GitLab 16.0. + +### Removed `external` field from Releases and Release link APIs + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +From GitLab 15.9, all Release links are external. The `external` field in the Releases and Release link APIs was deprecated in 15.9, and removed in GitLab 16.0. + +### Security report schemas version 14.x.x + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +Version 14.x.x [security report schemas](https://gitlab.com/gitlab-org/security-products/security-report-schemas) have been removed. +Security reports that use schema version 14.x.x will cause an error in the pipeline's **Security** tab. For more information, refer to [security report validation](https://docs.gitlab.com/ee/user/application_security/#security-report-validation). + +### Self-monitoring project is removed + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +GitLab self-monitoring project was meant to enable self-hosted GitLab administrators to visualize performance metrics of GitLab within GitLab itself. This feature relied on GitLab Metrics dashboards. With metrics dashboard being removed, self-monitoring project is also removed. We recommended that self-hosted users monitor their GitLab instance with alternative visualization tools, such as Grafana. + +### Starboard directive in the config for the GitLab agent for Kubernetes removed + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The GitLab operational container scanning feature no longer requires you to install Starboard. The `starboard:` directive in configuration files for the GitLab agent for Kubernetes has been removed. Use the `container_scanning:` directive instead. + +### Stop publishing GitLab Runner images based on Windows Server 2004 and 20H2 + +As of GitLab 16.0, GitLab Runner images based on Windows Server 2004 and 20H2 will not be provided as these operating systems are end-of-life. + +### The Phabricator task importer + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The [Phabricator task importer](https://docs.gitlab.com/ee/user/project/import/phabricator.html) was deprecated in +GitLab 15.7 and is removed in 16.0. + +The Phabricator project hasn't been actively maintained since June 1, 2021. We haven't observed imports using this +tool. There has been no activity on the open related issues on GitLab. + +### The Security Code Scan-based GitLab SAST analyzer is now removed + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +GitLab SAST uses various [analyzers](https://docs.gitlab.com/ee/user/application_security/sast/analyzers/) to scan code for vulnerabilities. +We've reduced the number of supported analyzers used by default in GitLab SAST. +This is part of our long-term strategy to deliver a faster, more consistent user experience across different programming languages. + +As of GitLab 16.0, the [SAST CI/CD template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml) no longer uses the [Security Code Scan](https://gitlab.com/gitlab-org/security-products/analyzers/security-code-scan)-based analyzer for .NET. +We've removed this analyzer from the SAST CI/CD template and replaced it with GitLab-supported detection rules for C# in the [Semgrep-based analyzer](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep). + +Because this analyzer has reached End of Support in GitLab 16.0, we won't provide further updates to it. +However, we won't delete any container images we previously published for this analyzer or remove the ability to run it by using a [custom CI/CD pipeline job](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportssast). + +If you've already dismissed a vulnerability finding from the deprecated analyzer, the replacement attempts to respect your previous dismissal. See [Vulnerability translation documentation](https://docs.gitlab.com/ee/user/application_security/sast/analyzers.html#vulnerability-translation) for further details. + +If you customize the behavior of GitLab SAST by disabling the Semgrep-based analyzer or depending on specific SAST jobs in your pipelines, you must take action as detailed in the [deprecation issue for this change](https://gitlab.com/gitlab-org/gitlab/-/issues/390416#actions-required). + +### The stable Terraform CI/CD template has been replaced with the latest template + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +With every major GitLab version, we update the stable Terraform templates with the current latest templates. +This change affects the [quickstart](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml) +and the [base](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml) templates. + +The new templates do not change the directory to `$TF_ROOT` explicitly: `gitlab-terraform` gracefully +handles directory changing. If you altered the job scripts to assume that the current working directory is `$TF_ROOT`, you must manually add `cd "$TF_ROOT"` now. + +Because the latest template introduces Merge Request Pipeline support which is not supported in Auto DevOps, +those rules are not yet integrated into the stable template. +However, we may introduce them later on, which may break your Terraform pipelines in regards to which jobs are executed. + +To accommodate the changes, you might need to adjust the [`rules`](https://docs.gitlab.com/ee/ci/yaml/#rules) in your +`.gitlab-ci.yml` file. + +### Two DAST API variables have been removed + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The variables `DAST_API_HOST_OVERRIDE` and `DAST_API_SPECIFICATION` have been removed from use for DAST API scans. + +`DAST_API_HOST_OVERRIDE` has been removed in favor of using the `DAST_API_TARGET_URL` to automatically override the host in the OpenAPI specification. + +`DAST_API_SPECIFICATION` has been removed in favor of `DAST_API_OPENAPI`. To continue using an OpenAPI specification to guide the test, users must replace the `DAST_API_SPECIFICATION` variable with the `DAST_API_OPENAPI` variable. The value can remain the same, but the variable name must be replaced. + +### Use of `id` field in vulnerabilityFindingDismiss mutation + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +You can use the vulnerabilityFindingDismiss GraphQL mutation to set the status of a vulnerability finding to `Dismissed`. Previously, this mutation used the `id` field to identify findings uniquely. However, this did not work for dismissing findings from the pipeline security tab. Therefore, using the `id` field as an identifier has been dropped in favor of the `uuid` field. Using the 'uuid' field as an identifier allows you to dismiss the finding from the pipeline security tab. + +### Vulnerability confidence field + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +In GitLab 15.3, [security report schemas below version 15 were deprecated](https://docs.gitlab.com/ee/update/deprecations.html#security-report-schemas-version-14xx). +The `confidence` attribute on vulnerability findings exists only in schema versions before `15-0-0` and in GitLab prior to 15.4. To maintain consistency +between the reports and our public APIs, the `confidence` attribute on any vulnerability-related components of our GraphQL API is now removed. + +### `CI_BUILD_*` predefined variables removed + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The predefined CI/CD variables that start with `CI_BUILD_*` were deprecated in GitLab 9.0, and removed in GitLab 16.0. If you still use these variables, you must change to the replacement [predefined variables](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html) which are functionally identical: + +| Removed variable | Replacement variable | +| --------------------- |------------------------ | +| `CI_BUILD_BEFORE_SHA` | `CI_COMMIT_BEFORE_SHA` | +| `CI_BUILD_ID` | `CI_JOB_ID` | +| `CI_BUILD_MANUAL` | `CI_JOB_MANUAL` | +| `CI_BUILD_NAME` | `CI_JOB_NAME` | +| `CI_BUILD_REF` | `CI_COMMIT_SHA` | +| `CI_BUILD_REF_NAME` | `CI_COMMIT_REF_NAME` | +| `CI_BUILD_REF_SLUG` | `CI_COMMIT_REF_SLUG` | +| `CI_BUILD_REPO` | `CI_REPOSITORY_URL` | +| `CI_BUILD_STAGE` | `CI_JOB_STAGE` | +| `CI_BUILD_TAG` | `CI_COMMIT_TAG` | +| `CI_BUILD_TOKEN` | `CI_JOB_TOKEN` | +| `CI_BUILD_TRIGGERED` | `CI_PIPELINE_TRIGGERED` | + +### `POST ci/lint` API endpoint removed + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The `POST ci/lint` API endpoint was deprecated in 15.7, and removed in 16.0. This endpoint did not validate the full range of CI/CD configuration options. Instead, use [`POST /projects/:id/ci/lint`](https://docs.gitlab.com/ee/api/lint.html#validate-a-ci-yaml-configuration-with-a-namespace), which properly validates CI/CD configuration. + +### vulnerabilityFindingDismiss GraphQL mutation + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The `VulnerabilityFindingDismiss` GraphQL mutation has been removed. This mutation was not used often as the Vulnerability Finding ID was not available to users (this field was [deprecated in 15.3](https://docs.gitlab.com/ee/update/deprecations.html#use-of-id-field-in-vulnerabilityfindingdismiss-mutation)). Instead of `VulnerabilityFindingDismiss`, you should use `VulnerabilityDismiss` to dismiss vulnerabilities in the Vulnerability Report or `SecurityFindingDismiss` for security findings in the CI Pipeline Security tab. + +## Removed in 15.11 + +### Exporting and importing projects in JSON format not supported + +GitLab previously created project file exports in JSON format. In GitLab 12.10, NDJSON was added as the preferred format. + +To support transitions, importing JSON-formatted project file exports was still possible if you configured the +relevant feature flags. + +From GitLab 15.11, importing a JSON-formatted project file exports is not supported. + +### openSUSE Leap 15.3 packages + +Distribution support and security updates for openSUSE Leap 15.3 [ended December 2022](https://en.opensuse.org/Lifetime#Discontinued_distributions). + +Starting in GitLab 15.7 we started providing packages for openSUSE Leap 15.4, and in GitLab 15.11 we stopped providing packages for openSUSE Leap 15.3. + +To continue using GitLab, [upgrade](https://en.opensuse.org/SDB:System_upgrade) to openSUSE Leap 15.4. + +## Removed in 15.9 + +### Live Preview no longer available in the Web IDE + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The Live Preview feature of the Web IDE was intended to provide a client-side preview of static web applications. However, complex configuration steps and a narrow set of supported project types have limited its utility. With the introduction of the Web IDE Beta in GitLab 15.7, you can now connect to a full server-side runtime environment. With upcoming support for installing extensions in the Web IDE, we’ll also support more advanced workflows than those available with Live Preview. As of GitLab 15.9, Live Preview is no longer available in the Web IDE. + +### `omniauth-authentiq` gem no longer available + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +`omniauth-authentiq` is an OmniAuth strategy gem that was part of GitLab. The company providing authentication services, Authentiq, has shut down. Therefore the gem is being removed. + +### `omniauth-shibboleth` gem no longer available + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +`omniauth-shibboleth` is an OmniAuth strategy gem that was part of GitLab. The gem has not received security updates and does not meet GitLab quality guidance criteria. This gem was originally scheduled for removal by 14.1, but it was not removed at that time. The gem is being removed now. + ## Removed in 15.8 ### CiliumNetworkPolicy within the auto deploy Helm chart is removed @@ -45,6 +717,27 @@ If you want to preserve this functionality, you can follow one of these two path 1. Fork the [GitLab Auto Deploy Helm chart](https://gitlab.com/gitlab-org/cluster-integration/auto-deploy-image/-/tree/master/assets/auto-deploy-app) into the `chart/` path within your project 1. Set `AUTO_DEPLOY_IMAGE_VERSION` and `DAST_AUTO_DEPLOY_IMAGE_VERSION` to the most recent version of the image that included the CiliumNetworkPolicy +### Exporting and importing groups in JSON format not supported + +GitLab previously created group file exports in JSON format. In GitLab 13.10, NDJSON was added as the preferred format. + +To support transitions, importing JSON-formatted group file exports was still possible if you configured the +relevant feature flags. + +From GitLab 15.8, importing a JSON-formatted group file exports is not supported. + +### `artifacts:public` CI/CD keyword refactored + +WARNING: +This is a [breaking change](https://docs.gitlab.com/ee/development/deprecation_guidelines/). +Review the details carefully before upgrading. + +The [`artifacts:public` CI/CD keyword](https://docs.gitlab.com/ee/ci/yaml/#artifactspublic) was discovered to be not working properly since GitLab 15.8 and needed to be refactored. This feature is disabled on GitLab.com, and disabled by default on self-managed instances. If an administrator for an instance running GitLab 15.8 or 15.9 enabled this feature via the `non_public_artifacts` feature flag, it is likely that artifacts created with the `public:false` setting are being treated as `public:true`. + +If you have projects that use this setting, you should delete artifacts that must not be public, or [change the visibility](https://docs.gitlab.com/ee/user/public_access.html#change-project-visibility) of affected projects to private, before updating to GitLab 15.8 or later. + +In GitLab 15.10, this feature's code was refactored. On instances with this feature enabled, new artifacts created with `public:false` are now working as expected, though still disabled by default. Avoid testing this feature with production data until it is enabled by default and made generally available. + ## Removed in 15.7 ### File Type variable expansion in `.gitlab-ci.yml` @@ -189,7 +882,7 @@ If your object storage provider does not support `background_upload`, please [mi #### Encrypted S3 buckets -Additionally, this also breaks the use of [encrypted S3 buckets](https://docs.gitlab.com/ee/administration/object_storage.html#encrypted-s3-buckets) with [storage-specific configuration form](https://docs.gitlab.com/ee/administration/object_storage.html#storage-specific-configuration). +Additionally, this also breaks the use of [encrypted S3 buckets](https://docs.gitlab.com/ee/administration/object_storage.html#encrypted-s3-buckets) with [storage-specific configuration form](https://docs.gitlab.com/ee/administration/object_storage.html#configure-each-object-type-to-define-its-own-storage-connection-storage-specific-form). If your S3 buckets have [SSE-S3 or SSE-KMS encryption enabled](https://docs.aws.amazon.com/kms/latest/developerguide/services-s3.html), please [migrate your configuration to use consolidated object storage form](https://docs.gitlab.com/ee/administration/object_storage.html#transition-to-consolidated-form) before upgrading to GitLab 15.0. Otherwise, you may start getting `ETag mismatch` errors during objects upload. @@ -207,7 +900,7 @@ Some users found that they could specify a path prefix to the bucket. In direct If you have set a prefix, you can use a workaround to revert to background uploads: -1. Continue to use [storage-specific configuration](https://docs.gitlab.com/ee/administration/object_storage.html#storage-specific-configuration). +1. Continue to use [storage-specific configuration](https://docs.gitlab.com/ee/administration/object_storage.html#configure-each-object-type-to-define-its-own-storage-connection-storage-specific-form). 1. In Omnibus GitLab, set the `GITLAB_LEGACY_BACKGROUND_UPLOADS` to re-enable background uploads: ```ruby diff --git a/doc/update/restore_after_failure.md b/doc/update/restore_after_failure.md deleted file mode 100644 index 92b68410dca..00000000000 --- a/doc/update/restore_after_failure.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -stage: Systems -group: Distribution -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -remove_date: '2023-02-28' -redirect_to: '../raketasks/backup_restore.md' ---- - -# Restoring from backup after a failed upgrade (removed) **(FREE SELF)** - -This content was removed in GitLab 15.7. -Use the [backup and restore](../raketasks/backup_restore.md) documentation instead. diff --git a/doc/update/upgrading_from_ce_to_ee.md b/doc/update/upgrading_from_ce_to_ee.md index b99bb3d7992..8a66da507ec 100644 --- a/doc/update/upgrading_from_ce_to_ee.md +++ b/doc/update/upgrading_from_ce_to_ee.md @@ -2,7 +2,6 @@ stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false --- # Upgrading from Community Edition to Enterprise Edition from source **(FREE SELF)** diff --git a/doc/update/upgrading_from_source.md b/doc/update/upgrading_from_source.md index b5ce0e74100..7e2c9bf53dd 100644 --- a/doc/update/upgrading_from_source.md +++ b/doc/update/upgrading_from_source.md @@ -2,12 +2,11 @@ stage: Systems group: Distribution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -comments: false --- # Upgrading Community Edition and Enterprise Edition from source **(FREE SELF)** -Make sure you view this update guide from the branch (version) of GitLab you +Make sure you view this upgrade guide from the branch (version) of GitLab you would like to install (for example, `11.8`). You can select the required version of documentation in the dropdown list in the upper-right corner of GitLab documentation page. In each of the following examples, replace `BRANCH` with the branch of the version you upgrading to (for example, `11-8-stable` for `11.8`). Replace `PREVIOUS_BRANCH` with the @@ -59,24 +58,10 @@ sudo service gitlab stop ### 3. Update Ruby -NOTE: -Beginning in GitLab 13.6, we only support Ruby 2.7 or higher, and dropped -support for Ruby 2.6. Be sure to upgrade if necessary. - +From GitLab 15.10, we only support Ruby 3.0.x and dropped support for Ruby 2.7. Be sure to upgrade if necessary. You can check which version you are running with `ruby -v`. -Download Ruby and compile it: - -```shell -mkdir /tmp/ruby && cd /tmp/ruby -curl --remote-name --location --progress-bar "https://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.6.tar.gz" -echo 'e7203b0cc09442ed2c08936d483f8ac140ec1c72e37bb5c401646b7866cb5d10 ruby-2.7.6.tar.gz' | sha256sum -c - && tar xzf ruby-2.7.6.tar.gz -cd ruby-2.7.6 - -./configure --disable-install-rdoc --enable-shared -make -sudo make install -``` +[Install Ruby](https://www.ruby-lang.org/en/documentation/installation/). ### 4. Update Node.js @@ -146,7 +131,7 @@ Remember to set `git -> bin_path` to `/usr/local/bin/git` in `config/gitlab.yml` ### 7. Update PostgreSQL WARNING: -GitLab 14.0 requires at least PostgreSQL 12. +GitLab 16.0 requires at least PostgreSQL 13. The latest version of GitLab might depend on a more recent PostgreSQL version than what you are running. You may also have to enable some @@ -197,6 +182,8 @@ git diff origin/PREVIOUS_BRANCH:config/gitlab.yml.example origin/BRANCH:config/g #### New configuration options for `database.yml` +> [Changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/119139) in GitLab 16.0 to have `ci:` section in `config/database.yml.postgresql`. + There might be configuration options available for [`database.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/database.yml.postgresql). View them with the command below and apply them manually to your current `database.yml`: @@ -338,8 +325,8 @@ sudo -u git -H bundle exec rake "gitlab:workhorse:install[/home/git/gitlab-workh ``` NOTE: -If you get any errors concerning Rack attack, see the [13.0](#1301) specific -upgrade instructions. +If you get any errors concerning Rack attack, see the [13.1](index.md#custom-rack-attack-initializers) +specific changes. ### 13. Update Gitaly @@ -408,85 +395,12 @@ If all items are green, then congratulations, the upgrade is complete! This is an optional step. If you [installed the product documentation](../install/installation.md#install-the-product-documentation), see how to [upgrade to a later version](../administration/docs_self_host.md#upgrade-the-product-documentation-to-a-later-version). -## Version specific upgrading instructions - -This section contains upgrading instructions for specific versions. When -present, first follow the upgrading guidelines for all versions. If the version -you are upgrading to is not listed here, then no additional steps are required. - - - -### 15.9.0 - -With the addition of `gitlab-sshd` the Kerberos headers are needed to build GitLab Shell. - -```shell -sudo apt install libkrb5-dev -``` - -### 15.0.0 - -Support for more than one database has been added to GitLab. [As part of this](https://gitlab.com/gitlab-org/gitlab/-/issues/338182), -`config/database.yml` must include a database name in the database configuration. -The `main: database` must be first. If an invalid or deprecated syntax is used, an error is generated -during application start: - -```plaintext -ERROR: This installation of GitLab uses unsupported 'config/database.yml'. -The main: database needs to be defined as a first configuration item instead of primary. (RuntimeError) -``` - -Previously, the `config/database.yml` file looked like the following: - -```yaml -production: - adapter: postgresql - encoding: unicode - database: gitlabhq_production - ... -``` - -Starting with GitLab 15.0, it must define a `main` database first: - -```yaml -production: - main: - adapter: postgresql - encoding: unicode - database: gitlabhq_production - ... -``` - -### 14.5.0 - -As part of [enabling real-time issue assignees](https://gitlab.com/gitlab-org/gitlab/-/issues/330117), Action Cable is now enabled by default, and requires `config/cable.yml` to be present. -You can configure this by running: +## Version specific changes -```shell -cd /home/git/gitlab - -sudo -u git -H cp config/cable.yml.example config/cable.yml - -# Change the Redis socket path if you are not using the default Debian / Ubuntu configuration -sudo -u git -H editor config/cable.yml -``` - -### 13.0.1 - -As part of [deprecating Rack Attack throttles on Omnibus GitLab](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4750), the Rack Attack initializer on GitLab -was renamed from [`config/initializers/rack_attack_new.rb` to `config/initializers/rack_attack.rb`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/33072). -If this file exists on your installation, consider creating a backup before updating: - -```shell -cd /home/git/gitlab -cp config/initializers/rack_attack.rb ~/config/initializers/rack_attack_backup.rb -``` +Upgrading versions might need some manual intervention. For more information, +[check the version you are upgrading to](index.md#version-specific-upgrading-instructions) +for additional steps required for all GitLab installations, and for +steps that apply to self-compiled (source) installations. ## Troubleshooting diff --git a/doc/update/upgrading_postgresql_using_slony.md b/doc/update/upgrading_postgresql_using_slony.md deleted file mode 100644 index 6d2abee3fc6..00000000000 --- a/doc/update/upgrading_postgresql_using_slony.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -stage: Data Stores -group: Database -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -remove_date: '2023-02-28' -redirect_to: '../administration/postgresql/replication_and_failover.md' ---- - -# Upgrading PostgreSQL Using Slony (removed) **(FREE SELF)** - -This content was removed in GitLab 15.7. -Patroni has been used for database replication since GitLab 14.0. To perform upgrades, use the [Patroni replication documentation](../administration/postgresql/replication_and_failover.md) instead. diff --git a/doc/update/zero_downtime.md b/doc/update/zero_downtime.md index bb5cd195e5d..0eb7a520850 100644 --- a/doc/update/zero_downtime.md +++ b/doc/update/zero_downtime.md @@ -93,8 +93,8 @@ meet the other online upgrade requirements mentioned above. ## Multi-node / HA deployment WARNING: -You can only upgrade one minor release at a time. So from 13.6 to 13.7, not to 13.8. -If you attempt more than one minor release, the upgrade may fail. +You can only upgrade one minor release at a time. So from 15.6 to 15.7, not to 15.8. +If you attempt more than one minor release, the upgrade may fail. ### Use a load balancer in front of web (Puma) nodes @@ -116,7 +116,7 @@ continue to accept connections but mark their respective health check endpoints to be unhealthy. On seeing this, the load balancer should disconnect them gracefully. -Puma restarts only after completing all the currently processing requests. +Puma restarts only after completing all the currently-processing requests. This ensures data and service integrity. Once they have restarted, the health check end points are marked healthy. @@ -180,6 +180,9 @@ Before you update the main GitLab application you must (in order): 1. Upgrade the Gitaly nodes that reside on separate servers. 1. Upgrade Praefect if using Gitaly Cluster. +Because of a [known issue](https://gitlab.com/groups/gitlab-org/-/epics/10328), Gitaly and Gitaly Cluster upgrades +cause some downtime. + #### Upgrade Gitaly nodes [Upgrade the GitLab package](package/index.md#upgrade-to-a-specific-version-using-the-official-repositories) on the Gitaly nodes one at a time to ensure access to Git repositories is maintained. @@ -465,6 +468,9 @@ Log in to your **primary** node, executing the following: sudo SKIP_POST_DEPLOYMENT_MIGRATIONS=true gitlab-rake db:migrate ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the primary site to the secondary site if they're different. + The file must be the same on all of a site's nodes. + ### Update the Geo secondary site On each **secondary** node, executing the following: @@ -538,7 +544,7 @@ setting `gitlab_rails['auto_migrate'] = false` in ## Multi-node / HA deployment with Geo **(PREMIUM SELF)** WARNING: -You can only upgrade one minor release at a time. +You can only upgrade one minor release at a time. You also must first start with the Gitaly cluster, updating Gitaly one node one at a time. This will ensure access to the Git repositories for the remainder of the upgrade process. This section describes the steps required to upgrade a multi-node / HA deployment with Geo. Some steps must be performed on a particular node. This @@ -665,6 +671,9 @@ sudo gitlab-ctl hup puma sudo gitlab-ctl restart sidekiq ``` +1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the primary site to the secondary site if they're different. The + file must be the same on all of a site's nodes. + ### Step 3: Update each Geo secondary multi-node deployment Only proceed if you have successfully completed all steps on the Geo **primary** multi-node deployment. diff --git a/doc/user/admin_area/analytics/dev_ops_reports.md b/doc/user/admin_area/analytics/dev_ops_reports.md index 2d19c0a0058..31cc9825452 100644 --- a/doc/user/admin_area/analytics/dev_ops_reports.md +++ b/doc/user/admin_area/analytics/dev_ops_reports.md @@ -39,7 +39,7 @@ feature is available. ## DevOps Adoption **(ULTIMATE SELF)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/247112) in GitLab 13.7 as a [Beta feature](../../../policy/alpha-beta-support.md#beta-features). +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/247112) in GitLab 13.7 as a [Beta feature](../../../policy/alpha-beta-support.md#beta). > - The Overview tab [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/330401) in GitLab 14.1. > - DAST and SAST metrics [added](https://gitlab.com/gitlab-org/gitlab/-/issues/328033) in GitLab 14.1. > - Fuzz Testing metrics [added](https://gitlab.com/gitlab-org/gitlab/-/issues/330398) in GitLab 14.2. diff --git a/doc/user/admin_area/analytics/index.md b/doc/user/admin_area/analytics/index.md index 4304e612e4a..2ac8941b286 100644 --- a/doc/user/admin_area/analytics/index.md +++ b/doc/user/admin_area/analytics/index.md @@ -14,7 +14,7 @@ Instance-level analytics provide insights into the feature and data usage of you Prerequisite: -- You must have administrator access for your instance. +- You must have administrator access to the instance. To view instance-level analytics: diff --git a/doc/user/admin_area/appearance.md b/doc/user/admin_area/appearance.md index a1fae7e8712..a5311b083c3 100644 --- a/doc/user/admin_area/appearance.md +++ b/doc/user/admin_area/appearance.md @@ -2,7 +2,6 @@ stage: none group: unassigned info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -disqus_identifier: 'https://docs.gitlab.com/ee/customization/branded_login_page.html' --- # GitLab Appearance **(FREE SELF)** @@ -71,6 +70,27 @@ to review the saved appearance settings: NOTE: You can add also add a [customized help message](settings/help_page.md) below the sign in message or add [a Sign in text message](settings/sign_in_restrictions.md#sign-in-information). +## Progressive Web App + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/375708) in GitLab 15.9. + +GitLab can be installed as a [Progressive Web App](https://developer.mozilla.org/en-US/docs/Web/Progressive_web_apps) (PWA). +Use the Progressive Web App settings to customize its appearance, including its name, +description, and icon. + +### Configure the PWA settings + +To configure the PWA settings: + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Settings > Appearance**. +1. Scroll to the **Progressive Web App (PWA)** section. +1. Complete the fields. + - **Icon**: If you use the standard GitLab icon, it is available in sizes 192x192 pixels, + 512x512 pixels, also as a maskable icon. If you use a custom icon, it must be in either size + 192x192 pixels, or 512x512 pixels. +1. Select **Update appearance settings**. + ## New project pages You can add a new project guidelines message to the **New project page** in GitLab. diff --git a/doc/user/admin_area/custom_project_templates.md b/doc/user/admin_area/custom_project_templates.md index 847f687d051..9d360539595 100644 --- a/doc/user/admin_area/custom_project_templates.md +++ b/doc/user/admin_area/custom_project_templates.md @@ -1,8 +1,7 @@ --- -stage: Manage -group: Import +stage: Create +group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -type: reference --- # Custom instance-level project templates **(PREMIUM SELF)** @@ -18,7 +17,7 @@ is created, based on the user's access permissions: - Public projects can be selected by any authenticated user as a template for a new project, if all enabled [project features](../project/settings/index.md#configure-project-visibility-features-and-permissions) - except for **GitLab Pages** and **Security & Compliance** are set to **Everyone With Access**. + except for **GitLab Pages** and **Security and Compliance** are set to **Everyone With Access**. The same applies to internal projects. - Private projects can be selected only by users who are members of the projects. @@ -41,6 +40,24 @@ To select the group to use as the source for the project templates: Projects in subgroups of the template group are **not** included in the template list. +## What is copied from the templates + +The entire custom instance-level project templates repository is copied, including: + +- Branches +- Commits +- Tags + +If the user: + +- Has the Owner role on the custom instance-level project templates project or is a GitLab administrator, all project settings are copied over to the new + project. +- Doesn't have the Owner role or is not a GitLab administrator, project [deploy keys](../project/deploy_keys/index.md#view-deploy-keys) and project + [webhooks](../project/integrations/webhooks.md) aren't copied over because they contain sensitive data. + +To learn more about what is migrated, see +[Items that are exported](../project/settings/import_export.md#items-that-are-exported). + - - - diff --git a/doc/user/admin_area/monitoring/health_check.md b/doc/user/admin_area/monitoring/health_check.md index 668d34af024..f3b09c61532 100644 --- a/doc/user/admin_area/monitoring/health_check.md +++ b/doc/user/admin_area/monitoring/health_check.md @@ -92,8 +92,6 @@ Example response: On failure, the endpoint returns a `503` HTTP status code. -This check does hit the database and Redis if authenticated via `token`. - This check is being exempt from Rack Attack. ## Liveness diff --git a/doc/user/admin_area/reporting/git_abuse_rate_limit.md b/doc/user/admin_area/reporting/git_abuse_rate_limit.md index 66d1173058e..83b28404714 100644 --- a/doc/user/admin_area/reporting/git_abuse_rate_limit.md +++ b/doc/user/admin_area/reporting/git_abuse_rate_limit.md @@ -4,20 +4,16 @@ group: Anti-Abuse info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Git abuse rate limit (administration) **(ULTIMATE)** +# Git abuse rate limit (administration) **(ULTIMATE SELF)** -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/8066) in GitLab 15.2 [with a flag](../../../administration/feature_flags.md) named `git_abuse_rate_limit_feature_flag`. Disabled by default. +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/8066) in GitLab 15.2 [with a flag](../../../administration/feature_flags.md) named `git_abuse_rate_limit_feature_flag`. Disabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/394996) in GitLab 15.10. Feature flag `git_abuse_rate_limit_feature_flag` removed. -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `git_abuse_rate_limit_feature_flag`. On GitLab.com, this feature is available. +This is the administration documentation. For information about Git abuse rate limiting at the group level, see the [group-level documentation](../../group/reporting/git_abuse_rate_limit.md). -Git abuse rate limiting is a feature to automatically [ban users](../moderate_users.md#ban-and-unban-users) who download or clone more than a specified number of repositories in any project in the instance within a given time frame. Banned users cannot sign in to the instance and cannot access any non-public group via HTTP or SSH. +Git abuse rate limiting is a feature to automatically [ban users](../moderate_users.md#ban-and-unban-users) who download, clone, or fork more than a specified number of repositories in any project in the instance in a given time frame. Banned users cannot sign in to the instance and cannot access any non-public group via HTTP or SSH. The rate limit also applies to users who authenticate with a [personal](../../../user/profile/personal_access_tokens.md) or [group access token](../../../user/group/settings/group_access_tokens.md). -If the `git_abuse_rate_limit_feature_flag` feature flag is enabled, all application administrators receive an email when a user is about to be banned. - -If automatic banning is disabled, a user is not banned automatically when they exceed the limit. However, administrators are still notified. You can use this setup to determine the correct values of the rate limit settings before enabling automatic banning. - -If automatic banning is enabled, administrators receive an email when a user is about to be banned, and the user is automatically banned from the GitLab instance. +Git abuse rate limiting does not apply to instance administrators, [deploy tokens](../../../user/project/deploy_tokens/index.md), or [deploy keys](../../../user/project/deploy_keys/index.md). ## Configure Git abuse rate limiting @@ -28,9 +24,16 @@ If automatic banning is enabled, administrators receive an email when a user is 1. Enter a number in the **Number of repositories** field, greater than or equal to `0` and less than or equal to `10,000`. This number specifies the maximum amount of unique repositories a user can download in the specified time period before they're banned. When set to `0`, Git abuse rate limiting is disabled. 1. Enter a number in the **Reporting time period (seconds)** field, greater than or equal to `0` and less than or equal to `86,400` (10 days). This number specifies the time in seconds a user can download the maximum amount of repositories before they're banned. When set to `0`, Git abuse rate limiting is disabled. 1. Optional. Exclude up to `100` users by adding them to the **Excluded users** field. Excluded users are not automatically banned. + 1. Add up to `100` users to the **Send notifications to** field. You must select at least one user. All application administrators are selected by default. 1. Optional. Turn on the **Automatically ban users from this namespace when they exceed the specified limits** toggle to enable automatic banning. 1. Select **Save changes**. +## Automatic ban notifications + +If automatic banning is disabled, a user is not banned automatically when they exceed the limit. However, notifications are still sent to the users listed under **Send notifications to**. You can use this setup to determine the correct values of the rate limit settings before enabling automatic banning. + +If automatic banning is enabled, an email notification is sent when a user is about to be banned, and the user is automatically banned from the GitLab instance. + ## Unban a user 1. On the top bar, select **Main menu > Admin**. diff --git a/doc/user/admin_area/reporting/spamcheck.md b/doc/user/admin_area/reporting/spamcheck.md index 5c305eff4fa..16c144d2469 100644 --- a/doc/user/admin_area/reporting/spamcheck.md +++ b/doc/user/admin_area/reporting/spamcheck.md @@ -21,15 +21,15 @@ Spamcheck is only available for package-based installations: 1. Edit `/etc/gitlab/gitlab.rb` and enable Spamcheck: - ```ruby - spamcheck['enable'] = true - ``` + ```ruby + spamcheck['enable'] = true + ``` 1. Reconfigure GitLab: - ```shell - sudo gitlab-ctl reconfigure - ``` + ```shell + sudo gitlab-ctl reconfigure + ``` 1. Verify that the new services `spamcheck` and `spam-classifier` are up and running: diff --git a/doc/user/admin_area/review_abuse_reports.md b/doc/user/admin_area/review_abuse_reports.md index b8531fded18..314e0c77f36 100644 --- a/doc/user/admin_area/review_abuse_reports.md +++ b/doc/user/admin_area/review_abuse_reports.md @@ -1,6 +1,6 @@ --- -stage: Manage -group: Authentication and Authorization +stage: Anti-Abuse +group: Anti-Abuse info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: reference, howto --- diff --git a/doc/user/admin_area/settings/account_and_limit_settings.md b/doc/user/admin_area/settings/account_and_limit_settings.md index 35a4c0aeea7..5c730375f98 100644 --- a/doc/user/admin_area/settings/account_and_limit_settings.md +++ b/doc/user/admin_area/settings/account_and_limit_settings.md @@ -36,7 +36,9 @@ can create in their personal namespace: ## Max attachment size -The maximum file size for attachments in GitLab comments and replies is 10 MB. +> [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/20061) from 10 MB to 100 MB in GitLab 15.7. + +The maximum file size for attachments in GitLab comments and replies is 100 MB. To change the maximum attachment size: 1. On the top bar, select **Main menu > Admin**. @@ -174,7 +176,32 @@ wiki, packages, or snippets. The repository size limit applies to both private a For details on manually purging files, see [reducing the repository size using Git](../../project/repository/reducing_the_repo_size_using_git.md). -## Customize session duration for Git Operations when 2FA is enabled **(PREMIUM SELF)** +## Session duration + +### Customize the default session duration + +You can change how long users can remain signed in without activity. + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Settings > General**. +1. Expand **Account and limit**. The set duration is in **Session duration (minutes)**. + +If [Remember me](#turn-remember-me-on-or-off) is enabled, users' sessions can remain active for an indefinite period of time. + +For details, see [cookies used for sign-in](../../profile/index.md#cookies-used-for-sign-in). + +### Turn **Remember me** on or off + +> Ability to turn the **Remember me** setting on and off [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/369133) in GitLab 16.0. + +Users can select the **Remember me** checkbox on sign-in, and their session will remain active for an indefinite period of time when accessed from that specific browser. You can turn off this setting if you need sessions to expire for security or compliance purposes. Turning off this setting will ensure users' sessions expire after the number of minutes of inactivity set when you [customize your session duration](#customize-the-default-session-duration). + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Settings > General**. +1. Expand **Account and limit**. +1. Select or clear the **Remember me** checkbox to turn this setting on or off. + +### Customize session duration for Git Operations when 2FA is enabled **(PREMIUM SELF)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/296669) in GitLab 13.9. > - It's deployed behind a feature flag, disabled by default. @@ -305,6 +332,17 @@ By default, newly created users have a public profile. GitLab administrators can 1. On the left sidebar, select **Settings > General**, then expand **Account and limit**. 1. Select the **Make new users' profiles private by default** checkbox. +## Prevent users from deleting their accounts **(PREMIUM SELF)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/26053) in GitLab 16.0 [with a flag](../../../administration/feature_flags.md) named `deleting_account_disabled_for_users`. Disabled by default. + +By default, users can delete their own accounts. GitLab administrators can prevent +users from deleting their own accounts: + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Settings > General**, then expand **Account and limit**. +1. Clear the **Allows users to delete their own accounts** checkbox. + ## Troubleshooting ### 413 Request Entity Too Large diff --git a/doc/user/admin_area/settings/continuous_integration.md b/doc/user/admin_area/settings/continuous_integration.md index aa171fe4536..27af64cd0e8 100644 --- a/doc/user/admin_area/settings/continuous_integration.md +++ b/doc/user/admin_area/settings/continuous_integration.md @@ -54,7 +54,7 @@ To enable a project runner for more than one project: 1. On the top bar, select **Main menu > Admin**. 1. From the left sidebar, select **CI/CD > Runners**. 1. Select the runner you want to edit. -1. In the upper right, select **Edit** (**{pencil}**). +1. In the upper-right corner, select **Edit** (**{pencil}**). 1. Under **Restrict projects for this runner**, search for a project. 1. To the left of the project, select **Enable**. 1. Repeat this process for each additional project. @@ -148,7 +148,7 @@ are locked against deletion and kept regardless of the expiry time. When disabled, the latest artifacts for any **new** successful or fixed pipelines are allowed to expire. -This setting takes precedence over the [project level setting](../../../ci/pipelines/job_artifacts.md#keep-artifacts-from-most-recent-successful-jobs). +This setting takes precedence over the [project level setting](../../../ci/jobs/job_artifacts.md#keep-artifacts-from-most-recent-successful-jobs). If disabled at the instance level, you cannot enable this per-project. To disable the setting: @@ -194,6 +194,16 @@ To set all new [CI/CD variables](../../../ci/variables/index.md) as 1. On the left sidebar, select **Settings > CI/CD**. 1. Select **Protect CI/CD variables by default**. +## Maximum includes + +The maximum number of [includes](../../../ci/yaml/includes.md) per pipeline can be set at the instance level. +The default is `150`. + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Settings > CI/CD**. +1. Change the value of **Maximum includes**. +1. Select **Save changes** for the changes to take effect. + ## Default CI/CD configuration file > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18073) in GitLab 12.5. @@ -227,6 +237,7 @@ from the Admin Area: - **Maximum number of DAG dependencies that a job can have** - **Maximum number of runners registered per group** - **Maximum number of runners registered per project** + - **Maximum number of downstream pipelines in a pipeline's hierarchy tree** ## Enable or disable the pipeline suggestion banner @@ -244,12 +255,13 @@ To enable or disable the banner: ## Required pipeline configuration **(ULTIMATE SELF)** -> [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/352316) from GitLab Premium to GitLab Ultimate in 15.0. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/352316) from GitLab Premium to GitLab Ultimate in 15.0. +> - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/389467) in GitLab 15.9. -NOTE: -An alternative [compliance solution](../../group/compliance_frameworks.md#compliance-pipelines) -is available. We recommend this alternative solution because it provides greater flexibility, -allowing required pipelines to be assigned to specific compliance framework labels. +WARNING: +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/389467) in GitLab 15.9 +and is planned for removal in 17.0. Use [compliance pipelines](../../group/compliance_frameworks.md#compliance-pipelines) +instead. This change is a breaking change. You can set a [CI/CD template](../../../ci/examples/index.md#cicd-templates) as a required pipeline configuration for all projects on a GitLab instance. You can @@ -267,7 +279,7 @@ use a template from: The project CI/CD configuration merges into the required pipeline configuration when a pipeline runs. The merged configuration is the same as if the required pipeline configuration added the project configuration with the [`include` keyword](../../../ci/yaml/index.md#include). -To view a project's full merged configuration, [View the merged YAML](../../../ci/pipeline_editor/index.md#view-expanded-configuration) +To view a project's full merged configuration, [View full configuration](../../../ci/pipeline_editor/index.md#view-full-configuration) in the pipeline editor. To select a CI/CD template for the required pipeline configuration: @@ -344,9 +356,9 @@ To restrict all users in an instance from registering runners: 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Settings > CI/CD**. -1. Expand **Runner registration**. -1. Clear the checkbox if you don't want to display runner registration - information in the UI for group or project members. +1. Expand **Runners**. +1. In the **Runner registration** section, clear the **Members of the project can register runners** and + **Members of the group can register runners** checkboxes to remove runner registration from the UI. 1. Select **Save changes**. NOTE: @@ -370,6 +382,20 @@ To restrict runner registration by members in a specific group: 1. Clear the **New group runners can be registered** checkbox if you want to disable runner registration by all members in the group. If the setting is read-only, you must enable runner registration for the [instance](#restrict-runner-registration-by-all-users-in-an-instance). 1. Select **Save changes**. +## Disable runner version management + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/114041) in GitLab 15.10. + +By default, GitLab instances periodically fetch official runner version data from GitLab.com to [determine whether the runners need upgrades](../../../ci/runners/configure_runners.md#determine-which-runners-need-to-be-upgraded). + +To disable your instance fetching this data: + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **Runners**. +1. In the **Runner version management** section, clear the **Fetch GitLab Runner release version data from GitLab.com** checkbox. +1. Select **Save changes**. + ## Troubleshooting ### 413 Request Entity Too Large diff --git a/doc/user/admin_area/settings/deprecated_api_rate_limits.md b/doc/user/admin_area/settings/deprecated_api_rate_limits.md index 8bf0ffd21a5..13f8bc008e3 100644 --- a/doc/user/admin_area/settings/deprecated_api_rate_limits.md +++ b/doc/user/admin_area/settings/deprecated_api_rate_limits.md @@ -28,9 +28,9 @@ the general user and IP rate limits for requests to deprecated endpoints. You ca and IP rate limits already in place, and increase or decrease the rate limits for deprecated API endpoints. No other new features are provided by this override. -Prerequisites: +Prerequisite: -- You must have administrator access for your instance. +- You must have administrator access to the instance. To override the general user and IP rate limits for requests to deprecated API endpoints: diff --git a/doc/user/admin_area/settings/email.md b/doc/user/admin_area/settings/email.md index 484f51d8739..90852463e9d 100644 --- a/doc/user/admin_area/settings/email.md +++ b/doc/user/admin_area/settings/email.md @@ -86,12 +86,13 @@ To disable these notifications: ### Custom additional text in deactivation emails **(FREE SELF)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355964) in GitLab 15.9 [with a flag](../../../administration/feature_flags.md) named `deactivation_email_additional_text`. Disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355964) in GitLab 15.9 [with a flag](../../../administration/feature_flags.md) named `deactivation_email_additional_text`. Disabled by default. +> - [Enabled on self-managed and GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/111882) in GitLab 15.9. FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an -administrator to [enable the feature flag](../../../administration/feature_flags.md) named -`deactivation_email_additional_text`. On GitLab.com, this feature is unavailable. +On self-managed GitLab, by default this feature is available. To hide the feature, ask an +administrator to [disable the feature flag](../../../administration/feature_flags.md) named +`deactivation_email_additional_text`. You can add additional text at the bottom of the email that GitLab sends to users when their account is deactivated. This email text is separate from the [custom additional text](#custom-additional-text) diff --git a/doc/user/admin_area/settings/external_authorization.md b/doc/user/admin_area/settings/external_authorization.md index 94d9ec73640..072873ba7f6 100644 --- a/doc/user/admin_area/settings/external_authorization.md +++ b/doc/user/admin_area/settings/external_authorization.md @@ -13,8 +13,6 @@ controlled by an external service that permits access based on project classification and user access. GitLab provides a way to check project authorization with your own defined service. -## Overview - After the external service is configured and enabled, when a project is accessed, a request is made to the external service with the user information and project classification label assigned to the project. When the service @@ -39,13 +37,10 @@ the [Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/logs When using TLS Authentication with a self signed certificate, the CA certificate needs to be trusted by the OpenSSL installation. When using GitLab installed using Omnibus, learn to install a custom CA in the -[Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/ssl.html). +[Omnibus GitLab documentation](https://docs.gitlab.com/omnibus/settings/ssl/index.html). Alternatively, learn where to install custom certificates by using `openssl version -d`. -When external authorization is enabled, [deploy tokens](../../project/deploy_tokens/index.md) - and [deploy keys](../../project/deploy_keys/index.md) can't be used for Git operations. - ## Configuration The external authorization service can be enabled by an administrator: @@ -56,6 +51,30 @@ The external authorization service can be enabled by an administrator: 1. Complete the fields. 1. Select **Save changes**. +### Allow external authorization with deploy tokens and deploy keys + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/386656) in GitLab 15.9. +> - Deploy tokens no longer being able to access container or package registries [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/387721) in GitLab 16.0. + +You can set your instance to allow external authorization for Git operations with +[deploy tokens](../../project/deploy_tokens/index.md) or [deploy keys](../../project/deploy_keys/index.md). + +Prerequisites: + +- You must be using classification labels without a service URL for external authorization. + +To allow authorization with deploy tokens and keys: + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Settings > General**. +1. Expand **External authorization**, and: + - Leave the service URL field empty. + - Select **Allow deploy tokens and deploy keys to be used with external authorization**. +1. Select **Save changes**. + +WARNING: +If you enable external authorization, deploy tokens cannot access container or package registries. If you use deploy tokens to access these registries, this measure breaks this use of these tokens. Disable external authorization to use tokens with container or package registries. + ## How it works When GitLab requests access, it sends a JSON POST request to the external @@ -106,7 +125,7 @@ You can use your own classification label in the project's label" box. When no classification label is specified on a project, the default label defined in the [global settings](#configuration) is used. -The label is shown on all project pages in the upper right corner. +On all project pages, in the upper-right corner, the label appears. ![classification label on project page](img/classification_label_on_project_page_v14_8.png) diff --git a/doc/user/admin_area/settings/files_api_rate_limits.md b/doc/user/admin_area/settings/files_api_rate_limits.md index ef9a3674c49..8677e3d86bf 100644 --- a/doc/user/admin_area/settings/files_api_rate_limits.md +++ b/doc/user/admin_area/settings/files_api_rate_limits.md @@ -26,7 +26,7 @@ for the Files API. No other new features are provided by this override. Prerequisite: -- You must have administrator access for your instance. +- You must have administrator access to the instance. To override the general user and IP rate limits for requests to the Repository files API: diff --git a/doc/user/admin_area/settings/help_page.md b/doc/user/admin_area/settings/help_page.md index 07d3ae83d74..5d9fc23aaff 100644 --- a/doc/user/admin_area/settings/help_page.md +++ b/doc/user/admin_area/settings/help_page.md @@ -71,14 +71,7 @@ You can specify a custom URL to which users are directed when they: > - [Feature flag `help_page_documentation_redirect`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71737) removed in GitLab 14.4. > - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71737) in GitLab 14.4. -The `/help` URL of a GitLab instance displays a basic version of the documentation sourced from the -[`doc` directory](https://gitlab.com/gitlab-org/gitlab/-/tree/master/doc) of GitLab. `/help` links -are often used for contextual help. - -You can redirect these `/help` links to either: - -- The more navigable and searchable version published at [`docs.gitlab.com`](https://docs.gitlab.com). -- A destination that meets [necessary requirements](#destination-requirements). +You can redirect all `/help` links to a destination that meets the [necessary requirements](#destination-requirements). 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Settings > Preferences**. @@ -86,17 +79,19 @@ You can redirect these `/help` links to either: 1. In the **Documentation pages URL** field, enter the URL. 1. Select **Save changes**. +If the "Documentation pages URL" field is empty, the GitLab instance displays a basic version of the documentation sourced from the [`doc` directory](https://gitlab.com/gitlab-org/gitlab/-/tree/master/doc) of GitLab. + ### Destination requirements When redirecting `/help`, GitLab: - Redirects requests to the specified URL. -- Appends `ee` and the documentation path to the URL. +- Appends `ee` and the documentation path, which includes the version number, to the URL. - Appends `.html` to the URL, and removes `.md` if necessary. For example, if the URL is set to `https://docs.gitlab.com`, requests for `/help/user/admin_area/settings/help_page.md` redirect to: -`https://docs.gitlab.com/ee/user/admin_area/settings/help_page.html`. +`https://docs.gitlab.com/${VERSION}/ee/user/admin_area/settings/help_page.html`. + + + diff --git a/doc/user/analytics/value_streams_dashboard.md b/doc/user/analytics/value_streams_dashboard.md index f8cfb1bf06b..384f4ce3455 100644 --- a/doc/user/analytics/value_streams_dashboard.md +++ b/doc/user/analytics/value_streams_dashboard.md @@ -4,21 +4,18 @@ group: Optimize info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Value Streams Dashboard **(PREMIUM)** +# Value Streams Dashboard (Beta) **(ULTIMATE)** -> Introduced in GitLab 15.8 as a Closed [Beta](../../policy/alpha-beta-support.md#beta-features) feature. +> - Introduced in GitLab 15.8 as a Closed [Beta](../../policy/alpha-beta-support.md#beta) feature [with a flag](../../administration/feature_flags.md) named `group_analytics_dashboards_page`. Disabled by default. +> - Released in GitLab 15.11 as an Open [Beta](../../policy/alpha-beta-support.md#beta) feature [with a flag](../../administration/feature_flags.md) named `group_analytics_dashboards_page`. Enabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/392734) in GitLab 16.0. Feature flag `group_analytics_dashboards_page` removed. You can leave feedback on dashboard bugs or functionality in [issue 381787](https://gitlab.com/gitlab-org/gitlab/-/issues/381787). -This feature is not ready for production use. - -The Value Streams Dashboard is a customizable dashboard to enable decision-makers to identify trends, patterns, and opportunities for digital transformation improvements. +The Value Streams Dashboard is a customizable dashboard that enables decision-makers to identify trends, patterns, and opportunities for digital transformation improvements. This page is a work in progress, and we're updating the information as we add more features. For more information, see the [Value Stream Management category direction page](https://about.gitlab.com/direction/plan/value_stream_management/). -After the feature flag is enabled, to open the new page, append this path `/analytics/dashboards` to the group URL -(for example, `https://gitlab.com/groups/gitlab-org/-/analytics/dashboards`). - ## Initial use case Our initial use case is focused on providing the ability to compare software delivery metrics. @@ -27,7 +24,7 @@ This comparison can help decision-makers understand whether projects and groups The beta version of the Value Streams Dashboard includes the following metrics: - [DORA metrics](dora_metrics.md) -- [Value Stream Analytics (VSA) - flow metrics](value_stream_analytics.md) +- [Value Stream Analytics (VSA) - flow metrics](../group/value_stream_analytics/index.md) The Value Streams Dashboard allows you to: @@ -49,20 +46,91 @@ that are the largest value contributors, overperforming, or underperforming. You can also drill down the metrics for further analysis. When you hover over a metric, a tooltip displays an explanation of the metric and a link to the related documentation page. -## Customize the dashboard widgets +## View the value streams dashboard + +Prerequisite: + +- To view the value streams dashboard for a group, you must have at least the Reporter role for the group. + +To view the value streams dashboard: + +1. On the top bar, select **Main menu**, and: + - For a project, select **Projects** and find your project. + - For a group, select **Groups** and find your group. +1. On the left sidebar, select **Analytics > Value stream**. +1. Below the **Filter results** text box, in the **Key metrics** row, select **Value Streams Dashboard / DORA**. +1. Optional. To open the new page, append this path `/analytics/dashboards/value_streams_dashboard` to the group URL +(for example, `https://gitlab.com/groups/gitlab-org/-/analytics/dashboards/value_streams_dashboard`). + +## Customize the dashboard panels You can customize the Value Streams Dashboard and configure what subgroups and projects to include in the page. A view can display maximum four subgroups or projects. +### Using query parameters + To display multiple subgroups and projects, specify their path as a URL parameter. -For example, the parameter `query=gitlab-org/gitlab-foss,gitlab-org/gitlab,gitlab-org/gitlab-design,gitlab-org/gitlab-docs` displays three separate widgets, one each for the: +For example, the parameter `query=gitlab-org/gitlab-ui,gitlab-org/plan-stage` displays three separate panels, one each for the: - `gitlab-org` group - `gitlab-ui` project - `gitlab-org/plan-stage` subgroup +### Using YAML configuration + +To change the default content of the page, you need to create a YAML configuration file in a project of your choice. Query parameters can still be used to override the YAML configuration. + +First, you need to set up the project. + +Prerequisite: + +- You must have at least the Maintainer role for the group. + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Settings > General**. +1. Scroll to **Analytics Dashboards** and select **Expand**. +1. Select the project where you would like to store your YAML configuration file. +1. Select **Save changes**. + +After you have set up the project, set up the configuration file: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. In the default branch, create the configuration file: `.gitlab/analytics/dashboards/value_streams/value_streams.yaml`. +1. In the `value_streams.yaml` configuration file, fill in the configuration options: + +```yaml +# title - Change the title of the Value Streams Dashboard. [optional] +title: 'Custom Dashboard title' + +# description - Change the description of the Value Streams Dashboard. [optional] +description: 'Custom description' + +# panels - List of panels that contain panel settings. +# title - Change the title of the panel. [optional] +# data.namespace - The Group or Project path to use for the chart panel. +panels: + - title: 'My Custom Project' + data: + namespace: group/my-custom-project + - data: + namespace: group/another-project + - title: 'My Custom Group' + data: + namespace: group/my-custom-group + - data: + namespace: group/another-group +``` + + The following example has an option configuration for a panel for the `my-group` namespace: + + ```yaml + panels: + - data: + namespace: my-group + ``` + ## Dashboard metrics and drill-down reports | Metric | Description | Drill-down report | Documentation page | @@ -71,7 +139,9 @@ For example, the parameter `query=gitlab-org/gitlab-foss,gitlab-org/gitlab,gitla | Lead time for changes | The time to successfully deliver a commit into production. This metric reflects the efficiency of CI/CD pipelines. | [Lead time tab](https://gitlab.com/groups/gitlab-org/-/analytics/ci_cd?tab=lead-time) | [Lead time for changes](dora_metrics.md#lead-time-for-changes) | | Time to restore service | The time it takes an organization to recover from a failure in production. | [Time to restore service tab](https://gitlab.com/groups/gitlab-org/-/analytics/ci_cd?tab=time-to-restore-service) | [Time to restore service](dora_metrics.md#time-to-restore-service) | | Change failure rate | Percentage of deployments that cause an incident in production. | [Change failure rate tab](https://gitlab.com/groups/gitlab-org/-/analytics/ci_cd?tab=change-failure-rate) | [Change failure rate](dora_metrics.md#change-failure-rate) | -| VSA Lead time | Median time from issue created to issue closed. | [Value Stream Analytics](https://gitlab.com/groups/gitlab-org/-/analytics/value_stream_analytics) | [View the lead time and cycle time for issues](value_stream_analytics.md#view-the-lead-time-and-cycle-time-for-issues) | -| VSA Cycle time | Median time from the earliest commit of a linked issue's merge request to when that issue is closed. | [VSA overview](https://gitlab.com/groups/gitlab-org/-/analytics/value_stream_analytics) | [View the lead time and cycle time for issues](value_stream_analytics.md#view-the-lead-time-and-cycle-time-for-issues) | +| Lead time | Median time from issue created to issue closed. | [Value Stream Analytics](https://gitlab.com/groups/gitlab-org/-/analytics/value_stream_analytics) | [View the lead time and cycle time for issues](../group/value_stream_analytics/index.md#key-metrics) | +| Cycle time | Median time from the earliest commit of a linked issue's merge request to when that issue is closed. | [VSA overview](https://gitlab.com/groups/gitlab-org/-/analytics/value_stream_analytics) | [View the lead time and cycle time for issues](../group/value_stream_analytics/index.md#key-metrics) | | New issues | Number of new issues created. | [Issue Analytics](https://gitlab.com/groups/gitlab-org/-/issues_analytics) | Issue analytics [for projects](issue_analytics.md) and [for groups](../../user/group/issues_analytics/index.md) | | Number of deploys | Total number of deploys to production. | [Merge Request Analytics](https://gitlab.com/gitlab-org/gitlab/-/analytics/merge_request_analytics) | [Merge request analytics](merge_request_analytics.md) | +| Critical vulnerabilities over time | Critical vulnerabilities over time in project or group | [Vulnerability report](https://gitlab.com/gitlab-org/gitlab/-/security/vulnerability_report) | [Vulnerability report](../application_security/vulnerability_report/index.md) | +| High vulnerabilities over time | High vulnerabilities over time in project or group | [Vulnerability report](https://gitlab.com/gitlab-org/gitlab/-/security/vulnerability_report) | [Vulnerability report](../application_security/vulnerability_report/index.md) | diff --git a/doc/user/application_security/api_fuzzing/index.md b/doc/user/application_security/api_fuzzing/index.md index b55c5a1b299..b613b0cc33e 100644 --- a/doc/user/application_security/api_fuzzing/index.md +++ b/doc/user/application_security/api_fuzzing/index.md @@ -16,6 +16,9 @@ We recommend that you use fuzz testing in addition to [GitLab Secure](../index.m other security scanners and your own test processes. If you're using [GitLab CI/CD](../../../ci/index.md), you can run fuzz tests as part your CI/CD workflow. + +For an overview, see [Web API Fuzzing](https://www.youtube.com/watch?v=oUHsfvLGhDk). + ## When Web API fuzzing runs Web API fuzzing runs in the `fuzz` stage of the CI/CD pipeline. To ensure API fuzzing scans the @@ -50,6 +53,7 @@ Example projects using these methods are available: - [Example Postman Collection project](https://gitlab.com/gitlab-org/security-products/demos/api-fuzzing/postman-api-fuzzing-example) - [Example GraphQL project](https://gitlab.com/gitlab-org/security-products/demos/api-fuzzing/graphql-api-fuzzing-example) - [Example SOAP project](https://gitlab.com/gitlab-org/security-products/demos/api-fuzzing/soap-api-fuzzing-example) +- [Authentication Token using Selenium](https://gitlab.com/gitlab-org/security-products/demos/api-fuzzing/auth-token-selenium) ## Enable Web API fuzzing @@ -98,7 +102,7 @@ a YAML snippet that you can paste in your GitLab CI/CD configuration. To generate an API Fuzzing configuration snippet: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Security & Compliance > Configuration**. +1. On the left sidebar, select **Security and Compliance > Security configuration**. 1. In the **API Fuzzing** row, select **Enable API Fuzzing**. 1. Complete the fields. For details see [Available CI/CD variables](#available-cicd-variables). 1. Select **Generate code snippet**. @@ -330,6 +334,8 @@ This example is a minimal configuration for API Fuzzing. From here you can: #### API Fuzzing with a GraphQL Schema file +API Fuzzing can use a GraphQL schema file to understand and test a GraphQL endpoint that has introspection disabled. To use a GraphQL schema file, it must be in the introspection JSON format. A GraphQL schema can be converted to a the introspection JSON format using an online 3rd party tool: [https://transform.tools/graphql-to-introspection-json](https://transform.tools/graphql-to-introspection-json). + To configure API Fuzzing to use a GraphQl schema file that provides information about the target API to test: 1. [Include](../../../ci/yaml/index.md#includetemplate) @@ -1994,7 +2000,7 @@ When configured correctly, a CI/CD pipeline contains a `fuzz` stage and an `apif typical operation, the job always succeeds even if faults are identified during fuzz testing. Faults are displayed on the **Security** pipeline tab with the suite name. When testing against the -repositories default branch, the fuzzing faults are also shown on the Security & Compliance's +repositories default branch, the fuzzing faults are also shown on the Security and Compliance's Vulnerability Report page. To prevent an excessive number of reported faults, the API fuzzing scanner limits the number of @@ -2026,7 +2032,7 @@ Follow these steps to view details of a fuzzing fault: 1. You can view faults in a project, or a merge request: - - In a project, go to the project's **Security & Compliance > Vulnerability Report** + - In a project, go to the project's **Security and Compliance > Vulnerability Report** page. This page shows all vulnerabilities from the default branch only. - In a merge request, go the merge request's **Security** section and select the **Expand** button. API Fuzzing faults are available in a section labeled @@ -2345,9 +2351,12 @@ apifuzzer_v1: FUZZAPI_EXCLUDE_PATHS: /api/v1/** rules: rules: - - if: $API_FUZZING_DISABLED + - if: $API_FUZZING_DISABLED == 'true' || $API_FUZZING_DISABLED == '1' when: never - - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && + - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH == 'true' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH == '1' && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - if: $CI_COMMIT_BRANCH && @@ -2361,7 +2370,7 @@ apifuzzer_v2: FUZZAPI_EXCLUDE_PATHS: /api/v2/** rules: rules: - - if: $API_FUZZING_DISABLED + - if: $API_FUZZING_DISABLED == 'true' || $API_FUZZING_DISABLED == '1' when: never - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME @@ -2396,7 +2405,7 @@ apifuzzer_branch: FUZZAPI_EXCLUDE_PATHS: /api/large_response_json rules: rules: - - if: $API_FUZZING_DISABLED + - if: $API_FUZZING_DISABLED == 'true' || $API_FUZZING_DISABLED == '1' when: never - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME @@ -2414,7 +2423,7 @@ apifuzzer_branch: apifuzzer_main: extends: apifuzzer_fuzz rules: - - if: $API_FUZZING_DISABLED + - if: $API_FUZZING_DISABLED == 'true' || $API_FUZZING_DISABLED == '1' when: never - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME @@ -2697,14 +2706,75 @@ You can expedite the investigation with the [testssl.sh tool](https://testssl.sh 1. Run `./testssl.sh --log https://specs`. 1. Attach the log file to your support ticket. +### `ERROR: Job failed: failed to pull image` + +This error message occurs when pulling an image from a container registry that requires authentication to access (it is not public). + +In the job console output the error looks like: + +```log +Running with gitlab-runner 15.6.0~beta.186.ga889181a (a889181a) + on blue-2.shared.runners-manager.gitlab.com/default XxUrkriX +Resolving secrets +00:00 +Preparing the "docker+machine" executor +00:06 +Using Docker executor with image registry.gitlab.com/security-products/api-security:2 ... +Starting service registry.example.com/my-target-app:latest ... +Pulling docker image registry.example.com/my-target-app:latest ... +WARNING: Failed to pull image with policy "always": Error response from daemon: Get https://registry.example.com/my-target-app/manifests/latest: unauthorized (manager.go:237:0s) +ERROR: Job failed: failed to pull image "registry.example.com/my-target-app:latest" with specified policies [always]: Error response from daemon: Get https://registry.example.com/my-target-app/manifests/latest: unauthorized (manager.go:237:0s) +``` + +**Error message** + +- In GitLab 15.9 and earlier, `ERROR: Job failed: failed to pull image` followed by `Error response from daemon: Get IMAGE: unauthorized`. + +**Solution** + +Authentication credentials are provided using the methods outlined in the [Access an image from a private Container Registry](../../../ci/docker/using_docker_images.md#access-an-image-from-a-private-container-registry) documentation section. The method used is dictated by your container registry provider and its configuration. If your using a container registry provided by a third party, such as a cloud provider (Azure, Google Could (GCP), AWS and so on), check the providers documentation for information on how to authenticate to their container registries. + +The following example uses the [statically defined credentials](../../../ci/docker/using_docker_images.md#use-statically-defined-credentials) authentication method. In this example the container registry is `registry.example.com` and image is `my-target-app:latest`. + +1. Read how to [Determine your `DOCKER_AUTH_CONFIG` data](../../../ci/docker/using_docker_images.md#determine-your-docker_auth_config-data) to understand how to compute the variable value for `DOCKER_AUTH_CONFIG`. The configuration variable `DOCKER_AUTH_CONFIG` contains the Docker JSON configuration to provide the appropriate authentication information. For example, to access private container registry: `registry.example.com` with the credentials `aGVsbG8gd29ybGQK`, the Docker JSON looks like: + + ```json + { + "auths": { + "registry.example.com": { + "auth": "aGVsbG8gd29ybGQK" + } + } + } + ``` + +1. Add the `DOCKER_AUTH_CONFIG` as a CI/CD variable. Instead of adding the configuration variable directly in your `.gitlab-ci.yml` file you should create a project [CI/CD variable](../../../ci/variables/index.md#for-a-project). +1. Rerun your job, and the statically-defined credentials are now used to sign in to the private container registry `registry.example.com`, and let you pull the image `my-target-app:latest`. If succeeded the job console shows an output like: + + ```log + Running with gitlab-runner 15.6.0~beta.186.ga889181a (a889181a) + on blue-4.shared.runners-manager.gitlab.com/default J2nyww-s + Resolving secrets + 00:00 + Preparing the "docker+machine" executor + 00:56 + Using Docker executor with image registry.gitlab.com/security-products/api-security:2 ... + Starting service registry.example.com/my-target-app:latest ... + Authenticating with credentials from $DOCKER_AUTH_CONFIG + Pulling docker image registry.example.com/my-target-app:latest ... + Using docker image sha256:139c39668e5e4417f7d0eb0eeb74145ba862f4f3c24f7c6594ecb2f82dc4ad06 for registry.example.com/my-target-app:latest with digest registry.example.com/my-target- + app@sha256:2b69fc7c3627dbd0ebaa17674c264fcd2f2ba21ed9552a472acf8b065d39039c ... + Waiting for services to be up and running (timeout 30 seconds)... + ``` + ## Get support or request an improvement To get support for your particular problem use the [getting help channels](https://about.gitlab.com/get-help/). The [GitLab issue tracker on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues) is the right place for bugs and feature proposals about API Security and API Fuzzing. -Use `~"Category:API Security"` [label](../../../development/contributing/issue_workflow.md#labels) when opening a new issue regarding API fuzzing to ensure it is quickly reviewed by the right people. Refer to our [review response SLO](https://about.gitlab.com/handbook/engineering/workflow/code-review/#review-response-slo) to understand when you should receive a response. +Use `~"Category:API Security"` [label](../../../development/labels/index.md) when opening a new issue regarding API fuzzing to ensure it is quickly reviewed by the right people. Refer to our [review response SLO](https://about.gitlab.com/handbook/engineering/workflow/code-review/#review-response-slo) to understand when you should receive a response. -[Search the issue tracker](https://gitlab.com/gitlab-org/gitlab/-/issues) for similar entries before submitting your own, there's a good chance somebody else had the same issue or feature proposal. Show your support with an award emoji and or join the discussion. +[Search the issue tracker](https://gitlab.com/gitlab-org/gitlab/-/issues) for similar entries before submitting your own, there's a good chance somebody else had the same issue or feature proposal. Show your support with an emoji reaction or join the discussion. When experiencing a behavior not working as expected, consider providing contextual information: diff --git a/doc/user/application_security/api_security/api_discovery/index.md b/doc/user/application_security/api_security/api_discovery/index.md new file mode 100644 index 00000000000..95b53249653 --- /dev/null +++ b/doc/user/application_security/api_security/api_discovery/index.md @@ -0,0 +1,181 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +type: reference, howto +--- + +# API Discovery **(ULTIMATE)** + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/9302) in GitLab 15.9. The API Discovery feature is in [Beta](../../../../policy/alpha-beta-support.md). + +API Discovery analyzes your application and produces an OpenAPI document describing the web APIs it exposes. This schema document can then be used by [DAST API](../../dast_api/index.md) or [API Fuzzing](../../api_fuzzing/index.md) to perform security scans of the web API. + +## Supported frameworks + +- [Java Spring-Boot](#java-spring-boot) + +## When does API Discovery run? + +API Discovery runs as a standalone job in your pipeline. The resulting OpenAPI document is captured as a job artifact so it can be used by other jobs in later stages. + +API Discovery runs in the `test` stage by default. The `test` stage was chosen as it typically executes before the stages used by other API Security features such as DAST API and API Fuzzing. + +## Example API Discovery configurations + +The following projects demonstrate API Discovery: + +- [Example Java Spring Boot v2 Pet Store](https://gitlab.com/gitlab-org/security-products/demos/api-discovery/java-spring-boot-v2-petstore) + +## Java Spring-Boot + +[Spring Boot](https://spring.io/projects/spring-boot) is a popular framework for creating stand-alone, production-grade Spring-based applications. + +### Supported Applications + +- Spring Boot: v2.X (>= 2.1) +- Java: 11, 17 (LTS versions) +- Executable JARs + +API Discovery supports Spring Boot major version 2, minor versions 1 and later. Versions 2.0.X are not supported due to known bugs which affect API Discovery and were fixed in 2.1. + +Major version 3 is planned to be supported in the future. Support for major version 1 is not planned. + +API Discovery is tested with and officially supports LTS versions of the Java runtime. Other versions may work also, and bug reports from non-LTS versions are welcome. + +Only applications that are built as Spring Boot [executable JARs](https://docs.spring.io/spring-boot/docs/current/reference/html/executable-jar.html#appendix.executable-jar.nested-jars.jar-structure) are supported. + +### Configure as pipeline job + +The easiest way to run API Discovery is through a pipeline job based on our CI template. +When running in this method, you provide a container image that has the required dependencies installed (such as an appropriate Java runtime). See [Image Requirements](#image-requirements) for more information. + +1. A container image that meets the [image requirements](#image-requirements) is uploaded to a container registry. If the container registry requires authentication see [this help section](/ee/ci/docker/using_docker_images.md#access-an-image-from-a-private-container-registry). +1. In a job in the `build` stage, build your application and configure the resulting Spring Boot executable JAR as a job artifact. +1. Include the API Discovery template in your `.gitlab-ci.yml` file. + + ```yaml + include: + - template: Security/API-Discovery.gitlab-ci.yml + ``` + + Only a single `include` statement is allowed per `.gitlab-ci.yml` file. If you are including other files, combine them into a single `include` statement. + + ```yaml + include: + - template: Security/API-Discovery.gitlab-ci.yml + - template: Security/DAST-API.gitlab-ci.yml + ``` + +1. Create a new job that extends from `.api_discovery_java_spring_boot`. The default stage is `test` which can be optionally changed to any value. + + ```yaml + api_discovery: + extends: .api_discovery_java_spring_boot + ``` + +1. Configure the `image` for the job. + + ```yaml + api_discovery: + extends: .api_discovery_java_spring_boot + image: openjdk:11-jre-slim + ``` + +1. Provide the Java class path needed by your application. This includes your compatible build + artifact from step 2, along with any additional dependencies. For this example, the build artifact + is `build/libs/spring-boot-app-0.0.0.jar` and contains all needed dependencies. The variable + `API_DISCOVERY_JAVA_CLASSPATH` is used to provide the class path. + + ```yaml + api_discovery: + extends: .api_discovery_java_spring_boot + image: openjdk:11-jre-slim + variables: + API_DISCOVERY_JAVA_CLASSPATH: build/libs/spring-boot-app-0.0.0.jar + ``` + +1. Optional. If the image provided is missing a dependency needed by API Discovery, it can be added + using a `before_script`. In this example, the `openjdk:11-jre-slim` container doesn't include + `curl` which is required by API Discovery. The dependency can be installed using the Debian + package manager `apt`: + + ```yaml + api_discovery: + extends: .api_discovery_java_spring_boot + image: openjdk:11-jre-slim + variables: + API_DISCOVERY_JAVA_CLASSPATH: build/libs/spring-boot-app-0.0.0.jar + before_script: + - apt-get update && apt-get install -y curl + ``` + +1. Optional. If the image provided doesn't automatically set the `JAVA_HOME` environment variable, + or include `java` in the path, the `API_DISCOVERY_JAVA_HOME` variable can be used. + + ```yaml + api_discovery: + extends: .api_discovery_java_spring_boot + image: openjdk:11-jre-slim + variables: + API_DISCOVERY_JAVA_CLASSPATH: build/libs/spring-boot-app-0.0.0.jar + API_DISCOVERY_JAVA_HOME: /opt/java + ``` + +1. Optional. If the package registry at `API_DISCOVERY_PACKAGES` is not public, provide a token that + has read access to the GitLab API and registry using the `API_DISCOVERY_PACKAGE_TOKEN` variable. + This is not required if you are using `gitlab.com` and have not customized the `API_DISCOVERY_PACKAGES` + variable. The following example uses a + [custom CI/CD variable](../../../../ci/variables/index.md#define-a-cicd-variable-in-the-ui) named + `GITLAB_READ_TOKEN` to store the token. + + ```yaml + api_discovery: + extends: .api_discovery_java_spring_boot + image: openjdk:8-jre-alpine + variables: + API_DISCOVERY_JAVA_CLASSPATH: build/libs/spring-boot-app-0.0.0.jar + API_DISCOVERY_PACKAGE_TOKEN: $GITLAB_READ_TOKEN + ``` + +After the API Discovery job has successfully run, the OpenAPI document is available as a job artifact called `gl-api-discovery-openapi.json`. + +#### Image requirements + +- Linux container image. +- Java versions 11 or 17 are officially supported, but other versions are likely compatible as well. +- The `curl` command. +- A shell at `/bin/sh` (like `busybox`, `sh`, or `bash`). + +### Available CI/CD variables + +| CI/CD variable | Description | +|---------------------------------------------|--------------------| +| `API_DISCOVERY_DISABLED` | Disables the API Discovery job when using template job rules. | +| `API_DISCOVERY_DISABLED_FOR_DEFAULT_BRANCH` | Disables the API Discovery job for default branch pipelines when using template job rules. | +| `API_DISCOVERY_JAVA_CLASSPATH` | Java class-path that includes target Spring Boot application. (`build/libs/sample-0.0.0.jar`) | +| `API_DISCOVERY_JAVA_HOME` | If provided is used to set `JAVA_HOME`. | +| `API_DISCOVERY_PACKAGES` | GitLab Project Package API Prefix (defaults to `$CI_API_V4_URL/projects/42503323/packages`). | +| `API_DISCOVERY_PACKAGE_TOKEN` | GitLab token for calling the GitLab package API. Only needed when `API_DISCOVERY_PACKAGES` is set to a non-public project. | +| `API_DISCOVERY_VERSION` | API Discovery version to use (defaults to `1`). Can be used to pin a version by providing the full version number `1.1.0`. | + +## Get support or request an improvement + +To get support for your particular problem, use the [getting help channels](https://about.gitlab.com/get-help/). + +The [GitLab issue tracker on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues) is the right place for bugs and feature proposals about API Discovery. +Use `~"Category:API Security"` [label](../../../../development/labels/index.md) when opening a new issue regarding API Discovery to ensure it is quickly reviewed by the right people. Refer to our [review response SLO](https://about.gitlab.com/handbook/engineering/workflow/code-review/#review-response-slo) to understand when you should receive a response. + +[Search the issue tracker](https://gitlab.com/gitlab-org/gitlab/-/issues) for similar entries before submitting your own, there's a good chance somebody else had the same issue or feature proposal. Show your support with an emoji reaction or join the discussion. + +When experiencing a behavior not working as expected, consider providing contextual information: + +- GitLab version if using a self-managed instance. +- `.gitlab-ci.yml` job definition. +- Full job console output. +- Framework in use with version (for example Spring Boot v2.3.2). +- Language runtime with version (for example OpenJDK v17.0.1). + + +WARNING: +**Sanitize data attached to a support issue**. Remove sensitive information, including: credentials, passwords, tokens, keys, and secrets. diff --git a/doc/user/application_security/api_security/index.md b/doc/user/application_security/api_security/index.md new file mode 100644 index 00000000000..5c2e74bceae --- /dev/null +++ b/doc/user/application_security/api_security/index.md @@ -0,0 +1,21 @@ +--- +stage: Secure +group: Dynamic Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +type: reference, howto +--- + +# API Security **(ULTIMATE)** + +API Security refers to the measures taken to secure and protect web Application Programming Interfaces (APIs) from unauthorized access, misuse, and attacks. +APIs are a crucial component of modern application development as they allow applications to interact with each other and exchange data. +However, this also makes them attractive to attackers and vulnerable to security threats if not properly secured. +In this section, we discuss GitLab features that can be used to ensure the security of web APIs in your application. +Some of the features discussed are specific to web APIs and others are more general solutions that are also used with web API applications. + +- [SAST](../sast) identified vulnerabilities by analyzing the application's codebase. +- [Dependency Scanning](../dependency_scanning) reviews a project 3rd party dependencies for known vulnerabilities (for example CVEs). +- [Container Scanning](../container_scanning) analyzes container images to identify known OS package vulnerabilities and installed language dependencies. +- [API Discovery](api_discovery) examines an application containing a REST API and intuits an OpenAPI specification for that API. OpenAPI specification documents are used by other GitLab security tools. +- [DAST API](../dast_api) performs dynamic analysis security testing of web APIs. It can identify various security vulnerabilities in your application, including the OWASP Top 10. +- [API Fuzzing](../api_fuzzing) performs fuzz testing of a web API. Fuzz testing looks for issues in an application that are not previously known and don't map to classic vulnerability types such as SQL Injection. diff --git a/doc/user/application_security/breach_and_attack_simulation/index.md b/doc/user/application_security/breach_and_attack_simulation/index.md new file mode 100644 index 00000000000..bb67150d4fa --- /dev/null +++ b/doc/user/application_security/breach_and_attack_simulation/index.md @@ -0,0 +1,141 @@ +--- +stage: Secure +group: Incubation +info: Breach and Attack Simulation is a GitLab Incubation Engineering program. No technical writer assigned to this group. +type: reference, howto +--- + +# Breach and Attack Simulation **(ULTIMATE)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/402784) in GitLab 15.11 as an Incubating feature. +> - [Included](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/119981) in the `Security/BAS.latest.gitlab-ci.yml` in GitLab 16.0. + +DISCLAIMER: +Breach and Attack Simulation is a set of incubating features being developed by the Incubation Engineering Department and is subject to significant changes over time. + +Breach and Attack Simulation (BAS) uses additional security testing techniques to assess the risk of detected vulnerabilities and prioritize the remediation of exploitable vulnerabilities. + +For feedback, bug reports, and feature requests, see the [feedback issue](https://gitlab.com/gitlab-org/gitlab/-/issues/404809). + +WARNING: +Only run BAS scans against test servers. Testing attacker behavior can lead to modification or loss of data. + +## Extend Dynamic Application Security Testing (DAST) + +You can simulate attacks with [DAST](../dast/index.md) to detect vulnerabilities. +By default, DAST active checks match an expected response, or determine by response +time whether a vulnerability was exploited. + +To enable BAS extended DAST scanning for your application, use the `dast_with_bas` job defined +in the GitLab BAS CI/CD template file. Updates to the template are provided with GitLab +upgrades, allowing you to benefit from any improvements and additions. + +1. Include the appropriate CI/CD template: + + - [`BAS.latest.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/BAS.latest.gitlab-ci.yml): + Latest version of the BAS template. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/119981) + in GitLab 16.0). + + WARNING: + The latest version of the template may include breaking changes. Use the + stable template unless you need a feature provided only in the latest template. + + For more information about template versioning, see the [CI/CD documentation](../../../development/cicd/templates.md#latest-version). + +1. Choose one of the following options for running BAS extended DAST scans: + + - [Enable a separate BAS extended DAST job](#enable-a-separate-bas-extended-dast-job) + + - You're not using the latest DAST template yet. + - Continue using a stable version of the DAST security analyzer image for DAST scans. + - Create a duplicate `dast_with_bas` job which extends your existing DAST job configuration. + + - [Extend an existing DAST job](#extend-an-existing-dast-job) + - You're already using the latest DAST template rather than the stable template. + - Extend your existing DAST job to include the latest DAST security analyzer image tag from the Breach and Attack Simulation SEG. + +1. Setup a callback server to [enable callback attacks](#enable-callback-attacks). + +### Enable a separate BAS extended DAST job + +To maintain a separate DAST job while testing the BAS extended DAST image: + +1. Add a `dast` stage to your GitLab CI/CD stages configuration. + + ```yaml + stages: + - build + - test + - deploy + - dast + ``` + +1. Set the `DAST_WEBSITE` [CI/CD variable](../../../ci/yaml/index.md#variables). + + ```yaml + dast_with_bas: + variables: + DAST_WEBSITE: http://yourapp + ``` + +### Extend an existing DAST job + +To enable Breach and Attack Simulation features inside of an existing DAST job: + +1. Follow the steps in [Create a DAST CI/CD job](../dast/browser_based.md#create-a-dast-cicd-job). + +1. Extend DAST to using the [extends](../../../ci/yaml/yaml_optimization.md#use-extends-to-reuse-configuration-sections) keyword to your DAST job's configuration: + + ```yaml + dast: + extends: .dast_with_bas + ``` + +1. Disable the `dast+job` job included in the BAS template by setting `DAST_BAS_DISABLED`: + + ```yaml + variables: + DAST_BAS_DISABLED: "true" + ``` + +### Enable callback attacks + +DISCLAIMER: +This page contains information related to upcoming products, features, and functionality. +It is important to note that the information presented is for informational purposes only. +Please do not rely on this information for purchasing or planning purposes. +As with all projects, the items mentioned on this page are subject to change or delay. +The development, release, and timing of any products, features, or functionality remain at the +sole discretion of GitLab Inc. + +Perform Out-of-Band Application Security Testing (OAST) for certain [active checks](../dast/checks/index.md#active-checks). + +1. Extend the `.dast_with_bas_using_services` job configuration using the [extends](../../../ci/yaml/yaml_optimization.md#use-extends-to-reuse-configuration-sections) keyword: + + ```yaml + dast: + extends: .dast_with_bas_using_services + + dast_with_bas: + extends: + # NOTE: extends overwrites rather than merges so dast must be included in this list. + - dast + - .dast_with_bas_using_services + ``` + +1. Use a [!reference tag](../../../ci/yaml/yaml_optimization.md#reference-tags) to pull in the default `callback` service container in your `services`. + + ```yaml + services: + # NOTE: services overwrites rather than merges so it must be referenced to merge. + - !reference [.dast_with_bas_using_services, services] + - name: $CI_REGISTRY_IMAGE + alias: yourapp + ``` + +You can also manually enable callback attacks by making sure to: + +1. Set the `DAST_FF_ENABLE_BAS` [CI/CD variable](../dast/browser_based.md#available-cicd-variables) to `true`. +1. Enable both the application being tested and callback service container using [services](../../../ci/services/index.md). +1. Enable container-to-container networking [making the callback service accessible](../../../ci/services/index.md#connecting-services) in the job. +1. Set `DAST_BROWSER_CALLBACK` to include `Address:$YOUR_CALLBACK_URL` key/value pair where the callback service is accessible to the Runner/DAST container. diff --git a/doc/user/application_security/configuration/index.md b/doc/user/application_security/configuration/index.md index ea422f0b33c..d5a05477ddf 100644 --- a/doc/user/application_security/configuration/index.md +++ b/doc/user/application_security/configuration/index.md @@ -5,7 +5,7 @@ group: Static Analysis info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Security Configuration **(FREE)** +# Security configuration **(FREE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20711) in GitLab 12.6. > - SAST configuration was [enabled](https://gitlab.com/groups/gitlab-org/-/epics/3659) in 13.3 and [improved](https://gitlab.com/gitlab-org/gitlab/-/issues/232862) in 13.4. @@ -13,7 +13,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - A simplified version was made [available in all tiers](https://gitlab.com/gitlab-org/gitlab/-/issues/294076) in GitLab 13.10. > - [Redesigned](https://gitlab.com/gitlab-org/gitlab/-/issues/326926) in 14.2. -The Security Configuration page lists the following for the security testing and compliance tools: +The **Security configuration** page lists the following for the security testing and compliance tools: - Name, description, and a documentation link. - Whether or not it is available. @@ -41,7 +41,7 @@ all security features are configured by default. To view a project's security configuration: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Security & Compliance > Configuration**. +1. On the left sidebar, select **Security and Compliance > Security configuration**. Select **Configuration history** to see the `.gitlab-ci.yml` file's history. @@ -51,7 +51,7 @@ You can configure the following security controls: - [Static Application Security Testing](../sast/index.md) (SAST) - Select **Enable SAST** to configure SAST for the current project. - For more details, read [Configure SAST in the UI](../sast/index.md#configure-sast-in-the-ui). + For more details, read [Configure SAST in the UI](../sast/index.md#configure-sast-by-using-the-ui). - [Dynamic Application Security Testing](../dast/index.md) (DAST) - Select **Enable DAST** to configure DAST for the current project. - Select **Manage scans** to manage the saved DAST scans, site profiles, and scanner profiles. diff --git a/doc/user/application_security/container_scanning/index.md b/doc/user/application_security/container_scanning/index.md index 0a586a14cc4..7a82f98425a 100644 --- a/doc/user/application_security/container_scanning/index.md +++ b/doc/user/application_security/container_scanning/index.md @@ -15,12 +15,16 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86092) the major analyzer version from `4` to `5` in GitLab 15.0. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/86783) from GitLab Ultimate to GitLab Free in 15.0. > - Container Scanning variables that reference Docker [renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/357264) in GitLab 15.4. +> - Container Scanning template [moved](https://gitlab.com/gitlab-org/gitlab/-/issues/381665) from `Security/Container-Scanning.gitlab-ci.yml` to `Jobs/Container-Scanning.gitlab-ci.yml` in GitLab 15.6. Your application's Docker image may itself be based on Docker images that contain known vulnerabilities. By including an extra Container Scanning job in your pipeline that scans for those vulnerabilities and displays them in a merge request, you can use GitLab to audit your Docker-based apps. + +For an overview, see [Container Scanning](https://www.youtube.com/watch?v=C0jn2eN5MAs). + Container Scanning is often considered part of Software Composition Analysis (SCA). SCA can contain aspects of inspecting the items your code uses. These items typically include application and system dependencies that are almost always imported from external sources, rather than sourced from items @@ -71,14 +75,14 @@ information directly in the merge request. | [Access to Security Dashboard page](#security-dashboard) | No | Yes | | [Access to Dependency List page](../dependency_list/index.md) | No | Yes | -## Requirements +## Prerequisites To enable container scanning in your pipeline, you need the following: - GitLab CI/CD pipeline must include the `test` stage, which is available unless overridden with the [`stages`](../../../ci/yaml/index.md#stages) keyword. - [GitLab Runner](https://docs.gitlab.com/runner/) with the [`docker`](https://docs.gitlab.com/runner/executors/docker.html) or [`kubernetes`](https://docs.gitlab.com/runner/install/kubernetes.html) executor on Linux/amd64. -- Docker `18.09.03` or higher installed on the same computer as the runner. If you're using the +- Docker `18.09.03` or later installed on the same computer as the runner. If you're using the shared runners on GitLab.com, then this is already the case. - An image matching the [supported distributions](#supported-distributions). - [Build and push](../../packages/container_registry/build_and_push_images.md#use-gitlab-cicd) @@ -90,19 +94,19 @@ To enable container scanning in your pipeline, you need the following: ## Configuration To enable container scanning, add the -[`Container-Scanning.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml) +[`Container-Scanning.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml) to your `.gitlab-ci.yml` file: ```yaml include: - - template: Jobs/Container-Scanning.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml ``` The included template: - Creates a `container_scanning` job in your CI/CD pipeline. - Pulls the built Docker image from your project's [container registry](../../packages/container_registry/index.md) - (see [requirements](#requirements)) and scans it for possible vulnerabilities. + (see [prerequisites](#prerequisites)) and scans it for possible vulnerabilities. GitLab saves the results as a [Container Scanning report artifact](../../../ci/yaml/artifacts_reports.md#artifactsreportscontainer_scanning) @@ -117,7 +121,7 @@ registry, and scans the image: ```yaml include: - template: Jobs/Build.gitlab-ci.yml - - template: Jobs/Container-Scanning.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml container_scanning: variables: @@ -142,7 +146,7 @@ enables verbose output for the analyzer: ```yaml include: - - template: Jobs/Container-Scanning.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml variables: SECURE_LOG_LEVEL: 'debug' @@ -154,7 +158,7 @@ To scan images located in a registry other than the project's, use the following ```yaml include: - - template: Jobs/Container-Scanning.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml container_scanning: variables: @@ -178,7 +182,9 @@ container_scanning: - export AWS_ECR_PASSWORD=$(aws ecr get-login-password --region region) include: - - template: Jobs/Container-Scanning.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml + +variables: CS_IMAGE: .dkr.ecr..amazonaws.com/: CS_REGISTRY_USER: AWS CS_REGISTRY_PASSWORD: "$AWS_ECR_PASSWORD" @@ -199,7 +205,7 @@ For example: ```yaml include: - - template: Jobs/Container-Scanning.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml container_scanning: variables: @@ -223,7 +229,7 @@ By default, the report only includes packages managed by the Operating System (O ```yaml include: - - template: Jobs/Container-Scanning.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml container_scanning: variables: @@ -251,7 +257,7 @@ including a large number of false positives. | `ADDITIONAL_CA_CERT_BUNDLE` | `""` | Bundle of CA certs that you want to trust. See [Using a custom SSL CA certificate authority](#using-a-custom-ssl-ca-certificate-authority) for more details. | All | | `CI_APPLICATION_REPOSITORY` | `$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG` | Docker repository URL for the image to be scanned. | All | | `CI_APPLICATION_TAG` | `$CI_COMMIT_SHA` | Docker repository tag for the image to be scanned. | All | -| `CS_ANALYZER_IMAGE` | `registry.gitlab.com/security-products/container-scanning:5` | Docker image of the analyzer. | All | +| `CS_ANALYZER_IMAGE` | `registry.gitlab.com/security-products/container-scanning:6` | Docker image of the analyzer. | All | | `CS_DEFAULT_BRANCH_IMAGE` | `""` | The name of the `CS_IMAGE` on the default branch. See [Setting the default branch image](#setting-the-default-branch-image) for more details. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/338877) in GitLab 14.5. | All | | `CS_DISABLE_DEPENDENCY_LIST` | `"false"` | Disable Dependency Scanning for packages installed in the scanned image. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345434) in GitLab 14.6. | All | | `CS_DISABLE_LANGUAGE_VULNERABILITY_SCAN` | `"true"` | Disable scanning for language-specific packages installed in the scanned image. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345434) in GitLab 14.6. | All | @@ -260,10 +266,6 @@ including a large number of false positives. | `CS_IGNORE_UNFIXED` | `"false"` | Ignore vulnerabilities that are not fixed. | All | | `CS_REGISTRY_INSECURE` | `"false"` | Allow access to insecure registries (HTTP only). Should only be set to `true` when testing the image locally. Works with all scanners, but the registry must listen on port `80/tcp` for Trivy to work. | All | | `CS_SEVERITY_THRESHOLD` | `UNKNOWN` | Severity level threshold. The scanner outputs vulnerabilities with severity level higher than or equal to this threshold. Supported levels are `UNKNOWN`, `LOW`, `MEDIUM`, `HIGH`, and `CRITICAL`. | Trivy | -| `DOCKER_IMAGE` | `$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG` | **Deprecated** will be removed in GitLab 16.0. Replaced by `CS_IMAGE`. The Docker image to be scanned. If set, this variable overrides the `$CI_APPLICATION_REPOSITORY` and `$CI_APPLICATION_TAG` variables. | All | -| `DOCKER_PASSWORD` | `$CI_REGISTRY_PASSWORD` | **Deprecated** will be removed in GitLab 16.0. Replaced by `CS_REGISTRY_PASSWORD`. Password for accessing a Docker registry requiring authentication. The default is only set if `$DOCKER_IMAGE` resides at [`$CI_REGISTRY`](../../../ci/variables/predefined_variables.md). Not supported when [FIPS mode](../../../development/fips_compliance.md#enable-fips-mode) is enabled. | All | -| `DOCKER_USER` | `$CI_REGISTRY_USER` | **Deprecated** will be removed in GitLab 16.0. Replaced by `CS_REGISTRY_USER`. Username for accessing a Docker registry requiring authentication. The default is only set if `$DOCKER_IMAGE` resides at [`$CI_REGISTRY`](../../../ci/variables/predefined_variables.md). Not supported when [FIPS mode](../../../development/fips_compliance.md#enable-fips-mode) is enabled. | All | -| `DOCKERFILE_PATH` | `Dockerfile` | **Deprecated** will be removed in GitLab 16.0. Replaced by `CS_DOCKERFILE_PATH`. The path to the `Dockerfile` to use for generating remediations. By default, the scanner looks for a file named `Dockerfile` in the root directory of the project. You should configure this variable only if your `Dockerfile` is in a non-standard location, such as a subdirectory. See [Solutions for vulnerabilities](#solutions-for-vulnerabilities-auto-remediation) for more details. | All | | `CS_IMAGE` | `$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG` | The Docker image to be scanned. If set, this variable overrides the `$CI_APPLICATION_REPOSITORY` and `$CI_APPLICATION_TAG` variables. | All | | `CS_REGISTRY_PASSWORD` | `$CI_REGISTRY_PASSWORD` | Password for accessing a Docker registry requiring authentication. The default is only set if `$CS_IMAGE` resides at [`$CI_REGISTRY`](../../../ci/variables/predefined_variables.md). Not supported when [FIPS mode](../../../development/fips_compliance.md#enable-fips-mode) is enabled. | All | | `CS_REGISTRY_USER` | `$CI_REGISTRY_USER` | Username for accessing a Docker registry requiring authentication. The default is only set if `$CS_IMAGE` resides at [`$CI_REGISTRY`](../../../ci/variables/predefined_variables.md). Not supported when [FIPS mode](../../../development/fips_compliance.md#enable-fips-mode) is enabled. | All | @@ -303,9 +305,9 @@ standard tag plus the `-fips` extension. | Scanner name | `CS_ANALYZER_IMAGE` | | --------------- | ------------------- | -| Default (Trivy) | `registry.gitlab.com/security-products/container-scanning:5-fips` | -| Grype | `registry.gitlab.com/security-products/container-scanning/grype:5-fips` | -| Trivy | `registry.gitlab.com/security-products/container-scanning/trivy:5-fips` | +| Default (Trivy) | `registry.gitlab.com/security-products/container-scanning:6-fips` | +| Grype | `registry.gitlab.com/security-products/container-scanning/grype:6-fips` | +| Trivy | `registry.gitlab.com/security-products/container-scanning/trivy:6-fips` | NOTE: Prior to GitLab 15.0, the `-ubi` image extension is also available. GitLab 15.0 and later only @@ -326,7 +328,7 @@ To enable Container Scanning in a project, create a merge request from the Secur page: 1. In the project where you want to enable Container Scanning, go to - **Security & Compliance > Configuration**. + **Security and Compliance > Security configuration**. 1. In the **Container Scanning** row, select **Configure with a merge request**. This automatically creates a merge request with the changes necessary to enable Container Scanning. @@ -346,7 +348,7 @@ This example sets `GIT_STRATEGY` to `fetch`: ```yaml include: - - template: Jobs/Container-Scanning.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml container_scanning: variables: @@ -362,9 +364,9 @@ The following options are available: | Scanner name | `CS_ANALYZER_IMAGE` | |----------------------------------------------------------|--------------------------------------------------------------------| -| Default ([Trivy](https://github.com/aquasecurity/trivy)) | `registry.gitlab.com/security-products/container-scanning:5` | -| [Grype](https://github.com/anchore/grype) | `registry.gitlab.com/security-products/container-scanning/grype:5` | -| Trivy | `registry.gitlab.com/security-products/container-scanning/trivy:5` | +| Default ([Trivy](https://github.com/aquasecurity/trivy)) | `registry.gitlab.com/security-products/container-scanning:6` | +| [Grype](https://github.com/anchore/grype) | `registry.gitlab.com/security-products/container-scanning/grype:6` | +| Trivy | `registry.gitlab.com/security-products/container-scanning/trivy:6` | ### Setting the default branch image @@ -392,7 +394,7 @@ duplicated: ```yaml include: - - template: Jobs/Container-Scanning.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml container_scanning: variables: @@ -525,7 +527,7 @@ successfully run. For more information, see [Offline environments](../offline_de To use container scanning in an offline environment, you need: -- GitLab Runner with the [`docker` or `kubernetes` executor](#requirements). +- GitLab Runner with the [`docker` or `kubernetes` executor](#prerequisites). - To configure a local Docker container registry with copies of the container scanning images. You can find these images in their respective registries: @@ -555,9 +557,9 @@ For container scanning, import the following images from `registry.gitlab.com` i [local Docker container registry](../../packages/container_registry/index.md): ```plaintext -registry.gitlab.com/security-products/container-scanning:5 -registry.gitlab.com/security-products/container-scanning/grype:5 -registry.gitlab.com/security-products/container-scanning/trivy:5 +registry.gitlab.com/security-products/container-scanning:6 +registry.gitlab.com/security-products/container-scanning/grype:6 +registry.gitlab.com/security-products/container-scanning/trivy:6 ``` The process for importing Docker images into a local offline Docker registry depends on @@ -578,7 +580,7 @@ For details on saving and transporting Docker images as a file, see the Docker d ```yaml include: - - template: Jobs/Container-Scanning.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml container_scanning: image: $CI_REGISTRY/namespace/container-scanning @@ -597,7 +599,7 @@ following `.gitlab-ci.yml` example as a template. ```yaml variables: - SOURCE_IMAGE: registry.gitlab.com/security-products/container-scanning:5 + SOURCE_IMAGE: registry.gitlab.com/security-products/container-scanning:6 TARGET_IMAGE: $CI_REGISTRY/namespace/container-scanning image: docker:stable @@ -629,7 +631,7 @@ This example shows the configuration needed to scan images in a private [Google ```yaml include: - - template: Jobs/Container-Scanning.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml container_scanning: variables: @@ -693,6 +695,14 @@ These reports must follow a format defined in the For more information, see [Security scanner integration](../../../development/integrations/secure.md). +### CycloneDX Software Bill of Materials + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/396381) in GitLab 15.11. + +In addition to the [JSON report file](#reports-json-format), the [Container Scanning](https://gitlab.com/gitlab-org/security-products/analyzers/container-scanning) tool outputs a [CycloneDX](https://cyclonedx.org/) Software Bill of Materials (SBOM) for the scanned image. This CycloneDX SBOM is named `gl-sbom-report.cdx.json` and is saved in the same directory as the `JSON report file`. This feature is only supported when the `Trivy` analyzer is used. + +You can download CycloneDX SBOMs [the same way as other job artifacts](../../../ci/jobs/job_artifacts.md#download-job-artifacts). + ## Security Dashboard The [Security Dashboard](../security_dashboard/index.md) shows you an overview of all @@ -771,7 +781,7 @@ To prevent the error, ensure the Docker version that the runner is using is ### Getting warning message `gl-container-scanning-report.json: no matching files` -For information on this, see the [general Application Security troubleshooting section](../../../ci/pipelines/job_artifacts.md#error-message-no-files-to-upload). +For information on this, see the [general Application Security troubleshooting section](../../../ci/jobs/job_artifacts_troubleshooting.md#error-message-no-files-to-upload). ## Changes diff --git a/doc/user/application_security/coverage_fuzzing/index.md b/doc/user/application_security/coverage_fuzzing/index.md index 5d2593a1bed..09370a9a7f5 100644 --- a/doc/user/application_security/coverage_fuzzing/index.md +++ b/doc/user/application_security/coverage_fuzzing/index.md @@ -15,6 +15,9 @@ We recommend that you use fuzz testing in addition to the other security scanner and your own test processes. If you're using [GitLab CI/CD](../../../ci/index.md), you can run your coverage-guided fuzz testing as part your CI/CD workflow. + +For an overview, see [Coverage Fuzzing](https://www.youtube.com/watch?v=bbIenVVcjW0). + ## Coverage-guided fuzz testing process The fuzz testing process: @@ -40,7 +43,7 @@ You can use the following fuzzing engines to test the specified languages. | Language | Fuzzing Engine | Example | |---------------------------------------------|------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------| | C/C++ | [libFuzzer](https://llvm.org/docs/LibFuzzer.html) | [c-cpp-example](https://gitlab.com/gitlab-org/security-products/demos/coverage-fuzzing/c-cpp-fuzzing-example) | -| GoLang | [go-fuzz (libFuzzer support)](https://github.com/dvyukov/go-fuzz) | [go-fuzzing-example](https://gitlab.com/gitlab-org/security-products/demos/coverage-fuzzing/go-fuzzing-example) | +| Go | [go-fuzz (libFuzzer support)](https://github.com/dvyukov/go-fuzz) | [go-fuzzing-example](https://gitlab.com/gitlab-org/security-products/demos/coverage-fuzzing/go-fuzzing-example) | | Swift | [libFuzzer](https://github.com/apple/swift/blob/master/docs/libFuzzerIntegration.md) | [swift-fuzzing-example](https://gitlab.com/gitlab-org/security-products/demos/coverage-fuzzing/swift-fuzzing-example) | | Rust | [cargo-fuzz (libFuzzer support)](https://github.com/rust-fuzz/cargo-fuzz) | [rust-fuzzing-example](https://gitlab.com/gitlab-org/security-products/demos/coverage-fuzzing/rust-fuzzing-example) | | Java | [Javafuzz](https://gitlab.com/gitlab-org/security-products/analyzers/fuzzers/javafuzz) (recommended) | [javafuzz-fuzzing-example](https://gitlab.com/gitlab-org/security-products/demos/coverage-fuzzing/javafuzz-fuzzing-example) | @@ -54,7 +57,7 @@ You can use the following fuzzing engines to test the specified languages. To confirm the status of coverage-guided fuzz testing: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Security & Compliance > Configuration**. +1. On the left sidebar, select **Security and Compliance > Security configuration**. 1. In the **Coverage Fuzzing** section the status is: - **Not configured** - **Enabled** @@ -145,7 +148,7 @@ Each fuzzing step outputs these artifacts: previous jobs. You can download the JSON report file from the CI/CD pipelines page. For more information, see -[Downloading artifacts](../../../ci/pipelines/job_artifacts.md#download-job-artifacts). +[Downloading artifacts](../../../ci/jobs/job_artifacts.md#download-job-artifacts). ## Corpus registry @@ -168,7 +171,7 @@ artifacts files you can download from the CI/CD pipeline. Also, a project member To view details of the corpus registry: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Security & Compliance > Configuration**. +1. On the left sidebar, select **Security and Compliance > Security configuration**. 1. In the **Coverage Fuzzing** section, select **Manage corpus**. ### Create a corpus in the corpus registry @@ -196,7 +199,7 @@ provided by the `COVFUZZ_CORPUS_NAME` variable. The corpus is updated on every p To upload an existing corpus file: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Security & Compliance > Configuration**. +1. On the left sidebar, select **Security and Compliance > Security configuration**. 1. In the **Coverage Fuzzing** section, select **Manage corpus**. 1. Select **New corpus**. 1. Complete the fields. @@ -222,7 +225,7 @@ Prerequisites: ## Coverage-guided fuzz testing report -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220062) in GitLab 13.3 as an [Alpha feature](../../../policy/alpha-beta-support.md#alpha-features). +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/220062) in GitLab 13.3 as an [Experiment](../../../policy/alpha-beta-support.md#experiment). For detailed information about the `gl-coverage-fuzzing-report.json` file's format, read the [schema](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/coverage-fuzzing-report-format.json). diff --git a/doc/user/application_security/dast/authentication.md b/doc/user/application_security/dast/authentication.md index 77732ab532c..1a68abd01f6 100644 --- a/doc/user/application_security/dast/authentication.md +++ b/doc/user/application_security/dast/authentication.md @@ -32,7 +32,7 @@ Authentication supports single-step login forms, multi-step login forms, single ## Getting started NOTE: -We recommend periodically confirming that the analyzer's authentication is still working, as this tends to break over +You should periodically confirming that the analyzer's authentication is still working, as this tends to break over time due to changes to the application. To run a DAST authenticated scan: @@ -45,34 +45,45 @@ To run a DAST authenticated scan: ### Prerequisites -- You are using either the [DAST proxy-based analyzer](proxy-based.md) or the [DAST browser-based analyzer](browser_based.md). -- You know the URL of the login form of your application. Alternatively, you know how to navigate to the login form from the authentication URL (see [clicking to navigate to the login form](#clicking-to-navigate-to-the-login-form)). - You have the username and password of the user you would like to authenticate as during the scan. +- You have checked the [known limitations](#known-limitations) to ensure DAST can authenticate to your application. +- You have satisfied the prerequisites depending on whether you're using [form authentication](#form-authentication) or [HTTP authentication](#http-authentication). +- You have thought about how you can [verify](#verifying-authentication-is-successful) whether or not authentication was successful. + +#### Form authentication + +- You are using either the [DAST proxy-based analyzer](proxy-based.md) or the [DAST browser-based analyzer](browser_based.md). +- You know the URL of the login form of your application. Alternatively, you know how to go to the login form from the authentication URL (see [clicking to go to the login form](#clicking-to-go-to-the-login-form)). - You know the [selectors](#finding-an-elements-selector) of the username and password HTML fields that DAST uses to input the respective values. - You know the element's [selector](#finding-an-elements-selector) that submits the login form when selected. -- You have thought about how you can [verify](#verifying-authentication-is-successful) whether or not authentication was successful. -- You have checked the [known limitations](#known-limitations) to ensure DAST can authenticate to your application. + +#### HTTP authentication + +- You must be using the [DAST browser-based analyzer](browser_based.md). ### Available CI/CD variables | CI/CD variable | Type | Description | |:-----------------------------------------------|:------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `DAST_AUTH_COOKIES` | string | Set to a comma-separated list of cookie names to specify which cookies are used for authentication. | -| `DAST_AUTH_REPORT` | boolean | Used in combination with exporting the `gl-dast-debug-auth-report.html` artifact to aid in debugging authentication issues. | +| `DAST_AUTH_REPORT` | boolean | Set to `true` to generate a report detailing steps taken during the authentication process. You must also define `gl-dast-debug-auth-report.html` as a CI job artifact to be able to access the generated report. The report's content aids when debugging authentication failures. | +| `DAST_AUTH_TYPE` 2 | string | The authentication type to use. Example: `basic-digest`. | | `DAST_AUTH_URL` 1 | URL | The URL of the page containing the sign-in HTML form on the target website. `DAST_USERNAME` and `DAST_PASSWORD` are submitted with the login form to create an authenticated scan. Example: `https://login.example.com`. | -| `DAST_AUTH_VERIFICATION_LOGIN_FORM` | boolean | Verifies successful authentication by checking for the absence of a login form once the login form has been submitted. | -| `DAST_AUTH_VERIFICATION_SELECTOR` | [selector](#finding-an-elements-selector) | Verifies successful authentication by checking for presence of a selector once the login form has been submitted. Example: `css:.user-photo`. | -| `DAST_AUTH_VERIFICATION_URL` 1 | URL | Verifies successful authentication by checking the URL in the browser once the login form has been submitted. Example: `"https://example.com/loggedin_page"`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207335) in GitLab 13.8. | +| `DAST_AUTH_VERIFICATION_LOGIN_FORM` | boolean | Verifies successful authentication by checking for the absence of a login form after the login form has been submitted. | +| `DAST_AUTH_VERIFICATION_SELECTOR` | [selector](#finding-an-elements-selector) | Verifies successful authentication by checking for presence of a selector after the login form has been submitted. Example: `css:.user-photo`. | +| `DAST_AUTH_VERIFICATION_URL` 1 | URL | Verifies successful authentication by checking the URL in the browser after the login form has been submitted. Example: `"https://example.com/loggedin_page"`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207335) in GitLab 13.8. | | `DAST_BROWSER_PATH_TO_LOGIN_FORM` 1 | [selector](#finding-an-elements-selector) | Comma-separated list of selectors that are selected prior to attempting to enter `DAST_USERNAME` and `DAST_PASSWORD` into the login form. Example: `"css:.navigation-menu,css:.login-menu-item"`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/326633) in GitLab 14.1. | | `DAST_EXCLUDE_URLS` 1 | URLs | The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. | -| `DAST_FIRST_SUBMIT_FIELD` | string | The `id` or `name` of the element that when selected submits the username form of a multi-page login process. For example, `css:button[type='user-submit']`. [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/9894) in GitLab 12.4. | +| `DAST_FIRST_SUBMIT_FIELD` | string | The `id` or `name` of the element that when selected submits the username form of a multi-page login process. For example, `css:button[type='user-submit']`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9894) in GitLab 12.4. | | `DAST_PASSWORD` 1 | string | The password to authenticate to in the website. Example: `P@55w0rd!` | | `DAST_PASSWORD_FIELD` | string | The selector of password field at the sign-in HTML form. Example: `id:password` | -| `DAST_SUBMIT_FIELD` | string | The `id` or `name` of the element that when selected submits the login form or the password form of a multi-page login process. For example, `css:button[type='submit']`. [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/9894) in GitLab 12.4. | +| `DAST_SUBMIT_FIELD` | string | The `id` or `name` of the element that when selected submits the login form or the password form of a multi-page login process. For example, `css:button[type='submit']`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9894) in GitLab 12.4. | | `DAST_USERNAME` 1 | string | The username to authenticate to in the website. Example: `admin` | | `DAST_USERNAME_FIELD` 1 | string | The selector of username field at the sign-in HTML form. Example: `name:username` | +| `DAST_AUTH_DISABLE_CLEAR_FIELDS` | boolean | Disables clearing of username and password fields before attempting manual login. Set to `false` by default. | 1. Available to an on-demand proxy-based DAST scan. +1. Not available to proxy-based scans. ### Update the target website @@ -93,12 +104,34 @@ dast: DAST_AUTH_URL: "https://example.com/login" ``` +### Configuration for HTTP authentication + +To use an [HTTP authentication scheme](https://www.chromium.org/developers/design-documents/http-authentication/) such as Basic Authentication you can set the `DAST_AUTH_TYPE` value to `basic-digest`. +Other schemes such as Negotiate or NTLM may work but aren't officially supported due to current lack of automated test coverage. + +Configuration requires the CI/CD variables `DAST_AUTH_TYPE`, `DAST_AUTH_URL`, `DAST_USERNAME`, `DAST_PASSWORD` to be defined for the DAST job. If you don't have a unique login URL, set `DAST_AUTH_URL` to the same URL as `DAST_WEBSITE`. + +```yaml +include: + - template: DAST.gitlab-ci.yml + +dast: + variables: + DAST_WEBSITE: "https://example.com" + DAST_AUTH_TYPE: "basic-digest" + DAST_AUTH_URL: "https://example.com" +``` + +Do **not** define `DAST_USERNAME` and `DAST_PASSWORD` in the YAML job definition file as this could present a security risk. Instead, create them as masked CI/CD variables using the GitLab UI. +See [Custom CI/CD variables](../../../ci/variables/index.md#for-a-project) for more information. +The proxy-based analyzer does not support basic authentication as an authentication mechanism. A workaround could be to set `DAST_REQUEST_HEADERS` as a masked CI/CD variable with a value containing the appropriate `Authorization` header, for example, `Authorization: Basic dXNlcm5hbWU6cGFzc3dvcmQK`. + ### Configuration for a single-step login form A single-step login form has all login form elements on a single page. Configuration requires the CI/CD variables `DAST_AUTH_URL`, `DAST_USERNAME`, `DAST_USERNAME_FIELD`, `DAST_PASSWORD`, `DAST_PASSWORD_FIELD`, and `DAST_SUBMIT_FIELD` to be defined for the DAST job. -It is recommended to set up the URL and selectors of fields in the job definition YAML, for example: +You should set up the URL and selectors of fields in the job definition YAML, for example: ```yaml include: @@ -114,16 +147,24 @@ dast: ``` Do **not** define `DAST_USERNAME` and `DAST_PASSWORD` in the YAML job definition file as this could present a security risk. Instead, create them as masked CI/CD variables using the GitLab UI. -See [Custom CI/CI variables](../../../ci/variables/index.md#for-a-project) for more information. +See [Custom CI/CD variables](../../../ci/variables/index.md#for-a-project) for more information. ### Configuration for a multi-step login form A multi-step login form has two pages. The first page has a form with the username and a next submit button. If the username is valid, a second form on the subsequent page has the password and the form submit button. -Configuration requires the CI/CD variables `DAST_AUTH_URL`, `DAST_USERNAME`, `DAST_USERNAME_FIELD`, `DAST_FIRST_SUBMIT_FIELD`, `DAST_PASSWORD`, `DAST_PASSWORD_FIELD`, and `DAST_SUBMIT_FIELD` to be defined for the DAST job. +Configuration requires the CI/CD variables to be defined for the DAST job: + +- `DAST_AUTH_URL` +- `DAST_USERNAME` +- `DAST_USERNAME_FIELD` +- `DAST_FIRST_SUBMIT_FIELD` +- `DAST_PASSWORD` +- `DAST_PASSWORD_FIELD` +- `DAST_SUBMIT_FIELD`. -It is recommended to set up the URL and selectors of fields in the job definition YAML, for example: +You should set up the URL and selectors of fields in the job definition YAML, for example: ```yaml include: @@ -140,21 +181,21 @@ dast: ``` Do **not** define `DAST_USERNAME` and `DAST_PASSWORD` in the YAML job definition file as this could present a security risk. Instead, create them as masked CI/CD variables using the GitLab UI. -See [Custom CI/CI variables](../../../ci/variables/index.md#for-a-project) for more information. +See [Custom CI/CD variables](../../../ci/variables/index.md#for-a-project) for more information. ### Configuration for Single Sign-On (SSO) If a user can log into an application, then in most cases, DAST is also able to log in. -This is the case even when an application uses Single Sign-on. Applications using SSO solutions should configure DAST +Even when an application uses Single Sign-on. Applications using SSO solutions should configure DAST authentication using the [single-step](#configuration-for-a-single-step-login-form) or [multi-step](#configuration-for-a-multi-step-login-form) login form configuration guides. DAST supports authentication processes where a user is redirected to an external Identity Provider's site to log in. Check the [known limitations](#known-limitations) of DAST authentication to determine if your SSO authentication process is supported. -### Clicking to navigate to the login form +### Clicking to go to the login form Define `DAST_BROWSER_PATH_TO_LOGIN_FORM` to provide a path of elements to click on from the `DAST_AUTH_URL` so that DAST can access the -login form. This is useful for applications that show the login form in a pop-up (modal) window or when the login form does not +login form. This method is suitable for applications that show the login form in a pop-up (modal) window or when the login form does not have a unique URL. For example: @@ -204,12 +245,12 @@ Selectors have the format `type`:`search string`. DAST searches for the selector Chrome DevTools element selector tool is an effective way to find a selector. -1. Open Chrome and navigate to the page where you would like to find a selector, for example, the login page for your site. +1. Open Chrome and go to the page where you would like to find a selector, for example, the login page for your site. 1. Open the `Elements` tab in Chrome DevTools with the keyboard shortcut `Command + Shift + c` in macOS or `Ctrl + Shift + c` in Windows. 1. Select the `Select an element in the page to select it` tool. ![search-elements](img/dast_auth_browser_scan_search_elements.png) 1. Select the field on your page that you would like to know the selector for. -1. Once the tool is active, highlight a field you wish to view the details of. +1. After the tool is active, highlight a field you wish to view the details of. ![highlight](img/dast_auth_browser_scan_highlight.png) 1. Once highlighted, you can see the element's details, including attributes that would make a good candidate for a selector. @@ -220,15 +261,15 @@ In this example, the `id="user_login"` appears to be a good candidate. You can u Judicious choice of selector leads to a scan that is resilient to the application changing. -In order of preference, it is recommended to choose as selectors: +In order of preference, you should choose as selectors: -- `id` fields. These are generally unique on a page, and rarely change. -- `name` fields. These are generally unique on a page, and rarely change. +- `id` fields. These fields generally unique on a page, and rarely change. +- `name` fields. These fields generally unique on a page, and rarely change. - `class` values specific to the field, such as the selector `"css:.username"` for the `username` class on the username field. - Presence of field specific data attributes, such as the selector, `"css:[data-username]"` when the `data-username` field has any value on the username field. - Multiple `class` hierarchy values, such as the selector `"css:.login-form .username"` when there are multiple elements with class `username` but only one nested inside the element with the class `login-form`. -When using selectors to locate specific fields we recommend you avoid searching on: +When using selectors to locate specific fields you should avoid searching on: - Any `id`, `name`, `attribute`, `class` or `value` that is dynamically generated. - Generic class names, such as `column-10` and `dark-grey`. @@ -237,7 +278,7 @@ When using selectors to locate specific fields we recommend you avoid searching ## Verifying authentication is successful -Once DAST has submitted the login form, a verification process takes place +After DAST has submitted the login form, a verification process takes place to determine if authentication succeeded. The scan halts with an error if authentication is unsuccessful. Following the submission of the login form, authentication is determined to be unsuccessful when: @@ -255,7 +296,7 @@ DAST tests for the absence of a login form if no verification checks are configu #### Verify based on the URL -Define `DAST_AUTH_VERIFICATION_URL` as the URL displayed in the browser tab once the login form is successfully submitted. +Define `DAST_AUTH_VERIFICATION_URL` as the URL displayed in the browser tab after the login form is successfully submitted. DAST compares the verification URL to the URL in the browser after authentication. If they are not the same, authentication is unsuccessful. @@ -275,7 +316,7 @@ dast: #### Verify based on presence of an element Define `DAST_AUTH_VERIFICATION_SELECTOR` as a [selector](#finding-an-elements-selector) that finds one or many elements on the page -displayed once the login form is successfully submitted. If no element is found, authentication is unsuccessful. +displayed after the login form is successfully submitted. If no element is found, authentication is unsuccessful. Searching for the selector on the page displayed when login fails should return no elements. For example: @@ -293,7 +334,7 @@ dast: #### Verify based on absence of a login form Define `DAST_AUTH_VERIFICATION_LOGIN_FORM` as `"true"` to indicate that DAST should search for the login form on the -page displayed once the login form is successfully submitted. If a login form is still present after logging in, authentication is unsuccessful. +page displayed after the login form is successfully submitted. If a login form is still present after logging in, authentication is unsuccessful. For example: diff --git a/doc/user/application_security/dast/browser_based.md b/doc/user/application_security/dast/browser_based.md index bdc08988cc0..7b263e5817d 100644 --- a/doc/user/application_security/dast/browser_based.md +++ b/doc/user/application_security/dast/browser_based.md @@ -42,10 +42,10 @@ To crawl a navigation path, DAST opens a browser window and instructs it to perf When the browser has finished loading the result of the final action, DAST inspects the page for actions a user might take, creates a new navigation for each found, and adds them to the navigation path to form new navigation paths. For example: -- DAST processes navigation path `LoadURL[https://example.com]`. -- DAST finds two user actions, `LeftClick[class=menu]` and `LeftClick[id=users]`. -- DAST creates two new navigation paths, `LoadURL[https://example.com] -> LeftClick[class=menu]` and `LoadURL[https://example.com] -> LeftClick[id=users]`. -- Crawling begins on the two new navigation paths. +1. DAST processes navigation path `LoadURL[https://example.com]`. +1. DAST finds two user actions, `LeftClick[class=menu]` and `LeftClick[id=users]`. +1. DAST creates two new navigation paths, `LoadURL[https://example.com] -> LeftClick[class=menu]` and `LoadURL[https://example.com] -> LeftClick[id=users]`. +1. Crawling begins on the two new navigation paths. It's common for an HTML element to exist in multiple places in an application, such as a menu visible on every page. Duplicate elements can cause crawlers to crawl the same pages again or become stuck in a loop. @@ -170,13 +170,15 @@ For authentication CI/CD variables, see [Authentication](authentication.md). | `DAST_ADVERTISE_SCAN` | boolean | `true` | Set to `true` to add a `Via` header to every request sent, advertising that the request was sent as part of a GitLab DAST scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/334947) in GitLab 14.1. | | `DAST_BROWSER_ACTION_STABILITY_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `800ms` | The maximum amount of time to wait for a browser to consider a page loaded and ready for analysis after completing an action. | | `DAST_BROWSER_ACTION_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `7s` | The maximum amount of time to wait for a browser to complete an action. | -| `DAST_BROWSER_ALLOWED_HOSTS` | List of strings | `site.com,another.com` | Hostnames included in this variable are considered in scope when crawled. By default the `DAST_WEBSITE` hostname is included in the allowed hosts list. Headers set using `DAST_REQUEST_HEADERS` are added to every request made to these hostnames. | +| `DAST_BROWSER_ALLOWED_HOSTS` | List of strings | `site.com,another.com` | Hostnames included in this variable are considered in scope when crawled. By default the `DAST_WEBSITE` hostname is included in the allowed hosts list. Headers set using `DAST_REQUEST_HEADERS` are added to every request made to these hostnames. | | `DAST_BROWSER_COOKIES` | dictionary | `abtesting_group:3,region:locked` | A cookie name and value to be added to every request. | -| `DAST_BROWSER_CRAWL_GRAPH` | boolean | `true` | Set to `true` to generate an SVG graph of navigation paths visited during crawl phase of the scan. | -| `DAST_BROWSER_DEVTOOLS_LOG` | string | `Default:messageAndBody,truncate:2000` | Set to log protocol messages between DAST and the Chromium browser. | | +| `DAST_BROWSER_CRAWL_GRAPH` | boolean | `true` | Set to `true` to generate an SVG graph of navigation paths visited during crawl phase of the scan. You must also define `gl-dast-crawl-graph.svg` as a CI job artifact to be able to access the generated graph. | +| `DAST_BROWSER_CRAWL_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `5m` | The maximum amount of time to wait for the crawl phase of the scan to complete. Defaults to `24h`. | +| `DAST_BROWSER_DEVTOOLS_LOG` | string | `Default:messageAndBody,truncate:2000` | Set to log protocol messages between DAST and the Chromium browser. | +| `DAST_BROWSER_DOM_READY_AFTER_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `200ms` | Define how long to wait for updates to the DOM before checking a page is stable. Defaults to `500ms`. | | `DAST_BROWSER_ELEMENT_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `600ms` | The maximum amount of time to wait for an element before determining it is ready for analysis. | | `DAST_BROWSER_EXCLUDED_ELEMENTS` | selector | `a[href='2.html'],css:.no-follow` | Comma-separated list of selectors that are ignored when scanning. | -| `DAST_BROWSER_EXCLUDED_HOSTS` | List of strings | `site.com,another.com` | Hostnames included in this variable are considered excluded and connections are forcibly dropped. | +| `DAST_BROWSER_EXCLUDED_HOSTS` | List of strings | `site.com,another.com` | Hostnames included in this variable are considered excluded and connections are forcibly dropped. | | `DAST_BROWSER_EXTRACT_ELEMENT_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `5s` | The maximum amount of time to allow the browser to extract newly found elements or navigations. | | `DAST_BROWSER_FILE_LOG` | List of strings | `brows:debug,auth:debug` | A list of modules and their intended logging level for use in the file log. | | `DAST_BROWSER_FILE_LOG_PATH` | string | `/output/browserker.log` | Set to the path of the file log. | @@ -190,19 +192,21 @@ For authentication CI/CD variables, see [Authentication](authentication.md). | `DAST_BROWSER_NAVIGATION_STABILITY_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `7s` | The maximum amount of time to wait for a browser to consider a page loaded and ready for analysis after a navigation completes. | | `DAST_BROWSER_NAVIGATION_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `15s` | The maximum amount of time to wait for a browser to navigate from one page to another. | | `DAST_BROWSER_NUMBER_OF_BROWSERS` | number | `3` | The maximum number of concurrent browser instances to use. For shared runners on GitLab.com, we recommended a maximum of three. Private runners with more resources may benefit from a higher number, but are likely to produce little benefit after five to seven instances. | -| `DAST_BROWSER_PAGE_LOADING_SELECTOR` | selector | `css:#page-is-loading` | Selector that when is no longer visible on the page, indicates to the analyzer that the page has finished loading and the scan can continue. Cannot be used with `DAST_BROWSER_PAGE_READY_SELECTOR` | -| `DAST_BROWSER_PAGE_READY_SELECTOR` | selector | `css:#page-is-ready` | Selector that when detected as visible on the page, indicates to the analyzer that the page has finished loading and the scan can continue. Cannot be used with `DAST_BROWSER_PAGE_LOADING_SELECTOR` | +| `DAST_BROWSER_PAGE_LOADING_SELECTOR` | selector | `css:#page-is-loading` | Selector that when is no longer visible on the page, indicates to the analyzer that the page has finished loading and the scan can continue. Cannot be used with `DAST_BROWSER_PAGE_READY_SELECTOR`. | +| `DAST_BROWSER_PAGE_READY_SELECTOR` | selector | `css:#page-is-ready` | Selector that when detected as visible on the page, indicates to the analyzer that the page has finished loading and the scan can continue. Cannot be used with `DAST_BROWSER_PAGE_LOADING_SELECTOR`. | +| `DAST_BROWSER_PASSIVE_CHECK_WORKERS` | int | `5` | Number of workers that passive scan in parallel. Recommend setting to the number of available CPUs. | | `DAST_BROWSER_SCAN` | boolean | `true` | Required to be `true` to run a browser-based scan. | | `DAST_BROWSER_SEARCH_ELEMENT_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `3s` | The maximum amount of time to allow the browser to search for new elements or user actions. | | `DAST_BROWSER_STABILITY_TIMEOUT` | [Duration string](https://pkg.go.dev/time#ParseDuration) | `7s` | The maximum amount of time to wait for a browser to consider a page loaded and ready for analysis. | | `DAST_EXCLUDE_RULES` | string | `10020,10026` | Set to a comma-separated list of ZAP Vulnerability Rule IDs to exclude them from running during the scan. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://www.zaproxy.org/docs/alerts/). | | `DAST_EXCLUDE_URLS` | URLs | `https://example.com/.*/sign-out` | The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. | +| `DAST_FF_ENABLE_BAS` | boolean | `true` | Set to `true` to [enable Breach and Attack Simulation](../breach_and_attack_simulation/index.md#extend-dynamic-application-security-testing-dast) during this DAST scan. | | `DAST_FULL_SCAN_ENABLED` | boolean | `true` | Set to `true` to run both passive and active checks. Default: `false` | | `DAST_PATHS` | string | `/page1.html,/category1/page3.html` | Set to a comma-separated list of URL paths relative to `DAST_WEBSITE` for DAST to scan. | | `DAST_PATHS_FILE` | string | `/builds/project/urls.txt` | Set to a file path containing a list of URL paths relative to `DAST_WEBSITE` for DAST to scan. The file must be plain text with one path per line. | | `DAST_PKCS12_CERTIFICATE_BASE64` | string | `ZGZkZ2p5NGd...` | The PKCS12 certificate used for sites that require Mutual TLS. Must be encoded as base64 text. | -| `DAST_PKCS12_PASSWORD` | string | `password` | The password of the certificate used in `DAST_PKCS12_CERTIFICATE_BASE64`. Create sensitive [custom CI/CI variables](../../../ci/variables/index.md#define-a-cicd-variable-in-the-ui) using the GitLab UI. | -| `DAST_REQUEST_HEADERS` | string | `Cache-control:no-cache` | Set to a comma-separated list of request header names and values. | +| `DAST_PKCS12_PASSWORD` | string | `password` | The password of the certificate used in `DAST_PKCS12_CERTIFICATE_BASE64`. Create sensitive [custom CI/CI variables](../../../ci/variables/index.md#define-a-cicd-variable-in-the-ui) using the GitLab UI. | +| `DAST_REQUEST_HEADERS` | string | `Cache-control:no-cache` | Set to a comma-separated list of request header names and values. | | `DAST_SKIP_TARGET_CHECK` | boolean | `true` | Set to `true` to prevent DAST from checking that the target is available before scanning. Default: `false`. | | `DAST_TARGET_AVAILABILITY_TIMEOUT` | number | `60` | Time limit in seconds to wait for target availability. | | `DAST_WEBSITE` | URL | `https://example.com` | The URL of the website to scan. | @@ -232,9 +236,14 @@ This can come at a cost of increased scan time. You can manage the trade-off between coverage and scan time with the following measures: +- Vertically scale the runner and use a higher number of browsers with the [variable](#available-cicd-variables) `DAST_BROWSER_NUMBER_OF_BROWSERS`. The default is `3`. - Limit the number of actions executed by the browser with the [variable](#available-cicd-variables) `DAST_BROWSER_MAX_ACTIONS`. The default is `10,000`. - Limit the page depth that the browser-based crawler checks coverage on with the [variable](#available-cicd-variables) `DAST_BROWSER_MAX_DEPTH`. The crawler uses a breadth-first search strategy, so pages with smaller depth are crawled first. The default is `10`. -- Vertically scale the runner and use a higher number of browsers with [variable](#available-cicd-variables) `DAST_BROWSER_NUMBER_OF_BROWSERS`. The default is `3`. +- Limit the time taken to crawl the target application with the [variable](#available-cicd-variables) `DAST_BROWSER_CRAWL_TIMEOUT`. The default is `24h`. Scans continue with passive and active checks when the crawler times out. +- Build the crawl graph with the [variable](#available-cicd-variables) `DAST_BROWSER_CRAWL_GRAPH` to see what pages are being crawled. +- Prevent pages from being crawled using the [variable](#available-cicd-variables) `DAST_EXCLUDE_URLS`. +- Prevent elements being selected using the [variable](#available-cicd-variables) `DAST_BROWSER_EXCLUDED_ELEMENTS`. Use with caution, as defining this variable causes an extra lookup for each page crawled. +- If the target application has minimal or fast rendering, consider reducing the [variable](#available-cicd-variables) `DAST_BROWSER_DOM_READY_AFTER_TIMEOUT` to a smaller value. The default is `500ms`. ## Timeouts @@ -275,16 +284,6 @@ dast: NOTE: Adjusting these values may impact scan time because they adjust how long each browser waits for various activities to complete. -## Artifacts - -Using the latest version of the DAST [template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml) these artifacts are exposed for download by default. - -The list of artifacts includes the following files: - -- `gl-dast-debug-auth-report.html` -- `gl-dast-debug-crawl-report.html` -- `gl-dast-crawl-graph.svg` - ## Troubleshooting See [troubleshooting](browser_based_troubleshooting.md) for more information. diff --git a/doc/user/application_security/dast/checks/16.7.md b/doc/user/application_security/dast/checks/16.7.md index d407234d2c2..edaace407ae 100644 --- a/doc/user/application_security/dast/checks/16.7.md +++ b/doc/user/application_security/dast/checks/16.7.md @@ -22,8 +22,8 @@ Only three directives are applicable for the `Strict-Transport-Security` header. 1. `includeSubDomains`: This optional, valueless directive signals that the policy applies to this host as well as any subdomains found under this host's domain. 1. `preload`: While not part of the specification, setting this optional value allows major browser organizations to add this site into the browser's preloaded set of HTTPS sites. This requires further action on behalf of the website operator to submit their domain to the browser's HSTS preload list. See [hstspreload.org](https://hstspreload.org/) for more information. -Note that invalid directives, or the `Strict-Transport-Security` header appearing more than once (if the values are -different) is considered invalid. +Invalid directives, or the `Strict-Transport-Security` header appearing more than once (if the +values are different) is considered invalid. Prior to adding to this security configuration to your website, it is recommended you review the hstspreload.org [Deployment Recommendations](https://hstspreload.org/#deployment-recommendations). diff --git a/doc/user/application_security/dast/dast_troubleshooting.md b/doc/user/application_security/dast/dast_troubleshooting.md index da382920604..c2e7f153e02 100644 --- a/doc/user/application_security/dast/dast_troubleshooting.md +++ b/doc/user/application_security/dast/dast_troubleshooting.md @@ -70,7 +70,7 @@ tips for optimizing DAST scans in a [blog post](https://about.gitlab.com/blog/20 ## Getting warning message `gl-dast-report.json: no matching files` -For information on this, see the [general Application Security troubleshooting section](../../../ci/pipelines/job_artifacts.md#error-message-no-files-to-upload). +For information on this, see the [general Application Security troubleshooting section](../../../ci/jobs/job_artifacts_troubleshooting.md#error-message-no-files-to-upload). ## Getting error `dast job: chosen stage does not exist` when including DAST CI template diff --git a/doc/user/application_security/dast/index.md b/doc/user/application_security/dast/index.md index 3cb8967afd2..1f4506a22e5 100644 --- a/doc/user/application_security/dast/index.md +++ b/doc/user/application_security/dast/index.md @@ -15,6 +15,9 @@ visible from the source code. Dynamic Application Security Testing (DAST) examines applications for vulnerabilities like these in deployed environments. + +For an overview, see [Dynamic Application Security Testing (DAST)](https://www.youtube.com/watch?v=nbeDUoLZJTo). + NOTE: To learn how four of the top six attacks were application-based and how to protect your organization, download our diff --git a/doc/user/application_security/dast/proxy-based.md b/doc/user/application_security/dast/proxy-based.md index f70afac4c26..f65501712cc 100644 --- a/doc/user/application_security/dast/proxy-based.md +++ b/doc/user/application_security/dast/proxy-based.md @@ -80,10 +80,10 @@ To enable DAST to run automatically, either: - Enable [Auto DAST](../../../topics/autodevops/stages.md#auto-dast) (provided by [Auto DevOps](../../../topics/autodevops/index.md)). -- [Edit the `.gitlab.ci.yml` file manually](#edit-the-gitlabciyml-file-manually). -- [Use an automatically configured merge request](#configure-dast-using-the-ui). +- [Edit the `.gitlab-ci.yml` file manually](#edit-the-gitlab-ciyml-file-manually). +- [Configure DAST using the UI](#configure-dast-using-the-ui). -#### Edit the `.gitlab.ci.yml` file manually +#### Edit the `.gitlab-ci.yml` file manually In this method you manually edit the existing `.gitlab-ci.yml` file. Use this method if your GitLab CI/CD configuration file is complex. @@ -358,37 +358,36 @@ including a large number of false positives. | CI/CD variable | Type | Description | |:------------------------------------------------|:--------------|:------------------------------| -| `DAST_ADVERTISE_SCAN` | boolean | Set to `true` to add a `Via` header to every request sent, advertising that the request was sent as part of a GitLab DAST scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/334947) in GitLab 14.1. | -| `DAST_AGGREGATE_VULNERABILITIES` | boolean | Vulnerability aggregation is set to `true` by default. To disable this feature and see each vulnerability individually set to `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/254043) in GitLab 14.0. | +| `DAST_ADVERTISE_SCAN` | boolean | Set to `true` to add a `Via` header to every request sent, advertising that the request was sent as part of a GitLab DAST scan. | +| `DAST_AGGREGATE_VULNERABILITIES` | boolean | Vulnerability aggregation is set to `true` by default. To disable this feature and see each vulnerability individually set to `false`. | | `DAST_ALLOWED_HOSTS` | Comma-separated list of strings | Hostnames included in this variable are considered in scope when crawled. By default the `DAST_WEBSITE` hostname is included in the allowed hosts list. Headers set using `DAST_REQUEST_HEADERS` are added to every request made to these hostnames. Example, `site.com,another.com`. | -| `DAST_API_HOST_OVERRIDE` 1 | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/383467)** in GitLab 15.7. Replaced by [DAST API scan](../dast_api/index.md#available-cicd-variables). Used to override domains defined in API specification files. Only supported when importing the API specification from a URL. Example: `example.com:8080`. | -| `DAST_API_SPECIFICATION` 1 | URL or string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/383467)** in GitLab 15.7. Replaced by [DAST API scan](../dast_api/index.md#available-cicd-variables). The API specification to import. The specification can be hosted at a URL, or the name of a file present in the `/zap/wrk` directory. The variable `DAST_WEBSITE` must be specified if this is omitted. | +| `DAST_API_HOST_OVERRIDE` 1 | string | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/383467)** in GitLab 16.0. Replaced by [DAST API scan](../dast_api/index.md#available-cicd-variables). | +| `DAST_API_SPECIFICATION` 1 | URL or string | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/383467)** in GitLab 16.0. Replaced by [DAST API scan](../dast_api/index.md#available-cicd-variables). | | `DAST_AUTH_EXCLUDE_URLS` | URLs | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/289959)** in GitLab 14.0. Replaced by `DAST_EXCLUDE_URLS`. The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. | | `DAST_AUTO_UPDATE_ADDONS` | boolean | ZAP add-ons are pinned to specific versions in the DAST Docker image. Set to `true` to download the latest versions when the scan starts. Default: `false`. | -| `DAST_DEBUG` 1 | boolean | Enable debug message output. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_EXCLUDE_RULES` | string | Set to a comma-separated list of Vulnerability Rule IDs to exclude them from running during the scan. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://www.zaproxy.org/docs/alerts/). For example, `HTTP Parameter Override` has a rule ID of `10026`. Cannot be used when `DAST_ONLY_INCLUDE_RULES` is set. **Note:** In earlier versions of GitLab the excluded rules were executed but vulnerabilities they generated were suppressed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/118641) in GitLab 12.10. | +| `DAST_DEBUG` 1 | boolean | Enable debug message output. Default: `false`. | +| `DAST_EXCLUDE_RULES` | string | Set to a comma-separated list of Vulnerability Rule IDs to exclude them from running during the scan. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://www.zaproxy.org/docs/alerts/). For example, `HTTP Parameter Override` has a rule ID of `10026`. Cannot be used when `DAST_ONLY_INCLUDE_RULES` is set. **Note:** In earlier versions of GitLab the excluded rules were executed but vulnerabilities they generated were suppressed. | | `DAST_EXCLUDE_URLS` 1 | URLs | The URLs to skip during the authenticated scan; comma-separated. Regular expression syntax can be used to match multiple URLs. For example, `.*` matches an arbitrary character sequence. Example, `http://example.com/sign-out`. | -| `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` | boolean | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/293595)** in GitLab 14.0. Set to `true` to require domain validation when running DAST full scans. Default: `false` | | `DAST_FULL_SCAN_ENABLED` 1 | boolean | Set to `true` to run a [ZAP Full Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Full-Scan) instead of a [ZAP Baseline Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan). Default: `false` | -| `DAST_HTML_REPORT` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/384340)** in GitLab 15.7. The filename of the HTML report written at the end of a scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_INCLUDE_ALPHA_VULNERABILITIES` | boolean | Set to `true` to include alpha passive and active scan rules. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_MARKDOWN_REPORT` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/384340)** in GitLab 15.7. The filename of the Markdown report written at the end of a scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_MASK_HTTP_HEADERS` | string | Comma-separated list of request and response headers to be masked (GitLab 13.1). Must contain **all** headers to be masked. Refer to [list of headers that are masked by default](#hide-sensitive-information). | -| `DAST_MAX_URLS_PER_VULNERABILITY` | number | The maximum number of URLs reported for a single vulnerability. `DAST_MAX_URLS_PER_VULNERABILITY` is set to `50` by default. To list all the URLs set to `0`. [Introduced](https://gitlab.com/gitlab-org/security-products/dast/-/merge_requests/433) in GitLab 13.12. | -| `DAST_ONLY_INCLUDE_RULES` | string | Set to a comma-separated list of Vulnerability Rule IDs to configure the scan to run only them. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://www.zaproxy.org/docs/alerts/). Cannot be used when `DAST_EXCLUDE_RULES` is set. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/250651) in GitLab 13.12. | -| `DAST_PATHS` | string | Set to a comma-separated list of URLs for DAST to scan. For example, `/page1.html,/category1/page3.html,/page2.html`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214120) in GitLab 13.4. | -| `DAST_PATHS_FILE` | string | The file path containing the paths within `DAST_WEBSITE` to scan. The file must be plain text with one path per line. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/258825) in GitLab 13.6. | +| `DAST_HTML_REPORT` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/384340)** in GitLab 15.7. The filename of the HTML report written at the end of a scan. | +| `DAST_INCLUDE_ALPHA_VULNERABILITIES` | boolean | Set to `true` to include alpha passive and active scan rules. Default: `false`. | +| `DAST_MARKDOWN_REPORT` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/384340)** in GitLab 15.7. The filename of the Markdown report written at the end of a scan. | +| `DAST_MASK_HTTP_HEADERS` | string | Comma-separated list of request and response headers to be masked. Must contain **all** headers to be masked. Refer to [list of headers that are masked by default](#hide-sensitive-information). | +| `DAST_MAX_URLS_PER_VULNERABILITY` | number | The maximum number of URLs reported for a single vulnerability. `DAST_MAX_URLS_PER_VULNERABILITY` is set to `50` by default. To list all the URLs set to `0`. | +| `DAST_ONLY_INCLUDE_RULES` | string | Set to a comma-separated list of Vulnerability Rule IDs to configure the scan to run only them. Rule IDs are numbers and can be found from the DAST log or on the [ZAP project](https://www.zaproxy.org/docs/alerts/). Cannot be used when `DAST_EXCLUDE_RULES` is set. | +| `DAST_PATHS` | string | Set to a comma-separated list of URLs for DAST to scan. For example, `/page1.html,/category1/page3.html,/page2.html`. | +| `DAST_PATHS_FILE` | string | The file path containing the paths within `DAST_WEBSITE` to scan. The file must be plain text with one path per line. | | `DAST_PKCS12_CERTIFICATE_BASE64` | string | The PKCS12 certificate used for sites that require Mutual TLS. Must be encoded as base64 text. | | `DAST_PKCS12_PASSWORD` | string | The password of the certificate used in `DAST_PKCS12_CERTIFICATE_BASE64`. | | `DAST_REQUEST_HEADERS` 1 | string | Set to a comma-separated list of request header names and values. Headers are added to every request made by DAST. For example, `Cache-control: no-cache,User-Agent: DAST/1.0` | -| `DAST_SKIP_TARGET_CHECK` | boolean | Set to `true` to prevent DAST from checking that the target is available before scanning. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229067) in GitLab 13.8. | -| `DAST_SPIDER_MINS` 1 | number | The maximum duration of the spider scan in minutes. Set to `0` for unlimited. Default: One minute, or unlimited when the scan is a full scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_SPIDER_START_AT_HOST` | boolean | Set to `false` to prevent DAST from resetting the target to its host before scanning. When `true`, non-host targets `http://test.site/some_path` is reset to `http://test.site` before scan. Default: `true`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/258805) in GitLab 13.6. | +| `DAST_SKIP_TARGET_CHECK` | boolean | Set to `true` to prevent DAST from checking that the target is available before scanning. Default: `false`. | +| `DAST_SPIDER_MINS` 1 | number | The maximum duration of the spider scan in minutes. Set to `0` for unlimited. Default: One minute, or unlimited when the scan is a full scan. | +| `DAST_SPIDER_START_AT_HOST` | boolean | Set to `false` to prevent DAST from resetting the target to its host before scanning. When `true`, non-host targets `http://test.site/some_path` is reset to `http://test.site` before scan. Default: `true`. | | `DAST_TARGET_AVAILABILITY_TIMEOUT` 1 | number | Time limit in seconds to wait for target availability. | -| `DAST_USE_AJAX_SPIDER` 1 | boolean | Set to `true` to use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Default: `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | -| `DAST_XML_REPORT` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/384340)** in GitLab 15.7. The filename of the XML report written at the end of a scan. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_USE_AJAX_SPIDER` 1 | boolean | Set to `true` to use the AJAX spider in addition to the traditional spider, useful for crawling sites that require JavaScript. Default: `false`. | +| `DAST_XML_REPORT` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/384340)** in GitLab 15.7. The filename of the XML report written at the end of a scan. | | `DAST_WEBSITE` 1 | URL | The URL of the website to scan. | -| `DAST_ZAP_CLI_OPTIONS` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/383467)** in GitLab 15.7. ZAP server command-line options. For example, `-Xmx3072m` would set the Java maximum memory allocation pool size. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12652) in GitLab 13.1. | +| `DAST_ZAP_CLI_OPTIONS` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/383467)** in GitLab 15.7. ZAP server command-line options. For example, `-Xmx3072m` would set the Java maximum memory allocation pool size. | | `DAST_ZAP_LOG_CONFIGURATION` | string | **{warning}** **[Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/383467)** in GitLab 15.7. Set to a semicolon-separated list of additional log4j properties for the ZAP Server. Example: `logger.httpsender.name=org.parosproxy.paros.network.HttpSender;logger.httpsender.level=debug;logger.sitemap.name=org.parosproxy.paros.model.SiteMap;logger.sitemap.level=debug;` | | `SECURE_ANALYZERS_PREFIX` | URL | Set the Docker registry base address from which to download the analyzer. | diff --git a/doc/user/application_security/dast_api/index.md b/doc/user/application_security/dast_api/index.md index 0cdbb879cfc..d494259ecc4 100644 --- a/doc/user/application_security/dast_api/index.md +++ b/doc/user/application_security/dast_api/index.md @@ -48,6 +48,7 @@ The following projects demonstrate DAST API scanning: - [Example Postman Collection project](https://gitlab.com/gitlab-org/security-products/demos/api-dast/postman-example) - [Example GraphQL project](https://gitlab.com/gitlab-org/security-products/demos/api-dast/graphql-example) - [Example SOAP project](https://gitlab.com/gitlab-org/security-products/demos/api-dast/soap-example) +- [Authentication Token using Selenium](https://gitlab.com/gitlab-org/security-products/demos/api-dast/auth-token-selenium) ## Targeting API for DAST scanning @@ -97,8 +98,6 @@ The environment variables `DAST_API_OPENAPI_ALL_MEDIA_TYPES` and `DAST_API_OPENA #### Configure DAST API with an OpenAPI Specification -To configure DAST API scanning with an OpenAPI specification: - To configure DAST API scanning with an OpenAPI Specification: 1. [Include](../../../ci/yaml/index.md#includetemplate) @@ -264,6 +263,8 @@ This example is a minimal configuration for DAST API. From here you can: #### DAST API scanning with a GraphQL Schema file +DAST API can use a GraphQL schema file to understand and test a GraphQL endpoint that has introspection disabled. To use a GraphQL schema file, it must be in the introspection JSON format. A GraphQL schema can be converted to a the introspection JSON format using an online 3rd party tool: [https://transform.tools/graphql-to-introspection-json](https://transform.tools/graphql-to-introspection-json). + To configure DAST API to use a GraphQL schema file that provides information about the target API to test: 1. [Include](../../../ci/yaml/index.md#includetemplate) @@ -997,6 +998,14 @@ repository's root as `.gitlab/gitlab-dast-api-config.yml`. The following profiles are pre-defined in the default configuration file. Profiles can be added, removed, and modified by creating a custom configuration. +#### Passive + +- Application Information Check +- Cleartext Authentication Check +- JSON Hijacking Check +- Sensitive Information Check +- Session Cookie Check + #### Quick - Application Information Check @@ -1924,7 +1933,7 @@ variables: When configured correctly, a CI/CD pipeline contains a `dast` stage and an `dast_api` job. The job only fails when an invalid configuration is provided. During typical operation, the job always succeeds even if vulnerabilities are identified during testing. -Vulnerabilities are displayed on the **Security** pipeline tab with the suite name. When testing against the repositories default branch, the DAST API vulnerabilities are also shown on the Security & Compliance's Vulnerability Report page. +Vulnerabilities are displayed on the **Security** pipeline tab with the suite name. When testing against the repositories default branch, the DAST API vulnerabilities are also shown on the Security and Compliance's Vulnerability Report page. To prevent an excessive number of reported vulnerabilities, the DAST API scanner limits the number of vulnerabilities it reports per operation. @@ -1941,7 +1950,7 @@ Follow these steps to view details of a vulnerability: 1. You can view vulnerabilities in a project, or a merge request: - - In a project, go to the project's **Security & Compliance > Vulnerability Report** + - In a project, go to the project's **Security and Compliance > Vulnerability Report** page. This page shows all vulnerabilities from the default branch only. - In a merge request, go the merge request's **Security** section and select the **Expand** button. DAST API vulnerabilities are available in a section labeled @@ -2243,9 +2252,12 @@ dast_api_v1: variables: DAST_API_EXCLUDE_PATHS: /api/v1/** rules: - - if: $DAST_API_DISABLED + - if: $DAST_API_DISABLED == 'true' || $DAST_API_DISABLED == '1' when: never - - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH && + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == 'true' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == '1' && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - if: $CI_COMMIT_BRANCH && @@ -2258,9 +2270,12 @@ dast_api_v2: variables: DAST_API_EXCLUDE_PATHS: /api/v2/** rules: - - if: $DAST_API_DISABLED + - if: $DAST_API_DISABLED == 'true' || $DAST_API_DISABLED == '1' + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == 'true' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH && + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == '1' && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - if: $CI_COMMIT_BRANCH && @@ -2292,9 +2307,12 @@ dast_api_branch: variables: DAST_API_EXCLUDE_PATHS: /api/large_response_json rules: - - if: $DAST_API_DISABLED + - if: $DAST_API_DISABLED == 'true' || $DAST_API_DISABLED == '1' when: never - - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH && + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == 'true' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == '1' && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - if: $CI_COMMIT_BRANCH && @@ -2310,9 +2328,12 @@ dast_api_branch: dast_api_main: extends: dast_api rules: - - if: $DAST_API_DISABLED + - if: $DAST_API_DISABLED == 'true' || $DAST_API_DISABLED == '1' + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == 'true' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH && + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == '1' && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - if: $CI_COMMIT_BRANCH && @@ -2553,14 +2574,75 @@ You can expedite the investigation with the [testssl.sh tool](https://testssl.sh 1. Run `./testssl.sh --log https://specs`. 1. Attach the log file to your support ticket. +### `ERROR: Job failed: failed to pull image` + +This error message occurs when pulling an image from a container registry that requires authentication to access (it is not public). + +In the job console output the error looks like: + +```log +Running with gitlab-runner 15.6.0~beta.186.ga889181a (a889181a) + on blue-2.shared.runners-manager.gitlab.com/default XxUrkriX +Resolving secrets +00:00 +Preparing the "docker+machine" executor +00:06 +Using Docker executor with image registry.gitlab.com/security-products/api-security:2 ... +Starting service registry.example.com/my-target-app:latest ... +Pulling docker image registry.example.com/my-target-app:latest ... +WARNING: Failed to pull image with policy "always": Error response from daemon: Get https://registry.example.com/my-target-app/manifests/latest: unauthorized (manager.go:237:0s) +ERROR: Job failed: failed to pull image "registry.example.com/my-target-app:latest" with specified policies [always]: Error response from daemon: Get https://registry.example.com/my-target-app/manifests/latest: unauthorized (manager.go:237:0s) +``` + +**Error message** + +- In GitLab 15.9 and earlier, `ERROR: Job failed: failed to pull image` followed by `Error response from daemon: Get IMAGE: unauthorized`. + +**Solution** + +Authentication credentials are provided using the methods outlined in the [Access an image from a private Container Registry](../../../ci/docker/using_docker_images.md#access-an-image-from-a-private-container-registry) documentation section. The method used is dictated by your container registry provider and its configuration. If your using a container registry provided by a 3rd party, such as a cloud provider (Azure, Google Could (GCP), AWS and so on), check the providers documentation for information on how to authenticate to their container registries. + +The following example uses the [statically defined credentials](../../../ci/docker/using_docker_images.md#use-statically-defined-credentials) authentication method. In this example the container registry is `registry.example.com` and image is `my-target-app:latest`. + +1. Read how to [Determine your `DOCKER_AUTH_CONFIG` data](../../../ci/docker/using_docker_images.md#determine-your-docker_auth_config-data) to understand how to compute the variable value for `DOCKER_AUTH_CONFIG`. The configuration variable `DOCKER_AUTH_CONFIG` contains the Docker JSON configuration to provide the appropriate authentication information. For example, to access private container registry: `registry.example.com` with the credentials `aGVsbG8gd29ybGQK`, the Docker JSON looks like: + + ```json + { + "auths": { + "registry.example.com": { + "auth": "aGVsbG8gd29ybGQK" + } + } + } + ``` + +1. Add the `DOCKER_AUTH_CONFIG` as a CI/CD variable. Instead of adding the configuration variable directly in your `.gitlab-ci.yml`file you should create a project [CI/CD variable](../../../ci/variables/index.md#for-a-project). +1. Rerun your job, and the statically-defined credentials are now used to sign in to the private container registry `registry.example.com`, and let you pull the image `my-target-app:latest`. If succeeded the job console shows an output like: + + ```log + Running with gitlab-runner 15.6.0~beta.186.ga889181a (a889181a) + on blue-4.shared.runners-manager.gitlab.com/default J2nyww-s + Resolving secrets + 00:00 + Preparing the "docker+machine" executor + 00:56 + Using Docker executor with image registry.gitlab.com/security-products/api-security:2 ... + Starting service registry.example.com/my-target-app:latest ... + Authenticating with credentials from $DOCKER_AUTH_CONFIG + Pulling docker image registry.example.com/my-target-app:latest ... + Using docker image sha256:139c39668e5e4417f7d0eb0eeb74145ba862f4f3c24f7c6594ecb2f82dc4ad06 for registry.example.com/my-target-app:latest with digest registry.example.com/my-target- + app@sha256:2b69fc7c3627dbd0ebaa17674c264fcd2f2ba21ed9552a472acf8b065d39039c ... + Waiting for services to be up and running (timeout 30 seconds)... + ``` + ## Get support or request an improvement To get support for your particular problem, use the [getting help channels](https://about.gitlab.com/get-help/). The [GitLab issue tracker on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues) is the right place for bugs and feature proposals about API Security and DAST API. -Use `~"Category:API Security"` [label](../../../development/contributing/issue_workflow.md#labels) when opening a new issue regarding DAST API to ensure it is quickly reviewed by the right people. Refer to our [review response SLO](https://about.gitlab.com/handbook/engineering/workflow/code-review/#review-response-slo) to understand when you should receive a response. +Use `~"Category:API Security"` [label](../../../development/labels/index.md) when opening a new issue regarding DAST API to ensure it is quickly reviewed by the right people. Refer to our [review response SLO](https://about.gitlab.com/handbook/engineering/workflow/code-review/#review-response-slo) to understand when you should receive a response. -[Search the issue tracker](https://gitlab.com/gitlab-org/gitlab/-/issues) for similar entries before submitting your own, there's a good chance somebody else had the same issue or feature proposal. Show your support with an award emoji and or join the discussion. +[Search the issue tracker](https://gitlab.com/gitlab-org/gitlab/-/issues) for similar entries before submitting your own, there's a good chance somebody else had the same issue or feature proposal. Show your support with an emoji reaction or join the discussion. When experiencing a behavior not working as expected, consider providing contextual information: diff --git a/doc/user/application_security/dependency_list/index.md b/doc/user/application_security/dependency_list/index.md index b86d98471ac..afed5b3b0ca 100644 --- a/doc/user/application_security/dependency_list/index.md +++ b/doc/user/application_security/dependency_list/index.md @@ -13,11 +13,12 @@ info: To determine the technical writer assigned to the Stage/Group associated w Use the dependency list to review your project's dependencies and key details about those dependencies, including their known vulnerabilities. It is a collection of dependencies in your project, including existing and new findings. -To see the dependency list, go to your project and select **Security & Compliance > Dependency list**. + +For an overview, see [Project Dependency](https://www.youtube.com/watch?v=ckqkn9Tnbw4). -This information is sometimes referred to as a Software Bill of Materials, SBOM, or BOM. +To see the dependency list, go to your project and select **Security and Compliance > Dependency list**. -The dependency list only shows the results of the last successful pipeline to run on the default branch. This is why we recommend not changing the default behavior of allowing the secure jobs to fail. +This information is sometimes referred to as a Software Bill of Materials, SBOM, or BOM. ## Prerequisites @@ -29,6 +30,9 @@ To view your project's dependencies, ensure you meet the following requirements: - Your project uses at least one of the [languages and package managers](../dependency_scanning/index.md#supported-languages-and-package-managers) supported by Gemnasium. +- A successful pipeline was run on the default branch. + You should not change the default behavior of allowing the + [application security jobs](../../application_security/index.md#application-coverage) to fail. ## View a project's dependencies diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 579dd4dfc4f..c510be55981 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -35,6 +35,9 @@ vulnerability. ![Dependency scanning Widget](img/dependency_scanning_v13_2.png) + +For an overview, see [Dependency Scanning](https://www.youtube.com/watch?v=TBnfbGk4c4o). + ## Dependency Scanning compared to Container Scanning GitLab offers both Dependency Scanning and Container Scanning @@ -147,8 +150,8 @@ table.supported-languages ul {
- - + + @@ -187,14 +190,10 @@ table.supported-languages ul { - + @@ -212,8 +211,8 @@ table.supported-languages ul { - - + + - + + + + + @@ -238,7 +241,7 @@ table.supported-languages ul { - + @@ -258,8 +261,8 @@ table.supported-languages ul { @@ -295,28 +298,30 @@ table.supported-languages ul {

  • - Support for these versions of Java is deprecated and is planned to be removed in the GitLab 16.0 release. Additionally, these versions of Java are not supported by the FIPS-enabled image of gemnasium-maven. Official support is limited to LTS versions only. Although it may be possible to use Dependency Scanning with other versions by building a custom dependency scanning image, this approach is not officially supported by GitLab. + Support for Kotlin projects for Android is tracked in issue 336866.

  • - Although Gradle with Java 8 is supported, there are other issues such that Android project builds are not supported at this time. - See the backlog issue Android support for Dependency - Scanning (gemnasium-maven) for more details. Also, Gradle is not supported when FIPS mode is enabled. + Gradle is not supported when FIPS mode is enabled.

  • - +

    - The presence of a Pipfile.lock file alone will not trigger the analyzer; the presence of a Pipfile is - still required in order for the analyzer to be executed. However, if a Pipfile.lock file is found, it is used by - Gemnasium to scan the exact package versions listed in this file. + Support for pnpm lockfiles was introduced in GitLab 15.11. pnpm lockfiles do not store bundled dependencies, so the reported dependencies may differ from npm or yarn.

    +
    • +
  • +

    - Support for Pipfile.lock files without requiring the presence of a Pipfile is tracked in - issue: Dependency Scanning of Pipfile.lock without - installing project dependencies. + For support of Python 3.10, add the following stanza to the GitLab CI/CD configuration file. This specifies that the Python 3.10 image is to be used, instead of the default Python 3.9. +

    +
    +
    gemnasium-dependency_scanning:
+  image:
+    name: $CI_TEMPLATE_REGISTRY_HOST/security-products/gemnasium-python:4-python-3.10

  • @@ -355,7 +360,8 @@ The following package managers use lockfiles that GitLab analyzers are capable o | Go | Not applicable | [1.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/go-modules/gosum/default/go.sum) 1 | | NuGet | v1 | [4.9](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/csharp-nuget-dotnetcore/default/src/web.api/packages.lock.json#L2) | | npm | v1, v2, v32 | [6.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/js-npm/default/package-lock.json#L4), [7.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/js-npm/lockfileVersion2/package-lock.json#L4), [9.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/scanner/parser/npm/fixtures/lockfile-v3/simple/package-lock.json#L4) | -| yarn | v1 | [1.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/js-yarn/default/yarn.lock#L2) | +| pnpm | v5.3, v5.4, v6 | [7.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/js-pnpm/default/pnpm-lock.yaml#L1), [8.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/scanner/parser/pnpm/fixtures/v6/simple/pnpm-lock.yaml#L1) | +| yarn | v1, v23, v33 | [1.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/js-yarn/classic/default/yarn.lock#L2), [2.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/js-yarn/berry/v2/default/yarn.lock), [3.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/js-yarn/berry/v3/default/yarn.lock) | | Poetry | v1 | [1.x](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium/-/blob/master/qa/fixtures/python-poetry/default/poetry.lock) | @@ -373,6 +379,26 @@ The following package managers use lockfiles that GitLab analyzers are capable o Support for lockfileVersion = 3 was introduced in GitLab 15.7.

    • +
  • + +

    + Support for Yarn v2 and v3 was introduced in GitLab 15.11. However, this feature is also available to versions of GitLab 15.0 and later. +

    +

    + The following features are not supported for Yarn v2 or v3: +

    + +

    + Yarn files that contain a patch, a workspace, or both, are still processed, but these features are ignored. +

    +
    • @@ -398,7 +424,7 @@ To support the following package managers, the GitLab analyzers proceed in two s

  • - This test uses the default version of maven specified by the .tool-versions file. + This test uses the default version of maven specified by the `.tool-versions` file.

  • @@ -417,7 +443,7 @@ To support the following package managers, the GitLab analyzers proceed in two s By default, the analyzer uses Java 17 and Gradle 7.3.3.

    - For Java versions 8 and 11, Gradle 6.7.1 is automatically selected, and for Java versions 13 to 17, Gradle 7.3.3 is automatically selected. + For Java versions 8 and 11, Gradle 6.7.1 is automatically selected, and for Java version 17, Gradle 7.3.3 is automatically selected.

  • @@ -553,7 +579,7 @@ always take the latest dependency scanning artifact available. To enable Dependency Scanning in a project, you can create a merge request: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Security & Compliance > Configuration**. +1. On the left sidebar, select **Security and Compliance > Security configuration**. 1. In the **Dependency Scanning** row, select **Configure with a merge request**. 1. Review and merge the merge request to enable Dependency Scanning. @@ -626,6 +652,7 @@ The following variables allow configuration of global dependency scanning settin | `DS_EXCLUDED_ANALYZERS` | Specify the analyzers (by name) to exclude from Dependency Scanning. For more information, see [Dependency Scanning Analyzers](analyzers.md). | | `DS_EXCLUDED_PATHS` | Exclude files and directories from the scan based on the paths. A comma-separated list of patterns. Patterns can be globs (see [`doublestar.Match`](https://pkg.go.dev/github.com/bmatcuk/doublestar/v4@v4.0.2#Match) for supported patterns), or file or folder paths (for example, `doc,spec`). Parent directories also match patterns. Default: `"spec, test, tests, tmp"`. | | `DS_IMAGE_SUFFIX` | Suffix added to the image name. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/354796) in GitLab 14.10.) Automatically set to `"-fips"` when FIPS mode is enabled. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/357922) in GitLab 15.0.) | +| `DS_MAX_DEPTH` | Defines how many directory levels deep that the analyzer should search for supported files to scan. A value of `-1` scans all directories regardless of depth. Default: `2`. | | `SECURE_ANALYZERS_PREFIX` | Override the name of the Docker registry providing the official default images (proxy). Read more about [customizing analyzers](analyzers.md). | | `SECURE_LOG_LEVEL` | Set the minimum logging level. Messages of this logging level or higher are output. From highest to lowest severity, the logging levels are: `fatal`, `error`, `warn`, `info`, `debug`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10880) in GitLab 13.1. Default: `info`. | @@ -642,16 +669,17 @@ The following variables are used for configuring specific analyzers (used for a | `DS_REMEDIATE` | `gemnasium` | `"true"`, `"false"` in FIPS mode | Enable automatic remediation of vulnerable dependencies. Not supported in FIPS mode. | | `DS_REMEDIATE_TIMEOUT` | `gemnasium` | `5m` | Timeout for auto-remediation. | | `GEMNASIUM_LIBRARY_SCAN_ENABLED` | `gemnasium` | `"true"` | Enable detecting vulnerabilities in vendored JavaScript libraries. For now, `gemnasium` leverages [`Retire.js`](https://github.com/RetireJS/retire.js) to do this job. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/350512) in GitLab 14.8. | -| `DS_JAVA_VERSION` | `gemnasium-maven` | `17` | Version of Java. Available versions: `8`, `11`, `13`, `14`, `15`, `16`, `17`. Available versions in FIPS-enabled image: `8`, `11`, `17`. | +| `DS_JAVA_VERSION` | `gemnasium-maven` | `17` | Version of Java. Available versions: `8`, `11`, `17`. | | `MAVEN_CLI_OPTS` | `gemnasium-maven` | `"-DskipTests --batch-mode"` | List of command line arguments that are passed to `maven` by the analyzer. See an example for [using private repositories](../index.md#using-private-maven-repositories). | | `GRADLE_CLI_OPTS` | `gemnasium-maven` | | List of command line arguments that are passed to `gradle` by the analyzer. | | `SBT_CLI_OPTS` | `gemnasium-maven` | | List of command-line arguments that the analyzer passes to `sbt`. | | `PIP_INDEX_URL` | `gemnasium-python` | `https://pypi.org/simple` | Base URL of Python Package Index. | | `PIP_EXTRA_INDEX_URL` | `gemnasium-python` | | Array of [extra URLs](https://pip.pypa.io/en/stable/reference/pip_install/#cmdoption-extra-index-url) of package indexes to use in addition to `PIP_INDEX_URL`. Comma-separated. **Warning:** Read [the following security consideration](#python-projects) when using this environment variable. | | `PIP_REQUIREMENTS_FILE` | `gemnasium-python` | | Pip requirements file to be scanned. | +| `PIPENV_PYPI_MIRROR` | `gemnasium-python` | | If set, overrides the PyPi index used by Pipenv with a [mirror](https://github.com/pypa/pipenv/blob/v2022.1.8/pipenv/environments.py#L263). | | `DS_PIP_VERSION` | `gemnasium-python` | | Force the install of a specific pip version (example: `"19.3"`), otherwise the pip installed in the Docker image is used. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12811) in GitLab 12.7) | | `DS_PIP_DEPENDENCY_PATH` | `gemnasium-python` | | Path to load Python pip dependencies from. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12412) in GitLab 12.2) | -| `DS_INCLUDE_DEV_DEPENDENCIES` | `gemnasium` | `"true"` | When set to `"false"`, development dependencies and their vulnerabilities are not reported. Only NPM and Poetry projects are supported. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227861) in GitLab 15.1. | +| `DS_INCLUDE_DEV_DEPENDENCIES` | `gemnasium` | `"true"` | When set to `"false"`, development dependencies and their vulnerabilities are not reported. Only Composer, NPM, and Poetry projects are supported. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227861) in GitLab 15.1. | | `GOOS` | `gemnasium` | `"linux"` | The operating system for which to compile Go code. | | `GOARCH` | `gemnasium` | `"amd64"` | The architecture of the processor for which to compile Go code. | | `GOFLAGS` | `gemnasium` | | The flags passed to the `go build` tool. | @@ -718,39 +746,8 @@ Gemnasium scanning jobs automatically use FIPS-enabled image when FIPS mode is e To manually switch to FIPS-enabled images, set the variable `DS_IMAGE_SUFFIX` to `"-fips"`. -To ensure compliance with FIPS, the FIPS-enabled image of `gemnasium-maven` uses the OpenJDK packages for RedHat UBI. -As a result, it only supports Java 8, 11, and 17. - Dependency scanning for Gradle projects and auto-remediation for Yarn projects are not supported in FIPS mode. -## Interacting with the vulnerabilities - -Once a vulnerability is found, you can interact with it. Read more on how to -[address the vulnerabilities](../vulnerabilities/index.md). - -## Solutions for vulnerabilities - -Some vulnerabilities can be fixed by applying the solution that GitLab -automatically generates. Read more about the -[solutions for vulnerabilities](../vulnerabilities/index.md#resolve-a-vulnerability). - -## Security Dashboard - -The Security Dashboard is a good place to get an overview of all the security -vulnerabilities in your groups, projects and pipelines. Read more about the -[Security Dashboard](../security_dashboard/index.md). - -## Vulnerabilities database update - -For more information about the vulnerabilities database update, see the -[maintenance table](../index.md#vulnerability-scanner-maintenance). - -## Dependency List - -An additional benefit of dependency scanning is the ability to view your -project's dependencies and their known vulnerabilities. Read more about -the [Dependency List](../dependency_list/index.md). - ## Reports JSON format The dependency scanning tool emits a JSON report file. For more information, see the @@ -867,7 +864,7 @@ Here's an example dependency scanning report: ### CycloneDX Software Bill of Materials -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/350509) in GitLab 14.8 in [Beta](../../../policy/alpha-beta-support.md#beta-features). +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/350509) in GitLab 14.8 in [Beta](../../../policy/alpha-beta-support.md#beta). > - Generally available in GitLab 15.7. In addition to the [JSON report file](#reports-json-format), the [Gemnasium](https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium) @@ -908,7 +905,7 @@ Then the Gemnasium scanner generates the following CycloneDX SBOMs: └── gl-sbom-go-go.cdx.json ``` -The CycloneDX SBOMs can be downloaded [the same way as other job artifacts](../../../ci/pipelines/job_artifacts.md#download-job-artifacts). +You can download CycloneDX SBOMs [the same way as other job artifacts](../../../ci/jobs/job_artifacts.md#download-job-artifacts). ### Merging multiple CycloneDX SBOMs @@ -926,10 +923,30 @@ include: merge cyclonedx sboms: stage: merge-cyclonedx-sboms image: - name: cyclonedx/cyclonedx-cli:0.24.0 + name: cyclonedx/cyclonedx-cli:0.24.2 entrypoint: [""] script: - - find . -name "gl-sbom-*.cdx.json" -exec /cyclonedx merge --output-file gl-sbom-all.cdx.json --input-files "{}" + + - apt-get update && apt-get install -y jq + - find . -name "gl-sbom-*.cdx.json" -exec cyclonedx merge --output-file gl-sbom-all.cdx.json --input-files "{}" + + # remove duplicates from merged file. See https://github.com/CycloneDX/cyclonedx-cli/issues/188 for details. + - | + jq '. | + { + "bomFormat": .bomFormat, + "specVersion": .specVersion, + "serialNumber": .serialNumber, + "version": .version, + "metadata": { + "tools": [ + (.metadata.tools | unique[]) + ] + }, + "components": [ + (.components | unique[]) + ] + }' "gl-sbom-all.cdx.json" > gl-sbom-all.cdx.json.tmp && mv gl-sbom-all.cdx.json.tmp gl-sbom-all.cdx.json + # optional: validate the merged sbom + - cyclonedx validate --input-version v1_4 --input-file gl-sbom-all.cdx.json artifacts: paths: - gl-sbom-all.cdx.json @@ -980,9 +997,9 @@ import the following default dependency scanning analyzer images from `registry. your [local Docker container registry](../../packages/container_registry/index.md): ```plaintext -registry.gitlab.com/security-products/gemnasium:3 -registry.gitlab.com/security-products/gemnasium-maven:3 -registry.gitlab.com/security-products/gemnasium-python:3 +registry.gitlab.com/security-products/gemnasium:4 +registry.gitlab.com/security-products/gemnasium-maven:4 +registry.gitlab.com/security-products/gemnasium-python:4 ``` The process for importing Docker images into a local offline Docker registry depends on @@ -1078,6 +1095,24 @@ ensure that it can reach your private repository. Here is an example configurati setuptools.ssl_support.cert_paths = ['internal.crt'] ``` +#### Python (Pipenv) + +If running in a limited network connectivity environment, you must configure the `PIPENV_PYPI_MIRROR` +variable to use a private PyPi mirror. This mirror must contain both default and development dependencies. + +```yaml +variables: + PIPENV_PYPI_MIRROR: https://pypi.example.com/simple +``` + + +Alternatively, if it's not possible to use a private registry, you can load the required packages +into the Pipenv virtual environment cache. For this option, the project must check in the +`Pipfile.lock` into the repository, and load both default and development packages into the cache. +See the example [python-pipenv](https://gitlab.com/gitlab-org/security-products/tests/python-pipenv/-/blob/41cc017bd1ed302f6edebcfa3bc2922f428e07b6/.gitlab-ci.yml#L20-42) +project for an example of how this can be done. + + ## Hosting a copy of the `gemnasium_db` advisory database The [`gemnasium_db`](https://gitlab.com/gitlab-org/security-products/gemnasium-db) Git repository is @@ -1188,7 +1223,7 @@ affected. Read more in ### Getting warning message `gl-dependency-scanning-report.json: no matching files` -For information on this, see the [general Application Security troubleshooting section](../../../ci/pipelines/job_artifacts.md#error-message-no-files-to-upload). +For information on this, see the [general Application Security troubleshooting section](../../../ci/jobs/job_artifacts_troubleshooting.md#error-message-no-files-to-upload). ### Limitation when using rules:exists diff --git a/doc/user/application_security/iac_scanning/index.md b/doc/user/application_security/iac_scanning/index.md index c2f1257f989..48b2b8c1f1a 100644 --- a/doc/user/application_security/iac_scanning/index.md +++ b/doc/user/application_security/iac_scanning/index.md @@ -119,7 +119,7 @@ that you can download and analyze. To enable IaC Scanning in a project, you can create a merge request: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Security & Compliance > Configuration**. +1. On the left sidebar, select **Security and Compliance > Security configuration**. 1. In the **Infrastructure as Code (IaC) Scanning** row, select **Configure with a merge request**. 1. Review and merge the merge request to enable IaC Scanning. @@ -153,6 +153,8 @@ To disable analyzer rules: - Find it in the [JSON report artifact](#reports-json-format). - Search for the rule name in the [list of KICS queries](https://docs.kics.io/latest/queries/all-queries/) and copy the alphanumeric identifier that's shown. The rule name is shown on the [Vulnerability Page](../vulnerabilities/index.md) when a rule violation is detected. +After you merge the `sast-ruleset.toml` file to the default branch, existing findings for disabled rules are [automatically resolved](#automatic-vulnerability-resolution). + In the following example `sast-ruleset.toml` file, the disabled rules are assigned to the `kics` analyzer by matching the `type` and `value` of identifiers: @@ -243,7 +245,8 @@ kics-iac-sast: ## Automatic vulnerability resolution -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/368284) in GitLab 15.9 [with a project-level flag](../../../administration/feature_flags.md) named `sec_mark_dropped_findings_as_resolved`. Enabled by default on GitLab.com; disabled by default in self-managed. On GitLab.com, [contact Support](https://about.gitlab.com/support/) if you need to disable the flag for your project. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/368284) in GitLab 15.9 [with a project-level flag](../../../administration/feature_flags.md) named `sec_mark_dropped_findings_as_resolved`. +> - Enabled by default in 15.10. On GitLab.com, [contact Support](https://about.gitlab.com/support/) if you need to disable the flag for your project. To help you focus on the vulnerabilities that are still relevant, GitLab IaC Scanning automatically [resolves](../vulnerabilities/index.md#vulnerability-status-values) vulnerabilities when: @@ -260,7 +263,7 @@ The IaC tool emits a JSON report file in the existing SAST report format. For mo [schema for this report](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/sast-report-format.json). The JSON report file can be downloaded from the CI pipelines page, or the -pipelines tab on merge requests by [setting `artifacts: paths`](../../../ci/yaml/index.md#artifactspaths) to `gl-sast-report.json`. For more information see [Downloading artifacts](../../../ci/pipelines/job_artifacts.md). +pipelines tab on merge requests by [setting `artifacts: paths`](../../../ci/yaml/index.md#artifactspaths) to `gl-sast-report.json`. For more information see [Downloading artifacts](../../../ci/jobs/job_artifacts.md). ## Troubleshooting diff --git a/doc/user/application_security/index.md b/doc/user/application_security/index.md index 492b200d22b..a3c512a813c 100644 --- a/doc/user/application_security/index.md +++ b/doc/user/application_security/index.md @@ -101,7 +101,7 @@ The following vulnerability scanners and their databases are regularly updated: | Secure scanning tool | Vulnerabilities database updates | |:----------------------------------------------------------------|:---------------------------------| -| [Container Scanning](container_scanning/index.md) | A job runs on a daily basis to build new images with the latest vulnerability database updates from the upstream scanner. For more details, see [Vulnerabilities database update](container_scanning/index.md#vulnerabilities-database). | +| [Container Scanning](container_scanning/index.md) | A job runs on a daily basis to build new images with the latest vulnerability database updates from the upstream scanner. GitLab monitors this job through an internal alert that tells the engineering team when the database becomes more than 48 hours old. For more information, see the [Vulnerabilities database update](container_scanning/index.md#vulnerabilities-database). | | [Dependency Scanning](dependency_scanning/index.md) | Relies on the [GitLab Advisory Database](https://gitlab.com/gitlab-org/security-products/gemnasium-db). It is updated on a daily basis using [data from NVD, the `ruby-advisory-db` and the GitHub Advisory Database as data sources](https://gitlab.com/gitlab-org/security-products/gemnasium-db/-/blob/master/SOURCES.md). See our [current measurement of time from CVE being issued to our product being updated](https://about.gitlab.com/handbook/engineering/development/performance-indicators/#cve-issue-to-update). | | [Dynamic Application Security Testing (DAST)](dast/index.md) | The scanning engine is updated on a periodic basis. See the [version of the underlying tool `zaproxy`](https://gitlab.com/gitlab-org/security-products/dast/blob/main/Dockerfile#L1). The scanning rules are downloaded at scan runtime. | | [Static Application Security Testing (SAST)](sast/index.md) | The source of scan rules depends on which [analyzer](sast/analyzers.md) is used for each [supported programming language](sast/index.md#supported-languages-and-frameworks). GitLab maintains a ruleset for the Semgrep-based analyzer and updates it regularly based on internal research and user feedback. For other analyzers, the ruleset is sourced from the upstream open-source scanner. Each analyzer is updated at least once per month if a relevant update is available. | @@ -240,9 +240,9 @@ reports are available to download. To download a report, select ### Ultimate -A merge request contains a security widget which displays a summary of the new results. New results are determined by comparing the findings of the merge request against the findings of the most recent completed pipeline (`success`, `failed`, `canceled` or `skipped`) for the latest commit in the target branch. +A merge request contains a security widget which displays a summary of the _new_ results. New results are determined by comparing the findings of the merge request against the findings of the most recent completed pipeline (`success`, `failed`, `canceled` or `skipped`) for the commit when the feature branch was created from the target branch. -If security scans have not run for the most recent completed pipeline in the target branch there is no base for comparison. The vulnerabilities from the merge request findings are listed as new in the merge request security widget. We recommend you run a scan of the `default` (target) branch before enabling feature branch scans for your developers. +If security scans have not run for the completed pipeline in the target branch when the feature branch was created, there is no base for comparison. The vulnerabilities from the merge request findings are listed as new in the merge request security widget. We recommend you run a scan of the `default` (target) branch before enabling feature branch scans for your developers. The merge request security widget displays only a subset of the vulnerabilities in the generated JSON artifact because it contains both new and existing findings. @@ -274,13 +274,12 @@ By default, the vulnerability report does not show vulnerabilities of `dismissed > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9928) in GitLab 12.2. > - [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/357300) the Vulnerability-Check feature in GitLab 15.0. +> - [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/397067) the License-Check feature in GitLab 16.0. You can enforce an additional approval for merge requests that would introduce one of the following security issues: - A security vulnerability. For more details, read [Scan result policies](policies/scan-result-policies.md). -- A software license compliance violation. For more details, read - [Enabling license approvals within a project](../compliance/license_check_rules.md#enabling-license-approvals-within-a-project). ## Using private Maven repositories @@ -577,7 +576,7 @@ Debug logging can be a serious security risk. The output may contain the content environment variables and other secrets available to the job. The output is uploaded to the GitLab server and visible in job logs. -This message is often followed by the [error `No files to upload`](../../ci/pipelines/job_artifacts.md#error-message-no-files-to-upload), +This message is often followed by the [error `No files to upload`](../../ci/jobs/job_artifacts_troubleshooting.md#error-message-no-files-to-upload), and preceded by other errors or warnings that indicate why the JSON report wasn't generated. Check the entire job log for such messages. If you don't find these messages, retry the failed job after setting `SECURE_LOG_LEVEL: "debug"` as a [custom CI/CD variable](../../ci/variables/index.md#for-a-project). diff --git a/doc/user/application_security/offline_deployments/index.md b/doc/user/application_security/offline_deployments/index.md index 5a075bf731b..1ee4d9b2a19 100644 --- a/doc/user/application_security/offline_deployments/index.md +++ b/doc/user/application_security/offline_deployments/index.md @@ -121,7 +121,7 @@ include: ``` The pipeline downloads the Docker images needed for the Security Scanners and saves them as -[job artifacts](../../../ci/pipelines/job_artifacts.md) or pushes them to the [Container Registry](../../packages/container_registry/index.md) +[job artifacts](../../../ci/jobs/job_artifacts.md) or pushes them to the [Container Registry](../../packages/container_registry/index.md) of the project where the pipeline is executed. These archives can be transferred to another location and [loaded](https://docs.docker.com/engine/reference/commandline/load/) in a Docker daemon. This method requires a runner with access to both `gitlab.com` (including diff --git a/doc/user/application_security/policies/img/association_diagram.png b/doc/user/application_security/policies/img/association_diagram.png index d082e297c68..3a56aeba91b 100644 Binary files a/doc/user/application_security/policies/img/association_diagram.png and b/doc/user/application_security/policies/img/association_diagram.png differ diff --git a/doc/user/application_security/policies/img/policy_rule_mode_v14_9.png b/doc/user/application_security/policies/img/policy_rule_mode_v14_9.png deleted file mode 100644 index 8ca7547a33c..00000000000 Binary files a/doc/user/application_security/policies/img/policy_rule_mode_v14_9.png and /dev/null differ diff --git a/doc/user/application_security/policies/img/policy_rule_mode_v15_9.png b/doc/user/application_security/policies/img/policy_rule_mode_v15_9.png new file mode 100644 index 00000000000..8cb2e82ac05 Binary files /dev/null and b/doc/user/application_security/policies/img/policy_rule_mode_v15_9.png differ diff --git a/doc/user/application_security/policies/img/policy_yaml_mode_v14_9.png b/doc/user/application_security/policies/img/policy_yaml_mode_v14_9.png deleted file mode 100644 index 1d71e8684e9..00000000000 Binary files a/doc/user/application_security/policies/img/policy_yaml_mode_v14_9.png and /dev/null differ diff --git a/doc/user/application_security/policies/img/policy_yaml_mode_v15_9.png b/doc/user/application_security/policies/img/policy_yaml_mode_v15_9.png new file mode 100644 index 00000000000..95b637efef3 Binary files /dev/null and b/doc/user/application_security/policies/img/policy_yaml_mode_v15_9.png differ diff --git a/doc/user/application_security/policies/img/scan_execution_policy_rule_mode_v15_11.png b/doc/user/application_security/policies/img/scan_execution_policy_rule_mode_v15_11.png new file mode 100644 index 00000000000..76b9a971172 Binary files /dev/null and b/doc/user/application_security/policies/img/scan_execution_policy_rule_mode_v15_11.png differ diff --git a/doc/user/application_security/policies/img/scan_execution_policy_rule_mode_v15_5.png b/doc/user/application_security/policies/img/scan_execution_policy_rule_mode_v15_5.png deleted file mode 100644 index 5ae7c2e065a..00000000000 Binary files a/doc/user/application_security/policies/img/scan_execution_policy_rule_mode_v15_5.png and /dev/null differ diff --git a/doc/user/application_security/policies/img/scheduled_scan_execution_policies_diagram.png b/doc/user/application_security/policies/img/scheduled_scan_execution_policies_diagram.png new file mode 100644 index 00000000000..fcf7a8352fd Binary files /dev/null and b/doc/user/application_security/policies/img/scheduled_scan_execution_policies_diagram.png differ diff --git a/doc/user/application_security/policies/index.md b/doc/user/application_security/policies/index.md index a214d0d2cec..0d821f8e47c 100644 --- a/doc/user/application_security/policies/index.md +++ b/doc/user/application_security/policies/index.md @@ -13,7 +13,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w Policies in GitLab provide security teams a way to require scans of their choice to be run whenever a project pipeline runs according to the configuration specified. Security teams can therefore be confident that the scans they set up have not been changed, altered, or disabled. You -can access these by navigating to your project's **Security & Compliance > Policies** page. +can access these by navigating to your project's **Security and Compliance > Policies** page. GitLab supports the following security policies: @@ -23,9 +23,13 @@ GitLab supports the following security policies: ## Security policy project All security policies are stored as YAML in a separate security policy project that gets linked to -the development project. This association can be a one-to-many relationship, allowing one security -policy project to apply to multiple development projects. Linked projects are not required to be in -the same group as the development projects to which they are linked. +the development project, group, or sub-group. This association can be a one-to-many relationship, allowing one security +policy project to apply to multiple development projects, groups, or sub-groups: + +- For self-managed GitLab instances, linked projects are not required to be in the same group + or the same subgroup as the development projects to which they are linked. +- For GitLab SaaS, the security policy project is required to be in the same top-level group + as the development project, although, it is not necessary for the project to be in the same subgroup. ![Security Policy Project Linking Diagram](img/association_diagram.png) @@ -34,10 +38,6 @@ project and the security policy project, this is not recommended. Keeping the se project separate from the development project allows for complete separation of duties between security/compliance teams and development teams. -You should not link a security policy project to a development project and to the group -or sub-group the development project belongs to at the same time. Linking this way will result in -approval rules from the Scan Result Policy not being applied to merge requests in the development project. - All security policies are stored in the `.gitlab/security-policies/policy.yml` YAML file inside the linked security policy project. The format for this YAML is specific to the type of policy that is stored there. Examples and schema information are available for the following policy types: @@ -59,7 +59,7 @@ As a project owner, take the following steps to create or edit an association be project and a project that you would like to designate as the security policy project: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Security & Compliance > Policies**. +1. On the left sidebar, select **Security and Compliance > Policies**. 1. Select **Edit Policy Project**, and search for and select the project you would like to link from the dropdown list. 1. Select **Save**. @@ -83,7 +83,7 @@ policy's information (for example, description or enforcement status), and create and edit deployed policies: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Security & Compliance > Policies**. +1. On the left sidebar, select **Security and Compliance > Policies**. ![Policies List Page](img/policies_list_v15_1.png) @@ -94,7 +94,7 @@ status), and create and edit deployed policies: You can use the policy editor to create, edit, and delete policies: 1. On the top bar, select **Main menu > Projects** and find your group. -1. On the left sidebar, select **Security & Compliance > Policies**. +1. On the left sidebar, select **Security and Compliance > Policies**. - To create a new policy, select **New policy** which is located in the **Policies** page's header. You can then select which type of policy to create. - To edit an existing policy, select **Edit policy** in the selected policy drawer. @@ -104,13 +104,13 @@ The policy editor has two modes: - The visual _Rule_ mode allows you to construct and preview policy rules using rule blocks and related controls. - ![Policy Editor Rule Mode](img/policy_rule_mode_v14_9.png) + ![Policy Editor Rule Mode](img/policy_rule_mode_v15_9.png) - YAML mode allows you to enter a policy definition in `.yaml` format and is aimed at expert users and cases that the Rule mode doesn't support. - ![Policy Editor YAML Mode](img/policy_yaml_mode_v14_9.png) + ![Policy Editor YAML Mode](img/policy_yaml_mode_v15_9.png) You can use both modes interchangeably and switch between them at any time. If a YAML resource is incorrect or contains data not supported @@ -129,19 +129,6 @@ time that the first policy merge request is created. You can use the [Vulnerability-Check Migration](https://gitlab.com/gitlab-org/gitlab/-/snippets/2328089) script to bulk create policies or associate security policy projects with development projects. For instructions and a demonstration of how to use the Vulnerability-Check Migration script, see [this video](https://youtu.be/biU1N26DfBc). -## Scan execution policies - -See [Scan execution policies](scan-execution-policies.md). - -## Scan result policy editor - -See [Scan result policies](scan-result-policies.md). - -## Roadmap - -See the [Category Direction page](https://about.gitlab.com/direction/govern/security_policies/security_policy_management/) -for more information on the product direction of security policies within GitLab. - ## Troubleshooting ### `Branch name 'update-policy-' does not follow the pattern ''` @@ -151,3 +138,17 @@ When you create a new security policy or change an existing policy, a new branch If you have group or instance [push rules that do not allow branch name patterns](../../project/repository/push_rules.md#validate-branch-names) that contain the text `update-policy-`, you will get an error that states `Branch name 'update-policy-' does not follow the pattern ''`. The workaround is to amend your group or instance push rules to allow branches following the pattern `update-policy-` followed by an integer timestamp. + +### Troubleshooting common issues configuring security policies + +- Confirm that projects contain a `.gitlab-ci.yml` file. This file is required for scan execution policies. +- Confirm that scanners are properly configured and producing results for the latest branch. Security Policies are designed to require approval when there are no results (no security report), as this ensures that no vulnerabilities are introduced. We cannot know if there are any vulnerabilities unless the scans enforced by the policy complete successfully and are evaluated. +- When running scan execution policies based on a SAST action, ensure target repositories contain proper code files. SAST runs different analyzers [based on the types of files in the repo](../sast/index.md#supported-languages-and-frameworks), and if no supported files are found it will not run any jobs. See the [SAST CI template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml) for more details. +- Check for any branch configuration conflicts. If your policy is configured to enforce rules on `main` but some projects within the scope are using `master` as their default branch, the policy is not applied for the latter. Support for specifying the `default` branch in your policies is proposed in [epic 9468](https://gitlab.com/groups/gitlab-org/-/epics/9468). +- Scan result policies created at the group or sub-group level can take some time to apply to all the merge requests in the group. +- Scheduled scan execution policies run with a minimum 15 minute cadence. Learn more [about the schedule rule type](../policies/scan-execution-policies.md#schedule-rule-type). +- When scheduling pipelines, keep in mind that CRON scheduling is based on UTC on GitLab SaaS and is based on your server time for self managed instances. When testing new policies, it may appear pipelines are not running properly when in fact they are scheduled in your server's timezone. +- When enforcing scan execution policies, the target project's pipeline is triggered by the user who last updated the security policy project's `policy.yml` file. The user must have permission to trigger the pipeline in the project for the policy to be enforced, and the pipeline to run. Work to address this is being tracked in [issue 394958](https://gitlab.com/gitlab-org/gitlab/-/issues/394958). +- You should not link a security policy project to a development project and to the group or sub-group the development project belongs to at the same time. Linking this way will result in approval rules from the Scan Result Policy not being applied to merge requests in the development project. + +If you are still experiencing issues, you can [view recent reported bugs](https://gitlab.com/gitlab-org/gitlab/-/issues/?sort=popularity&state=opened&label_name%5B%5D=group%3A%3Asecurity%20policies&label_name%5B%5D=type%3A%3Abug&first_page_size=20) and raise new unreported issues. diff --git a/doc/user/application_security/policies/scan-execution-policies.md b/doc/user/application_security/policies/scan-execution-policies.md index 26c7e1d9c77..61671efab63 100644 --- a/doc/user/application_security/policies/scan-execution-policies.md +++ b/doc/user/application_security/policies/scan-execution-policies.md @@ -33,7 +33,7 @@ NOTE: Only group, subgroup, or project Owners have the [permissions](../../permissions.md#project-members-permissions) to select Security Policy Project. -Once your policy is complete, save it by selecting **Create via merge request** +Once your policy is complete, save it by selecting **Configure with a merge request** at the bottom of the editor. You are redirected to the merge request on the project's configured security policy project. If one does not link to your project, a security policy project is automatically created. Existing policies can also be @@ -44,7 +44,7 @@ Most policy changes take effect as soon as the merge request is merged. Any chan do not go through a merge request and are committed directly to the default branch may require up to 10 minutes before the policy changes take effect. -![Scan Execution Policy Editor Rule Mode](img/scan_execution_policy_rule_mode_v15_5.png) +![Scan Execution Policy Editor Rule Mode](img/scan_execution_policy_rule_mode_v15_11.png) ## Scan execution policies schema @@ -88,7 +88,7 @@ This rule enforces the defined actions and schedules a scan on the provided date |------------|------|-----------------|-------------| | `type` | `string` | `schedule` | The rule's type. | | `branches` | `array` of `string` | `*` or the branch's name | The branch the given policy applies to (supports wildcard). This field is required if the `agents` field is not set. | -| `cadence` | `string` | CRON expression (for example, `0 0 * * *`) | A whitespace-separated string containing five fields that represents the scheduled time. | +| `cadence` | `string` | CRON expression (for example, `0 0 * * *`) | A whitespace-separated string containing five fields that represents the scheduled time. Minimum of 15 minute intervals when used together with the `branches` field. | | `agents` | `object` | | The name of the [GitLab agents](../../clusters/agent/index.md) where [Operational Container Scanning](../../clusters/agent/vulnerabilities.md) runs. The object key is the name of the Kubernetes agent configured for your project in GitLab. This field is required if the `branches` field is not set. | GitLab supports the following types of CRON syntax for the `cadence` field: @@ -99,8 +99,18 @@ GitLab supports the following types of CRON syntax for the `cadence` field: NOTE: Other elements of the [CRON syntax](https://docs.oracle.com/cd/E12058_01/doc/doc.1014/e12030/cron_expressions.htm) may work in the cadence field if supported by the [cron](https://github.com/robfig/cron) we are using in our implementation, however, GitLab does not officially test or support them. -NOTE: -If using the `agents` field, required for `Operational Container Scanning`, the CRON expression is evaluated in [UTC](https://www.timeanddate.com/worldclock/timezone/utc) using the system-time of the Kubernetes-agent pod. If not using the `agents` field, the CRON expression is evaluated in standard [UTC](https://www.timeanddate.com/worldclock/timezone/utc) time from GitLab.com. If you have a self-managed GitLab instance and have [changed the server time zone](../../../administration/timezone.md), the CRON expression is evaluated with the new time zone. +When using the `schedule` rule type in conjunction with the `agents` field, note the following: + +- The GitLab Agent for Kubernetes checks every 30 seconds to see if there is an applicable policy. When a policy is found, the scans are executed according to the `cadence` defined. +- The CRON expression is evaluated using the system-time of the Kubernetes-agent pod. + +When using the `schedule` rule type in conjunction with the `branches` field, note the following: + +- The cron worker runs on 15 minute intervals and starts any pipelines that were scheduled to run during the previous 15 minutes. +- Based on your rule, you might expect scheduled pipelines to run with an offset of up to 15 minutes. +- The CRON expression is evaluated in standard [UTC](https://www.timeanddate.com/worldclock/timezone/utc) time from GitLab.com. If you have a self-managed GitLab instance and have [changed the server time zone](../../../administration/timezone.md), the CRON expression is evaluated with the new time zone. + +![CRON worker diagram](img/scheduled_scan_execution_policies_diagram.png) ### `agent` schema @@ -140,7 +150,7 @@ rule in the defined policy are met. | Field | Type | Possible values | Description | |-------|------|-----------------|-------------| -| `scan` | `string` | `dast`, `secret_detection`, `sast`, `container_scanning`, `dependency_scanning` | The action's type. | +| `scan` | `string` | `sast`, `sast_iac`, `dast`, `secret_detection`, `container_scanning`, `dependency_scanning` | The action's type. | | `site_profile` | `string` | Name of the selected [DAST site profile](../dast/proxy-based.md#site-profile). | The DAST site profile to execute the DAST scan. This field should only be set if `scan` type is `dast`. | | `scanner_profile` | `string` or `null` | Name of the selected [DAST scanner profile](../dast/proxy-based.md#scanner-profile). | The DAST scanner profile to execute the DAST scan. This field should only be set if `scan` type is `dast`.| | `variables` | `object` | | A set of CI variables, supplied as an array of `key: value` pairs, to apply and enforce for the selected scan. The `key` is the variable name, with its `value` provided as a string. This parameter supports any variable that the GitLab CI job supports for the specified scan. | @@ -247,5 +257,5 @@ developer may want to try running a SAST scan with different variables than the this case, two SAST jobs run in the pipeline, one with the developer's variables and one with the security and compliance team's variables. If you want to avoid running duplicate scans, you can either remove the scans from the project's `.gitlab-ci.yml` file or disable your -local jobs by setting `SAST_DISABLED: true`. Disabling jobs this way does not prevent the security jobs defined by scan execution +local jobs by setting `SAST_DISABLED: "true"`. Disabling jobs this way does not prevent the security jobs defined by scan execution policies from running. diff --git a/doc/user/application_security/policies/scan-result-policies.md b/doc/user/application_security/policies/scan-result-policies.md index bc74b8bdfb1..ecbbf4703b0 100644 --- a/doc/user/application_security/policies/scan-result-policies.md +++ b/doc/user/application_security/policies/scan-result-policies.md @@ -10,8 +10,12 @@ info: To determine the technical writer assigned to the Stage/Group associated w You can use scan result policies to take action based on scan results. For example, one type of scan result policy is a security approval policy that allows approval to be required based on the -findings of one or more security scan jobs. Scan result policies are evaluated after a CI scanning -job is fully executed. The following video gives you an overview of GitLab scan result policies: +findings of one or more security scan jobs. Scan result policies are evaluated after a CI scanning job is fully executed. + +NOTE: +Scan result policies are applicable only to [protected](../../project/protected_branches.md) target branches. + +The following video gives you an overview of GitLab scan result policies:
    See the video: Overview of GitLab Scan Result Policies. @@ -29,7 +33,7 @@ NOTE: Only project Owners have the [permissions](../../permissions.md#project-members-permissions) to select Security Policy Project. -Once your policy is complete, save it by selecting **Create merge request** at the bottom of the +Once your policy is complete, save it by selecting **Configure with a merge request** at the bottom of the editor. This redirects you to the merge request on the project's configured security policy project. If a security policy project doesn't link to your project, GitLab creates such a project for you. Existing policies can also be removed from the editor interface by selecting **Delete policy** at @@ -72,22 +76,19 @@ the following sections and tables provide an alternative. This rule enforces the defined actions based on security scan findings. -| Field | Type | Possible values | Description | -|------------|------|-----------------|-------------| -| `type` | `string` | `scan_finding` | The rule's type. | -| `branches` | `array` of `string` | `[]` or the branch's name | Applicable only to protected target branches. An empty array, `[]`, applies the rule to all protected target branches. | -| `scanners` | `array` of `string` | `sast`, `secret_detection`, `dependency_scanning`, `container_scanning`, `dast`, `coverage_fuzzing`, `api_fuzzing` | The security scanners for this rule to consider. | -| `vulnerabilities_allowed` | `integer` | Greater than or equal to zero | Number of vulnerabilities allowed before this rule is considered. | -| `severity_levels` | `array` of `string` | `info`, `unknown`, `low`, `medium`, `high`, `critical`| The severity levels for this rule to consider. | -| `vulnerability_states` | `array` of `string` | `newly_detected`, `detected`, `confirmed`, `resolved`, `dismissed` | All vulnerabilities fall into two categories:

    **Newly Detected Vulnerabilities** - the `newly_detected` policy option covers vulnerabilities identified in the merge request branch itself but that do not currently exist on the default branch. This policy option requires a pipeline to complete before the rule is evaluated so that it knows whether vulnerabilities are newly detected or not. Merge requests are blocked until the pipeline and necessary security scans are complete. The `newly_detected` option considers both of the following statuses:

    • Detected
    • Dismissed

    **Pre-Existing Vulnerabilities** - these policy options are evaluated immediately and do not require a pipeline complete as they consider only vulnerabilities previously detected in the default branch.

    • `Detected` - the policy looks for vulnerabilities in the detected state.
    • `Confirmed` - the policy looks for vulnerabilities in the confirmed state.
    • `Dismissed` - the policy looks for vulnerabilities in the dismissed state.
    • `Resolved` - the policy looks for vulnerabilities in the resolved state. | +| Field | Type | Possible values | Description | +|------------|------|--------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `type` | `string` | `scan_finding` | The rule's type. | +| `branches` | `array` of `string` | `[]` or the branch's name | Applicable only to protected target branches. An empty array, `[]`, applies the rule to all protected target branches. | +| `scanners` | `array` of `string` | `sast`, `secret_detection`, `dependency_scanning`, `container_scanning`, `dast`, `coverage_fuzzing`, `api_fuzzing` | The security scanners for this rule to consider. `sast` includes results from both SAST and SAST IaC scanners. | +| `vulnerabilities_allowed` | `integer` | Greater than or equal to zero | Number of vulnerabilities allowed before this rule is considered. | +| `severity_levels` | `array` of `string` | `info`, `unknown`, `low`, `medium`, `high`, `critical` | The severity levels for this rule to consider. | +| `vulnerability_states` | `array` of `string` | `newly_detected`, `detected`, `confirmed`, `resolved`, `dismissed`, `new_needs_triage`, `new_dismissed` | All vulnerabilities fall into two categories:

    **Newly Detected Vulnerabilities** - the `newly_detected` policy option covers vulnerabilities identified in the merge request branch itself but that do not currently exist on the default branch. This policy option requires a pipeline to complete before the rule is evaluated so that it knows whether vulnerabilities are newly detected or not. Merge requests are blocked until the pipeline and necessary security scans are complete. The `newly_detected` option considers both of the following statuses:

    • Detected
    • Dismissed

    The `new_needs_triage` option considers the status

    • Detected

    The `new_dismissed` option considers the status

    • Dismissed

    **Pre-Existing Vulnerabilities** - these policy options are evaluated immediately and do not require a pipeline complete as they consider only vulnerabilities previously detected in the default branch.

    • `Detected` - the policy looks for vulnerabilities in the detected state.
    • `Confirmed` - the policy looks for vulnerabilities in the confirmed state.
    • `Dismissed` - the policy looks for vulnerabilities in the dismissed state.
    • `Resolved` - the policy looks for vulnerabilities in the resolved state. | ## `license_finding` rule type -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/8092) in GitLab 15.9 [with a flag](../../../administration/feature_flags.md) named `license_scanning_policies`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `license_scanning_policies`. -On GitLab.com, this feature is not available. +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/8092) in GitLab 15.9 [with a flag](../../../administration/feature_flags.md) named `license_scanning_policies`. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/397644) in GitLab 15.11. Feature flag `license_scanning_policies` removed. This rule enforces the defined actions based on license findings. @@ -112,6 +113,7 @@ the defined policy. | `user_approvers_ids` | `array` of `integer` | ID of one of more users | The IDs of users to consider as approvers. Users must have access to the project to be eligible to approve. | | `group_approvers` | `array` of `string` | Path of one of more groups | The groups to consider as approvers. Users with [direct membership in the group](../../project/merge_requests/approvals/rules.md#group-approvers) are eligible to approve. | | `group_approvers_ids` | `array` of `integer` | ID of one of more groups | The IDs of groups to consider as approvers. Users with [direct membership in the group](../../project/merge_requests/approvals/rules.md#group-approvers) are eligible to approve. | +| `role_approvers` | `array` of `string` | One or more [roles](../../../user/permissions.md#roles) (for example: `owner`, `maintainer`) | The roles to consider as approvers that are eligible to approve. | Requirements and limitations: @@ -119,6 +121,7 @@ Requirements and limitations: Otherwise, scan result policies do not have any effect. - The maximum number of policies is five. - Each policy can have a maximum of five rules. +- All configured scanners must be present in the merge request's latest pipeline. If not, approvals are required even if some vulnerability criteria have not been met. ## Example security scan result policies project @@ -167,6 +170,8 @@ scan_result_policy: approvals_required: 1 user_approvers: - sam.white + role_approvers: + - owner ``` In this example: diff --git a/doc/user/application_security/sast/analyzers.md b/doc/user/application_security/sast/analyzers.md index c8542142830..f52ff7f9a55 100644 --- a/doc/user/application_security/sast/analyzers.md +++ b/doc/user/application_security/sast/analyzers.md @@ -33,7 +33,6 @@ SAST supports the following official analyzers: - [`nodejs-scan`](https://gitlab.com/gitlab-org/security-products/analyzers/nodejs-scan) (NodeJsScan) - [`phpcs-security-audit`](https://gitlab.com/gitlab-org/security-products/analyzers/phpcs-security-audit) (PHP CS security-audit) - [`pmd-apex`](https://gitlab.com/gitlab-org/security-products/analyzers/pmd-apex) (PMD (Apex only)) -- [`security-code-scan`](https://gitlab.com/gitlab-org/security-products/analyzers/security-code-scan) (Security Code Scan (.NET)) - [`semgrep`](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) (Semgrep) - [`sobelow`](https://gitlab.com/gitlab-org/security-products/analyzers/sobelow) (Sobelow (Elixir Phoenix)) - [`spotbugs`](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) (SpotBugs with the Find Sec Bugs plugin (Ant, Gradle and wrapper, Grails, Maven and wrapper, SBT)) @@ -43,6 +42,7 @@ SAST has used other analyzers in previous versions. These analyzers reached End - [`bandit`](https://gitlab.com/gitlab-org/security-products/analyzers/bandit) (Bandit); [End of Support](https://gitlab.com/gitlab-org/gitlab/-/issues/352554) in GitLab 15.4. Replaced by the `semgrep` analyzer with GitLab-managed rules. - [`eslint`](https://gitlab.com/gitlab-org/security-products/analyzers/eslint) (ESLint (JavaScript and React)); [End of Support](https://gitlab.com/gitlab-org/gitlab/-/issues/352554) in GitLab 15.4. Replaced by the `semgrep` analyzer with GitLab-managed rules. - [`gosec`](https://gitlab.com/gitlab-org/security-products/analyzers/gosec) (Gosec); [End of Support](https://gitlab.com/gitlab-org/gitlab/-/issues/352554) in GitLab 15.4. Replaced by the `semgrep` analyzer with GitLab-managed rules. +- [`security-code-scan`](https://gitlab.com/gitlab-org/security-products/analyzers/security-code-scan) (Security Code Scan (.NET)); [End of Support](https://gitlab.com/gitlab-org/gitlab/-/issues/390416) in GitLab 16.0. Replaced by the `semgrep` analyzer with GitLab-managed rules. ## SAST analyzer features @@ -252,7 +252,7 @@ Each analyzer provides data about the vulnerabilities it detects. The following data available from each analyzer. The values provided by these tools are heterogeneous so they are sometimes normalized into common values, for example, `severity` and `confidence`. -| Property / tool | Apex | Bandit1 | Brakeman | ESLint security1 | SpotBugs | Flawfinder | Gosec1 | Kubesec Scanner | MobSF | NodeJsScan | PHP CS Security Audit | Security code Scan (.NET) | Semgrep | Sobelow | +| Property / tool | Apex | Bandit1 | Brakeman | ESLint security1 | SpotBugs | Flawfinder | Gosec1 | Kubesec Scanner | MobSF | NodeJsScan | PHP CS Security Audit | Security code Scan (.NET)1 | Semgrep | Sobelow | |--------------------------------|------|--------|----------|-----------------|----------|------------|-------|-----------------|-------|------------|-----------------------|---------------------------|---------|---------| | Affected item (for example, class or package) | ✓ | ✗ | ✓ | ✗ | ✓ | ✓ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | | Confidence | ✗ | ✓ | ✓ | ✗ | ✓ | x | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | ⚠ | ✓ | diff --git a/doc/user/application_security/sast/customize_rulesets.md b/doc/user/application_security/sast/customize_rulesets.md index d070883df9a..ddf8db4e489 100644 --- a/doc/user/application_security/sast/customize_rulesets.md +++ b/doc/user/application_security/sast/customize_rulesets.md @@ -23,12 +23,18 @@ repository being scanned. There are two kinds of customization: ## Disable predefined rules -You can disable predefined rules for any SAST analyzer. Disabled rules don't appear -on the [Pipeline Security](../index.md#view-security-scan-information-in-the-pipeline-security-tab) -tab or the [Vulnerability Report](../index.md#view-security-scan-information-in-the-vulnerability-report). +You can disable predefined rules for any SAST analyzer. -Disabling rules has a retroactive effect. The analyzer continues to scan for the -vulnerability, but findings are omitted from the [`gl-sast-report.json` artifact](index.md#reports-json-format). +When you disable a rule: + +- Most analyzers still scan for the vulnerability. The results are removed as a processing step after the scan completes, and they don't appear in the [`gl-sast-report.json` artifact](index.md#reports-json-format). +- Findings for the disabled rule no longer appear in the [Pipeline Security tab](../index.md#view-security-scan-information-in-the-pipeline-security-tab). +- Existing findings for the disabled rule on the default branch are marked ["No longer detected"](../vulnerability_report/index.md#activity-filter) in the [Vulnerability Report](../index.md#view-security-scan-information-in-the-vulnerability-report). + +The Semgrep-based analyzer handles disabled rules differently: + +- To improve performance, the Semgrep-based analyzer doesn't scan for disabled rules at all. +- If you disable a rule in the Semgrep-based analyzer, existing vulnerability findings for that rule are [automatically resolved](index.md#automatic-vulnerability-resolution) after you merge the `sast-ruleset.toml` file to the default branch. See the [Schema](#schema) and [Examples](#examples) sections for information on how to configure this behavior. @@ -323,7 +329,7 @@ With the following custom ruleset configuration, vulnerabilities found with [semgrep] [[semgrep.ruleset]] [semgrep.ruleset.identifier] - type = "CWE" + type = "cwe" value = "322" [semgrep.ruleset.override] severity = "Critical" diff --git a/doc/user/application_security/sast/index.md b/doc/user/application_security/sast/index.md index b0d84e4cff9..64c0f3440c5 100644 --- a/doc/user/application_security/sast/index.md +++ b/doc/user/application_security/sast/index.md @@ -77,45 +77,46 @@ For more information about our plans for language support in SAST, see the [cate | Language / framework | [Analyzer](analyzers.md) used for scanning | Minimum supported GitLab version | |------------------------------|--------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------| -| .NET Core | [Security Code Scan](https://gitlab.com/gitlab-org/security-products/analyzers/security-code-scan) | 11.0 | -| .NET Framework1 | [Security Code Scan](https://gitlab.com/gitlab-org/security-products/analyzers/security-code-scan) | 13.0 | -| .NET (all versions, C# only) | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) with GitLab-managed rules | 15.4 | +| .NET Core3 | [Security Code Scan](https://gitlab.com/gitlab-org/security-products/analyzers/security-code-scan) | 11.0 | +| .NET Framework3 | [Security Code Scan](https://gitlab.com/gitlab-org/security-products/analyzers/security-code-scan) | 13.0 | +| .NET (all versions, C# only) | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) with [GitLab-managed rules](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep/-/blob/main/RULES.md) | 15.4 | | Apex (Salesforce) | [PMD](https://gitlab.com/gitlab-org/security-products/analyzers/pmd-apex) | 12.1 | -| C | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) with GitLab-managed rules | 14.2 | +| C | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) with [GitLab-managed rules](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep/-/blob/main/RULES.md) | 14.2 | | C/C++ | [Flawfinder](https://gitlab.com/gitlab-org/security-products/analyzers/flawfinder) | 10.7 | | Elixir (Phoenix) | [Sobelow](https://gitlab.com/gitlab-org/security-products/analyzers/sobelow) | 11.1 | -| Go3 | [Gosec](https://gitlab.com/gitlab-org/security-products/analyzers/gosec) | 10.7 | -| Go | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) with GitLab-managed rules | 14.4 | -| Groovy2 | [SpotBugs](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) with the find-sec-bugs plugin | 11.3 (Gradle) & 11.9 (Maven, SBT) | +| Go2 | [Gosec](https://gitlab.com/gitlab-org/security-products/analyzers/gosec) | 10.7 | +| Go | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) with [GitLab-managed rules](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep/-/blob/main/RULES.md) | 14.4 | +| Groovy1 | [SpotBugs](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) with the find-sec-bugs plugin | 11.3 (Gradle) & 11.9 (Maven, SBT) | | Helm Charts | [Kubesec](https://gitlab.com/gitlab-org/security-products/analyzers/kubesec) | 13.1 | -| Java (any build system) | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) with GitLab-managed rules | 14.10 | -| Java2, 3 | [SpotBugs](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) with the find-sec-bugs plugin | 10.6 (Maven), 10.8 (Gradle) & 11.9 (SBT) | +| Java (any build system) | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) with [GitLab-managed rules](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep/-/blob/main/RULES.md) | 14.10 | +| Java1, 2 | [SpotBugs](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) with the find-sec-bugs plugin | 10.6 (Maven), 10.8 (Gradle) & 11.9 (SBT) | | Java (Android) | [MobSF (beta)](https://gitlab.com/gitlab-org/security-products/analyzers/mobsf) | 13.5 | -| JavaScript3 | [ESLint security plugin](https://gitlab.com/gitlab-org/security-products/analyzers/eslint) | 11.8 | -| JavaScript | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) with GitLab-managed rules | 13.10 | +| JavaScript2 | [ESLint security plugin](https://gitlab.com/gitlab-org/security-products/analyzers/eslint) | 11.8 | +| JavaScript | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) with [GitLab-managed rules](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep/-/blob/main/RULES.md) | 13.10 | | Kotlin (Android) | [MobSF (beta)](https://gitlab.com/gitlab-org/security-products/analyzers/mobsf) | 13.5 | -| Kotlin (General)2 | [SpotBugs](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) with the find-sec-bugs plugin | 13.11 | +| Kotlin (General)1 | [SpotBugs](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) with the find-sec-bugs plugin | 13.11 | | Kubernetes manifests | [Kubesec](https://gitlab.com/gitlab-org/security-products/analyzers/kubesec) | 12.6 | | Node.js | [NodeJsScan](https://gitlab.com/gitlab-org/security-products/analyzers/nodejs-scan) | 11.1 | | Objective-C (iOS) | [MobSF (beta)](https://gitlab.com/gitlab-org/security-products/analyzers/mobsf) | 13.5 | | PHP | [phpcs-security-audit](https://gitlab.com/gitlab-org/security-products/analyzers/phpcs-security-audit) | 10.8 | -| Python3 | [bandit](https://gitlab.com/gitlab-org/security-products/analyzers/bandit) | 10.3 | -| Python | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) with GitLab-managed rules | 13.9 | -| React3 | [ESLint react plugin](https://gitlab.com/gitlab-org/security-products/analyzers/eslint) | 12.5 | -| React | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) with GitLab-managed rules | 13.10 | +| Python2 | [bandit](https://gitlab.com/gitlab-org/security-products/analyzers/bandit) | 10.3 | +| Python | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) with [GitLab-managed rules](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep/-/blob/main/RULES.md) | 13.9 | +| React2 | [ESLint react plugin](https://gitlab.com/gitlab-org/security-products/analyzers/eslint) | 12.5 | +| React | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) with [GitLab-managed rules](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep/-/blob/main/RULES.md) | 13.10 | | Ruby | [brakeman](https://gitlab.com/gitlab-org/security-products/analyzers/brakeman) | 13.9 | | Ruby on Rails | [brakeman](https://gitlab.com/gitlab-org/security-products/analyzers/brakeman) | 10.3 | -| Scala2 | [SpotBugs](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) with the find-sec-bugs plugin | 11.0 (SBT) & 11.9 (Gradle, Maven) | +| Scala (any build system) | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) with [GitLab-managed rules](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep/-/blob/main/RULES.md) | 16.0 | +| Scala1 | [SpotBugs](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) with the find-sec-bugs plugin | 11.0 (SBT) & 11.9 (Gradle, Maven) | | Swift (iOS) | [MobSF (beta)](https://gitlab.com/gitlab-org/security-products/analyzers/mobsf) | 13.5 | -| TypeScript3 | [ESLint security plugin](https://gitlab.com/gitlab-org/security-products/analyzers/eslint) | 11.9, [merged](https://gitlab.com/gitlab-org/gitlab/-/issues/36059) with ESLint in 13.2 | -| TypeScript | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) with GitLab-managed rules | 13.10 | +| TypeScript2 | [ESLint security plugin](https://gitlab.com/gitlab-org/security-products/analyzers/eslint) | 11.9, [merged](https://gitlab.com/gitlab-org/gitlab/-/issues/36059) with ESLint in 13.2 | +| TypeScript | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) with [GitLab-managed rules](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep/-/blob/main/RULES.md) | 13.10 | -1. .NET 4 support is limited. The analyzer runs in a Linux container and does not have access to Windows-specific libraries or features. Use the Semgrep-based scanner if you need .NET 4 support. 1. The SpotBugs-based analyzer supports [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/), and [SBT](https://www.scala-sbt.org/). It can also be used with variants like the [Gradle wrapper](https://docs.gradle.org/current/userguide/gradle_wrapper.html), [Grails](https://grails.org/), -and the [Maven wrapper](https://github.com/takari/maven-wrapper). However, SpotBugs has [limitations](https://gitlab.com/gitlab-org/gitlab/-/issues/350801) when used against [Ant](https://ant.apache.org/)-based projects. We recommend using the Semgrep-based analyzer for Ant-based Java projects. +and the [Maven wrapper](https://github.com/takari/maven-wrapper). However, SpotBugs has [limitations](https://gitlab.com/gitlab-org/gitlab/-/issues/350801) when used against [Ant](https://ant.apache.org/)-based projects. We recommend using the Semgrep-based analyzer for Ant-based Java or Scala projects. 1. These analyzers reached [End of Support](https://about.gitlab.com/handbook/product/gitlab-the-product/#end-of-support) status [in GitLab 15.4](https://gitlab.com/gitlab-org/gitlab/-/issues/352554). +1. Security Code Scan reached [End of Support](https://about.gitlab.com/handbook/product/gitlab-the-product/#end-of-support) status [in GitLab 16.0](https://gitlab.com/gitlab-org/gitlab/-/issues/390416). ## Multi-project support @@ -183,7 +184,8 @@ For more information, see the confidential project `https://gitlab.com/gitlab-or ## Automatic vulnerability resolution -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/368284) in GitLab 15.9 [with a project-level flag](../../../administration/feature_flags.md) named `sec_mark_dropped_findings_as_resolved`. Enabled by default on GitLab.com; disabled by default in self-managed. On GitLab.com, [contact Support](https://about.gitlab.com/support/) if you need to disable the flag for your project. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/368284) in GitLab 15.9 [with a project-level flag](../../../administration/feature_flags.md) named `sec_mark_dropped_findings_as_resolved`. +> - Enabled by default in GitLab 15.10. On GitLab.com, [contact Support](https://about.gitlab.com/support/) if you need to disable the flag for your project. To help you focus on the vulnerabilities that are still relevant, GitLab SAST automatically [resolves](../vulnerabilities/index.md#vulnerability-status-values) vulnerabilities when: @@ -237,7 +239,7 @@ as shown in the following table: | See new findings in merge request widget | **{dotted-circle}** | **{check-circle}** | | [Manage vulnerabilities](../vulnerabilities/index.md) | **{dotted-circle}** | **{check-circle}** | | [Access the Security Dashboard](../security_dashboard/index.md) | **{dotted-circle}** | **{check-circle}** | -| [Configure SAST in the UI](#configure-sast-in-the-ui) | **{dotted-circle}** | **{check-circle}** | +| [Configure SAST by using the UI](#configure-sast-by-using-the-ui) | **{dotted-circle}** | **{check-circle}** | | [Customize SAST rulesets](customize_rulesets.md) | **{dotted-circle}** | **{check-circle}** | | [Detect False Positives](#false-positive-detection) | **{dotted-circle}** | **{check-circle}** | | [Track moved vulnerabilities](#advanced-vulnerability-tracking) | **{dotted-circle}** | **{check-circle}** | @@ -256,7 +258,7 @@ To configure SAST for a project you can: - Use [Auto SAST](../../../topics/autodevops/stages.md#auto-sast), provided by [Auto DevOps](../../../topics/autodevops/index.md). - [Configure SAST in your CI/CD YAML](#configure-sast-in-your-cicd-yaml). -- [Configure SAST using the UI](#configure-sast-in-the-ui) (introduced in GitLab 13.3). +- [Configure SAST by using the UI](#configure-sast-by-using-the-ui). You can enable SAST across many projects by [enforcing scan execution](../index.md#enforce-scan-execution). @@ -281,15 +283,12 @@ The results are saved as a that you can later download and analyze. When downloading, you always receive the most recent SAST artifact available. -### Configure SAST in the UI +### Configure SAST by using the UI -You can enable and configure SAST in the UI, either with default settings, or with customizations. +You can enable and configure SAST by using the UI, either with the default settings or with customizations. The method you can use depends on your GitLab license tier. -- [Configure SAST in the UI with customizations](#configure-sast-in-the-ui-with-customizations). **(ULTIMATE)** -- [Configure SAST in the UI with default settings only](#configure-sast-in-the-ui-with-default-settings-only). - -### Configure SAST in the UI with customizations **(ULTIMATE)** +#### Configure SAST with customizations **(ULTIMATE)** > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3659) in GitLab 13.3. > - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/232862) in GitLab 13.4. @@ -303,7 +302,7 @@ successfully, and an error may occur. To enable and configure SAST with customizations: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Security & Compliance > Configuration**. +1. On the left sidebar, select **Security and Compliance > Security configuration**. 1. If the project does not have a `.gitlab-ci.yml` file, select **Enable SAST** in the Static Application Security Testing (SAST) row, otherwise select **Configure SAST**. 1. Enter the custom SAST values. @@ -318,7 +317,7 @@ To enable and configure SAST with customizations: Pipelines now include a SAST job. -### Configure SAST in the UI with default settings only +#### Configure SAST with default settings only > [Introduced](https://about.gitlab.com/releases/2021/02/22/gitlab-13-9-released/#security-configuration-page-for-all-users) in GitLab 13.9 @@ -330,7 +329,7 @@ successfully, and an error may occur. To enable and configure SAST with default settings: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Security & Compliance** > **Configuration**. +1. On the left sidebar, select **Security and Compliance** > **Configuration**. 1. In the SAST section, select **Configure with a merge request**. 1. Review and merge the merge request to enable SAST. @@ -600,7 +599,7 @@ Some analyzers can be customized with CI/CD variables. | `SAST_GOSEC_CONFIG` | Gosec | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/328301)** in GitLab 14.0 - use custom rulesets instead. Path to configuration for Gosec (optional). | | `PHPCS_SECURITY_AUDIT_PHP_EXTENSIONS` | phpcs-security-audit | Comma separated list of additional PHP Extensions. | | `SAST_DISABLE_BABEL` | NodeJsScan | **{warning}** **[Removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64025)** in GitLab 13.5 | -| `SAST_SEMGREP_METRICS` | Semgrep | Set to `"false"` to disable sending anonymized scan metrics to [r2c](https://r2c.dev/). Default: `true`. Introduced in GitLab 14.0 from the [confidential issue](../../project/issues/confidential_issues.md) `https://gitlab.com/gitlab-org/gitlab/-/issues/330565`. | +| `SAST_SEMGREP_METRICS` | Semgrep | Set to `"false"` to disable sending anonymized scan metrics to [r2c](https://semgrep.dev). Default: `true`. Introduced in GitLab 14.0 from the [confidential issue](../../project/issues/confidential_issues.md) `https://gitlab.com/gitlab-org/gitlab/-/issues/330565`. | | `SAST_SCANNER_ALLOWED_CLI_OPTS` | Semgrep | CLI options (arguments with value, or flags) that are passed to the underlying security scanner when running scan operation. Only a limited set of [options](#security-scanner-configuration) are accepted. Separate a CLI option and its value using either a blank space or equals (`=`) character. For example: `name1 value1` or `name1=value1`. Multiple options must be separated by blank spaces. For example: `name1 value1 name2 value2`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/368565) in GitLab 15.3. | #### Security scanner configuration @@ -617,6 +616,7 @@ flags are added to the scanner's CLI options. | Analyzer | CLI option | Description | |------------------------------------------------------------------------------|--------------------|------------------------------------------------------------------------------| | [Semgrep](https://gitlab.com/gitlab-org/security-products/analyzers/semgrep) | `--max-memory` | Sets the maximum system memory to use when running a rule on a single file. Measured in MB. | +| [Flawfinder](https://gitlab.com/gitlab-org/security-products/analyzers/flawfinder) | `--neverignore` | Never ignore security issues, even if they have an “ignore” directive in a comment. Adding this option is likely to result in the analyzer detecting additional vulnerability findings which cannot be automatically resolved. | #### Custom CI/CD variables @@ -664,7 +664,7 @@ To download the report file, you can either: - Download the file from the CI/CD pipelines page. - In the pipelines tab on merge requests, set [`artifacts: paths`](../../../ci/yaml/index.md#artifactspaths) to `gl-sast-report.json`. -For information, see [Download job artifacts](../../../ci/pipelines/job_artifacts.md#download-job-artifacts). +For information, see [Download job artifacts](../../../ci/jobs/job_artifacts.md#download-job-artifacts). For details of the report file's schema, see [SAST report file schema](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/sast-report-format.json). @@ -819,7 +819,7 @@ affected. Read more in ### Getting warning message `gl-sast-report.json: no matching files` -For information on this, see the [general Application Security troubleshooting section](../../../ci/pipelines/job_artifacts.md#error-message-no-files-to-upload). +For information on this, see the [general Application Security troubleshooting section](../../../ci/jobs/job_artifacts_troubleshooting.md#error-message-no-files-to-upload). ### Error: `sast is used for configuration only, and its script should not be executed` diff --git a/doc/user/application_security/secret_detection/automatic_response.md b/doc/user/application_security/secret_detection/automatic_response.md new file mode 100644 index 00000000000..a898a63e33b --- /dev/null +++ b/doc/user/application_security/secret_detection/automatic_response.md @@ -0,0 +1,242 @@ +--- +stage: Secure +group: Static Analysis +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Automatic response to leaked secrets **(ULTIMATE)** + +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/4639) in GitLab 13.6. + +GitLab Secret Detection automatically responds when it finds certain types of leaked secrets. +Automatic responses can: + +- Automatically revoke the secret. +- Notify the partner that issued the secret. The partner can then revoke the secret, notify its owner, or otherwise protect against abuse. + +## Supported secret types and actions + +GitLab supports automatic response for the following types of secrets: + +| Secret type | Action taken | Supported on GitLab.com | Supported in self-managed | +| ----- | --- | --- | --- | +| GitLab [Personal access tokens](../../profile/personal_access_tokens.md) | Immediately revoke token, send email to owner | ✅ | ✅ [15.9 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/371658) | +| Amazon Web Services (AWS) [IAM access keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) | Notify AWS | ✅ | ⚙ | + +**Component legend** + +- ✅ - Available by default +- ⚙ - Requires manual integration using a [Token Revocation API](../../../development/sec/token_revocation_api.md) + +## Feature availability + +> [Enabled for non-default branches](https://gitlab.com/gitlab-org/gitlab/-/issues/299212) in GitLab 15.11. + +Credentials are only post-processed when Secret Detection finds them: + +- In public projects, because publicly exposed credentials pose an increased threat. Expansion to private projects is considered in [issue 391379](https://gitlab.com/gitlab-org/gitlab/-/issues/391379). +- In projects with GitLab Ultimate, for technical reasons. Expansion to all tiers is tracked in [issue 391763](https://gitlab.com/gitlab-org/gitlab/-/issues/391763). + +## High-level architecture + +This diagram describes how a post-processing hook revokes a secret in the GitLab application: + +```mermaid +sequenceDiagram + autonumber + GitLab Rails-->+GitLab Rails: gl-secret-detection-report.json + GitLab Rails->>+GitLab Sidekiq: StoreScansService + GitLab Sidekiq-->+GitLab Sidekiq: ScanSecurityReportSecretsWorker + GitLab Sidekiq-->+GitLab Token Revocation API: GET revocable keys types + GitLab Token Revocation API-->>-GitLab Sidekiq: OK + GitLab Sidekiq->>+GitLab Token Revocation API: POST revoke revocable keys + GitLab Token Revocation API-->>-GitLab Sidekiq: ACCEPTED + GitLab Token Revocation API-->>+Partner API: revoke revocable keys + Partner API-->>+GitLab Token Revocation API: ACCEPTED +``` + +1. A pipeline with a Secret Detection job completes, producing a scan report (**1**). +1. The report is processed (**2**) by a service class, which schedules an asynchronous worker if token revocation is possible. +1. The asynchronous worker (**3**) communicates with an externally deployed HTTP service + (**4** and **5**) to determine which kinds of secrets can be automatically revoked. +1. The worker sends (**6** and **7**) the list of detected secrets which the GitLab Token Revocation API is able to + revoke. +1. The GitLab Token Revocation API sends (**8** and **9**) each revocable token to their respective vendor's [Partner API](#implement-a-partner-api). See the [GitLab Token Revocation API](../../../development/sec/token_revocation_api.md) + documentation for more information. + +## Partner program for leaked-credential notifications + +GitLab notifies partners when credentials they issue are leaked in public repositories on GitLab.com. +If you operate a cloud or SaaS product and you're interested in receiving these notifications, learn more in [epic 4944](https://gitlab.com/groups/gitlab-org/-/epics/4944). +Partners must [implement a Partner API](#implement-a-partner-api), which is called by the GitLab Token Revocation API. + +### Implement a Partner API + +A Partner API integrates with the GitLab Token Revocation API to receive and respond to leaked token revocation +requests. The service should be a publicly accessible HTTP API that is idempotent and rate-limited. + +Requests to your service can include one or more leaked tokens, and a header with the signature of the request +body. We strongly recommend that you verify incoming requests using this signature, to prove it's a genuine +request from GitLab. The diagram below details the necessary steps to receive, verify, and revoke leaked tokens: + +```mermaid +sequenceDiagram + autonumber + GitLab Token Revocation API-->>+Partner API: Send new leaked credentials + Partner API-->>+GitLab Public Keys endpoint: Get active public keys + GitLab Public Keys endpoint-->>+Partner API: One or more public keys + Partner API-->>+Partner API: Verify request is signed by GitLab + Partner API-->>+Partner API: Respond to leaks + Partner API-->>+GitLab Token Revocation API: HTTP status +``` + +1. The GitLab Token Revocation API sends (**1**) a [revocation request](#revocation-request) to the Partner API. The request + includes headers containing a public key identifier and signature of the request body. +1. The Partner API requests (**2**) a list of [public keys](#public-keys-endpoint) from GitLab. The response (**3**) + may include multiple public keys in the event of key rotation and should be filtered with the identifier in the request header. +1. The Partner API [verifies the signature](#verifying-the-request) against the actual request body, using the public key (**4**). +1. The Partner API processes the leaked tokens, which may involve automatic revocation (**5**). +1. The Partner API responds to the GitLab Token Revocation API (**6**) with the appropriate HTTP status code: + - A successful response code (HTTP 200 through 299) acknowledges that the partner has received and processed the request. + - An error code (HTTP 400 or higher) causes the GitLab Token Revocation API to retry the request. + +#### Revocation request + +This JSON schema document describes the body of the revocation request: + +```json +{ + "type": "array", + "items": { + "description": "A leaked token", + "type": "object", + "properties": { + "type": { + "description": "The type of token. This is vendor-specific and can be customised to suit your revocation service", + "type": "string", + "examples": [ + "my_api_token" + ] + }, + "token": { + "description": "The substring that was matched by the Secret Detection analyser. In most cases, this is the entire token itself", + "type": "string", + "examples": [ + "XXXXXXXXXXXXXXXX" + ] + }, + "url": { + "description": "The URL to the raw source file hosted on GitLab where the leaked token was detected", + "type": "string", + "examples": [ + "https://gitlab.example.com/some-repo/-/raw/abcdefghijklmnop/compromisedfile1.java" + ] + } + } + } +} +``` + +Example: + +```json +[{"type": "my_api_token", "token": "XXXXXXXXXXXXXXXX", "url": "https://example.com/some-repo/-/raw/abcdefghijklmnop/compromisedfile1.java"}] +``` + +In this example, Secret Detection has determined that an instance of `my_api_token` has been leaked. The +value of the token is provided to you, in addition to a publicly accessible URL to the raw content of the +file containing the leaked token. + +The request includes two special headers: + +| Header | Type | Description | +|--------|------|-------------| +| `Gitlab-Public-Key-Identifier` | string | A unique identifier for the key pair used to sign this request. Primarily used to aid in key rotation. | +| `Gitlab-Public-Key-Signature` | string | A base64-encoded signature of the request body. | + +You can use these headers along with the GitLab Public Keys endpoint to verify that the revocation request was genuine. + +#### Public Keys endpoint + +GitLab maintains a publicly-accessible endpoint for retrieving public keys used to verify revocation +requests. The endpoint can be provided on request. + +This JSON schema document describes the response body of the public keys endpoint: + +```json +{ + "type": "object", + "properties": { + "public_keys": { + "description": "An array of public keys managed by GitLab used to sign token revocation requests.", + "type": "array", + "items": { + "type": "object", + "properties": { + "key_identifier": { + "description": "A unique identifier for the keypair. Match this against the value of the Gitlab-Public-Key-Identifier header", + "type": "string" + }, + "key": { + "description": "The value of the public key", + "type": "string" + }, + "is_current": { + "description": "Whether the key is currently active and signing new requests", + "type": "boolean" + } + } + } + } + } +} +``` + +Example: + +```json +{ + "public_keys": [ + { + "key_identifier": "6917d7584f0fa65c8c33df5ab20f54dfb9a6e6ae", + "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEN05/VjsBwWTUGYMpijqC5pDtoLEf\nuWz2CVZAZd5zfa/NAlSFgWRDdNRpazTARndB2+dHDtcHIVfzyVPNr2aznw==\n-----END PUBLIC KEY-----\n", + "is_current": true + } + ] +} +``` + +#### Verifying the request + +You can check whether a revocation request is genuine by verifying the `Gitlab-Public-Key-Signature` header +against the request body, using the corresponding public key taken from the API response above. We use +[ECDSA](https://en.wikipedia.org/wiki/Elliptic_Curve_Digital_Signature_Algorithm) with SHA256 hashing to +produce the signature, which is then base64-encoded into the header value. + +The Python script below demonstrates how the signature can be verified. It uses the popular +[pyca/cryptography](https://cryptography.io/en/latest/) module for cryptographic operations: + +```python +import hashlib +import base64 +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.serialization import load_pem_public_key +from cryptography.hazmat.primitives.asymmetric import ec + +public_key = str.encode("") # obtained from the public keys endpoint +signature_header = "" # obtained from the `Gitlab-Public-Key-Signature` header +request_body = str.encode(r'') # obtained from the revocation request body + +pk = load_pem_public_key(public_key) +decoded_signature = base64.b64decode(signature_header) + +pk.verify(decoded_signature, request_body, ec.ECDSA(hashes.SHA256())) # throws if unsuccessful + +print("Signature verified!") +``` + +The main steps are: + +1. Loading the public key into a format appropriate for the crypto library you're using. +1. Base64-decoding the `Gitlab-Public-Key-Signature` header value. +1. Verifying the body against the decoded signature, specifying ECDSA with SHA256 hashing. diff --git a/doc/user/application_security/secret_detection/index.md b/doc/user/application_security/secret_detection/index.md index d6aab71a2c6..5a1dcc840ca 100644 --- a/doc/user/application_security/secret_detection/index.md +++ b/doc/user/application_security/secret_detection/index.md @@ -15,40 +15,42 @@ info: To determine the technical writer assigned to the Stage/Group associated w > `secret_detection_default_branch` and `secret_detection` were consolidated into one job, > `secret_detection`. -People may accidentally commit secrets (such as keys, passwords, and API tokens) to remote Git repositories. +People sometimes accidentally commit secrets like keys or API tokens to Git repositories. +After a sensitive value is pushed to a remote repository, anyone with access to the repository can impersonate the authorized user of the secret for malicious purposes. +Most organizations require exposed secrets to be revoked and replaced to address this risk. -Anyone with access to the repository could use the secrets for malicious purposes. Exposed secrets -must be considered compromised and be replaced, which can be costly. +Secret Detection scans your repository to help prevent your secrets from being exposed. +Secret Detection scanning works on all text files, regardless of the language or framework used. -To help prevent secrets from being committed to a Git repository, you can use Secret Detection to -scan your repository for secrets. Scanning is language and framework agnostic, but does not support -scanning binary files. +After you [enable Secret Detection](#enable-secret-detection), scans run in a CI/CD job named `secret_detection`. +You can run scans and view [Secret Detection JSON report artifacts](../../../ci/yaml/artifacts_reports.md#artifactsreportssecret_detection) in any GitLab tier. -Secret Detection uses an analyzer containing the [Gitleaks](https://github.com/zricethezav/gitleaks) -tool to scan the repository for secrets. Detection occurs in the `secret-detection` job. The results -are saved as a -[Secret Detection report artifact](../../../ci/yaml/artifacts_reports.md#artifactsreportssecret_detection) -that you can later download and analyze. Due to implementation limitations, we always take the -latest Secret Detection artifact available. +With GitLab Ultimate, Secret Detection results are also processed so you can: -GitLab SaaS supports post-processing hooks, so you can take action when a secret is found. For -more information, see [Post-processing and revocation](post_processing.md). +- See them in the [merge request widget](../index.md#view-security-scan-information-in-merge-requests), [pipeline security report](../vulnerability_report/pipeline.md), and [Vulnerability Report](../vulnerability_report/index.md). +- Use them in approval workflows. +- Review them in the security dashboard. +- [Automatically respond](automatic_response.md) to leaks in public repositories. -All identified secrets are reported in the: +## Detected secrets -- Merge request widget -- Pipelines' **Security** tab -- [Vulnerability Report](../vulnerability_report/index.md) +GitLab maintains the detection rules used in Secret Detection. +The [default ruleset](https://gitlab.com/gitlab-org/security-products/analyzers/secrets/-/blob/master/gitleaks.toml) +contains more than 100 patterns. -![Secret Detection in merge request widget](img/secret_detection_v13_2.png) +Most Secret Detection patterns search for specific types of secrets. +Many services add prefixes or other structural details to their secrets so they can be identified if they're leaked. +For example, GitLab [adds a `glpat-` prefix](../../admin_area/settings/account_and_limit_settings.md#personal-access-token-prefix) to project, group, and personal access tokens by default. -## Detected secrets +To provide more reliable, high-confidence results, Secret Detection only looks for passwords or other unstructured secrets in specific contexts like URLs. + +### Adding new patterns + +To search for other types of secrets in your repositories, you can configure a [custom ruleset](#custom-rulesets). -Secret Detection uses a [default ruleset](https://gitlab.com/gitlab-org/security-products/analyzers/secrets/-/blob/master/gitleaks.toml) -containing more than 90 secret detection patterns. You can also customize the secret detection -patterns using [custom rulesets](#custom-rulesets). If you want to contribute rulesets for -"well-identifiable" secrets, follow the steps detailed in the -[community contributions guidelines](https://gitlab.com/gitlab-org/gitlab/-/issues/345453). +To propose a new detection rule for all users of Secret Detection, create a merge request against the [file containing the default rules](https://gitlab.com/gitlab-org/security-products/analyzers/secrets/-/blob/master/gitleaks.toml). + +If you operate a cloud or SaaS product and you're interested in partnering with GitLab to better protect your users, learn more about our [partner program for leaked credential notifications](automatic_response.md#partner-program-for-leaked-credential-notifications). ## Features per tier @@ -59,6 +61,7 @@ Different features are available in different [GitLab tiers](https://about.gitla | [Configure Secret Detection scanner](#enable-secret-detection) | **{check-circle}** Yes | **{check-circle}** Yes | | [Customize Secret Detection settings](#configure-scan-settings) | **{check-circle}** Yes | **{check-circle}** Yes | | Download [JSON Report](../sast/index.md#reports-json-format) | **{check-circle}** Yes | **{check-circle}** Yes | +| [Check text for potential secrets](#warnings-for-potential-leaks-in-text-content) before it's posted | **{check-circle}** Yes | **{check-circle}** Yes | | See new findings in the merge request widget | **{dotted-circle}** No | **{check-circle}** Yes | | View identified secrets in the pipelines' **Security** tab | **{dotted-circle}** No | **{check-circle}** Yes | | [Manage vulnerabilities](../vulnerability_report/index.md) | **{dotted-circle}** No | **{check-circle}** Yes | @@ -141,12 +144,12 @@ To enable Secret Detection, either: - Enable [Auto DevOps](../../../topics/autodevops/index.md), which includes [Auto Secret Detection](../../../topics/autodevops/stages.md#auto-secret-detection). -- [Edit the `.gitlab.ci.yml` file manually](#edit-the-gitlabciyml-file-manually). Use this method if - your `.gitlab-ci.yml` file is complex. +- [Edit the `.gitlab-ci.yml` file manually](#edit-the-gitlab-ciyml-file-manually). Use this method + if your `.gitlab-ci.yml` file is complex. - [Use an automatically configured merge request](#use-an-automatically-configured-merge-request). -### Edit the `.gitlab.ci.yml` file manually +### Edit the `.gitlab-ci.yml` file manually This method requires you to manually edit the existing `.gitlab-ci.yml` file. Use this method if your GitLab CI/CD configuration file is complex. @@ -180,12 +183,12 @@ the `.gitlab-ci.yml` file. You then merge the merge request to enable Secret Det NOTE: This method works best with no existing `.gitlab-ci.yml` file, or with a minimal configuration file. If you have a complex GitLab configuration file it may not be parsed successfully, and an -error may occur. In that case, use the [manual](#edit-the-gitlabciyml-file-manually) method instead. +error may occur. In that case, use the [manual](#edit-the-gitlab-ciyml-file-manually) method instead. -To enable Secret Detection automatically: +To enable Secret Detection: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Security & Compliance > Configuration**. +1. On the left sidebar, select **Security and Compliance > Security configuration**. 1. In the **Secret Detection** row, select **Configure with a merge request**. 1. Optional. Complete the fields. 1. Select **Create merge request**. @@ -195,12 +198,13 @@ Pipelines now include a Secret Detection job. ## Responding to a leaked secret -Secrets detected by the analyzer should be immediately rotated. -[Purging a file from the repository's history](../../project/repository/reducing_the_repo_size_using_git.md#purge-files-from-repository-history) -may not be effective in removing all references to the file. Additionally, the secret will remain in any existing -forks or clones of the repository. +When a secret is detected, you should rotate it immediately. GitLab attempts to +[automatically revoke](automatic_response.md) some types of leaked secrets. For those that are not +automatically revoked, you must do so manually. -GitLab will attempt to [automatically revoke](post_processing.md) some types of leaked secrets. +[Purging a secret from the repository's history](../../project/repository/reducing_the_repo_size_using_git.md#purge-files-from-repository-history) +does not fully address the leak. The original secret remains in any existing forks or +clones of the repository. ## Pinning to specific analyzer version @@ -413,18 +417,16 @@ In the following example `secret-detection-ruleset.toml` file, rules are matched ### Synthesize a custom configuration -To create a custom configuration, you can use passthrough chains. Passthroughs can also be chained -to build more complex configurations. For more details, see -[SAST Customize ruleset](../sast/customize_rulesets.md). - -Only the following passthrough types are supported by the `secrets` analyzer: +You can use passthroughs to override the default Secret Detection ruleset. The +following passthrough types are supported by the `secrets` analyzer: -- `file` - `raw` +- `file` -In the `secret-detection-ruleset.toml` file, do one of the following: +To define a passthrough, add _one_ of the following to the +`secret-detection-ruleset.toml` file: -- Define a custom ruleset, for example: +- Using an inline (`raw`) value: ```toml [secrets] @@ -442,7 +444,7 @@ In the `secret-detection-ruleset.toml` file, do one of the following: """ ``` -- Provide the name of the file containing a custom ruleset, for example: +- Using an external `file` committed to the current repository: ```toml [secrets] @@ -454,6 +456,45 @@ In the `secret-detection-ruleset.toml` file, do one of the following: value = "config/gitleaks.toml" ``` +For more information on the syntax of passthroughs, see the +[passthroughs section on the SAST customize rulesets](../sast/customize_rulesets.md#the-analyzerpassthrough-section) +page. + +#### Extending the default configuration + +You can extend the default configuration with additional changes by using [Gitleaks `extend` support](https://github.com/gitleaks/gitleaks#configuration). + +In the following `file` passthrough example, the string `glpat-1234567890abcdefghij` is ignored by Secret Detection. That GitLab personal access token (PAT) is used in test cases. Detection of it would be a false positive. + +The `secret-detection-ruleset.toml` file defines that the configuration in `extended-gitleaks-config.toml` file is to be included. The `extended-gitleaks-config.toml` file defines the custom Gitleaks configuration. The `allowlist` stanza defines a regular expression that matches the secret that is to be ignored ("allowed"). + +```toml +# .gitlab/secret-detection-ruleset.toml +[secrets] + description = 'secrets custom rules configuration' + + [[secrets.passthrough]] + type = "file" + target = "gitleaks.toml" + value = "extended-gitleaks-config.toml" +``` + +```toml +# extended-gitleaks-config.toml +title = "extension of gitlab's default gitleaks config" + +[extend] +# Extends default packaged path +path = "/gitleaks.toml" + +[allowlist] + description = "allow list of test tokens to ignore in detection" + regexTarget = "match" + regexes = [ + '''glpat-1234567890abcdefghij''', + ] +``` + ## Running Secret Detection in an offline environment **(PREMIUM SELF)** An offline environment has limited, restricted, or intermittent access to external resources through @@ -532,6 +573,26 @@ variable, or as a CI/CD variable. - If using a variable, set the value of `ADDITIONAL_CA_CERT_BUNDLE` to the text representation of the certificate. +## Warnings for potential leaks in text content + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/368434) in GitLab 15.11. + +When you create an issue, propose a merge request, or write a comment, you might accidentally post a sensitive value. +For example, you might paste in the details of an API request or an environment variable that contains an authentication token. + +GitLab checks if the text of your issue description, merge request description, comment, or reply contains a sensitive token. +If a token is found, a warning message is displayed. You can then edit your message before posting it. +This check happens in your browser before the message is sent to the server. +The check is always on; you don't have to set it up. + +Your text is checked for the following secret types: + +- GitLab [personal access tokens](../../../security/token_overview.md#personal-access-tokens) +- GitLab [feed tokens](../../../security/token_overview.md#feed-token) + +This feature is separate from Secret Detection scanning, which checks your Git repository for leaked secrets. +[Issue 405147](https://gitlab.com/gitlab-org/gitlab/-/issues/405147) tracks efforts to align these two types of protection. + ## Troubleshooting ### Set the logging level @@ -552,7 +613,7 @@ visible in job logs. ### Warning: `gl-secret-detection-report.json: no matching files` -For information on this, see the [general Application Security troubleshooting section](../../../ci/pipelines/job_artifacts.md#error-message-no-files-to-upload). +For information on this, see the [general Application Security troubleshooting section](../../../ci/jobs/job_artifacts_troubleshooting.md#error-message-no-files-to-upload). ### Error: `Couldn't run the gitleaks command: exit status 2` diff --git a/doc/user/application_security/secret_detection/post_processing.md b/doc/user/application_security/secret_detection/post_processing.md index 22d7a8ba5af..3a6cf7f7e37 100644 --- a/doc/user/application_security/secret_detection/post_processing.md +++ b/doc/user/application_security/secret_detection/post_processing.md @@ -1,97 +1,11 @@ --- -stage: Secure -group: Static Analysis -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: 'automatic_response.md' +remove_date: '2023-08-08' --- -# Secret Detection post-processing and revocation **(ULTIMATE SAAS)** +This document was moved to [another location](automatic_response.md). -> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/4639) in GitLab 13.6. -> - [Disabled by default for GitLab personal access tokens](https://gitlab.com/gitlab-org/gitlab/-/issues/371658) in GitLab 15.6 [with a flag](../../../administration/feature_flags.md) named `gitlab_pat_auto_revocation`. Available to GitLab.com only. -> - [Enabled by default for GitLab personal access tokens](https://gitlab.com/gitlab-org/gitlab/-/issues/371658) in GitLab 15.9 - -GitLab.com and self-managed supports running post-processing hooks after detecting a secret. These -hooks can perform actions, like notifying the vendor that issued the secret. -The vendor can then confirm the credentials and take remediation actions, like: - -- Revoking a secret. -- Reissuing a secret. -- Notifying the creator of the secret. - -GitLab supports post-processing for the following vendors and secrets: - -| Vendor | Secret | GitLab.com | Self-managed | -| ----- | --- | --- | --- | -| GitLab | [Personal access tokens](../../profile/personal_access_tokens.md) | ✅ | ✅ 15.9 and later | -| Amazon Web Services (AWS) | [IAM access keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) | ✅ | ⚙ | - -**Component legend** - -- ✅ - Available by default -- ⚙ - Requires manual integration using a [Token Revocation API](../../../development/sec/token_revocation_api.md) - -## Feature availability - -Credentials are only post-processed when Secret Detection finds them: - -- In public projects, because publicly exposed credentials pose an increased threat. Expansion to private projects is considered in [issue 391379](https://gitlab.com/gitlab-org/gitlab/-/issues/391379). -- On the project [default branch](../../project/repository/branches/default.md), for technical reasons. Expansion to all branches is tracked in [issue 299212](https://gitlab.com/gitlab-org/gitlab/-/issues/299212). -- In projects with GitLab Ultimate, for technical reasons. Expansion to all tiers is tracked in [issue 391763](https://gitlab.com/gitlab-org/gitlab/-/issues/391763). - -## High-level architecture - -This diagram describes how a post-processing hook revokes a secret within the GitLab application: - -```mermaid -sequenceDiagram - autonumber - GitLab Rails->>+Sidekiq: gl-secret-detection-report.json - Sidekiq-->+Sidekiq: StoreSecurityReportsWorker - Sidekiq-->+Token Revocation API: GET revocable keys types - Token Revocation API-->>-Sidekiq: OK - Sidekiq->>+Token Revocation API: POST revoke revocable keys - Token Revocation API-->>-Sidekiq: ACCEPTED - Token Revocation API-->>+Receiver Service: revoke revocable keys - Receiver Service-->>+Token Revocation API: ACCEPTED -``` - -1. A pipeline with a Secret Detection job completes on the project's default branch, producing a scan - report (**1**). -1. The report is processed (**2**) by an asynchronous worker, which communicates with an externally - deployed HTTP service (**3** and **4**) to determine which kinds of secrets can be automatically - revoked. -1. The worker sends (**5** and **6**) the list of detected secrets which the Token Revocation API is able to - revoke. -1. The Token Revocation API sends (**7** and **8**) each revocable token to their respective vendor's [receiver service](#integrate-your-cloud-provider-service-with-gitlabcom). - -See the [Token Revocation API](../../../development/sec/token_revocation_api.md) documentation for more -information. - -## Integrate your cloud provider service with GitLab.com - -Third-party cloud and SaaS vendors interested in automated token revocation can -[express integration interest by filling out this form](https://forms.gle/wWpvrtLRK21Q2WJL9). -Vendors must [implement a revocation receiver service](#implement-a-revocation-receiver-service) -which will be called by the Token Revocation API. - -### Implement a revocation receiver service - -A revocation receiver service integrates with a GitLab instance's Token Revocation API to receive and respond -to leaked token revocation requests. The service should be a publicly accessible HTTP API that is -idempotent and rate-limited. Requests to your service from the Token Revocation API will follow the example -below: - -```plaintext -POST / HTTP/2 -Accept: */* -Content-Type: application/json -X-Gitlab-Token: MYSECRETTOKEN - -[ - {"type": "my_api_token", "token":"XXXXXXXXXXXXXXXX","url": "https://example.com/some-repo/~/raw/abcdefghijklmnop/compromisedfile1.java"} -] -``` - -In this example, Secret Detection has determined that an instance of `my_api_token` has been leaked. The -value of the token is provided to you, in addition to a publicly accessible URL to the raw content of the -file containing the leaked token. + + + + diff --git a/doc/user/application_security/secure_your_application.md b/doc/user/application_security/secure_your_application.md index fb10efff2c6..8dd1168a0d9 100644 --- a/doc/user/application_security/secure_your_application.md +++ b/doc/user/application_security/secure_your_application.md @@ -9,7 +9,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w GitLab can check your applications for security vulnerabilities. - [Get started](get-started-security.md) -- [Security Configuration](configuration/index.md) +- [Security configuration](configuration/index.md) - [Container Scanning](container_scanning/index.md) - [Dependency Scanning](dependency_scanning/index.md) - [Static Application Security Testing](sast/index.md) diff --git a/doc/user/application_security/security_dashboard/img/security_center_dashboard_v13_4.png b/doc/user/application_security/security_dashboard/img/security_center_dashboard_v13_4.png deleted file mode 100644 index 5379b5c6e5d..00000000000 Binary files a/doc/user/application_security/security_dashboard/img/security_center_dashboard_v13_4.png and /dev/null differ diff --git a/doc/user/application_security/security_dashboard/img/security_center_dashboard_v15_10.png b/doc/user/application_security/security_dashboard/img/security_center_dashboard_v15_10.png new file mode 100644 index 00000000000..c2780fce787 Binary files /dev/null and b/doc/user/application_security/security_dashboard/img/security_center_dashboard_v15_10.png differ diff --git a/doc/user/application_security/security_dashboard/index.md b/doc/user/application_security/security_dashboard/index.md index 7388ef80e68..e6cb4cef4b2 100644 --- a/doc/user/application_security/security_dashboard/index.md +++ b/doc/user/application_security/security_dashboard/index.md @@ -19,6 +19,9 @@ To use the Security Dashboards, you must: shared runners on GitLab.com, you are using the correct version. - Have the [correct role](../../permissions.md) for the project or group. + +For an overview, see [Security Dashboard](https://www.youtube.com/watch?v=QHQHN4luNpc). + ## When Security Dashboards are updated The Security Dashboards show results of scans from the most recent completed pipeline on the @@ -64,7 +67,7 @@ Project Security Dashboards show statistics for all vulnerabilities with a curre To view total number of vulnerabilities over time: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Security & Compliance > Security Dashboard**. +1. On the left sidebar, select **Security and Compliance > Security Dashboard**. 1. Filter and search for what you need. - To filter the chart by severity, select the legend name. - To view a specific time frame, use the time range handles (**{scroll-handle}**). @@ -77,7 +80,7 @@ To view total number of vulnerabilities over time: To download an SVG image of the vulnerabilities chart: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Security & Compliance > Security dashboard**. +1. On the left sidebar, select **Security and Compliance > Security dashboard**. 1. Select **Save chart as an image** (**{download}**). ## View vulnerabilities over time for a group @@ -133,7 +136,7 @@ The Security Center includes: - A [vulnerability report](../vulnerability_report/index.md). - A settings area to configure which projects to display. -![Security Center Dashboard with projects](img/security_center_dashboard_v13_4.png) +![Security Center Dashboard with projects](img/security_center_dashboard_v15_10.png) ### View the Security Center @@ -151,6 +154,9 @@ To add projects to the Security Center: After you add projects, the security dashboard and vulnerability report show the vulnerabilities found in those projects' default branches. +You can add a maximum of 1,000 projects, however the **Project** filter in the **Vulnerability +Report** is limited to 100 projects. + | Icon | Severity level | |:----------------------------------------------|:---------------| @@ -46,75 +66,69 @@ The following is a list of available violation severity levels, ranked from most | **{severity-low, 18, gl-fill-orange-300}** | Low | | **{severity-info, 18, gl-fill-blue-400}** | Info | -### Violation types + -The following is a list of violations that are either: +#### Violation types -- Already available. -- Aren't available, but which we are tracking in issues. +From [GitLab 14.10](https://gitlab.com/groups/gitlab-org/-/epics/6870), these are the available compliance violations. -| Violation | Severity level | Category | Description | Availability | -|:-------------------------------------|:----------------|:---------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------| -| Author approved merge request | High | [Separation of duties](#separation-of-duties) | The author of the merge request approved their own merge request. For more information, see [Prevent approval by author](../../project/merge_requests/approvals/settings.md#prevent-approval-by-author). | [Available in GitLab 14.10](https://gitlab.com/groups/gitlab-org/-/epics/6870) | -| Committers approved merge request | High | [Separation of duties](#separation-of-duties) | The committers of the merge request approved the merge request they contributed to. For more information, see [Prevent approvals by users who add commits](../../project/merge_requests/approvals/settings.md#prevent-approvals-by-users-who-add-commits). | [Available in GitLab 14.10](https://gitlab.com/groups/gitlab-org/-/epics/6870) | -| Fewer than two approvals | High | [Separation of duties](#separation-of-duties) | The merge request was merged with fewer than two approvals. For more information, see [Merge request approval rules](../../project/merge_requests/approvals/rules.md). | [Available in GitLab 14.10](https://gitlab.com/groups/gitlab-org/-/epics/6870) | -| Pipeline failed | Medium | [Pipeline results](../../../ci/pipelines/index.md) | The merge requests pipeline failed and was merged. | [Unavailable](https://gitlab.com/gitlab-org/gitlab/-/issues/346011) | -| Pipeline passed with warnings | Info | [Pipeline results](../../../ci/pipelines/index.md) | The merge request pipeline passed with warnings and was merged. | [Unavailable](https://gitlab.com/gitlab-org/gitlab/-/issues/346011) | -| Code coverage down more than 10% | High | [Code coverage](../../../ci/pipelines/settings.md#merge-request-test-coverage-results) | The code coverage report for the merge request indicates a reduction in coverage of more than 10%. | [Unavailable](https://gitlab.com/gitlab-org/gitlab/-/issues/346011) | -| Code coverage down between 5% to 10% | Medium | [Code coverage](../../../ci/pipelines/settings.md#merge-request-test-coverage-results) | The code coverage report for the merge request indicates a reduction in coverage of between 5% to 10%. | [Unavailable](https://gitlab.com/gitlab-org/gitlab/-/issues/346011) | -| Code coverage down between 1% to 5% | Low | [Code coverage](../../../ci/pipelines/settings.md#merge-request-test-coverage-results) | The code coverage report for the merge request indicates a reduction in coverage of between 1% to 5%. | [Unavailable](https://gitlab.com/gitlab-org/gitlab/-/issues/346011) | -| Code coverage down less than 1% | Info | [Code coverage](../../../ci/pipelines/settings.md#merge-request-test-coverage-results) | The code coverage report for the merge request indicates a reduction in coverage of less than 1%. | [Unavailable](https://gitlab.com/gitlab-org/gitlab/-/issues/346011) | +| Violation | Severity level | Category | Description | +|:----------------------------------|:---------------|:----------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Author approved merge request | High | [Separation of duties](#separation-of-duties) | Author of the merge request approved their own merge request. For more information, see [Prevent approval by author](../../project/merge_requests/approvals/settings.md#prevent-approval-by-author). | +| Committers approved merge request | High | [Separation of duties](#separation-of-duties) | Committers of the merge request approved the merge request they contributed to. For more information, see [Prevent approvals by users who add commits](../../project/merge_requests/approvals/settings.md#prevent-approvals-by-users-who-add-commits). | +| Fewer than two approvals | High | [Separation of duties](#separation-of-duties) | Merge request was merged with fewer than two approvals. For more information, see [Merge request approval rules](../../project/merge_requests/approvals/rules.md). | -## Merge request drawer +The following are unavailable compliance violations that are tracked in [epic 5237](https://gitlab.com/groups/gitlab-org/-/epics/5237). -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/299357) in GitLab 14.1. + -When you select a row, a drawer is shown that provides further details about the merge -request: +| Violation | Severity level | Category | Description | +|:-------------------------------------|:---------------|:---------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------| +| Pipeline failed | Medium | [Pipeline results](../../../ci/pipelines/index.md) | Merge requests pipeline failed and was merged. | +| Pipeline passed with warnings | Info | [Pipeline results](../../../ci/pipelines/index.md) | Merge request pipeline passed with warnings and was merged. | +| Code coverage down more than 10% | High | [Code coverage](../../../ci/testing/code_coverage.md#view-code-coverage-results-in-the-mr) | Code coverage report for the merge request indicates a reduction in coverage of more than 10%. | +| Code coverage down between 5% to 10% | Medium | [Code coverage](../../../ci/testing/code_coverage.md#view-code-coverage-results-in-the-mr) | Code coverage report for the merge request indicates a reduction in coverage of between 5% to 10%. | +| Code coverage down between 1% to 5% | Low | [Code coverage](../../../ci/testing/code_coverage.md#view-code-coverage-results-in-the-mr) | Code coverage report for the merge request indicates a reduction in coverage of between 1% to 5%. | +| Code coverage down less than 1% | Info | [Code coverage](../../../ci/testing/code_coverage.md#view-code-coverage-results-in-the-mr) | Code coverage report for the merge request indicates a reduction in coverage of less than 1%. | -- Project name and [compliance framework label](../../project/settings/index.md#add-a-compliance-framework-to-a-project), - if the project has one assigned. -- Link to the merge request. -- The merge request's branch path in the format `[source] into [target]`. -- A list of users that committed changes to the merge request. -- A list of users that commented on the merge request. -- A list of users that approved the merge request. -- The user that merged the merge request. + -## Separation of duties +##### Separation of duties -We support a separation of duties policy between users who create and approve merge requests. -Our criteria for the separation of duties is as follows: +GitLab supports a separation of duties policy between users who create and approve merge requests. Our criteria for the +separation of duties is: -- [A merge request author is **not** allowed to approve their merge request](../../project/merge_requests/approvals/settings.md#prevent-approval-by-author) -- [A merge request committer is **not** allowed to approve a merge request they have added commits to](../../project/merge_requests/approvals/settings.md#prevent-approvals-by-users-who-add-commits) -- [The minimum number of approvals required to merge a merge request is **at least** two](../../project/merge_requests/approvals/rules.md) +- [A merge request author is **not** allowed to approve their merge request](../../project/merge_requests/approvals/settings.md#prevent-approval-by-author). +- [A merge request committer is **not** allowed to approve a merge request they have added commits to](../../project/merge_requests/approvals/settings.md#prevent-approvals-by-users-who-add-commits). +- [The minimum number of approvals required to merge a merge request is **at least** two](../../project/merge_requests/approvals/rules.md). -## Chain of Custody report +### Chain of Custody report > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213364) in GitLab 13.3. > - Chain of Custody reports sent using email [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/342594) in GitLab 15.3 with a flag named `async_chain_of_custody_report`. Disabled by default. > - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/370100) in GitLab 15.5. Feature flag `async_chain_of_custody_report` removed. > - Chain of Custody report includes all commits (instead of just merge commits) [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267601) in GitLab 15.9 with a flag named `all_commits_compliance_report`. Disabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/112092) in GitLab 15.9. Feature flag `all_commits_compliance_report` removed. -FLAG: -On self-managed GitLab, by default the Chain of Custody report only contains information on merge commits. To make the report contain information on all commits to projects within a group, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `all_commits_compliance_report`. On GitLab.com, this feature is not available. - -The Chain of Custody report allows customers to export a list of merge commits within the group. -The data provides a comprehensive view with respect to merge commits. It includes the merge commit SHA, -merge request author, merge request ID, merge user, date merged, pipeline ID, group name, project name, and merge request approvers. -Depending on the merge strategy, the merge commit SHA can be a merge commit, squash commit, or a diff head commit. - -With the `all_commits_compliance_report` flag enabled, the Chain of Custody report provides a 1 month trailing window of any commit into a project under the group. +The Chain of Custody report provides a 1 month trailing window of all commits to a project under the group. To generate the report for all commits, GitLab: 1. Fetches all projects under the group. -1. For each project, fetches the last 1 month of commits. Each project is capped at 1024 commits. If there are more than 1024 commits in the 1-month window, they - are truncated. -1. Writes the commits to a CSV file. The file is truncated at 15 MB because the report is emailed as an attachment. +1. For each project, fetches the last 1 month of commits. Each project is capped at 1024 commits. If there are more than + 1024 commits in the 1-month window, they are truncated. +1. Writes the commits to a CSV file. The file is truncated at 15 MB because the report is emailed as an attachment + (GitLab 15.5 and later). + +The report includes: + +- Commit SHA. +- Commit author. +- Committer. +- Date committed. +- Group. +- Project. -The expanded report includes the commit SHA, commit author, committer, date committed, the group, and the project. If the commit has a related merge commit, then the following are also included: - Merge commit SHA. @@ -124,39 +138,152 @@ If the commit has a related merge commit, then the following are also included: - Pipeline ID. - Merge request approvers. +#### Generate Chain of Custody report + To generate the Chain of Custody report: 1. On the top bar, select **Main menu > Groups** and find your group. -1. On the left sidebar, select **Security & Compliance > Compliance report**. +1. On the left sidebar, select **Security and Compliance > Compliance report**. 1. Select **List of all merge commits**. -The Chain of Custody report is either: +Depending on your version of GitLab, the Chain of Custody report is either sent through email or available for download. + +#### Generate commit-specific Chain of Custody report + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267629) in GitLab 13.6. +> - Support for including all commits instead of only merge commits [added](https://gitlab.com/gitlab-org/gitlab/-/issues/393446) in GitLab 15.10. + +You can generate a commit-specific Chain of Custody report for a given commit SHA. This report provides only the +details for the provided commit SHA. + +To generate a commit-specific Chain of Custody report: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Security and Compliance > Compliance report**. +1. At the top of the compliance report, to the right of **List of all commits**, select the down arrow + (**{chevron-lg-down}**). +1. Enter the commit SHA, and then select **Export commit custody report**. + +Depending on your version of GitLab, the Chain of Custody report is either sent through email or available for download. + +Alternatively, use a direct link: `https://gitlab.com/groups//-/security/merge_commit_reports.csv?commit_sha={optional_commit_sha}`, +passing in an optional value to the `commit_sha` query parameter. + +## Compliance frameworks report + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/387910) in GitLab 15.10. -- Available for download. -- Sent through email. Requires GitLab 15.5 and later. +With compliance frameworks report, you can see the compliance frameworks that are applied to projects in a group. Each row of the report shows: -### Commit-specific Chain of Custody report +- Project name. +- Project path. +- Compliance framework label if the project has one assigned. -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267629) in GitLab 13.6. +The default framework for the group has a **default** badge. -Authenticated group owners can generate a commit-specific Chain of Custody report for a given commit SHA, either: +### View the compliance frameworks report for a group -- Using the GitLab UI: +Prerequisites: + +- You must be an administrator or have the Owner role for the group. - 1. On the top bar, select **Main menu > Groups** and find your group. - 1. On the left sidebar, select **Security & Compliance > Compliance report**. - 1. At the top of the compliance report, to the right of **List of all merge commits**, select the down arrow (**{chevron-lg-down}**). - 1. Enter the merge commit SHA, and then select **Export commit custody report**. - SHA and then select **Export commit custody report**. +To view the compliance frameworks report: -The Chain of Custody report is either: +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Security & Compliance > Compliance report**. +1. On the page, select the **Frameworks** tab. -- Available for download. -- Sent through email. Requires GitLab 15.5 and later. +### Apply a compliance framework to projects in a group -- Using a direct link: `https://gitlab.com/groups//-/security/merge_commit_reports.csv?commit_sha={optional_commit_sha}`, passing in an optional value to the - `commit_sha` query parameter. +> - Adding compliance frameworks using bulk actions [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/383209) in GitLab 15.11. +> - Adding compliance frameworks without using bulk actions [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/394795) in GitLab 16.0. -NOTE: -The Chain of Custody report download is a CSV file, with a maximum size of 15 MB. -The remaining records are truncated when this limit is reached. +You can apply a compliance framework to projects in a group. + +Prerequisites: + +- You must have the Owner role for the group. + +To apply a compliance framework to one project in a group: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Security and Compliance > Compliance report**. +1. On the page, select the **Frameworks** tab. +1. Next to the project you want to add the compliance framework to, select **{plus}** **Add framework**. +1. Select an existing compliance framework or create a new one. + +To apply a compliance framework to multiple projects in a group: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Security and Compliance > Compliance report**. +1. On the page, select the **Frameworks** tab. +1. Select multiple projects. +1. From the **Choose one bulk action** dropdown list, select **Apply framework to selected projects**. +1. Select framework to apply. +1. Select **Apply**. + +### Remove a compliance framework from projects in a group + +> - Removing compliance frameworks using bulk actions [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/383209) in GitLab 15.11. +> - Removing compliance frameworks without using bulk actions [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/394795) in GitLab 16.0. + +You can remove a compliance framework from projects in a group. + +Prerequisites: + +- You must have the Owner role for the group. + +To remove a compliance framework from one project in a group: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Security and Compliance > Compliance report**. +1. On the page, select the **Frameworks** tab. +1. Next to the compliance framework to remove from the project, select **{close}** on the framework label. + +To remove a compliance framework from multiple projects in a group: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Security and Compliance > Compliance report**. +1. On the page, select the **Frameworks** tab. +1. Select multiple projects. +1. From the **Choose one bulk action** dropdown list, select **Remove framework from selected projects**. +1. Select **Remove**. + +### Export a report of compliance frameworks on projects in a group + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/387912) in GitLab 16.0. + +Export a report of compliance frameworks that are applied to projects in a group. Reports: + +- Do not use filters on the framework report. +- Are truncated at 15 MB so the email attachment too large. + +Prerequisites: + +- You must be an administrator or have the Owner role for the group. + +To export a report of compliance frameworks on projects in a group: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Security and Compliance > Compliance report**. +1. On the page, select the **Frameworks** tab. +1. On the Frameworks tab, select the **Export as CSV** action in the top right corner + +A report is compiled and delivered to your email inbox as an attachment. + +#### Filter the compliance frameworks report + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/387911) in GitLab 15.11. + +To filter the list of compliance frameworks: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Security & Compliance > Compliance report**. +1. On the page, select the **Frameworks** tab. +1. In the search field: + 1. Select the attribute you want to filter by. + 1. Select an operator. + 1. Select from the list of options or enter text for the search. +1. Select **Search** (**{search}**). + +Repeat this process to filter by multiple attributes. diff --git a/doc/user/compliance/img/license-check_v13_4.png b/doc/user/compliance/img/license-check_v13_4.png deleted file mode 100644 index bc80f938395..00000000000 Binary files a/doc/user/compliance/img/license-check_v13_4.png and /dev/null differ diff --git a/doc/user/compliance/img/license_approval_policy_v15_9.png b/doc/user/compliance/img/license_approval_policy_v15_9.png index 43b1f89a07c..208643b47ae 100644 Binary files a/doc/user/compliance/img/license_approval_policy_v15_9.png and b/doc/user/compliance/img/license_approval_policy_v15_9.png differ diff --git a/doc/user/compliance/img/policies_maintainer_add_v14_3.png b/doc/user/compliance/img/policies_maintainer_add_v14_3.png deleted file mode 100644 index 7a27899f8c9..00000000000 Binary files a/doc/user/compliance/img/policies_maintainer_add_v14_3.png and /dev/null differ diff --git a/doc/user/compliance/img/policies_maintainer_edit_v14_3.png b/doc/user/compliance/img/policies_maintainer_edit_v14_3.png deleted file mode 100644 index 256c66bf7d8..00000000000 Binary files a/doc/user/compliance/img/policies_maintainer_edit_v14_3.png and /dev/null differ diff --git a/doc/user/compliance/img/policies_v13_0.png b/doc/user/compliance/img/policies_v13_0.png deleted file mode 100644 index 4918a0e6b62..00000000000 Binary files a/doc/user/compliance/img/policies_v13_0.png and /dev/null differ diff --git a/doc/user/compliance/index.md b/doc/user/compliance/index.md index 7981ab44bf9..ad36684d987 100644 --- a/doc/user/compliance/index.md +++ b/doc/user/compliance/index.md @@ -7,6 +7,12 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Compliance **(ULTIMATE)** -The compliance tools provided by GitLab help you keep an eye on various aspects of your project. For more information -on GitLab compliance features for projects, groups, and instances, see +The compliance tools provided by GitLab help you keep an eye on various aspects of your project, including: + +- [Compliance report](compliance_report/index.md). +- [License approval policies](license_approval_policies.md). +- [License list](license_list.md). +- [License scanning of CycloneDX files](license_scanning_of_cyclonedx_files/index.md). + +For more information on other GitLab compliance features for projects, groups, and instances, see [Compliance features](../../administration/compliance.md). diff --git a/doc/user/compliance/license_approval_policies.md b/doc/user/compliance/license_approval_policies.md index 32c90a1d317..860c2008021 100644 --- a/doc/user/compliance/license_approval_policies.md +++ b/doc/user/compliance/license_approval_policies.md @@ -5,11 +5,24 @@ group: Security Policies info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# License Approval Policies **(ULTIMATE)** +# License approval policies **(ULTIMATE)** -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/8092) in GitLab 15.9 [with a flag](../../administration/feature_flags.md) named `license_scanning_policies`. Disabled by default. +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/8092) in GitLab 15.9 [with a flag](../../administration/feature_flags.md) named `license_scanning_policies`. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/397644) in GitLab 15.11. Feature flag `license_scanning_policies` removed. -License Approval Policies allow you to specify multiple types of criteria that define when approval is required before a merge request can be merged in. +License approval policies allow you to specify multiple types of criteria that define when approval is required before a merge request can be merged in. + +NOTE: +License approval policies are applicable only to [protected](../project/protected_branches.md) target branches. + +The following video provides an overview of these policies. + +
    + See the video: Overview of GitLab License Approval Policies. +
    +
    + +
    ## Create a new license approval policy diff --git a/doc/user/compliance/license_check_rules.md b/doc/user/compliance/license_check_rules.md index 968cf49ffdf..3007d04a8c1 100644 --- a/doc/user/compliance/license_check_rules.md +++ b/doc/user/compliance/license_check_rules.md @@ -1,84 +1,15 @@ --- -type: reference, howto -stage: Govern -group: Security Policies -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: 'license_approval_policies.md' +remove_date: '2023-07-25' --- -# License Check Policies (DEPRECATED) **(ULTIMATE)** +# License Check Policies (removed) **(ULTIMATE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/390417) in GitLab 15.9. +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/390417) in GitLab 15.9 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/397067) in 16.0. +Use [License Approval Policies](license_approval_policies.md) instead. -WARNING: -This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/390417) in GitLab 15.9. Users should migrate over to use [License Approval Policies](license_approval_policies.md) prior to GitLab 16.0. - -License check policies allow you to specify licenses that are `allowed` or `denied` in a project. If a `denied` -license is newly committed it blocks the merge request and instructs the developer to remove it. -Note, the merge request is not able to be merged until the `denied` license is removed. -You may add a [`License-Check` approval rule](#enabling-license-approvals-within-a-project), -which enables a designated approver that can approve and then merge a merge request with `denied` license. - -These policies can be configured by using the [Managed Licenses API](../../api/managed_licenses.md). - -![Merge request with denied licenses](img/denied_licenses_v15_3.png) - -The **Policies** tab in the project's license compliance section displays your project's license -policies. Project maintainers can specify policies in this section. - -![Edit Policy](img/policies_maintainer_edit_v14_3.png) - -![Add Policy](img/policies_maintainer_add_v14_3.png) - -Developers of the project can view the policies configured in a project. - -![View Policies](img/policies_v13_0.png) - -## Enabling License Approvals within a project - -Prerequisites: - -- Maintainer or Owner role. - -`License-Check` is a [merge request approval](../project/merge_requests/approvals/index.md) rule -you can enable to allow an individual or group to approve a merge request that contains a `denied` -license. - -You can enable `License-Check` one of two ways: - -1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Settings > General**. -1. Expand **Merge request approvals**. -1. Select **Enable** or **Edit**. -1. Add or change the **Rule name** to `License-Check` (case sensitive). - -![License Check Approver Rule](img/license-check_v13_4.png) - -- Create an approval group in the [project policies section for License Compliance](license_check_rules.md#license-check-policies-deprecated). - You must set this approval group's number of approvals required to greater than zero. After you - enable this group in your project, the approval rule is enabled for all merge requests. - -Any code changes cause the approvals required to reset. - -An approval is required when a license report: - -- Contains a dependency that includes a software license that is `denied`. -- Is not generated during pipeline execution. - -An approval is optional when a license report: - -- Contains no software license violations. -- Contains only new licenses that are `allowed` or unknown. - -## Troubleshooting - -### The License Compliance widget is stuck in a loading state - -A loading spinner is displayed in the following scenarios: - -- While the pipeline is in progress. -- If the pipeline is complete, but still parsing the results in the background. -- If the license scanning job is complete, but the pipeline is still running. - -The License Compliance widget polls every few seconds for updated results. When the pipeline is complete, the first poll after pipeline completion triggers the parsing of the results. This can take a few seconds depending on the size of the generated report. - -The final state is when a successful pipeline run has been completed, parsed, and the licenses displayed in the widget. + + + + diff --git a/doc/user/compliance/license_compliance/index.md b/doc/user/compliance/license_compliance/index.md index 43e88e89c18..b7a68317fba 100644 --- a/doc/user/compliance/license_compliance/index.md +++ b/doc/user/compliance/license_compliance/index.md @@ -5,13 +5,13 @@ group: Composition Analysis info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# License compliance (DEPRECATED) **(ULTIMATE)** +# License Compliance (deprecated) **(ULTIMATE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5483) in GitLab 11.0. > - [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387561) in GitLab 15.9. WARNING: -This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387561) in GitLab 15.9. Users should migrate over to use the [new method of license scanning](../license_scanning_of_cyclonedx_files/index.md) prior to GitLab 16.0. +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387561) in GitLab 15.9. You should instead migrate to use [License approval policies](../license_approval_policies.md) and the [new method of license scanning](../license_scanning_of_cyclonedx_files/index.md) prior to GitLab 16.1. If you're using [GitLab CI/CD](../../../ci/index.md), you can use License Compliance to search your project's dependencies for their licenses. You can then decide whether to allow or deny the use of @@ -25,8 +25,8 @@ For the job to activate, License Finder needs to find a compatible package defin GitLab checks the License Compliance report, compares the licenses between the source and target branches, and shows the information right on the merge request. Denied licenses are indicated by a `x` red icon next to them as well as new licenses that -need a decision from you. In addition, you can [manually allow or deny](../license_check_rules.md) licenses in your -project's license compliance policy section. If a denied license is detected in a new commit, +need a decision from you. In addition, you can [manually allow or deny](../license_approval_policies.md) licenses in your +project's security policies section. If a denied license is detected in a new commit, GitLab blocks any merge requests containing that commit and instructs the developer to remove the license. @@ -49,11 +49,43 @@ that you can later download and analyze. WARNING: License Compliance Scanning does not support run-time installation of compilers and interpreters. +## Enable License Compliance + +To enable License Compliance in your project's pipeline, either: + +- Enable [Auto License Compliance](../../../topics/autodevops/stages.md#auto-license-compliance) + (provided by [Auto DevOps](../../../topics/autodevops/index.md)). +- Include the [`License-Scanning.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml) in your `.gitlab-ci.yml` file. + +License Compliance is not supported when GitLab is run with FIPS mode enabled. + +### Include the License Scanning template + +Prerequisites: + +- [GitLab Runner](../../../ci/runners/index.md) available, with the + [`docker` executor](https://docs.gitlab.com/runner/executors/docker.html). If you're using the + shared runners on GitLab.com, this is enabled by default. +- License Scanning runs in the `test` stage, which is available by default. If you redefine the stages in the + `.gitlab-ci.yml` file, the `test` stage is required. +- [FIPS mode](../../../development/fips_compliance.md#enable-fips-mode) must be disabled. + +To [include](../../../ci/yaml/index.md#includetemplate) the +[`License-Scanning.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml), add it to your `.gitlab-ci.yml` file: + +```yaml +include: + - template: Security/License-Scanning.gitlab-ci.yml +``` + +The included template creates a `license_scanning` job in your CI/CD pipeline and scans your +dependencies to find their licenses. + ## License expressions GitLab has limited support for [composite licenses](https://spdx.github.io/spdx-spec/v2-draft/SPDX-license-expressions/). License compliance can read multiple licenses, but always considers them combined using the `AND` operator. For example, -if a dependency has two licenses, and one of them is allowed and the other is denied by the project [policy](../license_check_rules.md), +if a dependency has two licenses, and one of them is allowed and the other is denied by the project [license approval policy](../license_approval_policies.md), GitLab evaluates the composite license as _denied_, as this is the safer option. The ability to support other license expression operators (like `OR`, `WITH`) is tracked in [this epic](https://gitlab.com/groups/gitlab-org/-/epics/6571). @@ -89,39 +121,7 @@ The reported licenses might be incomplete or inaccurate. | Rust | [Cargo](https://crates.io/) | | PHP | [Composer](https://getcomposer.org/) | -## Enable License Compliance - -To enable License Compliance in your project's pipeline, either: - -- Enable [Auto License Compliance](../../../topics/autodevops/stages.md#auto-license-compliance) - (provided by [Auto DevOps](../../../topics/autodevops/index.md)). -- Include the [`License-Scanning.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml) in your `.gitlab-ci.yml` file. - -License Compliance is not supported when GitLab is run with FIPS mode enabled. - -### Include the License Scanning template - -Prerequisites: - -- [GitLab Runner](../../../ci/runners/index.md) available, with the - [`docker` executor](https://docs.gitlab.com/runner/executors/docker.html). If you're using the - shared runners on GitLab.com, this is enabled by default. -- License Scanning runs in the `test` stage, which is available by default. If you redefine the stages in the - `.gitlab-ci.yml` file, the `test` stage is required. -- [FIPS mode](../../../development/fips_compliance.md#enable-fips-mode) must be disabled. - -To [include](../../../ci/yaml/index.md#includetemplate) the -[`License-Scanning.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml), add it to your `.gitlab-ci.yml` file: - -```yaml -include: - - template: Security/License-Scanning.gitlab-ci.yml -``` - -The included template creates a `license_scanning` job in your CI/CD pipeline and scans your -dependencies to find their licenses. - -### Available CI/CD variables +## Available CI/CD variables License Compliance can be configured using CI/CD variables. @@ -141,7 +141,7 @@ License Compliance can be configured using CI/CD variables. | `SECURE_ANALYZERS_PREFIX` | no | Set the Docker registry base address to download the analyzer from. | | `SETUP_CMD` | no | Custom setup for the dependency installation (experimental). | -### Installing custom dependencies +## Installing custom dependencies > Introduced in GitLab 11.4. @@ -169,7 +169,7 @@ variables: In this example, `my-custom-install-script.sh` is a shell script at the root directory of your project. -### Working with Monorepos +## Working with Monorepos Depending on your language, you may need to specify the path to the individual projects of a monorepo using the `LICENSE_FINDER_CLI_OPTS` variable. Passing in @@ -184,7 +184,7 @@ variables: LICENSE_FINDER_CLI_OPTS: "--aggregate_paths=relative-path/to/sub-project/one relative-path/to/sub-project/two" ``` -### Overriding the template +## Overriding the template WARNING: Beginning in GitLab 13.0, the use of [`only` and `except`](../../../ci/yaml/index.md#only--except) @@ -203,7 +203,7 @@ license_scanning: CI_DEBUG_TRACE: "true" ``` -### Configuring Maven projects +## Configuring Maven projects The License Compliance tool provides a `MAVEN_CLI_OPTS` CI/CD variable which can hold the command line arguments to pass to the `mvn install` command which is executed under the hood. @@ -227,7 +227,7 @@ to explicitly add `-DskipTests` to your options. If you still need to run tests during `mvn install`, add `-DskipTests=false` to `MAVEN_CLI_OPTS`. -#### Using private Maven repositories +### Using private Maven repositories If you have a private Maven repository which requires login credentials, you can use the `MAVEN_CLI_OPTS` CI/CD variable. @@ -250,13 +250,13 @@ Alternatively, you can use a Java key store to verify the TLS connection. For in generate a key store file, see the [Maven Guide to Remote repository access through authenticated HTTPS](https://maven.apache.org/guides/mini/guide-repository-ssl.html). -### Selecting the version of Java +## Selecting the version of Java License Compliance uses Java 8 by default. You can specify a different Java version using `LM_JAVA_VERSION`. `LM_JAVA_VERSION` only accepts versions: 8, 11, 14, 15. -### Selecting the version of Python +## Selecting the version of Python > - [Introduced](https://gitlab.com/gitlab-org/security-products/license-management/-/merge_requests/36) in GitLab 12.0. > - In [GitLab 12.2](https://gitlab.com/gitlab-org/gitlab/-/issues/12032), Python 3.5 became the default. @@ -277,22 +277,22 @@ license_scanning: `LM_PYTHON_VERSION` or `ASDF_PYTHON_VERSION` can be used to specify the desired version of Python. When both variables are specified `LM_PYTHON_VERSION` takes precedence. -### Custom root certificates for Python +## Custom root certificates for Python You can supply a custom root certificate to complete TLS verification by using the `ADDITIONAL_CA_CERT_BUNDLE` [CI/CD variable](#available-cicd-variables). -#### Using private Python repositories +### Using private Python repositories If you have a private Python repository you can use the `PIP_INDEX_URL` [CI/CD variable](#available-cicd-variables) to specify its location. -### Configuring npm projects +## Configuring npm projects You can configure npm projects by using an [`.npmrc`](https://docs.npmjs.com/configuring-npm/npmrc.html/) file. -#### Using private npm registries +### Using private npm registries If you have a private npm registry you can use the [`registry`](https://docs.npmjs.com/using-npm/config/#registry) @@ -304,7 +304,7 @@ For example: registry = https://npm.example.com ``` -#### Custom root certificates for npm +### Custom root certificates for npm You can supply a custom root certificate to complete TLS verification by using the `ADDITIONAL_CA_CERT_BUNDLE` [CI/CD variable](#available-cicd-variables). @@ -318,12 +318,12 @@ For example: strict-ssl = false ``` -### Configuring Yarn projects +## Configuring Yarn projects You can configure Yarn projects by using a [`.yarnrc.yml`](https://yarnpkg.com/configuration/yarnrc/) file. -#### Using private Yarn registries +### Using private Yarn registries If you have a private Yarn registry you can use the [`npmRegistryServer`](https://yarnpkg.com/configuration/yarnrc/#npmRegistryServer) @@ -335,17 +335,17 @@ For example: npmRegistryServer: "https://npm.example.com" ``` -#### Custom root certificates for Yarn +### Custom root certificates for Yarn You can supply a custom root certificate to complete TLS verification by using the `ADDITIONAL_CA_CERT_BUNDLE` [CI/CD variable](#available-cicd-variables). -### Configuring Bower projects +## Configuring Bower projects You can configure Bower projects by using a [`.bowerrc`](https://bower.io/docs/config/#bowerrc-specification) file. -#### Using private Bower registries +### Using private Bower registries If you have a private Bower registry you can use the [`registry`](https://bower.io/docs/config/#bowerrc-specification) @@ -359,16 +359,16 @@ For example: } ``` -#### Custom root certificates for Bower +### Custom root certificates for Bower You can supply a custom root certificate to complete TLS verification by using the `ADDITIONAL_CA_CERT_BUNDLE` [CI/CD variable](#available-cicd-variables), or by specifying a `ca` setting in a [`.bowerrc`](https://bower.io/docs/config/#bowerrc-specification) file. -### Configuring Bundler projects +## Configuring Bundler projects -#### Using private Bundler registries +### Using private Bundler registries If you have a private Bundler registry you can use the [`source`](https://bundler.io/man/gemfile.5.html#GLOBAL-SOURCES) @@ -380,7 +380,7 @@ For example: source "https://gems.example.com" ``` -#### Custom root certificates for Bundler +### Custom root certificates for Bundler You can supply a custom root certificate to complete TLS verification by using the `ADDITIONAL_CA_CERT_BUNDLE` [CI/CD variable](#available-cicd-variables), or by @@ -388,9 +388,9 @@ specifying a [`BUNDLE_SSL_CA_CERT`](https://bundler.io/v2.0/man/bundle-config.1. [variable](../../../ci/variables/index.md#define-a-cicd-variable-in-the-gitlab-ciyml-file) in the job definition. -### Configuring Cargo projects +## Configuring Cargo projects -#### Using private Cargo registries +### Using private Cargo registries If you have a private Cargo registry you can use the [`registries`](https://doc.rust-lang.org/cargo/reference/registries.html) @@ -403,7 +403,7 @@ For example: my-registry = { index = "https://my-intranet:8080/git/index" } ``` -#### Custom root certificates for Cargo +### Custom root certificates for Cargo To supply a custom root certificate to complete TLS verification, do one of the following: @@ -412,9 +412,9 @@ To supply a custom root certificate to complete TLS verification, do one of the [variable](../../../ci/variables/index.md#define-a-cicd-variable-in-the-gitlab-ciyml-file) in the job definition. -### Configuring Composer projects +## Configuring Composer projects -#### Using private Composer registries +### Using private Composer registries If you have a private Composer registry you can use the [`repositories`](https://getcomposer.org/doc/05-repositories.md) @@ -437,7 +437,7 @@ For example: } ``` -#### Custom root certificates for Composer +### Custom root certificates for Composer You can supply a custom root certificate to complete TLS verification by using the `ADDITIONAL_CA_CERT_BUNDLE` [CI/CD variable](#available-cicd-variables), or by @@ -445,7 +445,7 @@ specifying a [`COMPOSER_CAFILE`](https://getcomposer.org/doc/03-cli.md#composer- [variable](../../../ci/variables/index.md#define-a-cicd-variable-in-the-gitlab-ciyml-file) in the job definition. -### Configuring Conan projects +## Configuring Conan projects You can configure [Conan](https://conan.io/) projects by adding a `.conan` directory to your project root. The project root serves as the [`CONAN_USER_HOME`](https://docs.conan.io/en/latest/reference/env_vars.html#conan-user-home). @@ -474,7 +474,7 @@ NOTE: `license_scanning` image ships with [Mono](https://www.mono-project.com/) and [MSBuild](https://github.com/mono/msbuild#microsoftbuild-msbuild). Additional setup may be required to build packages for this project configuration. -#### Using private Conan registries +### Using private Conan registries By default, [Conan](https://conan.io/) uses the `conan-center` remote. For example: @@ -508,7 +508,7 @@ example: If credentials are required to authenticate then you can configure a [protected CI/CD variable](../../../ci/variables/index.md#protect-a-cicd-variable) following the naming convention described in the [`CONAN_LOGIN_USERNAME` documentation](https://docs.conan.io/en/latest/reference/env_vars.html#conan-login-username-conan-login-username-remote-name). -#### Custom root certificates for Conan +### Custom root certificates for Conan You can provide custom certificates by adding a `.conan/cacert.pem` file to the project root and setting [`CA_CERT_PATH`](https://docs.conan.io/en/latest/reference/env_vars.html#conan-cacert-path) @@ -518,7 +518,7 @@ If you specify the `ADDITIONAL_CA_CERT_BUNDLE` [CI/CD variable](#available-cicd- variable's X.509 certificates are installed in the Docker image's default trust store and Conan is configured to use this as the default `CA_CERT_PATH`. -### Configuring Go projects +## Configuring Go projects To configure [Go modules](https://github.com/golang/go/wiki/Modules) based projects, specify [CI/CD variables](https://pkg.go.dev/cmd/go#hdr-Environment_variables) @@ -528,14 +528,14 @@ If a project has [vendored](https://pkg.go.dev/cmd/go#hdr-Vendor_Directories) it then the combination of the `vendor` directory and `mod.sum` file are used to detect the software licenses associated with the Go module dependencies. -#### Using private Go registries +### Using private Go registries You can use the [`GOPRIVATE`](https://pkg.go.dev/cmd/go#hdr-Environment_variables) and [`GOPROXY`](https://pkg.go.dev/cmd/go#hdr-Environment_variables) environment variables to control where modules are sourced from. Alternatively, you can use [`go mod vendor`](https://go.dev/ref/mod#tmp_28) to vendor a project's modules. -#### Custom root certificates for Go +### Custom root certificates for Go You can specify the [`-insecure`](https://pkg.go.dev/cmd/go/internal/get) flag by exporting the [`GOFLAGS`](https://pkg.go.dev/cmd/go#hdr-Environment_variables) @@ -550,7 +550,7 @@ license_scanning: GOFLAGS: '-insecure' ``` -#### Using private NuGet registries +### Using private NuGet registries If you have a private NuGet registry you can add it as a source by adding it to the [`packageSources`](https://learn.microsoft.com/en-us/nuget/reference/nuget-config-file#package-source-sections) @@ -568,7 +568,7 @@ For example: ``` -#### Custom root certificates for NuGet +### Custom root certificates for NuGet You can supply a custom root certificate to complete TLS verification by using the `ADDITIONAL_CA_CERT_BUNDLE` [CI/CD variable](#available-cicd-variables). @@ -654,7 +654,7 @@ registry.gitlab.com/security-products/license-finder:latest The process for importing Docker images into a local offline Docker registry depends on **your network security policy**. Consult your IT staff to find an accepted and approved -process by which external resources can be imported or temporarily accessed. Note that these scanners are [updated periodically](../../application_security/index.md#vulnerability-scanner-maintenance) +process by which external resources can be imported or temporarily accessed. These scanners are [updated periodically](../../application_security/index.md#vulnerability-scanner-maintenance) with new definitions, so consider if you are able to make periodic updates yourself. For details on saving and transporting Docker images as a file, see the Docker documentation on @@ -693,8 +693,8 @@ Additional configuration may be needed for connecting to private registries for: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/212388) in GitLab 13.3. -Prior to GitLab 13.3, offline environments required an exact name match for [project policies](../license_check_rules.md). -In GitLab 13.3 and later, GitLab matches the name of [project policies](../license_check_rules.md) +Prior to GitLab 13.3, offline environments required an exact name match for [project policies](../license_approval_policies.md). +In GitLab 13.3 and later, GitLab matches the name of [project policies](../license_approval_policies.md) with identifiers from the [SPDX license list](https://spdx.org/licenses/). A local copy of the SPDX license list is distributed with the GitLab instance. If needed, the GitLab instance's administrator can manually update it with a [Rake task](../../../raketasks/spdx.md). diff --git a/doc/user/compliance/license_scanning_of_cyclonedx_files/index.md b/doc/user/compliance/license_scanning_of_cyclonedx_files/index.md index 483c15d648c..832f1007a91 100644 --- a/doc/user/compliance/license_scanning_of_cyclonedx_files/index.md +++ b/doc/user/compliance/license_scanning_of_cyclonedx_files/index.md @@ -7,25 +7,28 @@ info: To determine the technical writer assigned to the Stage/Group associated w # License scanning of CycloneDX files **(ULTIMATE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/384932) in GitLab 15.9 [with two flags](../../../administration/feature_flags.md) named `license_scanning_sbom_scanner` and `package_metadata_synchronization`. Both flags are disabled by default and both flags must be enabled for this feature to work. - -FLAG: -On self-managed GitLab, this feature is not available. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/384932) in GitLab 15.9 for GitLab SaaS [with two flags](../../../administration/feature_flags.md) named `license_scanning_sbom_scanner` and `package_metadata_synchronization`. Both flags are disabled by default and both flags must be enabled for this feature to work. +> - [Enabled](https://gitlab.com/gitlab-org/gitlab/-/issues/385173) in GitLab 15.10 for GitLab SaaS. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/385173) in GitLab 15.10 for self-managed GitLab [with two flags](../../../administration/feature_flags.md) named `license_scanning_sbom_scanner` and `package_metadata_synchronization`, both of which must be enabled for this feature to work. The flags are disabled by default due to the initial performance load when the license database is first imported. Work to improve performance is being tracked in [issue 397670](https://gitlab.com/gitlab-org/gitlab/-/issues/397670). +> - [Enabled](https://gitlab.com/gitlab-org/gitlab/-/issues/385173) in GitLab 15.11 for self-managed GitLab. To detect the licenses in use, License Compliance relies on running the [Dependency Scanning CI Jobs](../../application_security/dependency_scanning/index.md), and analyzing the [CycloneDX](https://cyclonedx.org/) Software Bill of Materials (SBOM) generated by those jobs. Other 3rd party scanners may also be used as long as they produce a CycloneDX file with a list of dependencies for [one of our supported languages](#supported-languages-and-package-managers). -This method of scanning is also capable of parsing and identifying over 500 different types of licenses -and can extract license information from packages that are dual-licensed or have multiple different licenses that apply. +This method of scanning is also capable of parsing and identifying over 500 different types of licenses, as defined in [the SPDX list](https://spdx.org/licenses/). +Licenses not in the SPDX list are reported as "Unknown". License information can also be extracted from packages that are dual-licensed, or have multiple different licenses that apply. + +## Enable license scanning -To enable license detection using Dependency Scanning in a project, -include the `Jobs/Dependency-Scanning.yml` template in its CI configuration, -but do not include the `Jobs/License-Scanning.yml` template. +Prerequisites: -## Requirements +- Enable [Synchronization with the GitLab License Database](../../admin_area/settings/security_and_compliance.md#choose-package-registry-metadata-to-sync) in Admin Area for the GitLab instance. +- Enable [Dependency Scanning](../../application_security/dependency_scanning/index.md#configuration) + and ensure that its prerequisites are met. -The license scanning requirements are the same as those for [Dependency Scanning](../../application_security/dependency_scanning/index.md#requirements). +From the `.gitlab-ci.yml` file, remove the deprecated line `Jobs/License-Scanning.gitlab-ci.yml`, if +it's present. ## Supported languages and package managers @@ -66,9 +69,12 @@ License scanning is supported for the following languages and package managers:
    • - + + + + @@ -104,11 +110,6 @@ License scanning is supported for the following languages and package managers: The supported files and versions are the ones supported by [Dependency Scanning](../../application_security/dependency_scanning/index.md#supported-languages-and-package-managers). -## Configuration - -To enable license scanning of CycloneDX files, -you must configure [Dependency Scanning](../../application_security/dependency_scanning/index.md#configuration). - ## License expressions GitLab has limited support for [composite licenses](https://spdx.github.io/spdx-spec/v2-draft/SPDX-license-expressions/). @@ -121,3 +122,23 @@ in [this epic](https://gitlab.com/groups/gitlab-org/-/epics/6571). ## Blocking merge requests based on detected licenses Users can require approval for merge requests based on the licenses that are detected by configuring a [license approval policy](../license_approval_policies.md). + +## Running in an offline environment + +For self-managed GitLab instances in an environment with limited, restricted, or intermittent access to external resources through the internet, some adjustments are required to successfully scan +CycloneDX reports for licenses. For more information, see the offline [quick start guide](../../../topics/offline/quick_start_guide.md#enabling-the-package-metadata-database). + +## Troubleshooting + +### A CycloneDX file is not being scanned and appears to provide no results + +Ensure that the CycloneDX file adheres to the [CycloneDX JSON specification](https://cyclonedx.org/docs/latest/json). This specification does [not permit duplicate entries](https://cyclonedx.org/docs/latest/json/#components). Projects that contain multiple SBOM files should either report each SBOM file up as individual CI report artifacts or they should ensure that duplicates are removed if the SBOMs are merged as part of the CI pipeline. + +You can validate CycloneDX SBOM files against the `CycloneDX JSON specification` as follows: + +```shell +$ docker run -it --rm -v "$PWD:/my-cyclonedx-sboms" -w /my-cyclonedx-sboms cyclonedx/cyclonedx-cli:latest cyclonedx validate --input-version v1_4 --input-file gl-sbom-all.cdx.json + +Validating JSON BOM... +BOM validated successfully. +``` diff --git a/doc/user/crm/index.md b/doc/user/crm/index.md index ebacda506b4..54e87118361 100644 --- a/doc/user/crm/index.md +++ b/doc/user/crm/index.md @@ -1,6 +1,6 @@ --- -stage: Plan -group: Certify +stage: Monitor +group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/discussions/img/index_notes_filters.png b/doc/user/discussions/img/index_notes_filters.png deleted file mode 100644 index 977a3770c05..00000000000 Binary files a/doc/user/discussions/img/index_notes_filters.png and /dev/null differ diff --git a/doc/user/discussions/index.md b/doc/user/discussions/index.md index 9c9b5301460..096b4dc2cc5 100644 --- a/doc/user/discussions/index.md +++ b/doc/user/discussions/index.md @@ -36,6 +36,8 @@ You can create comments in places like: - Issues - Merge requests - Snippets +- Tasks +- OKRs Each object can have as many as 5,000 comments. @@ -46,12 +48,12 @@ instance with `@username` or `@groupname`. All mentioned users are notified with Users can change this setting for themselves in the [notification settings](../profile/notifications.md). You can quickly see which comments involve you, because -mentions for yourself (the user currently signed in) are highlighted +mentions for yourself (the user who is signed in) are highlighted in a different color. Avoid mentioning `@all` in issues and merge requests. It sends an email notification -to all members of that project's parent group, not only the participants of the project, -and may be interpreted as spam. +to all members of that project's parent group, not only the participants of the project. +It might be interpreted as spam. Notifications and mentions can be disabled in [a group's settings](../group/manage.md#disable-email-notifications). @@ -89,8 +91,8 @@ The comment is not displayed on your project's **Repository > Commits** page. NOTE: When your comment contains a reference to a commit included in the merge request, -it's automatically converted to a link in the context of the current merge request. -For example, `28719b171a056960dfdc0012b625d0b47b123196` becomes +it's converted to a link in the context of the merge request. +For example, `28719b171a056960dfdc0012b625d0b47b123196` becomes `28719b17` that links to `https://gitlab.example.com/example-group/example-project/-/merge_requests/12345/diffs?commit_id=28719b171a056960dfdc0012b625d0b47b123196`. ## Add a comment to a commit @@ -203,41 +205,35 @@ You can also mark an [issue as confidential](../project/issues/confidential_issu ## Show only comments -For issues and merge requests with many comments, you can filter the page to show comments only. +In discussions with many comments, filter the discussion to show only comments or history of +changes (system notes). System notes include changes to the description, mentions in other GitLab +objects, or changes to labels, assignees, and the milestone. +GitLab saves your preference, and applies it to every issue, merge request, or epic you view. 1. Open the **Overview** tab in a merge request, issue, or epic. -1. On the right side of the page, select from the filter: +1. On the right side of the page, from the **Sort or filter** dropdown list, select a filter: - **Show all activity**: Display all user comments and system notes. - (issue updates, mentions from other issues, changes to the description, and so on). - **Show comments only**: Display only user comments. - **Show history only**: Display only activity notes. -![Notes filters dropdown list options](img/index_notes_filters.png) +## Change activity sort order -GitLab saves your preference, so it persists when you visit the same page again -from any device you're logged into. +Reverse the default order and interact with the activity feed sorted by most recent items +at the top. GitLab saves your preference in local storage and applies it to every issue, +merge request, or epic you view. -## View description change history **(PREMIUM)** +To change the activity sort order: -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10103) in GitLab 12.6. +1. Open the **Overview** tab in a merge request, issue, or epic. +1. On the right side of the page, from the **Sort or filter** dropdown list, select the sort order + **Newest first** or **Oldest first** (default). + +## View description change history **(PREMIUM)** You can see changes to the description listed in the history. To compare the changes, select **Compare with previous version**. -## Change activity sort order - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14588) in GitLab 12.10. - -You can reverse the default order and interact with the activity feed sorted by most recent items -at the top. Your preference is saved in local storage and automatically applies to every issue, -merge request, or epic you view. - -To change the activity sort order: - -1. Select the **Oldest first** (or **Newest first**) dropdown list. -1. Select either oldest or newest items to be shown first. - ## Assign an issue to the commenting user > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/191455) in GitLab 13.1. @@ -260,7 +256,7 @@ Prerequisites: To create a thread by replying to a comment: -1. In the upper right of the comment, select **Reply to comment** (**{comment}**). +1. In the upper-right corner of the comment, select **Reply to comment** (**{comment}**). ![Reply to comment button](img/reply_to_comment_button.png) @@ -283,7 +279,7 @@ Prerequisites: To create a thread: 1. Enter a comment. -1. Below the comment, to the right of the **Comment** button, select the down arrow (**{chevron-down}**). +1. Below the comment, to the right of **Comment**, select the down arrow (**{chevron-down}**). 1. From the list, select **Start thread**. 1. Select **Start thread** again. @@ -308,7 +304,7 @@ To resolve a thread: 1. Go to the thread. 1. Do one of the following: - - In the upper right of the original comment, select **Resolve thread** (**{check-circle}**). + - In the upper-right corner of the original comment, select **Resolve thread** (**{check-circle}**). - Below the last reply, in the **Reply** field, select **Resolve thread**. - Below the last reply, in the **Reply** field, enter text, select the **Resolve thread** checkbox, and select **Add comment now**. @@ -320,7 +316,7 @@ At the top of the page, the number of unresolved threads is updated: If you have multiple unresolved threads in a merge request, you can create an issue to resolve them separately. In the merge request, at the top of the page, -select the ellipsis icon button (**{ellipsis_v}**) in the threads control and then select **Create issue to resolve all threads**: +select the ellipsis icon button (**{ellipsis_v}**) in the threads control and then select **Resolve all with new issue**: ![Open new issue for all unresolved threads](img/create_new_issue_v15_4.png) diff --git a/doc/user/enterprise_user/index.md b/doc/user/enterprise_user/index.md index 3daee460956..ffeaaa8c591 100644 --- a/doc/user/enterprise_user/index.md +++ b/doc/user/enterprise_user/index.md @@ -22,7 +22,7 @@ A user account is considered an enterprise account when: - [SCIM](../group/saml_sso/scim_setup.md) creates the user account on behalf of the group. -A user can also [manually connect an identity provider (IdP) to a GitLab account whose email address matches the subscribing organization's domain](../group/saml_sso/index.md#linking-saml-to-your-existing-gitlabcom-account). +A user can also [manually connect an identity provider (IdP) to a GitLab account whose email address matches the subscribing organization's domain](../group/saml_sso/index.md#link-saml-to-your-existing-gitlabcom-account). By selecting **Authorize** when connecting these two accounts, the user account with the matching email address is classified as an enterprise user. However, this user account does not have an **Enterprise** badge in GitLab. @@ -31,6 +31,86 @@ Although a user can be a member of more than one group, each user account can be provisioned by only one group. As a result, a user is considered an enterprise user under one top-level group only. +## Verified domains for groups + +The following automated processes use [verified domains](../project/pages/custom_domains_ssl_tls_certification/index.md) to run: + +- [Bypass email confirmation for provisioned users](#bypass-email-confirmation-for-provisioned-users). + +### Set up a verified domain + +Prerequisites: + +- A custom domain name `example.com` or subdomain `subdomain.example.com`. +- Access to your domain's server control panel to set up a DNS `TXT` record to verify your domain's ownership. + +Setting up a verified domain is similar to [setting up a custom domain on GitLab Pages](../project/pages/custom_domains_ssl_tls_certification/index.md). However, you must: + +- Link the domain to a project. For more information on group-level domain verification, see [issue 5299](https://gitlab.com/groups/gitlab-org/-/epics/5299). +- Configure the DNS `TXT` record to verify the domain's ownership. + +Steps: + +#### 1. Add a custom domain for the matching email domain + +The custom domain must match the email domain exactly. For example, if your email is `username@example.com`, verify the `example.com` domain. + +1. On the top bar, select **Main menu > Groups** and find your top group. +1. On the left sidebar, select **Settings > Domain Verification**. +1. In the upper-right corner, select **Add Domain**. +1. In **Domain**, enter the domain name. +1. In **Project**, link to a project. +1. Optional. In **Certificate**, switch the **Manually enter certificate information** toggle to add an SSL/TLS + certificate. You can also add the certificate and key later. +1. Select **Add Domain**. + +#### 2. Get a verification code + +After you create a new domain, the verification code prompts you. Copy the values from GitLab +and paste them in your domain's control panel as a `TXT` record. + +![Get the verification code](../img/get_domain_verification_code_v16_0.png) + +#### 3. Verify the domain's ownership + +After you have added all the DNS records: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Settings > Domain Verification**. +1. On the domain table row, Select **Retry verification** (**{retry}**). + +![Verify your domain](../img/retry_domain_verification_v16_0.png) + +WARNING: +For GitLab instances with domain verification enabled, +if the domain cannot be verified for 7 days, that domain is removed +from the GitLab project. + +> **Notes:** +> +> - Domain verification is **required for GitLab.com users**; + for GitLab self-managed instances, your GitLab administrator has the option + to [disabled custom domain verification](../../administration/pages/index.md#custom-domain-verification). +> - [DNS propagation may take some time (up to 24 hours)](https://www.inmotionhosting.com/support/domain-names/dns-nameserver-changes/complete-guide-to-dns-records/), + although it's usually a matter of minutes to complete. Until it does, verification + fails, and attempts to visit your domain result in a 404. +> - Once your domain has been verified, leave the verification record + in place. Your domain is periodically reverified, and may be + disabled if the record is removed. + +### View domains in group + +To view all configured domains in your group: + +1. On the top bar, select **Main menu > Groups** and find your top-level group. +1. On the left sidebar, select **Settings > Domain Verification**. + +You then see: + +- A list of added domains. +- The domains' status of **Verified** or **Unverified**. +- The project where the domain has been configured. + ## Manage enterprise users in a namespace A top-level Owner of a namespace on a paid plan can retrieve information about and diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index fb32c64f06c..fae45e4b2d3 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -65,7 +65,7 @@ The IP addresses for `mg.gitlab.com` are subject to change at any time. On GitLab.com, there's a mailbox configured for Service Desk with the email address: `contact-project+%{key}@incoming.gitlab.com`. To use this mailbox, configure the -[custom suffix](../project/service_desk.md#configuring-a-custom-email-address-suffix) in project +[custom suffix](../project/service_desk.md#configure-a-custom-email-address-suffix) in project settings. ## Backups @@ -89,15 +89,21 @@ Similarly, you can clone a project's wiki to back it up. All files [uploaded after August 22, 2020](../project/wiki/index.md#create-a-new-wiki-page) are included when cloning. +## Delayed group deletion **(PREMIUM SAAS)** + +After May 08, 2023, all groups have delayed deletion enabled by default. + +Groups are permanently deleted after a seven-day delay. + +If you are on the Free tier, your groups are immediately deleted, and you will not be able to restore them. + ## Delayed project deletion **(PREMIUM SAAS)** -Top-level groups created after August 12, 2021 have delayed project deletion enabled by default. -Projects are permanently deleted after a seven-day delay. +After May 08, 2023, all groups have delayed project deletion enabled by default. -If you are on: +Projects are permanently deleted after a seven-day delay. -- Premium tier and above, you can disable this by changing the [group setting](../group/manage.md#enable-delayed-project-deletion). -- Free tier, you cannot disable this setting or restore projects. +If you are on the Free tier, your projects are immediately deleted, and you will not be able to restore them. ## Inactive project deletion @@ -125,20 +131,22 @@ Host gitlab.com ## GitLab Pages -Below are the settings for [GitLab Pages](https://about.gitlab.com/stages-devops-lifecycle/pages/). +Some settings for [GitLab Pages](../project/pages/index.md) differ from the +[defaults for self-managed instances](../../administration/pages/index.md): -| Setting | GitLab.com | Default | -|---------------------------|------------------------|------------------------| -| Domain name | `gitlab.io` | - | -| IP address | `35.185.44.232` | - | -| Custom domains support | **{check-circle}** Yes | **{dotted-circle}** No | -| TLS certificates support | **{check-circle}** Yes | **{dotted-circle}** No | -| [Maximum size](../../administration/pages/index.md#set-global-maximum-size-of-each-gitlab-pages-site) (compressed) | 1 GB | 100 MB | +| Setting | GitLab.com | +|:--------------------------------------------------|:-----------------------| +| Domain name | `gitlab.io` | +| IP address | `35.185.44.232` | +| Support for custom domains | **{check-circle}** Yes | +| Support for TLS certificates | **{check-circle}** Yes | +| Maximum site size | 1 GB | +| Number of custom domains per GitLab Pages website | 150 | -The maximum size of your Pages site is also regulated by the artifacts maximum size, +The maximum size of your Pages site depends on the maximum artifact size, which is part of [GitLab CI/CD](#gitlab-cicd). -There are also [rate limits set for GitLab Pages](#gitlabcom-specific-rate-limits). +[Rate limits](#gitlabcom-specific-rate-limits) also exist for GitLab Pages. ## GitLab CI/CD @@ -151,7 +159,7 @@ the related documentation. | Artifacts maximum size (compressed) | 1 GB | See [Maximum artifacts size](../../user/admin_area/settings/continuous_integration.md#maximum-artifacts-size). | | Artifacts [expiry time](../../ci/yaml/index.md#artifactsexpire_in) | From June 22, 2020, deleted after 30 days unless otherwise specified (artifacts created before that date have no expiry). | See [Default artifacts expiration](../admin_area/settings/continuous_integration.md#default-artifacts-expiration). | | Scheduled Pipeline Cron | `*/5 * * * *` | See [Pipeline schedules advanced configuration](../../administration/cicd.md#change-maximum-scheduled-pipeline-frequency). | -| Maximum jobs in active pipelines | `500` for Free tier, `1000` for all trial tiers, and unlimited otherwise. | See [Number of jobs in active pipelines](../../administration/instance_limits.md#number-of-jobs-in-active-pipelines). | +| Maximum jobs in active pipelines | `500` for Free tier, `1000` for all trial tiers, `20000` for Premium, and `100000` for Ultimate. | See [Number of jobs in active pipelines](../../administration/instance_limits.md#number-of-jobs-in-active-pipelines). | | Maximum CI/CD subscriptions to a project | `2` | See [Number of CI/CD subscriptions to a project](../../administration/instance_limits.md#number-of-cicd-subscriptions-to-a-project). | | Maximum number of pipeline triggers in a project | `25000` for Free tier, Unlimited for all paid tiers | See [Limit the number of pipeline triggers](../../administration/instance_limits.md#limit-the-number-of-pipeline-triggers). | | Maximum pipeline schedules in projects | `10` for Free tier, `50` for all paid tiers | See [Number of pipeline schedules](../../administration/instance_limits.md#number-of-pipeline-schedules). | @@ -175,7 +183,7 @@ varies by format: | Generic | 5 GB | | Helm | 5 MB | | Maven | 5 GB | -| npm: | 5 GB | +| npm | 5 GB | | NuGet | 5 GB | | PyPI | 5 GB | | Terraform | 1 GB | @@ -188,7 +196,7 @@ the default value [is the same as for self-managed instances](../admin_area/sett | Setting | GitLab.com default | |-------------------------------|--------------------| | [Repository size including LFS](../admin_area/settings/account_and_limit_settings.md#repository-size-limit) | 10 GB | -| [Maximum import size](../project/settings/import_export.md#maximum-import-file-size) | 5 GB | +| [Maximum import size](../project/settings/import_export.md#import-a-project-and-its-data) | 5 GB | | Maximum attachment size | 100 MB | If you are near or over the repository size limit, you can either @@ -200,6 +208,30 @@ NOTE: Cloudflare. Git LFS and imports other than a file upload are not affected by this limit. Repository limits apply to both public and private projects. +## Default import sources + +> Disabling all importers by default for new GitLab self-managed installations [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118970) in GitLab 16.0. + +The import sources that are available by default depend on which GitLab you use: + +- GitLab.com: all available import sources are enabled by default. +- GitLab self-managed: no import sources are enabled by default and must be + [enabled](../admin_area/settings/visibility_and_access_controls.md#configure-allowed-import-sources). + +| Import source | GitLab.com default | GitLab self-managed default | +|:----------------------------------------------------------------------------------------------------|:-----------------------|:----------------------------| +| [Bitbucket Cloud](../project/import/bitbucket.md) | **{check-circle}** Yes | **{dotted-circle}** No | +| [Bitbucket Server](../project/import/bitbucket_server.md) | **{check-circle}** Yes | **{dotted-circle}** No | +| [FogBugz](../project/import/fogbugz.md) | **{check-circle}** Yes | **{dotted-circle}** No | +| [Gitea](../project/import/gitea.md) | **{check-circle}** Yes | **{dotted-circle}** No | +| [GitLab by direct transfer](../group/import/index.md#migrate-groups-by-direct-transfer-recommended) | **{check-circle}** Yes | **{dotted-circle}** No | +| [GitLab using file exports](../project/settings/import_export.md) | **{check-circle}** Yes | **{dotted-circle}** No | +| [GitHub](../project/import/github.md) | **{check-circle}** Yes | **{dotted-circle}** No | +| [Manifest file](../project/import/manifest.md) | **{check-circle}** Yes | **{dotted-circle}** No | +| [Repository by URL](../project/import/repo_by_url.md) | **{check-circle}** Yes | **{dotted-circle}** No | + +[Other importers](../project/import/index.md#available-project-importers) are available. + ## IP range GitLab.com uses the IP ranges `34.74.90.64/28` and `34.74.226.0/24` for traffic from its Web/API @@ -247,7 +279,7 @@ The limit varies depending on your plan and the number of seats in your subscrip | Setting | Default for GitLab.com | |----------------------|-------------------------| -| Number of webhooks | `100` per project, `50` per group | +| Number of webhooks | `100` per project, `50` per group (subgroup webhooks are not counted towards parent group limits ) | | Maximum payload size | 25 MB | | Timeout | 10 seconds | @@ -263,73 +295,6 @@ Runner SaaS is the hosted, secure, and managed build environment you can use to For more information, see [Runner SaaS](../../ci/runners/index.md). -## Sidekiq - -GitLab.com runs [Sidekiq](https://sidekiq.org) with arguments `--timeout=4 --concurrency=4` -and the following environment variables: - -| Setting | GitLab.com | Default | -|----------------------------------------|------------|-----------| -| `GITLAB_MEMORY_WATCHDOG_ENABLED` | - | `true` | -| `SIDEKIQ_MEMORY_KILLER_MAX_RSS` | - | `2000000` | -| `SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS` | - | - | -| `SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL` | - | `3` | -| `SIDEKIQ_MEMORY_KILLER_GRACE_TIME` | - | `900` | -| `SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT` | - | `30` | -| `SIDEKIQ_LOG_ARGUMENTS` | `1` | `1` | - -For more information, see how to -[configure the environment variables](../../administration/sidekiq/sidekiq_memory_killer.md). - -NOTE: -The `SIDEKIQ_MEMORY_KILLER_MAX_RSS` setting is `16000000` on Sidekiq import -nodes and Sidekiq export nodes. - -## PostgreSQL - -GitLab.com being a fairly large installation of GitLab means we have changed -various PostgreSQL settings to better suit our needs. For example, we use -streaming replication and servers in hot-standby mode to balance queries across -different database servers. - -The list of GitLab.com specific settings (and their defaults) is as follows: - -| Setting | GitLab.com | Default | -|:--------------------------------------|:--------------------------------------------------------------------|:--------------------------------------| -| `archive_command` | `/usr/bin/envdir /etc/wal-e.d/env /opt/wal-e/bin/wal-e wal-push %p` | empty | -| `archive_mode` | on | off | -| `autovacuum_analyze_scale_factor` | 0.01 | 0.01 | -| `autovacuum_max_workers` | 6 | 3 | -| `autovacuum_vacuum_cost_limit` | 1000 | -1 | -| `autovacuum_vacuum_scale_factor` | 0.01 | 0.02 | -| `checkpoint_completion_target` | 0.7 | 0.9 | -| `checkpoint_segments` | 32 | 10 | -| `effective_cache_size` | 338688 MB | Based on how much memory is available | -| `hot_standby` | on | off | -| `hot_standby_feedback` | on | off | -| `log_autovacuum_min_duration` | 0 | -1 | -| `log_checkpoints` | on | off | -| `log_line_prefix` | `%t [%p]: [%l-1]` | empty | -| `log_min_duration_statement` | 1000 | -1 | -| `log_temp_files` | 0 | -1 | -| `maintenance_work_mem` | 2048 MB | 16 MB | -| `max_replication_slots` | 5 | 0 | -| `max_wal_senders` | 32 | 0 | -| `max_wal_size` | 5 GB | 1 GB | -| `shared_buffers` | 112896 MB | Based on how much memory is available | -| `shared_preload_libraries` | `pg_stat_statements` | empty | -| `shmall` | 30146560 | Based on the server's capabilities | -| `shmmax` | 123480309760 | Based on the server's capabilities | -| `wal_buffers` | 16 MB | -1 | -| `wal_keep_segments` | 512 | 10 | -| `wal_level` | replica | minimal | -| `statement_timeout` | 15 s | 60 s | -| `idle_in_transaction_session_timeout` | 60 s | 60 s | - -Some of these settings are in the process being adjusted. For example, the value -for `shared_buffers` is quite high, and we are -[considering adjusting it](https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/4985). - ## Puma GitLab.com uses the default of 60 seconds for [Puma request timeouts](../../administration/operations/puma.md#change-the-worker-timeout). @@ -358,21 +323,23 @@ are also informational headers with this response detailed in The following table describes the rate limits for GitLab.com, both before and after the limits change in January, 2021: -| Rate limit | From 2021-02-12 | From 2022-02-03 | -|:---------------------------------------------------------------------------|:------------------------------|:----------------------------------------| -| **Protected paths** (for a given **IP address**) | **10** requests per minute | **10** requests per minute | -| **Raw endpoint** traffic (for a given **project, commit, and file path**) | **300** requests per minute | **300** requests per minute | -| **Unauthenticated** traffic (from a given **IP address**) | **500** requests per minute | **500** requests per minute | -| **Authenticated** API traffic (for a given **user**) | **2,000** requests per minute | **2,000** requests per minute | -| **Authenticated** non-API HTTP traffic (for a given **user**) | **1,000** requests per minute | **1,000** requests per minute | -| **All** traffic (from a given **IP address**) | **2,000** requests per minute | **2,000** requests per minute | -| **Issue creation** | **300** requests per minute | **200** requests per minute | -| **Note creation** (on issues and merge requests) | **60** requests per minute | **60** requests per minute | -| **Advanced, project, and group search** API (for a given **IP address**) | **10** requests per minute | **10** requests per minute | -| **GitLab Pages** requests (for a given **IP address**) | | **1000** requests per **50 seconds** | -| **GitLab Pages** requests (for a given **GitLab Pages domain**) | | **5000** requests per **10 seconds** | -| **Pipeline creation** requests (for a given **project, user, and commit**) | | **25** requests per minute | -| **Alert integration endpoint** requests (for a given **project**) | | **3600** requests per hour | +| Rate limit | From 2021-02-12 | From 2022-02-03 | +|:---------------------------------------------------------------------------|:------------------------------|:-------------------------------------| +| **Protected paths** (for a given **IP address**) | **10** requests per minute | **10** requests per minute | +| **Raw endpoint** traffic (for a given **project, commit, and file path**) | **300** requests per minute | **300** requests per minute | +| **Unauthenticated** traffic (from a given **IP address**) | **500** requests per minute | **500** requests per minute | +| **Authenticated** API traffic (for a given **user**) | **2,000** requests per minute | **2,000** requests per minute | +| **Authenticated** non-API HTTP traffic (for a given **user**) | **1,000** requests per minute | **1,000** requests per minute | +| **All** traffic (from a given **IP address**) | **2,000** requests per minute | **2,000** requests per minute | +| **Issue creation** | **300** requests per minute | **200** requests per minute | +| **Note creation** (on issues and merge requests) | **60** requests per minute | **60** requests per minute | +| **Advanced, project, and group search** API (for a given **IP address**) | **10** requests per minute | **10** requests per minute | +| **GitLab Pages** requests (for a given **IP address**) | | **1000** requests per **50 seconds** | +| **GitLab Pages** requests (for a given **GitLab Pages domain**) | | **5000** requests per **10 seconds** | +| **GitLab Pages** TLS connections (for a given **IP address**) | | **1000** requests per **50 seconds** | +| **GitLab Pages** TLS connections (for a given **GitLab Pages domain**) | | **400** requests per **10 seconds** | +| **Pipeline creation** requests (for a given **project, user, and commit**) | | **25** requests per minute | +| **Alert integration endpoint** requests (for a given **project**) | | **3600** requests per hour | More details are available on the rate limits for [protected paths](#protected-paths-throttle) and @@ -497,7 +464,8 @@ and can't be configured on GitLab.com to expire. You can erase job logs In addition to the GitLab Enterprise Edition Omnibus install, GitLab.com uses the following applications and settings to achieve scale. All settings are -publicly available at [chef cookbooks](https://gitlab.com/gitlab-cookbooks). +publicly available, as [Kubernetes configuration](https://gitlab.com/gitlab-com/gl-infra/k8s-workloads/gitlab-com) +or [Chef cookbooks](https://gitlab.com/gitlab-cookbooks). ### Elastic cluster @@ -541,3 +509,10 @@ Service discovery: High Performance TCP/HTTP Load Balancer: - [`gitlab-cookbooks` / `gitlab-haproxy` · GitLab](https://gitlab.com/gitlab-cookbooks/gitlab-haproxy) + +## Sidekiq + +GitLab.com runs [Sidekiq](https://sidekiq.org) as an [external process](../../administration/sidekiq/index.md) +for Ruby job scheduling. + +The current settings are in the [GitLab.com Kubernetes pod configuration](https://gitlab.com/gitlab-com/gl-infra/k8s-workloads/gitlab-com/-/blob/master/releases/gitlab/values/gprd.yaml.gotmpl). diff --git a/doc/user/group/access_and_permissions.md b/doc/user/group/access_and_permissions.md index 4aecf016e56..50506d005b0 100644 --- a/doc/user/group/access_and_permissions.md +++ b/doc/user/group/access_and_permissions.md @@ -1,6 +1,6 @@ --- -stage: Manage -group: Organization +stage: Data Stores +group: Tenant Scale info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -37,12 +37,8 @@ The group's new subgroups have push rules set for them based on either: ## Restrict Git access protocols -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/365601) in GitLab 15.1 [with a flag](../../administration/feature_flags.md) named `group_level_git_protocol_control`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to -[enable the feature flag](../../administration/feature_flags.md) named `group_level_git_protocol_control`. On GitLab.com, -this feature is available. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/365601) in GitLab 15.1. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/365357) in GitLab 16.0. You can set the permitted protocols used to access a group's repositories to either SSH, HTTPS, or both. This setting is disabled when the [instance setting](../admin_area/settings/visibility_and_access_controls.md#configure-enabled-git-access-protocols) is @@ -64,7 +60,7 @@ To change the permitted Git access protocols for a group: To ensure only people from your organization can access particular resources, you can restrict access to groups by IP address. This group-level setting applies to: -- The GitLab UI, including subgroups, projects, and issues. +- The GitLab UI, including subgroups, projects, and issues. It does not apply to GitLab Pages. - [In GitLab 12.3 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/12874), the API. - In self-managed installations of GitLab 15.1 and later, you can also configure [globally-allowed IP address ranges](../admin_area/settings/visibility_and_access_controls.md#configure-globally-allowed-ip-address-ranges) @@ -90,8 +86,8 @@ To restrict group access by IP address: Keep in mind that restricting group access by IP address has the following implications: -- Administrators and group owners can access group settings from any IP address, regardless of IP restriction. However: - - Group owners can access the subgroups, but not the projects belonging to the group or subgroups, when accessing from a disallowed IP address. +- Administrators and group Owners can access group settings from any IP address, regardless of IP restriction. However: + - Group Owners can access the subgroups, but not the projects belonging to the group or subgroups, when accessing from a disallowed IP address. - Administrators can access projects belonging to the group when accessing from a disallowed IP address. Access to projects includes cloning code from them. - Users can still see group and project names and hierarchies. Only the following are restricted: @@ -102,8 +98,9 @@ Keep in mind that restricting group access by IP address has the following impli restricted IP address, the IP restriction prevents code from being cloned. - Users might still see some events from the IP-restricted groups and projects on their dashboard. Activity might include push, merge, issue, or comment events. -- IP access restrictions for Git operations via SSH are supported only on GitLab SaaS. - IP access restrictions applied to self-managed instances block SSH completely. +- IP access restrictions for Git operations via SSH are supported on GitLab SaaS. + IP access restrictions applied to self-managed instances are possible with [`gitlab-sshd`](../../administration/operations/gitlab_sshd.md) + with [PROXY protocol](../../administration/operations/gitlab_sshd.md#proxy-protocol-support) enabled. ## Restrict group access by domain **(PREMIUM)** @@ -181,12 +178,12 @@ prevent a project from being shared with other groups: 1. Select **Projects in `` cannot be shared with other groups**. 1. Select **Save changes**. -This setting applies to all subgroups unless overridden by a group owner. Groups already +This setting applies to all subgroups unless overridden by a group Owner. Groups already added to a project lose access when the setting is enabled. ## Prevent users from requesting access to a group -As a group owner, you can prevent non-members from requesting access to +As a group Owner, you can prevent non-members from requesting access to your group. 1. On the top bar, **Main menu > Groups** and find your group. @@ -202,7 +199,7 @@ your group. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216987) in GitLab 13.3. By default, projects in a group can be forked. -Optionally, on [GitLab Premium](https://about.gitlab.com/pricing/) or higher tiers, +Optionally, on [GitLab Premium and Ultimate tiers](https://about.gitlab.com/pricing/), you can prevent the projects in a group from being forked outside of the current top-level group. This setting is removed from the SAML setting page, and migrated to the @@ -221,13 +218,13 @@ Existing forks are not removed. ## Prevent members from being added to projects in a group **(PREMIUM)** -As a group owner, you can prevent any new project membership for all +As a group Owner, you can prevent any new project membership for all projects in a group, allowing tighter control over project membership. For example, if you want to lock the group for an [Audit Event](../../administration/audit_events.md), you can guarantee that project membership cannot be modified during the audit. -If group membership lock is enabled, the group owner can still: +If group membership lock is enabled, the group Owner can still: - Invite groups or add members to groups to give them access to projects in the **locked** group. - Change the role of group members. @@ -286,7 +283,15 @@ To create group links via filter: LDAP user permissions can be manually overridden by an administrator. To override a user's permissions: 1. On the top bar, select **Main menu > Groups** and find your group. -1. On the left sidebar, select **Group information > Members**. +1. On the left sidebar, select **Group information > Members**. If LDAP synchronization + has granted a user a role with: + - More permissions than the parent group membership, that user is displayed as having + [direct membership](../project/members/index.md#display-direct-members) of the group. + - The same or fewer permissions than the parent group membership, that user is displayed as having + [inherited membership](../project/members/index.md#display-inherited-members) of the group. +1. Optional. If the user you want to edit is displayed as having inherited membership, + [filter the subgroup to show direct members](manage.md#filter-a-group) before + overriding LDAP user permissions. 1. In the row for the user you are editing, select the pencil (**{pencil}**) icon. 1. Select **Edit permissions** in the modal. @@ -302,3 +307,17 @@ If a user sees a 404 when they would usually expect access, and the problem is l - `json.allowed`: `false` In viewing the log entries, compare `remote.ip` with the list of [allowed IP addresses](#restrict-group-access-by-ip-address) for the group. + +### Cannot update permissions for a group member + +If a group Owner cannot update permissions for a group member, check which memberships +are listed. Group Owners can only update direct memberships. + +If a parent group membership has the same or higher role than a subgroup, the +[inherited membership](../project/members/index.md#inherited-membership) is +listed on the subgroup members page, even if a [direct membership](../project/members/index.md#membership-types) +on the group exists. + +To view and update direct memberships, [filter the group to show direct members](manage.md#filter-a-group). + +The need to filter members by type through a redesigned members page that lists both direct and inherited memberships is proposed in [issue 337539](https://gitlab.com/gitlab-org/gitlab/-/issues/337539#note_1277786161). diff --git a/doc/user/group/clusters/index.md b/doc/user/group/clusters/index.md index cb760217487..4c448d8e5c2 100644 --- a/doc/user/group/clusters/index.md +++ b/doc/user/group/clusters/index.md @@ -1,11 +1,11 @@ --- type: reference -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Group-level Kubernetes clusters (certificate-based) (DEPRECATED) **(FREE)** +# Group-level Kubernetes clusters (certificate-based) (deprecated) **(FREE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/34758) in GitLab 11.6. > - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. diff --git a/doc/user/group/compliance_frameworks.md b/doc/user/group/compliance_frameworks.md index 84cca5800c2..2fca8b7b678 100644 --- a/doc/user/group/compliance_frameworks.md +++ b/doc/user/group/compliance_frameworks.md @@ -94,7 +94,8 @@ mutation { > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/331231) in GitLab 14.2. Group owners can configure a compliance pipeline in a project separate to other projects. By default, the compliance -pipeline configuration (`.gitlab-ci.yml` file) is run instead of the pipeline configuration of labeled projects. +pipeline configuration (for example, `.compliance-gitlab-ci.yml`) is run instead of the pipeline configuration (for example, `.gitlab-ci.yml`) of labeled +projects. However, the compliance pipeline configuration can reference the `.gitlab-ci.yml` file of the labeled projects so that: @@ -103,8 +104,11 @@ However, the compliance pipeline configuration can reference the `.gitlab-ci.yml - Jobs and variables defined in the compliance pipeline can't be changed by variables in the labeled project's `.gitlab-ci.yml` file. -See [example configuration](#example-configuration) for help configuring a compliance pipeline that runs jobs from -labeled project pipeline configuration. +For more information, see: + +- [Example configuration](#example-configuration) for help configuring a compliance pipeline that runs jobs from + labeled project pipeline configuration. +- The [Create a compliance pipeline](../../tutorials/compliance_pipeline/index.md) tutorial. ### Effect on labeled projects @@ -208,16 +212,47 @@ audit trail: - "# No after scripts." include: # Execute individual project's configuration (if project contains .gitlab-ci.yml) - project: '$CI_PROJECT_PATH' - file: '$CI_CONFIG_PATH' - ref: '$CI_COMMIT_SHA' # Must be defined or MR pipelines always use the use default branch - rules: - - if: $CI_PROJECT_PATH != "my-group/project-1" # Must be the hardcoded path to the project that hosts this configuration. + - project: '$CI_PROJECT_PATH' + file: '$CI_CONFIG_PATH' + ref: '$CI_COMMIT_SHA' # Must be defined or MR pipelines always use the use default branch + rules: + - if: $CI_PROJECT_PATH != "my-group/project-1" # Must be the hardcoded path to the project that hosts this configuration. ``` The `rules` configuration in the `include` definition avoids circular inclusion in case the compliance pipeline must be able to run in the host project itself. You can leave it out if your compliance pipeline only ever runs in labeled projects. +#### Compliance pipelines and custom pipeline configuration hosted externally + +The example above assumes that all projects host their pipeline configuration in the same project. +If any projects use [configuration hosted externally to the project](../../ci/pipelines/settings.md#specify-a-custom-cicd-configuration-file): + +- The `include` section in the example compliance pipeline configuration must be adjusted. + For example, using [`include:rules`](../../ci/yaml/includes.md#use-rules-with-include): + + ```yaml + include: + # If the custom path variables are defined, include the project's external config file. + - project: '$PROTECTED_PIPELINE_CI_PROJECT_PATH' + file: '$PROTECTED_PIPELINE_CI_CONFIG_PATH' + ref: '$PROTECTED_PIPELINE_CI_REF' + rules: + - if: $PROTECTED_PIPELINE_CI_PROJECT_PATH && $PROTECTED_PIPELINE_CI_CONFIG_PATH && $PROTECTED_PIPELINE_CI_REF + # If any custom path variable is not defined, include the project's internal config file as normal. + - project: '$CI_PROJECT_PATH' + file: '$CI_CONFIG_PATH' + ref: '$CI_COMMIT_SHA' + rules: + - if: $PROTECTED_PIPELINE_CI_PROJECT_PATH == null || $PROTECTED_PIPELINE_CI_CONFIG_PATH == null || $PROTECTED_PIPELINE_CI_REF == null + ``` + +- [CI/CD variables](../../ci/variables/index.md) must be added to projects with external + pipeline configuration. In this example: + + - `PROTECTED_PIPELINE_CI_PROJECT_PATH`: The path to the project hosting the configuration file, for example `group/subgroup/project`. + - `PROTECTED_PIPELINE_CI_CONFIG_PATH`: The path to the configuration file in the project, for example `path/to/.gitlab-ci.yml`. + - `PROTECTED_PIPELINE_CI_REF`: The ref to use when retrieving the configuration file, for example `main`. + #### Compliance pipelines in merge requests originating in project forks When a merge request originates in a fork, the branch to be merged usually only exists in the fork. @@ -312,3 +347,53 @@ mutation { } } ``` + +### Compliance jobs are overwritten by target repository + +If you use the `extends` statement in a compliance pipeline configuration, compliance jobs are overwritten by the target repository job. For example, +you could have the following `.compliance-gitlab-ci.yml` configuration: + +```yaml +"compliance job": + extends: + - .compliance_template + stage: build + +.compliance_template: + script: + - echo "take compliance action" +``` + +You could also have the following `.gitlab-ci.yml` configuration: + +```yaml +"compliance job": + stage: test + script: + - echo "overwriting compliance action" +``` + +This configuration results in the target repository pipeline overwriting the compliance pipeline, and you get the following message: +`overwriting compliance action`. + +To avoid overwriting a compliance job, don't use the `extends` keyword in compliance pipeline configuration. For example, +you could have the following `.compliance-gitlab-ci.yml` configuration: + +```yaml +"compliance job": + stage: build + script: + - echo "take compliance action" +``` + +You could also have the following `.gitlab-ci.yml` configuration: + +```yaml +"compliance job": + stage: test + script: + - echo "overwriting compliance action" +``` + +This configuration doesn't overwrite the compliance pipeline and you get the following message: +`take compliance action`. diff --git a/doc/user/group/contribution_analytics/img/group_stats_cal.png b/doc/user/group/contribution_analytics/img/group_stats_cal.png deleted file mode 100644 index 0238c7bf088..00000000000 Binary files a/doc/user/group/contribution_analytics/img/group_stats_cal.png and /dev/null differ diff --git a/doc/user/group/contribution_analytics/img/group_stats_table.png b/doc/user/group/contribution_analytics/img/group_stats_table.png deleted file mode 100644 index 1f58b9717d0..00000000000 Binary files a/doc/user/group/contribution_analytics/img/group_stats_table.png and /dev/null differ diff --git a/doc/user/group/custom_project_templates.md b/doc/user/group/custom_project_templates.md index 2716db27037..95d7ddb056e 100644 --- a/doc/user/group/custom_project_templates.md +++ b/doc/user/group/custom_project_templates.md @@ -1,7 +1,6 @@ --- -type: reference -stage: Manage -group: Import +stage: Create +group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -42,7 +41,7 @@ Projects in nested subgroups are not included in the template list. - Public and internal projects can be selected by any authenticated user as a template for a new project, if all [project features](../project/settings/index.md#configure-project-visibility-features-and-permissions) - except for **GitLab Pages** and **Security & Compliance** are set to **Everyone With Access**. + except for **GitLab Pages** and **Security and Compliance** are set to **Everyone With Access**. - Private projects can be selected only by users who are members of the projects. ## Example structure diff --git a/doc/user/group/devops_adoption/index.md b/doc/user/group/devops_adoption/index.md index a81b61c50ce..7105318e5df 100644 --- a/doc/user/group/devops_adoption/index.md +++ b/doc/user/group/devops_adoption/index.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Group DevOps Adoption **(ULTIMATE)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/321083) in GitLab 13.11 as a [Beta feature](../../../policy/alpha-beta-support.md#beta-features). +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/321083) in GitLab 13.11 as a [Beta feature](../../../policy/alpha-beta-support.md#beta). > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/333556) in GitLab 14.1. > - The Overview tab [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/330401) in GitLab 14.1. > - DAST and SAST metrics [added](https://gitlab.com/gitlab-org/gitlab/-/issues/328033) in GitLab 14.1. diff --git a/doc/user/group/epics/epic_boards.md b/doc/user/group/epics/epic_boards.md index 64addd524ad..6783e89779b 100644 --- a/doc/user/group/epics/epic_boards.md +++ b/doc/user/group/epics/epic_boards.md @@ -8,17 +8,27 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/5067) in GitLab 13.10. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/290039) in GitLab 14.1. +> - Displaying total weight on the top of lists [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/364503) in GitLab 15.11. Epic boards build on the existing [epic tracking functionality](index.md) and [labels](../../project/labels.md). Your epics appear as cards in vertical lists, organized by their assigned labels. +On the top of each list, you can see the number of epics in the list (**{epic}**) and the total weight of all its epics (**{weight}**). + +
    + See the video: Epics and Issue Boards - Project Management. +
    +
    + +
    + To view an epic board: 1. On the top bar, select **Main menu > Groups** and find your group. 1. On the left sidebar, select **Epics > Boards**. -![GitLab epic board - Premium](img/epic_board_v14_1.png) +![GitLab epic board - Premium](img/epic_board_v15_10.png) ## Create an epic board @@ -30,7 +40,7 @@ To create a new epic board: 1. On the top bar, select **Main menu > Groups** and find your group. 1. On the left sidebar, select **Epics > Boards**. -1. In the upper left corner, select the dropdown list with the current board name. +1. In the upper-left corner, select the dropdown list with the current board name. 1. Select **Create new board**. 1. Enter the new board's title. 1. Optional. To hide the Open or Closed lists, clear the **Show the Open list** and @@ -54,7 +64,7 @@ Prerequisites: To delete the active epic board: -1. Select the dropdown list with the current board name in the upper left corner of the epic boards page. +1. In the upper-left corner of the epic board page, select the dropdown list. 1. Select **Delete board**. 1. Select **Delete**. @@ -63,7 +73,7 @@ To delete the active epic board: - [Create a new list](#create-a-new-list). - [Remove an existing list](#remove-a-list). - [Filter epics](#filter-epics). -- Create workflows, like when using [issue boards](../../project/issue_board.md#create-workflows). +- Create workflows, like when using [issue boards](../../project/issue_board.md#issue-board-workflow-between-teams). - [Move epics and lists](#move-epics-and-lists). - Change epic labels (by dragging an epic between lists). - Close an epic (by dragging it to the **Closed** list). @@ -100,7 +110,7 @@ To remove a list from an epic board: 1. Select **Remove list**. A confirmation dialog appears. 1. Select **OK**. -## Create an epic from an epic board +### Create an epic from an epic board > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/233568) in GitLab 14.0. @@ -115,7 +125,7 @@ To create an epic from a list in epic board: 1. Enter the new epic's title. 1. Select **Create epic**. -![Create a GitLab epic from an epic board](img/epic_board_epic_create_v14_1.png) +![Create a GitLab epic from an epic board](img/epic_board_epic_create_v15_10.png) ### Filter epics diff --git a/doc/user/group/epics/img/button_close_epic.png b/doc/user/group/epics/img/button_close_epic.png deleted file mode 100644 index aa1a889ea23..00000000000 Binary files a/doc/user/group/epics/img/button_close_epic.png and /dev/null differ diff --git a/doc/user/group/epics/img/epic_board_epic_create_v14_1.png b/doc/user/group/epics/img/epic_board_epic_create_v14_1.png deleted file mode 100644 index 04017014885..00000000000 Binary files a/doc/user/group/epics/img/epic_board_epic_create_v14_1.png and /dev/null differ diff --git a/doc/user/group/epics/img/epic_board_epic_create_v15_10.png b/doc/user/group/epics/img/epic_board_epic_create_v15_10.png new file mode 100644 index 00000000000..71d1fc0a9fc Binary files /dev/null and b/doc/user/group/epics/img/epic_board_epic_create_v15_10.png differ diff --git a/doc/user/group/epics/img/epic_board_v14_1.png b/doc/user/group/epics/img/epic_board_v14_1.png deleted file mode 100644 index ccf1ef9559e..00000000000 Binary files a/doc/user/group/epics/img/epic_board_v14_1.png and /dev/null differ diff --git a/doc/user/group/epics/img/epic_board_v15_10.png b/doc/user/group/epics/img/epic_board_v15_10.png new file mode 100644 index 00000000000..03bc96d9623 Binary files /dev/null and b/doc/user/group/epics/img/epic_board_v15_10.png differ diff --git a/doc/user/group/epics/img/issue_list_v13_1.png b/doc/user/group/epics/img/issue_list_v13_1.png deleted file mode 100644 index ed0b4842bfe..00000000000 Binary files a/doc/user/group/epics/img/issue_list_v13_1.png and /dev/null differ diff --git a/doc/user/group/epics/img/issue_list_v15_11.png b/doc/user/group/epics/img/issue_list_v15_11.png new file mode 100644 index 00000000000..601406a8a33 Binary files /dev/null and b/doc/user/group/epics/img/issue_list_v15_11.png differ diff --git a/doc/user/group/epics/index.md b/doc/user/group/epics/index.md index 21c95f37aeb..32454693d71 100644 --- a/doc/user/group/epics/index.md +++ b/doc/user/group/epics/index.md @@ -59,7 +59,7 @@ have a start or due date, a visual - Link [related epics](linked_epics.md) based on a type of relationship. - Create workflows with [epic boards](epic_boards.md). - [Turn on notifications](../../profile/notifications.md) for about epic events. -- [Award an emoji](../../award_emojis.md) to an epic or its comments. +- [Add an emoji reaction](../../award_emojis.md) to an epic or its comments. - Collaborate on an epic by posting comments in a [thread](../../discussions/index.md). - Use [health status](../../project/issues/managing_issues.md#health-status) to track your progress. diff --git a/doc/user/group/epics/manage_epics.md b/doc/user/group/epics/manage_epics.md index 74cfa2bd6ed..98316188496 100644 --- a/doc/user/group/epics/manage_epics.md +++ b/doc/user/group/epics/manage_epics.md @@ -41,8 +41,6 @@ The newly created epic opens. ### Start and due date inheritance -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7332) in GitLab 12.5 to replace **From milestones**. - If you select **Inherited**: - For the **start date**: GitLab scans all child epics and issues assigned to the epic, @@ -52,7 +50,7 @@ If you select **Inherited**: and sets the due date to match the latest due date found in the child epics or the milestone assigned to the issues. -These are dynamic dates and recalculated if any of the following occur: +These dates are dynamic and recalculated if any of the following occur: - A child epic's dates change. - Milestones are reassigned to an issue. @@ -123,8 +121,6 @@ To reorder list items, when viewing an epic: ## Bulk edit epics -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7250) in GitLab 12.2. - Users with at least the Reporter role can manage epics. When bulk editing epics in a group, you can edit their labels. @@ -136,7 +132,7 @@ Prerequisites: To update multiple epics at the same time: 1. In a group, go to **Epics > List**. -1. Select **Edit epics**. A sidebar on the right appears with editable fields. +1. Select **Bulk edit**. A sidebar on the right appears with editable fields. 1. Select the checkboxes next to each epic you want to edit. 1. Select the appropriate fields and their values from the sidebar. 1. Select **Update all**. @@ -163,14 +159,13 @@ Prerequisites: - You must have at least the Reporter role for the epic's group. -Whenever you decide that there is no longer need for that epic, -close the epic by: - -- Selecting **Close epic**. +To close an epic, at the top of an epic, select **Close epic**. - ![close epic - button](img/button_close_epic.png) + +If you don't see this action at the top of an epic, your project or instance might have +enabled a feature flag for [moved actions](../../project/merge_requests/index.md#move-sidebar-actions) -- Using the `/close` [quick action](../../project/quick_actions.md). +You can also use the `/close` [quick action](../../project/quick_actions.md). ## Reopen a closed epic @@ -233,7 +228,6 @@ than 1000. The cached value is rounded to thousands or millions and updated ever ## Filter the list of epics -> - Filtering by epics was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/195704) in GitLab 12.9. > - Filtering by child epics was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9029) in GitLab 13.0. > - Filtering by the user's reaction emoji [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/325630) in GitLab 13.11. > - Sorting by epic titles [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/331625) in GitLab 14.1. @@ -341,7 +335,7 @@ in relation to epics. ### View issues assigned to an epic -On the **Epics and Issues** tab, you can see epics and issues assigned to this epic. +On the **Child issues and epics** section, you can see epics and issues assigned to this epic. Only epics and issues that you can access show on the list. You can always view the issues assigned to the epic if they are in the group's child project. @@ -350,7 +344,7 @@ of its parent group. ### View count of issues in an epic -On the **Epics and Issues** tab, under each epic name, hover over the total counts. +On the **Child issues and epics** section, under each epic name, hover over the total counts. The number indicates all epics associated with the project, including issues you might not have permission to. @@ -365,7 +359,7 @@ automatically added to the epic. > Minimum required role for the project [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/382506) from Reporter to Guest in GitLab 15.8. You can add existing issues to an epic, including issues in a project from a [different group hierarchy](index.md#child-issues-from-different-group-hierarchies). -Newly added issues appear at the top of the list of issues in the **Epics and Issues** tab. +Newly added issues appear at the top of the list of issues in the **Child issues and epics** section. An epic contains a list of issues and an issue can be associated with at most one epic. When you add a new issue that's already linked to an epic, the issue is automatically unlinked from its @@ -377,13 +371,13 @@ Prerequisites: To add an existing issue to an epic: -1. On the epic's page, under **Epics and Issues**, select **Add**. +1. On the epic's page, under **Child issues and epics**, select **Add**. 1. Select **Add an existing issue**. 1. Identify the issue to be added, using either of the following methods: - Paste the link of the issue. - Search for the desired issue by entering part of the issue's title, then selecting the desired - match ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9126) in GitLab 12.5). Issues - from different group hierarchies do not appear in search results. To add such an issue, enter its full URL. + match. Issues from different group hierarchies do not appear in search results. + To add such an issue, enter its full URL. If there are multiple issues to be added, press Space and repeat this step. 1. Select **Add**. @@ -401,7 +395,7 @@ Prerequisites: To create an issue from an epic: -1. On the epic's page, under **Epics and Issues**, select **Add**. +1. On the epic's page, under **Child issues and epics**, select **Add**. 1. Select **Add a new issue**. 1. Under **Title**, enter the title for the new issue. 1. From the **Project** dropdown list, select the project in which the issue should be created. @@ -426,14 +420,13 @@ To remove an issue from an epic: The **Remove issue** warning appears. 1. Select **Remove**. -![List of issues assigned to an epic](img/issue_list_v13_1.png) +![List of issues assigned to an epic](img/issue_list_v15_11.png) ### Reorder issues assigned to an epic -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9367) in GitLab 12.5. -> - Minimum required role for the project [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/382506) from Reporter to Guest in GitLab 15.8. +> Minimum required role for the project [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/382506) from Reporter to Guest in GitLab 15.8. -New issues appear at the top of the list in the **Epics and Issues** tab. +New issues appear at the top of the list in the **Child issues and epics** section. You can reorder the list of issues by dragging them. Prerequisites: @@ -442,7 +435,7 @@ Prerequisites: To reorder issues assigned to an epic: -1. Go to the **Epics and Issues** tab. +1. Go to the **Child issues and epics** section. 1. Drag issues into the desired order. ### Move issues between epics **(ULTIMATE)** @@ -450,7 +443,7 @@ To reorder issues assigned to an epic: > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33039) in GitLab 13.0. > - Minimum required role for the project [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/382506) from Reporter to Guest in GitLab 15.8. -New issues appear at the top of the list in the **Epics and Issues** +New issues appear at the top of the list in the **Child issues and epics** tab. You can move issues from one epic to another. Prerequisites: @@ -459,13 +452,12 @@ Prerequisites: To move an issue to another epic: -1. Go to the **Epics and Issues** tab. +1. Go to the **Child issues and epics** section. 1. Drag issues into the desired parent epic in the visible hierarchy. ### Promote an issue to an epic -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/3777) in GitLab 11.6. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/37081) from GitLab Ultimate to GitLab Premium in 12.8. +> [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/37081) from GitLab Ultimate to GitLab Premium in 12.8. Prerequisites: @@ -504,12 +496,12 @@ You can create a spreadsheet template to manage a pattern of consistently repeat For an introduction to epic templates, see [GitLab Epics and Epic Template Tip](https://www.youtube.com/watch?v=D74xKFNw8vg). -For more on epic templates, see [Epic Templates - Repeatable sets of issues](https://about.gitlab.com/handbook/marketing/strategic-marketing/getting-started/104/). +For more on epic templates, see [Epic Templates - Repeatable sets of issues](https://about.gitlab.com/handbook/marketing/brand-and-product-marketing/product-and-solution-marketing/getting-started/104/). ## Multi-level child epics **(ULTIMATE)** You can add any epic that belongs to a group or subgroup of the parent epic's group. -New child epics appear at the top of the list of epics in the **Epics and Issues** tab. +New child epics appear at the top of the list of epics in the **Child issues and epics** section. When you add an epic that's already linked to a parent epic, the link to its current parent is removed. @@ -522,11 +514,7 @@ The maximum number of direct child epics is 100. > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/8502) in GitLab 15.6 [with a flag](../../../administration/feature_flags.md) named `child_epics_from_different_hierarchies`. Disabled by default. > - Minimum required role for the group [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/382503) from Reporter to Guest in GitLab 15.7. > - Cross-group child epics [enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/375622) in GitLab 15.9. Enabled by default. - -FLAG: -On self-managed GitLab, by default this feature is available. To disable it, -ask an administrator to [disable the feature flag](../../../administration/feature_flags.md) named `child_epics_from_different_hierarchies`. -On GitLab.com, this feature is available. +> - [Feature flag `child_epics_from_different_hierarchies`](https://gitlab.com/gitlab-org/gitlab/-/issues/382719) removed in GitLab 15.10. You can add a child epic that belongs to a group that is different from the parent epic's group. @@ -548,7 +536,7 @@ Prerequisites: To add a new epic as child epic: 1. In an epic, in the **Child issues and epics** section, select **Add > Add a new epic**. -1. Select a group from the dropdown. The epic's group is selected by default. +1. Select a group from the dropdown list. The epic's group is selected by default. 1. Enter a title for the new epic. 1. Select **Create epic**. @@ -557,7 +545,7 @@ To add an existing epic as child epic: 1. In an epic, in the **Child issues and epics** section, select **Add > Add an existing epic**. 1. Identify the epic to be added, using either of the following methods: - Paste the link of the epic. - - Search for the desired issue by entering part of the epic's title, then selecting the desired match. This search is only available for epics within the same group hierarchy. + - Search for the desired issue by entering part of the epic's title, then selecting the desired match. This search is only available for epics in the same group hierarchy. If there are multiple epics to be added, press Space and repeat this step. 1. Select **Add**. @@ -567,7 +555,7 @@ To add an existing epic as child epic: > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/33039) in GitLab 13.0. > - Minimum required role for the group [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/382503) from Reporter to Guest in GitLab 15.7. -New child epics appear at the top of the list in the **Epics and Issues** tab. +New child epics appear at the top of the list in the **Child issues and epics** section. You can move child epics from one epic to another. When you add a new epic that's already linked to a parent epic, the link to its current parent is removed. Issues and child epics cannot be intermingled. @@ -578,15 +566,14 @@ Prerequisites: To move child epics to another epic: -1. Go to the **Epics and Issues** tab. +1. Go to the **Child issues and epics** section. 1. Drag epics into the desired parent epic. ### Reorder child epics assigned to an epic -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9367) in GitLab 12.5. -> - Minimum required role for the group [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/382503) from Reporter to Guest in GitLab 15.7. +> Minimum required role for the group [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/382503) from Reporter to Guest in GitLab 15.7. -New child epics appear at the top of the list in the **Epics and Issues** tab. +New child epics appear at the top of the list in the **Child issues and epics** section. You can reorder the list of child epics. Prerequisites: @@ -595,7 +582,7 @@ Prerequisites: To reorder child epics assigned to an epic: -1. Go to the **Epics and Issues** tab. +1. Go to the **Child issues and epics** section. 1. Drag epics into the desired order. ### Remove a child epic from a parent epic diff --git a/doc/user/group/import/index.md b/doc/user/group/import/index.md index eb32856ff79..7f55bf56102 100644 --- a/doc/user/group/import/index.md +++ b/doc/user/group/import/index.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -27,42 +27,55 @@ If you migrate from GitLab.com to self-managed GitLab, an administrator can crea > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267945) in GitLab 14.4 for project resources [with a flag](../../feature_flags.md) named `bulk_import_projects`. Disabled by default. > - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/339941) in GitLab 15.6. > - New application setting `bulk_import_enabled` [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/383268) in GitLab 15.8. `bulk_import` feature flag removed. +> - `bulk_import_projects` feature flag [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/339941) in GitLab 15.10. -FLAG: On self-managed GitLab, by default [migrating group items](#migrated-group-items) is not available. To show the feature, ask an administrator to [enable it in application settings](../../admin_area/settings/visibility_and_access_controls.md#enable-migration-of-groups-and-projects-by-direct-transfer). -Also on self-managed GitLab, by default [migrating project items](#migrated-project-items-beta) is not available. To show -this feature, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named -`bulk_import_projects`. The feature is not ready for production use. On GitLab.com, migration of both groups and projects is available. Migrating groups by direct transfer copies the groups from one place to another. You can: - Copy many groups at once. -- Copy top-level groups to: +- In the GitLab UI, copy top-level groups to: - Another top-level group. - The subgroup of any existing top-level group. - Another GitLab instance, including GitLab.com. -- Copy groups with projects (in [beta](../../../policy/alpha-beta-support.md#beta-features) and not ready for production +- In the [API](../../../api/bulk_imports.md), copy top-level groups and subgroups to these locations. +- Copy groups with projects (in [Beta](../../../policy/alpha-beta-support.md#beta) and not ready for production use) or without projects. Copying projects with groups is available: - On GitLab.com by default. - - On self-managed GitLab instances after an administrator first [enables the feature flag](../../../administration/feature_flags.md) named `bulk_import_projects`. Not all group and project resources are copied. See list of copied resources below: - [Migrated group items](#migrated-group-items). - [Migrated project items](#migrated-project-items-beta). +WARNING: +Importing groups with projects is in [Beta](../../../policy/alpha-beta-support.md#beta). This feature is not +ready for production use. + We invite you to leave your feedback about migrating by direct transfer in [the feedback issue](https://gitlab.com/gitlab-org/gitlab/-/issues/284495). If you want to move groups instead of copying groups, you can [transfer groups](../manage.md#transfer-a-group) if the groups are in the same GitLab instance. Transferring groups is a faster and more complete option. -### Rate limit +### Known issues + +See [epic 6629](https://gitlab.com/groups/gitlab-org/-/epics/6629) for a list of known issues for migrating by direct +transfer. -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/386452) in GitLab 15.9. +### Limits -Each user can perform up to six migrations per minute. +| Limit | Description | +|:------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 6 | Maximum number of migrations permitted by a destination GitLab instance per minute per user. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/386452) in GitLab 15.9. | +| 5 GB | Maximum relation size that can be downloaded from the source instance. | +| 10 GB | Maximum size of a decompressed archive. | +| 210 seconds | Maximum number of seconds to wait for decompressing an archive file. | +| 50 MB | Maximum length an NDJSON row can have. | +| 5 minutes | Maximum number of seconds until an empty export status on source instance is raised. | +| 8 hours | Time until migration times out. | +| 90 minutes | Time the destination is waiting for export to complete. | ### Visibility rules @@ -78,6 +91,8 @@ make sure to have a similar setup on the destination instance, or to import into ### Prerequisites +> Requirement for Maintainer role instead of Developer role introduced in GitLab 16.0 and backported to GitLab 15.11.1 and GitLab 15.10.5. + To migrate groups by direct transfer: - The network connection between instances or GitLab.com must support HTTPS. @@ -92,25 +107,22 @@ To migrate groups by direct transfer: - For GitLab 15.0 and earlier source instances, the personal access token must have both the `api` and `read_repository` scopes. - You must have the Owner role on the source group to migrate from. -- You must have at least the Maintainer role on the destination group to migrate to. Using the Developer role for this - purpose was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387891) in GitLab 15.8 and will be removed in - GitLab 16.0. +- You must have at least the Maintainer role on the destination group to migrate to. ### Prepare user accounts To ensure GitLab maps users and their contributions correctly: -1. Create the required users on the destination GitLab instance. When migrating to GitLab.com, you must create users - manually unless [SCIM](../../group/saml_sso/scim_setup.md) is used. Creating users with the API is only available to - self-managed instances because it requires administrator access. -1. Check that users have a public email on the source GitLab instance that matches their primary email on the - destination GitLab instance. -1. Ensure that users confirm their primary email addresses on the destination GitLab instance. Most users receive an - email asking them to confirm their email address. -1. If using an OmniAuth provider like SAML, link GitLab and SAML accounts of users on GitLab. All users on the - destination GitLab instance must sign in and verify their account on the destination GitLab instance. If using - [SAML SSO for GitLab.com groups](../../group/saml_sso/index.md), users must - [link their SAML identity to their GitLab.com account](../../group/saml_sso/index.md#linking-saml-to-your-existing-gitlabcom-account). +1. Create the required users on the destination GitLab instance. You can create users with the API only on self-managed instances because it requires + administrator access. When migrating to GitLab.com or a self-managed GitLab instance you can: + - Create users manually. + - Set up or use your existing [SAML SSO provider](../saml_sso/index.md) and leverage user synchronization of SAML SSO groups supported through + [SCIM](../../group/saml_sso/scim_setup.md). You can + [bypass the GitLab user account verification with verified email domains](../saml_sso/index.md#bypass-user-email-confirmation-with-verified-domains). +1. Ensure that users have a [public email](../../profile/index.md#set-your-public-email) on the source GitLab instance that matches any confirmed email address on the destination GitLab instance. Most + users receive an email asking them to confirm their email address. +1. If users already exist on the destination instance and you use [SAML SSO for GitLab.com groups](../../group/saml_sso/index.md), all users must + [link their SAML identity to their GitLab.com account](../../group/saml_sso/index.md#link-saml-to-your-existing-gitlabcom-account). ### Connect the source GitLab instance @@ -125,7 +137,7 @@ Create the group you want to import to and connect the source GitLab instance: 1. Enter the [personal access token](../../../user/profile/personal_access_tokens.md) for your source GitLab instance. 1. Select **Connect instance**. -### Select the groups to import +### Select the groups and projects to import > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/385689) in GitLab 15.8, option to import groups with or without projects. @@ -135,12 +147,15 @@ role. 1. By default, the proposed group namespaces match the names as they exist in source instance, but based on your permissions, you can choose to edit these names before you proceed to import any of them. 1. Next to the groups you want to import, select either: - - **Import with projects**. Importing groups with projects is in [Beta](../../../policy/alpha-beta-support.md#beta-features). This feature is not ready for production use. + - **Import with projects**. - **Import without projects**. - - **Import** on self-managed GitLab, when the `bulk_import_projects` feature flag is disabled and the feature is not available. 1. The **Status** column shows the import status of each group. If you leave the page open, it updates in real-time. 1. After a group has been imported, select its GitLab path to open its GitLab URL. +WARNING: +Importing groups with projects is in [Beta](../../../policy/alpha-beta-support.md#beta). This feature is not +ready for production use. + ### Group import history You can view all groups migrated by you by direct transfer listed on the group import history page. This list includes: @@ -157,108 +172,162 @@ To view group import history: 1. On the top bar, select **Create new…** (**{plus-square}**). 1. Select **New group**. 1. Select **Import group**. -1. Select **History** in the upper right corner. +1. In the upper-right corner, select **History**. 1. If there are any errors for a particular import, you can see them by selecting **Details**. ### Migrated group items -The [`import_export.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/import_export/group/import_export.yml) -file for groups lists many of the items imported when migrating groups by direct transfer. View this file in the branch -for your version of GitLab to see the list of items relevant to you. For example, -[`import_export.yml` on the `14-10-stable-ee` branch](https://gitlab.com/gitlab-org/gitlab/-/blob/14-10-stable-ee/lib/gitlab/import_export/group/import_export.yml). +The group items that are migrated depend on the version of GitLab you use on the destination. To determine if a +specific group item is migrated: + +1. Check the [`groups/stage.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/bulk_imports/groups/stage.rb) + file for all editions and the + [`groups/stage.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/ee/bulk_imports/groups/stage.rb) file + for Enterprise Edition for your version on the destination. For example, for version 15.9: + - (all editions). + - (Enterprise + Edition). +1. Check the + [`group/import_export.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/import_export/group/import_export.yml) + file for groups for your version on the destination. For example, for version 15.9: + . + +Any other group items are **not** migrated. Group items that are migrated to the destination GitLab instance include: -- Badges ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/292431) in 13.11) -- Board Lists -- Boards -- Epics ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/250281) in 13.7) - - Epic resource state events ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/291983) in GitLab 15.4) -- Finisher -- Group Labels ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/292429) in 13.9) -- Iterations ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/292428) in 13.10) -- Iterations cadences ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/96570) in 15.4) -- Members ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/299415) in 13.9) - Group members are associated with the imported group if: - - The user already exists in the destination GitLab instance and - - The user has a public email in the source GitLab instance that matches a - confirmed email in the destination GitLab instance -- Milestones ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/292427) in 13.10) -- Namespace Settings -- Releases - - Milestones ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339422) in GitLab 15.0). -- Subgroups -- Uploads - -Any other items are **not** migrated. - -### Migrated project items (beta) +| Group item | Introduced in | +|:---------------------|:----------------------------------------------------------------------------| +| Badges | [GitLab 13.11](https://gitlab.com/gitlab-org/gitlab/-/issues/292431) | +| Boards | [GitLab 13.7](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18938) | +| Board lists | [GitLab 13.7](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24863) | +| Epics 1 | [GitLab 13.7](https://gitlab.com/gitlab-org/gitlab/-/issues/250281) | +| Group labels | [GitLab 13.9](https://gitlab.com/gitlab-org/gitlab/-/issues/292429) | +| Iterations | [GitLab 13.10](https://gitlab.com/gitlab-org/gitlab/-/issues/292428) | +| Iteration cadences | [GitLab 15.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/96570) | +| Members 2 | [GitLab 13.9](https://gitlab.com/gitlab-org/gitlab/-/issues/299415) | +| Group milestones | [GitLab 13.10](https://gitlab.com/gitlab-org/gitlab/-/issues/292427) | +| Namespace settings | [GitLab 14.10](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85128) | +| Release milestones | [GitLab 15.0](https://gitlab.com/gitlab-org/gitlab/-/issues/339422) | +| Subgroups | [GitLab 13.7](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18938) | +| Uploads | [GitLab 13.7](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/18938) | + +1. Epic resource state events [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/291983) in GitLab 15.4, label + associations [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62074) in GitLab 13.12, state and + state ID [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28203) in GitLab 13.7, and system note + metadata [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63551) in GitLab 14.0. +1. Group members are associated with the imported group if the user: + - Already exists in the destination GitLab instance. + - Has a public email in the source GitLab instance that matches a confirmed email in the destination GitLab instance. + +### Migrated project items (Beta) > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267945) in GitLab 14.4 [with a flag](../../feature_flags.md) named `bulk_import_projects`. Disabled by default. > - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/339941) in GitLab 15.6. +> - `bulk_import_projects` feature flag [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/339941) in GitLab 15.10. +> - Project-only migrations using API [added](https://gitlab.com/gitlab-org/gitlab/-/issues/390515) in GitLab 15.11. + +If you choose to migrate projects when you [select groups to migrate](#select-the-groups-and-projects-to-import), +project items are migrated with the projects. + +The project items that are migrated depends on the version of GitLab you use on the destination. To determine if a +specific project item is migrated: -FLAG: -On self-managed GitLab, migrating project resources when migrating groups is not available by default. -To make it available ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named -`bulk_import_projects`. On GitLab.com, groups are migrated with all their projects by default. +1. Check the [`projects/stage.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/bulk_imports/projects/stage.rb) + file for all editions and the + [`projects/stage.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/ee/bulk_imports/projects/stage.rb) + file for Enterprise Edition for your version on the destination. For example, for version 15.9: + - (all editions). + - (Enterprise + Edition). +1. Check the + [`project/import_export.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/import_export/project/import_export.yml) + file for projects for your version on the destination. For example, for version 15.9: + . -The [`import_export.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/import_export/project/import_export.yml) -file for projects lists many of the items imported when migrating projects using group migration. View this file in the branch -for your version of GitLab to see the list of items relevant to you. For example, -[`import_export.yml` on the `14-10-stable-ee` branch](https://gitlab.com/gitlab-org/gitlab/-/blob/14-10-stable-ee/lib/gitlab/import_export/project/import_export.yml). +Any other project items are **not** migrated. + +If you choose not to migrate projects along with groups or if you want to retry a project migration, you can +initiate project-only migrations using the [API](../../../api/bulk_imports.md). WARNING: -Migrating projects when migrating groups by direct transfer is in [Beta](../../../policy/alpha-beta-support.md#beta-features) +Migrating projects when migrating groups by direct transfer is in [Beta](../../../policy/alpha-beta-support.md#beta) and is not ready for production use. Project items that are migrated to the destination GitLab instance include: -- Projects ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267945) in GitLab 14.4) - - Auto DevOps ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339410) in GitLab 14.6) - - Badges ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75029) in GitLab 14.6) - - Branches (including protected branches) ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339414) in GitLab 14.7) - - CI Pipelines ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339407) in GitLab 14.6) - - Designs ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339421) in GitLab 15.1) - - Issues ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267946) in GitLab 14.4) - - Issue iteration ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/96184) in 15.4) - - Issue resource state events ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/291983) in GitLab 15.4) - - Issue resource milestone events ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/291983) in GitLab 15.4) - - Issue resource iteration events ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/291983) in GitLab 15.4) - - Merge request URL references ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267947) in GitLab 15.6) - - Time Tracking ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267946) in GitLab 14.4) - - Issue boards ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71661) in GitLab 14.4) - - Labels ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339419) in GitLab 14.4) - - LFS Objects ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339405) in GitLab 14.8) - - Members ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/341886) in GitLab 14.8) - - Merge Requests ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339403) in GitLab 14.5) - - Multiple merge request assignees ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339520) in GitLab 15.3) - - Merge request reviewers ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339520) in GitLab 15.3) - - Merge request approvers ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339520) in GitLab 15.3) - - Merge request resource state events ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/291983) in GitLab 15.4) - - Merge request resource milestone events ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/291983) in GitLab 15.4) - - Issue URL references ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267947) in GitLab 15.6) - - Time Tracking ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339403) in GitLab 14.5) - - Push Rules ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339403) in GitLab 14.6) - - Milestones ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339417) in GitLab 14.5) - - External Pull Requests ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339409) in GitLab 14.5) - - Pipeline History ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339412) in GitLab 14.6) - - Pipeline Schedules ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339408) in GitLab 14.8) - - Project Features ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74722) in GitLab 14.6) - - Releases ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339422) in GitLab 15.1) - - Release Evidences ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/360567) in GitLab 15.1) - - Repositories ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267945) in GitLab 14.4) - - Snippets ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/343438) in GitLab 14.6) - - Settings ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339416) in GitLab 14.6) - - Avatar ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75249) in GitLab 14.6) - - Container Expiration Policy ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75653) in GitLab 14.6) - - Project Properties ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75898) in GitLab 14.6) - - Service Desk ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75653) in GitLab 14.6) - - Uploads ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/339401) in GitLab 14.5) - - Wikis ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345923) in GitLab 14.6) - -Items excluded from migration, because they contain sensitive information: - -- Pipeline Triggers. +| Project item | Introduced in | +|:----------------------------------------|:---------------------------------------------------------------------------| +| Projects | [GitLab 14.4](https://gitlab.com/gitlab-org/gitlab/-/issues/267945) | +| Auto DevOps | [GitLab 14.6](https://gitlab.com/gitlab-org/gitlab/-/issues/339410) | +| Badges | [GitLab 14.6](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75029) | +| Branches (including protected branches) | [GitLab 14.7](https://gitlab.com/gitlab-org/gitlab/-/issues/339414) | +| CI Pipelines | [GitLab 14.6](https://gitlab.com/gitlab-org/gitlab/-/issues/339407) | +| Commit comments | [GitLab 15.10](https://gitlab.com/gitlab-org/gitlab/-/issues/391601) | +| Designs | [GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/issues/339421) | +| Issues | [GitLab 14.4](https://gitlab.com/gitlab-org/gitlab/-/issues/267946) | +| Issue boards | [GitLab 14.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71661) | +| Labels | [GitLab 14.4](https://gitlab.com/gitlab-org/gitlab/-/issues/339419) | +| LFS Objects | [GitLab 14.8](https://gitlab.com/gitlab-org/gitlab/-/issues/339405) | +| Members | [GitLab 14.8](https://gitlab.com/gitlab-org/gitlab/-/issues/341886) | +| Merge requests | [GitLab 14.5](https://gitlab.com/gitlab-org/gitlab/-/issues/339403) | +| Push rules | [GitLab 14.6](https://gitlab.com/gitlab-org/gitlab/-/issues/339403) | +| Milestones | [GitLab 14.5](https://gitlab.com/gitlab-org/gitlab/-/issues/339417) | +| External pull requests | [GitLab 14.5](https://gitlab.com/gitlab-org/gitlab/-/issues/339409) | +| Pipeline history | [GitLab 14.6](https://gitlab.com/gitlab-org/gitlab/-/issues/339412) | +| Pipeline schedules | [GitLab 14.8](https://gitlab.com/gitlab-org/gitlab/-/issues/339408) | +| Project features | [GitLab 14.6](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74722) | +| Releases | [GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/issues/339422) | +| Release evidences | [GitLab 15.1](https://gitlab.com/gitlab-org/gitlab/-/issues/360567) | +| Repositories | [GitLab 14.4](https://gitlab.com/gitlab-org/gitlab/-/issues/267945) | +| Snippets | [GitLab 14.6](https://gitlab.com/gitlab-org/gitlab/-/issues/343438) | +| Settings | [GitLab 14.6](https://gitlab.com/gitlab-org/gitlab/-/issues/339416) | +| Uploads | [GitLab 14.5](https://gitlab.com/gitlab-org/gitlab/-/issues/339401) | +| Wikis | [GitLab 14.6](https://gitlab.com/gitlab-org/gitlab/-/issues/345923) | + +#### Issue-related items + +Issue-related project items that are migrated to the destination GitLab instance include: + +| Issue-related project item | Introduced in | +|:--------------------------------|:---------------------------------------------------------------------------| +| Issue iterations | [GitLab 15.4](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/96184) | +| Issue resource state events | [GitLab 15.4](https://gitlab.com/gitlab-org/gitlab/-/issues/291983) | +| Issue resource milestone events | [GitLab 15.4](https://gitlab.com/gitlab-org/gitlab/-/issues/291983) | +| Issue resource iteration events | [GitLab 15.4](https://gitlab.com/gitlab-org/gitlab/-/issues/291983) | +| Merge request URL references | [GitLab 15.6](https://gitlab.com/gitlab-org/gitlab/-/issues/267947) | +| Time tracking | [GitLab 14.4](https://gitlab.com/gitlab-org/gitlab/-/issues/267946) | + +#### Merge request-related items + +Merge request-related project items that are migrated to the destination GitLab instance include: + +| Merge request-related project item | Introduced in | +|:----------------------------------------|:--------------------------------------------------------------------| +| Multiple merge request assignees | [GitLab 15.3](https://gitlab.com/gitlab-org/gitlab/-/issues/339520) | +| Merge request reviewers | [GitLab 15.3](https://gitlab.com/gitlab-org/gitlab/-/issues/339520) | +| Merge request approvers | [GitLab 15.3](https://gitlab.com/gitlab-org/gitlab/-/issues/339520) | +| Merge request resource state events | [GitLab 15.4](https://gitlab.com/gitlab-org/gitlab/-/issues/291983) | +| Merge request resource milestone events | [GitLab 15.4](https://gitlab.com/gitlab-org/gitlab/-/issues/291983) | +| Issue URL references | [GitLab 15.6](https://gitlab.com/gitlab-org/gitlab/-/issues/267947) | +| Time tracking | [GitLab 14.5](https://gitlab.com/gitlab-org/gitlab/-/issues/339403) | + +#### Setting-related items + +Setting-related project items that are migrated to the destination GitLab instance include: + +| Setting-related project item | Introduced in | +|:-----------------------------|:---------------------------------------------------------------------------| +| Avatar | [GitLab 14.6](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75249) | +| Container expiration policy | [GitLab 14.6](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75653) | +| Project properties | [GitLab 14.6](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75898) | +| Service Desk | [GitLab 14.6](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75653) | + +#### Excluded items + +Project items excluded from migration because they contain sensitive information: + +- Pipeline triggers. ### Troubleshooting @@ -283,7 +352,7 @@ entities.where(status: [-1]).pluck(:destination_name, :destination_namespace, :s ``` You can also see all migrated entities with any failures related to them using an -[API endpoint](../../../api/bulk_imports.md#list-all-group-migrations-entities). +[API endpoint](../../../api/bulk_imports.md#list-all-group-or-project-migrations-entities). #### Stale imports @@ -316,6 +385,18 @@ To solve this, you must change the source group path to include a non-numerical - The [Groups API](../../../api/groups.md#update-group). +#### Other `404` errors + +You can receive other `404` errors when importing a group, for example: + +```json +"exception_message": "Unsuccessful response 404 from [FILTERED] Bo...", +"exception_class": "BulkImports::NetworkError", +``` + +This error indicates a problem transferring from the _source_ instance. To solve this, check that you have met the [prerequisites](#prerequisites) on the source +instance. + ## Migrate groups by uploading an export file (deprecated) > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2888) in GitLab 13.0 as an experimental feature. May change in future releases. @@ -323,8 +404,8 @@ To solve this, you must change the source group path to include a non-numerical WARNING: This feature was [deprecated](https://gitlab.com/groups/gitlab-org/-/epics/4619) in GitLab 14.6 and replaced by -[migrating groups by direct transfer](#migrate-groups-by-direct-transfer-recommended). To follow progress on a solution for -[offline environments](../../application_security/offline_deployments/index.md), see +[migrating groups by direct transfer](#migrate-groups-by-direct-transfer-recommended). However, this feature is still recommended for migrating groups between +offline systems. To follow progress on an alternative solution for [offline environments](../../application_security/offline_deployments/index.md), see [the relevant epic](https://gitlab.com/groups/gitlab-org/-/epics/8985). Prerequisites: @@ -356,13 +437,11 @@ Note the following: ### Compatibility -Group file exports are in NDJSON format. GitLab previously produced group file exports in JSON format, however: +> Support for JSON-formatted project file exports [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/383682) in GitLab 15.8. -- From GitLab 15.8, GitLab no longer supports importing a JSON-formatted group file export. -- Between GitLab 14.0 and GitLab 14.7, GitLab no longer produces group file exports in JSON format but, to support - transitions, can still import JSON-formatted group file exports. +Group file exports are in NDJSON format. -From GitLab 13.0, GitLab can import group file exports that were exported from a version of GitLab up to two +You can import group file exports that were exported from a version of GitLab up to two [minor](../../../policy/maintenance.md#versioning) versions behind, which is similar to our process for [security releases](../../../policy/maintenance.md#security-releases). @@ -377,7 +456,7 @@ For example: The [`import_export.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/import_export/group/import_export.yml) file for groups lists items exported and imported when migrating groups using file exports. View this file in the branch -for your version of GitLab to see the list of items relevant to you. For example, +for your version of GitLab to check which items can be imported to the destination GitLab instance. For example, [`import_export.yml` on the `14-10-stable-ee` branch](https://gitlab.com/gitlab-org/gitlab/-/blob/14-10-stable-ee/lib/gitlab/import_export/group/import_export.yml). Group items that are exported include: @@ -404,7 +483,7 @@ Items that are **not** exported include: - To preserve the member list and their respective permissions on imported groups, review the users in these groups. Make sure these users exist before importing the desired groups. -- Users must set a public email in the source GitLab instance that matches one of their verified emails in the target GitLab instance. +- Users must set a public email in the source GitLab instance that matches their confirmed primary email in the destination GitLab instance. Most users receive an email asking them to confirm their email address. ### Enable export for a group diff --git a/doc/user/group/index.md b/doc/user/group/index.md index db01358d899..7e093ccb01b 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -1,6 +1,6 @@ --- -stage: Manage -group: Organization +stage: Data Stores +group: Tenant Scale info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -20,6 +20,16 @@ For larger organizations, you can also create [subgroups](subgroups/index.md). For more information about creating and managing your groups, see [Manage groups](manage.md). +NOTE: +Self-managed customers should create a top-level group so you can see an overview of +your organization. For more information about efforts to create an +organization view of all groups, [see epic 9266](https://gitlab.com/groups/gitlab-org/-/epics/9266). +A top-level group can also have one complete +[Security Dashboard and Center](../application_security/security_dashboard/index.md), +[Vulnerability](../application_security/vulnerability_report/index.md#vulnerability-report) and +[Compliance Report](../compliance/compliance_report/index.md), and +[Value stream analytics](../group/value_stream_analytics/index.md). + ## Group visibility Like projects, a group can be configured to limit the visibility of it to: diff --git a/doc/user/group/insights/index.md b/doc/user/group/insights/index.md index 9eb6d4387c1..17aa6cb9655 100644 --- a/doc/user/group/insights/index.md +++ b/doc/user/group/insights/index.md @@ -67,7 +67,7 @@ To configure group insights: in a project that belongs to your group. 1. On the top bar, select **Main menu > Groups** and find your group. 1. On the left sidebar, select **Settings > General**. -1. Expand **Insights**. +1. Expand the **Analytics** tab and find the **Insights** section. 1. Select the project that contains your `.gitlab/insights.yml` configuration file. 1. Select **Save changes**. diff --git a/doc/user/group/iterations/index.md b/doc/user/group/iterations/index.md index 72d3bf65447..9b246e6ad47 100644 --- a/doc/user/group/iterations/index.md +++ b/doc/user/group/iterations/index.md @@ -62,6 +62,8 @@ To create an iteration cadence: - From the **Upcoming iterations** dropdown list, select how many upcoming iterations should be created and maintained by GitLab. - Optional. To move incomplete issues to the next iteration, select **Roll over issues**. + At the end of the current iteration, all open issues are added to the next iteration. + Issues are moved at midnight in the instance time zone (UTC by default). Administrators can change the instance time zone. 1. Select **Create cadence**. The cadence list page opens. If you want to manually manage the created cadence, read [Manual Iteration Management](#manual-iteration-management). diff --git a/doc/user/group/manage.md b/doc/user/group/manage.md index d21dbe357da..7c2c2eaa211 100644 --- a/doc/user/group/manage.md +++ b/doc/user/group/manage.md @@ -1,6 +1,6 @@ --- -stage: Manage -group: Organization +stage: Data Stores +group: Tenant Scale info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -8,6 +8,16 @@ info: To determine the technical writer assigned to the Stage/Group associated w Use groups to manage one or more related projects at the same time. +NOTE: +Self-managed customers should create a top-level group so you can see an overview of +your organization. For more information about efforts to create an +organization view of all groups, [see epic 9266](https://gitlab.com/groups/gitlab-org/-/epics/9266). +A top-level group can also have one complete +[Security Dashboard and Center](../application_security/security_dashboard/index.md), +[Vulnerability](../application_security/vulnerability_report/index.md#vulnerability-report) and +[Compliance Report](../compliance/compliance_report/index.md), and +[Value stream analytics](../group/value_stream_analytics/index.md). + ## View groups To view groups, on the top bar, select **Main menu > Groups > View all groups**. @@ -65,10 +75,10 @@ This action removes the group. It also adds a background job to delete all proje Specifically: -- In [GitLab 12.8 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/33257), on [GitLab Premium](https://about.gitlab.com/pricing/premium/) or higher tiers, this action adds a background job to mark a group for deletion. By default, the job schedules the deletion 7 days in the future. You can modify this waiting period through the [instance settings](../admin_area/settings/visibility_and_access_controls.md#deletion-protection). +- In [GitLab 12.8 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/33257), on [GitLab Premium or Ultimate tiers](https://about.gitlab.com/pricing/premium/), this action adds a background job to mark a group for deletion. By default, the job schedules the deletion 7 days in the future. You can modify this waiting period through the [instance settings](../admin_area/settings/visibility_and_access_controls.md#deletion-protection). - In [GitLab 13.6 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/39504), if the user who sets up the deletion is removed from the group before the -deletion happens, the job is cancelled, and the group is no longer scheduled for deletion. + deletion happens, the job is cancelled, and the group is no longer scheduled for deletion. ## Remove a group immediately **(PREMIUM)** @@ -155,7 +165,7 @@ You can sort members by **Account**, **Access granted**, **Max role**, or **Last 1. On the top bar, select **Main menu > Groups** and find your group. 1. On the left sidebar, select **Group information > Members**. -1. Above the list of members, in the upper right, from the **Account** list, select +1. Above the list of members, in the upper-right corner, from the **Account** list, select the criteria to filter by. 1. To switch the sort between ascending and descending, to the right of the **Account** list, select the arrow (**{sort-lowest}** or **{sort-highest}**). @@ -237,6 +247,23 @@ To change this setting for a specific group: To change this setting globally, see [Default project creation protection](../admin_area/settings/visibility_and_access_controls.md#define-which-roles-can-create-projects). +## Add Group README + +As a group owner or member, you can use a README to provide more information about your team, and invite users to contribute to your projects. +The README is displayed on the group overview page, and can be changed in the group settings. All group members can edit the README. + +Prerequisite: + +- To create the README from the group settings, you must have the Owner role for the group. + +To add a group README: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Settings > General**. +1. In the **Group README** section, select **Add README**. This action creates a new project `gitlab-profile` that contains the `README.md` file. +1. On the prompt for creating a README, select **Create and add README**. You're redirected to the Web IDE, where a README file is created. +1. In the Web IDE, edit and commit the `README.md` file. + ## Change the owner of a group You can change the owner of a group. Each group must always have at least one @@ -297,7 +324,7 @@ To change this setting for a specific group, see [group level default branch pro To change this setting globally, see [initial default branch protection](../project/repository/branches/default.md#instance-level-default-branch-protection). NOTE: -In [GitLab Premium or higher](https://about.gitlab.com/pricing/), GitLab administrators can choose to [disable group owners from updating the default branch protection](../project/repository/branches/default.md#prevent-overrides-of-default-branch-protection). +In [GitLab Premium or Ultimate](https://about.gitlab.com/pricing/), GitLab administrators can choose to [disable group owners from updating the default branch protection](../project/repository/branches/default.md#prevent-overrides-of-default-branch-protection). ## Use a custom name for the initial branch @@ -348,11 +375,14 @@ If you need to copy a group to a different GitLab instance, When transferring groups, note: - Changing a group's parent can have unintended side effects. See [what happens when a repository path changes](../project/repository/index.md#what-happens-when-a-repository-path-changes). -- You can only transfer groups to groups you manage. +- You must have the Owner role in the source and target group. - You must update your local repositories to point to the new location. - If the immediate parent group's visibility is lower than the group's current visibility, visibility levels for subgroups and projects change to match the new parent group's visibility. - Only explicit group membership is transferred, not inherited membership. If the group's owners have only inherited membership, this leaves the group without an owner. In this case, the user transferring the group becomes the group's owner. -- Transfers fail if [packages](../packages/index.md) exist in any of the projects in the group, or in any of its subgroups. +- Transfers fail if [npm packages](../packages/npm_registry/index.md) exist in any of the projects in the group, or in any of its subgroups. +- Existing packages that use a group-level endpoint (Maven, NuGet, PyPI, Composer, and Debian) need to be updated per the package's steps for setting up the group level endpoint. +- Existing package names need to be updated if the package uses an instance level endpoint ([Maven](../packages/maven_repository/index.md#naming-convention), [npm](../packages/npm_registry/index.md#naming-convention), [Conan](../packages/conan_repository/index.md#package-recipe-naming-convention-for-instance-remotes)) and the group was moved to another root level namespace. +- [Maven packages](../packages/maven_repository/index.md#naming-convention) follow a naming convention that prevent installing or publishing the respective package from a group level endpoint after group transfer. - Top-level groups that have a subscription on GitLab.com cannot be transferred. To make the transfer possible, the top-level group's subscription must be removed first. Then the top-level group can be transferred as a subgroup to another top-level group. To transfer a group: @@ -371,6 +401,9 @@ To transfer a group: > - [Instance setting to enable by default added](https://gitlab.com/gitlab-org/gitlab/-/issues/255449) in GitLab 14.2. > - [Instance setting is inherited and enforced when disabled](https://gitlab.com/gitlab-org/gitlab/-/issues/352960) in GitLab 15.1. > - [User interface changed](https://gitlab.com/gitlab-org/gitlab/-/issues/352961) in GitLab 15.1. +> - [Removed](https://gitlab.com/gitlab-org/gitlab/-/issues/389557) in GitLab 15.11 [with a flag](../../administration/feature_flags.md) named `always_perform_delayed_deletion`. Disabled by default. +> - Enabled delayed deletion by default and removed the option to delete immediately [on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/393622) on May 08, 2023. +> - Enabled delayed deletion by default and removed the option to delete immediately [on self-managed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/119606) in GitLab 16.0. [Delayed project deletion](../project/settings/index.md#delayed-project-deletion) is locked and disabled unless the instance-level settings for [deletion protection](../admin_area/settings/visibility_and_access_controls.md#deletion-protection) are enabled for either groups only or groups and projects. @@ -398,8 +431,10 @@ To enable delayed deletion of projects in a group: - (GitLab 15.0 and earlier), **Enforce for all subgroups**. 1. Select **Save changes**. -NOTE: -In GitLab 13.11 and above the group setting for delayed project deletion is inherited by subgroups. As discussed in [Cascading settings](../../development/cascading_settings.md) inheritance can be overridden, unless enforced by an ancestor. +In GitLab 13.11 and later, the group setting for delayed project deletion is inherited by subgroups. As discussed in [Cascading settings](../../development/cascading_settings.md), inheritance can be overridden unless enforced by an ancestor. + +In GitLab 15.11 with the `always_perform_delayed_deletion` feature flag enabled, this setting is removed +and all projects are deleted after the [retention period defined by the admin](../admin_area/settings/visibility_and_access_controls.md#retention-period). This will be the default behavior in GitLab 16.0 and later. ## Disable email notifications @@ -447,6 +482,10 @@ You can export a list of members in a group or subgroup as a CSV. 1. Select **Export as CSV**. 1. After the CSV file has been generated, it is emailed as an attachment to the user that requested it. +The output lists direct members and members inherited from the ancestor groups. +For members with `Minimal Access` in the selected group, their `Max Role` and `Source` are derived from their membership in subgroups. +[Issue 390358](https://gitlab.com/gitlab-org/gitlab/-/issues/390358) tracks the discussion about the group members CSV export list not matching the UI members list. + ## User cap for groups > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/330027) in GitLab 14.7. @@ -465,7 +504,7 @@ disabled for the group and its subgroups. Prerequisite: -- You must be assigned the Owner role) for the group. +- You must be assigned the Owner role for the group. To specify a user cap: @@ -640,9 +679,78 @@ To view the merge request approval settings for a group: Support for group-level settings for merge request approval rules is tracked in this [epic](https://gitlab.com/groups/gitlab-org/-/epics/4367). +## Group Code Suggestions **(FREE SAAS)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/405126) in GitLab 15.11. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/408158) from GitLab Ultimate to GitLab Premium in 16.0. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/410801) from GitLab Premium to GitLab Free in 16.0. + +WARNING: +This feature is in [Beta](../../policy/alpha-beta-support.md#beta). +Code Suggestions use generative AI to suggest code while you're developing. +Due to high demand, this feature will have unscheduled downtime and code suggestions in VS Code may be delayed. +Code Suggestions may produce +[low-quality or incomplete suggestions](../project/repository/code_suggestions.md#model-accuracy-and-quality). +Beta users should read about the [known limitations](../project/repository/code_suggestions.md#known-limitations). +We look forward to hearing your feedback. + +This setting enables users in the group to access [Code Suggestions](../project/repository/code_suggestions.md). +This setting [cascades to all projects](../project/merge_requests/approvals/settings.md#settings-cascading) +that belong to the group. + +To enable Code Suggestions for a group: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Settings > General**. +1. Expand the **Permissions and group features** section. +1. Find the **Code Suggestions** settings. +1. Select **Save changes**. + +## Group Experiment features setting **(ULTIMATE SAAS)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/404856) in GitLab 16.0. + +You can give all users in the group access to Experiment features. + +WARNING: +[Experiment features](../../policy/alpha-beta-support.md#experiment) may produce unexpected results +(for example, the results might be low-quality, incomplete, incoherent, offensive, or insensitive, +and might include insecure code or failed pipelines). + +This setting [cascades to all projects](../project/merge_requests/approvals/settings.md#settings-cascading) +that belong to the group. + +To enable Experiment features for a group: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Settings > General**. +1. Expand the **Permissions and group features** section. +1. Find the **Experiment features** settings. +1. Select **Save changes**. + +## Group third-party AI features setting **(ULTIMATE SAAS)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/404856) in GitLab 16.0. + +WARNING: +These Artifical Intelligence (AI) features use [third-party services](../ai_features.md#data-usage) +and require transmission of data, including personal data. + +You can give all users in the group access to third-party AI features. +This setting [cascades to all projects](../project/merge_requests/approvals/settings.md#settings-cascading) +that belong to the group. + +To enable third-party AI features for a group: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Settings > General**. +1. Expand the **Permissions and group features** section. +1. Find the **Third-party Artificial Intelligence (AI) features** settings. +1. Select **Save changes**. + ## Group activity analytics **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207164) in GitLab 12.10 as a [Beta feature](../../policy/alpha-beta-support.md#beta-features). +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207164) in GitLab 12.10 as a [Beta feature](../../policy/alpha-beta-support.md#beta). For a group, you can view how many merge requests, issues, and members were created in the last 90 days. @@ -745,3 +853,16 @@ Administrators can find a user's maximum permissions for a group or project. group = Group.find_by_full_path 'group' user.max_member_access_for_group group.id ``` + +### Unable to remove billable members with badge `Project Invite/Group Invite` + +```plaintext +Members who were invited via a group invitation cannot be removed. You can either remove the entire group, or ask an Owner of the invited group to remove the member. +``` + +This error typically occurs when the user you're trying to remove is part of an external group that has been [shared with one or more of your projects](../project/members/share_project_with_groups.md) or [groups](#share-a-group-with-another-group). To remove the user as a billable member, follow one of the options: + +- Remove the invited group membership from your project or group members page. +- Recommended. Remove the user directly from the invited group, if you have access to the group. + +The feature request to **Update billable_members endpoint to include invited group** is currently being worked on. For more information, see [issue 386583](https://gitlab.com/gitlab-org/gitlab/-/issues/386583) diff --git a/doc/user/group/moderate_users.md b/doc/user/group/moderate_users.md new file mode 100644 index 00000000000..4ec86893524 --- /dev/null +++ b/doc/user/group/moderate_users.md @@ -0,0 +1,48 @@ +--- +stage: Anti-Abuse +group: Anti-Abuse +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Moderate users **(FREE)** + +> [Introduced](https://gitlab.com/gitlab-org/modelops/anti-abuse/team-tasks/-/issues/155) in GitLab 15.8. + +This is the group-level documentation. For self-managed instances, see the [administration documentation](../admin_area/moderate_users.md). + +A group Owner can moderate user access by banning and unbanning users. + +## Unban a user + +To unban a user with the GraphQL API, see [`Mutation.namespaceBanDestroy`](../../api/graphql/reference/index.md#mutationnamespacebandestroy). + + +For a demo on unbanning a user at the group level, see [Namespace level ban - Unbanning a user](https://www.youtube.com/watch?v=mTQVbP3MQrs). + +Prerequisites: + +- In the top-level group, you must have the Owner role. + +To unban a user: + +1. Go to the top-level group. +1. On the left sidebar, select **Group information > Members**. +1. Select the **Banned** tab. +1. For the account you want to unban, select **Unban**. + +## Ban a user + + +For a demo on banning a user at the group level, see [Namespace level ban - Banning a user](https://youtu.be/1rbi1uEJmOI). + +Prerequisites: + +- In the top-level group, you must have the Owner role. +- In the top-level group, if the user you want to ban has the Owner role, you must [demote the user](manage.md#change-the-owner-of-a-group). + +To manually ban a user: + +1. Go to the top-level group. +1. On the left sidebar, select **Group information > Members**. +1. Next to the member you want to ban, select the vertical ellipsis (**{ellipsis_v}**). +1. From the dropdown list, select **Ban member**. diff --git a/doc/user/group/reporting/git_abuse_rate_limit.md b/doc/user/group/reporting/git_abuse_rate_limit.md index 14b188e1204..d5c44f4e245 100644 --- a/doc/user/group/reporting/git_abuse_rate_limit.md +++ b/doc/user/group/reporting/git_abuse_rate_limit.md @@ -11,13 +11,19 @@ info: To determine the technical writer assigned to the Stage/Group associated w FLAG: On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `limit_unique_project_downloads_per_namespace_user`. On GitLab.com, this feature is available. -Git abuse rate limiting is a feature to automatically ban users who download or clone more than a specified number of repositories in a group or any of its subgroups within a given time frame. Banned users cannot access the main group or any of its non-public subgroups via HTTP or SSH. Access to unrelated groups is unaffected. +This is the group-level documentation. For self-managed instances, see the [administration documentation](../../admin_area/reporting/git_abuse_rate_limit.md). -If the `limit_unique_project_downloads_per_namespace_user` feature flag is enabled, all users with the Owner role for the main group receive an email when a user is about to be banned. +Git abuse rate limiting is a feature to automatically ban users who download, clone, or fork more than a specified number of repositories of a group in a given time frame. Banned users cannot access the top-level group or any of its non-public subgroups via HTTP or SSH. The rate limit also applies to users who authenticate with a [personal](../../../user/profile/personal_access_tokens.md) or [group access token](../../../user/group/settings/group_access_tokens.md). Access to unrelated groups is unaffected. -If automatic banning is disabled, a user is not banned automatically when they exceed the limit. However, users with the Owner role for the main group are still notified. You can use this setup to determine the correct values of the rate limit settings before enabling automatic banning. +Git abuse rate limiting does not apply to top-level group owners, [deploy tokens](../../../user/project/deploy_tokens/index.md), or [deploy keys](../../../user/project/deploy_keys/index.md). -If automatic banning is enabled, users with the Owner role for the main group receive an email when a user is about to be banned, and the user is automatically banned from the group and its subgroups. +## Automatic ban notifications + +If the `limit_unique_project_downloads_per_namespace_user` feature flag is enabled, selected users receive an email when a user is about to be banned. + +If automatic banning is disabled, a user is not banned automatically when they exceed the limit. However, notifications are still sent. You can use this setup to determine the correct values of the rate limit settings before enabling automatic banning. + +If automatic banning is enabled, an email notification is sent when a user is about to be banned, and the user is automatically banned from the group and its subgroups. ## Configure Git abuse rate limiting @@ -26,29 +32,10 @@ If automatic banning is enabled, users with the Owner role for the main group re 1. Enter a number in the **Number of repositories** field, greater than or equal to `0` and less than or equal to `10,000`. This number specifies the maximum amount of unique repositories a user can download in the specified time period before they're banned. When set to `0`, Git abuse rate limiting is disabled. 1. Enter a number in the **Reporting time period (seconds)** field, greater than or equal to `0` and less than or equal to `86,400` (10 days). This number specifies the time in seconds a user can download the maximum amount of repositories before they're banned. When set to `0`, Git abuse rate limiting is disabled. 1. Optional. Exclude up to `100` users by adding them to the **Excluded users** field. Excluded users are not automatically banned. + 1. Add up to `100` users to the **Send notifications to** field. You must select at least one user. All users with the Owner role for the main group are selected by default. 1. Optional. Turn on the **Automatically ban users from this namespace when they exceed the specified limits** toggle to enable automatic banning. 1. Select **Save changes**. -## Unban a user - -Prerequisites: - -- You must have the Owner role. - -1. On the left sidebar, select **Group information > Members**. -1. Select the **Banned** tab. -1. For the account you want to unban, select **Unban**. - -## Ban a user - -> [Introduced](https://gitlab.com/gitlab-org/modelops/anti-abuse/team-tasks/-/issues/155) in GitLab 15.8. - -Prerequisites: - -- You must have the Owner role. - -To manually ban a user: +## Related topics -1. On the left sidebar, select **Group information > Members**. -1. Next to the member you want to ban, select the vertical ellipsis (**{ellipsis_v}**). -1. From the dropdown list, select **Ban member**. +- [Ban and unban users](../moderate_users.md). diff --git a/doc/user/group/repositories_analytics/index.md b/doc/user/group/repositories_analytics/index.md index 9971457f2ac..c2a4d8175b3 100644 --- a/doc/user/group/repositories_analytics/index.md +++ b/doc/user/group/repositories_analytics/index.md @@ -1,6 +1,6 @@ --- stage: Verify -group: Pipeline Insights +group: Pipeline Execution info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -42,7 +42,7 @@ To see the latest code coverage for each project in your group: 1. In the **Latest test coverage results** section, from the **Select projects** dropdown list, choose the projects you want to check. You can download code coverage data for specific projects using -[code coverage history](../../../ci/pipelines/settings.md#view-code-coverage-history). +[code coverage history](../../../ci/testing/code_coverage.md#view-history-of-project-code-coverage). ## Download historic test coverage data diff --git a/doc/user/group/saml_sso/example_saml_config.md b/doc/user/group/saml_sso/example_saml_config.md index 7778648e985..524a5d5a9bd 100644 --- a/doc/user/group/saml_sso/example_saml_config.md +++ b/doc/user/group/saml_sso/example_saml_config.md @@ -58,6 +58,8 @@ Attribute mapping: NOTE: Using the **Group ID** source attribute requires users to enter the group ID or object ID when configuring SAML group links. If available, use the **sAMAccountName** source attribute for the friendly group name instead. +[Azure AD limits the number of groups that can be sent in a SAML response to 150](https://support.esri.com/en-us/knowledge-base/000022190'). If a user is a member of more than 150 groups, Azure does not include that user's group claim in the SAML response. + ## Google Workspace Basic SAML app configuration: diff --git a/doc/user/group/saml_sso/group_sync.md b/doc/user/group/saml_sso/group_sync.md index 27482893bd6..ee59eeb98db 100644 --- a/doc/user/group/saml_sso/group_sync.md +++ b/doc/user/group/saml_sso/group_sync.md @@ -22,10 +22,7 @@ For a demo of Group Sync using Azure, see [Demo: SAML Group Sync](https://youtu. ## Configure SAML Group Sync NOTE: -You must include the SAML configuration block on all Sidekiq nodes in addition to Rails application nodes if you: - -- Use SAML Group Sync. -- Have multiple GitLab nodes, for example in a distributed or highly available architecture. +You must include the SAML configuration block on all Sidekiq nodes in addition to Rails application nodes if you use SAML Group Sync and have multiple GitLab nodes, for example in a distributed or highly available architecture. NOTE: SAML Group Sync is only supported for the [SAML provider named `saml`](../../../integration/saml.md#configure-gitlab-to-use-multiple-saml-idps). @@ -107,11 +104,37 @@ Users granted: - A lower or the same role with Group Sync are displayed as having [inherited membership](../../project/members/index.md#display-inherited-members) of the group. +SAML group membership is evaluated each time a user signs in. + +### Global SAML group memberships lock **(PREMIUM SELF)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/386390) in GitLab 15.10. + +GitLab administrators can use the global SAML group memberships lock to prevent group members from inviting new members to subgroups that have their membership synchronized with SAML Group Links. + +Global group memberships lock only applies to subgroups of a top-level group where SAML Group Links synchronization is configured. No user can modify the +membership of a top-level group configured for SAML Group Links synchronization. + +When global group memberships lock is enabled: + +- Only an administrator can manage memberships of any group including access levels. +- Users cannot: + - Share a project with other groups. + - Invite members to a project created in a group. + +To enable global group memberships lock: + +1. [Configure SAML](../../../integration/saml.md) for your self-managed GitLab instance. +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Settings > General**. +1. Expand the **Visibility and access controls** section. +1. Ensure the **Lock memberships to SAML synchronization** checkbox is selected. + ### Automatic member removal After a group sync, users who are not members of a mapped SAML group are removed from the group. On GitLab.com, users in the top-level group are assigned the -[default membership role](index.md#role) instead of being removed. +default membership role instead of being removed. For example, in the following diagram: diff --git a/doc/user/group/saml_sso/img/Azure-manage-group-claims.png b/doc/user/group/saml_sso/img/Azure-manage-group-claims.png index 2ff24733282..b08b6ab4907 100644 Binary files a/doc/user/group/saml_sso/img/Azure-manage-group-claims.png and b/doc/user/group/saml_sso/img/Azure-manage-group-claims.png differ diff --git a/doc/user/group/saml_sso/img/group_saml_configuration_information.png b/doc/user/group/saml_sso/img/group_saml_configuration_information.png deleted file mode 100644 index e03c50ceb01..00000000000 Binary files a/doc/user/group/saml_sso/img/group_saml_configuration_information.png and /dev/null differ diff --git a/doc/user/group/saml_sso/img/group_saml_settings_v13_12.png b/doc/user/group/saml_sso/img/group_saml_settings_v13_12.png deleted file mode 100644 index 2271f281655..00000000000 Binary files a/doc/user/group/saml_sso/img/group_saml_settings_v13_12.png and /dev/null differ diff --git a/doc/user/group/saml_sso/index.md b/doc/user/group/saml_sso/index.md index e650b2dd130..a54b3fea53e 100644 --- a/doc/user/group/saml_sso/index.md +++ b/doc/user/group/saml_sso/index.md @@ -8,249 +8,80 @@ info: To determine the technical writer assigned to the Stage/Group associated w > Introduced in GitLab 11.0. -This page describes SAML for groups. For instance-wide SAML on self-managed GitLab instances, see [SAML SSO for self-managed GitLab instances](../../../integration/saml.md). -[View the differences between SaaS and Self-Managed Authentication and Authorization Options](../../../administration/auth/index.md#saas-vs-self-managed-comparison). +Users can sign in to GitLab through their SAML identity provider. -SAML on GitLab.com allows users to sign in through their SAML identity provider. If the user is not already a member, the sign-in process automatically adds the user to the appropriate group. +[SCIM](scim_setup.md) synchronizes users with the group on GitLab.com. -User synchronization of SAML SSO groups is supported through [SCIM](scim_setup.md). SCIM supports adding and removing users from the GitLab group automatically. -For example, if you remove a user from the SCIM app, SCIM removes that same user from the GitLab group. +- When you add or remove a user from the SCIM app, SCIM adds or removes the user + from the GitLab group. +- If the user is not already a group member, the user is added to the group as part of the sign-in process. -SAML SSO is only configurable at the top-level group. +You can configure SAML SSO for the top-level group only. -If required, you can find [a glossary of common terms](../../../integration/saml.md#glossary-of-common-terms). +## Set up your identity provider -## Configure your identity provider - -1. Find the information in GitLab required for configuration: - 1. On the top bar, select **Main menu > Groups** and find your group. - 1. On the left sidebar, select **Settings > SAML SSO**. - 1. Note the **Assertion consumer service URL**, **Identifier**, and **GitLab single sign-on URL**. -1. Configure your SAML identity provider app using the noted details. - Alternatively, GitLab provides a [metadata XML configuration](#metadata-configuration). - See [specific identity provider documentation](#providers) for more details. -1. Configure the SAML response to include a [NameID](#nameid) that uniquely identifies each user. -1. Configure the required [user attributes](#user-attributes), ensuring you include the user's email address. -1. While the default is enabled for most SAML providers, ensure the app is set to have service provider - initiated calls to link existing GitLab accounts. -1. Once the identity provider is set up, move on to [configuring GitLab](#configure-gitlab). - -![Issuer and callback for configuring SAML identity provider with GitLab.com](img/group_saml_configuration_information.png) - -If your account is the only owner in the group after SAML is set up, you can't unlink the account. To [unlink the account](#unlinking-accounts), -set up another user as a group owner. - -### NameID - -GitLab.com uses the SAML NameID to identify users. The NameID element: - -- Is a required field in the SAML response. -- Must be unique to each user. -- Must be a persistent value that never changes, such as a randomly generated unique user ID. -- Is case sensitive. The NameID must match exactly on subsequent login attempts, so should not rely on user input that could change between upper and lower case. -- Should not be an email address or username. We strongly recommend against these as it's hard to - guarantee it doesn't ever change, for example, when a person's name changes. Email addresses are - also case-insensitive, which can result in users being unable to sign in. - -The relevant field name and recommended value for supported providers are in the [provider specific notes](#providers). - -WARNING: -Once users have signed into GitLab using the SSO SAML setup, changing the `NameID` breaks the configuration and potentially locks users out of the GitLab group. - -#### NameID Format - -We recommend setting the NameID format to `Persistent` unless using a field (such as email) that requires a different format. -Most NameID formats can be used, except `Transient` due to the temporary nature of this format. - -### User attributes - -To create users with the correct information for improved [user access and management](#user-access-and-management), -the user's details must be passed to GitLab as attributes in the SAML assertion. At a minimum, the user's email address -must be specified as an attribute named `email` or `mail`. - -You can configure the following attributes with GitLab.com Group SAML: +The SAML standard means that you can use a wide range of identity providers with GitLab. Your identity provider might have relevant documentation. It can be generic SAML documentation or specifically targeted for GitLab. -- `username` or `nickname`. We recommend you configure only one of these. -- The [attributes available](../../../integration/saml.md#configure-assertions) to self-managed GitLab instances. +When setting up your identity provider, use the following provider-specific documentation +to help avoid common issues and as a guide for terminology used. -### Metadata configuration +For identity providers not listed, you can refer to the [instance SAML notes on configuring an identity provider](../../../integration/saml.md#configure-saml-on-your-idp) +for additional guidance on information your provider may require. -GitLab provides metadata XML that can be used to configure your identity provider. +GitLab provides the following information for guidance only. +If you have any questions on configuring the SAML app, contact your provider's support. -1. On the top bar, select **Main menu > Groups** and find your group. -1. On the left sidebar, select **Settings > SAML SSO**. -1. Copy the provided **GitLab metadata URL**. -1. Follow your identity provider's documentation and paste the metadata URL when it's requested. +If you are having issues setting up your identity provider, see the +[troubleshooting documentation](#troubleshooting). -## Configure GitLab +### Azure -After you set up your identity provider to work with GitLab, you must configure GitLab to use it for authentication: +To set up SSO with Azure as your identity provider: -1. On the top bar, select **Main menu > Groups** and find your group. +1. In GitLab, on the top bar, select **Main menu > Groups** and find your group. 1. On the left sidebar, select **Settings > SAML SSO**. -1. Find the SSO URL from your identity provider and enter it the **Identity provider single sign-on URL** field. -1. Find and enter the fingerprint for the SAML token signing certificate in the **Certificate** field. -1. Select the access level to be applied to newly added users in the **Default membership role** field. The default access level is 'Guest'. -1. Select the **Enable SAML authentication for this group** checkbox. -1. Select the **Save changes** button. - -![Group SAML Settings for GitLab.com](img/group_saml_settings_v13_12.png) +1. Note the information on this page. +1. Go to Azure and [follow the instructions for configuring SSO for an application](https://learn.microsoft.com/en-us/azure/active-directory/manage-apps/add-application-portal-setup-sso). The following GitLab settings correspond to the Azure fields. -NOTE: -The certificate [fingerprint algorithm](../../../integration/saml.md#configure-saml-on-your-idp) must be in SHA1. When configuring the identity provider (such as [Google Workspace](#set-up-google-workspace)), use a secure signature algorithm. + | GitLab setting | Azure field | + | -----------------------------------------| ---------------------------------------------- | + | **Identifier** | **Identifier (Entity ID)** | + | **Assertion consumer service URL** | **Reply URL (Assertion Consumer Service URL)** | + | **GitLab single sign-on URL** | **Sign on URL** | + | **Identity provider single sign-on URL** | **Login URL** | + | **Certificate fingerprint** | **Thumbprint** | -### Additional configuration information +1. You should set the following attributes: + - **Unique User Identifier (Name identifier)** to `user.objectID`. + - **nameid-format** to `persistent`. For more information, see how to [manage user SAML identity](#manage-user-saml-identity). + - **Additional claims** to [supported attributes](#user-attributes). -Many SAML terms can vary between providers. It is possible that the information you are looking for is listed under another name. +1. Make sure the identity provider is set to have provider-initiated calls + to link existing GitLab accounts. -For more information, start with your identity provider's documentation. Look for their options and examples to see how they configure SAML. This can provide hints on what you need to configure GitLab to work with these providers. - -It can also help to look at our [more detailed docs for self-managed GitLab](../../../integration/saml.md). -SAML configuration for GitLab.com is mostly the same as for self-managed instances. -However, self-managed GitLab instances use a configuration file that supports more options as described in the external [OmniAuth SAML documentation](https://github.com/omniauth/omniauth-saml/). -Internally that uses the [`ruby-saml` library](https://github.com/onelogin/ruby-saml), so we sometimes check there to verify low level details of less commonly used options. - -It can also help to compare the XML response from your provider with our [example XML used for internal testing](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/spec/fixtures/saml/response.xml). - -### SSO enforcement - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5291) in GitLab 11.8. -> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/9255) in GitLab 11.11 with ongoing enforcement in the GitLab UI. -> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/292811) in GitLab 13.8, with an updated timeout experience. -> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/211962) in GitLab 13.8 with allowing group owners to not go through SSO. -> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/9152) in GitLab 13.11 with enforcing open SSO session to use Git if this setting is switched on. -> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/339888) in GitLab 14.7 to not enforce SSO checks for Git activity originating from CI/CD jobs. -> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/215155) in GitLab 15.5 [with a flag](../../../administration/feature_flags.md) named `transparent_sso_enforcement` to include transparent enforcement even when SSO enforcement is not enabled. Disabled on GitLab.com. -> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/375788) in GitLab 15.8 by enabling transparent SSO by default on GitLab.com. - -FLAG: -On self-managed GitLab, transparent SSO enforcement is unavailable. An -[issue exists](https://gitlab.com/gitlab-org/gitlab/-/issues/382917) to add -transparent SSO enforcement to self-managed GitLab. -On GitLab.com, transparent SSO enforcement is available by default. To turn off -transparent SSO, ask a support or production team to enable the -`transparent_sso_enforcement_override` feature flag for a specific customer -group. - -#### Transparent SSO enforcement - -By default, transparent SSO enforcement is enabled in GitLab.com. This means SSO is enforced: - -- When users access groups and projects in the organization's - group hierarchy. Users can view other groups and projects without SSO sign in. -- For each user with an existing SAML identity. - -When transparent SSO enforcement is enabled, users: - -- Are not prompted to sign in through SSO on each visit. GitLab checks - whether a user has authenticated through SSO. If the user last signed in more - than 24 hours ago, GitLab prompts the user to sign in again through SSO. -- Without SAML identities are not required to use SSO unless **Enforce - SSO-only authentication for web activity for this group** is enabled. - -A user has a SAML identity if one or both of the following are true: - -- They have signed in to GitLab by using their GitLab group's single sign-on URL. -- They were provisioned by SCIM. - -With transparent SSO enabled, SSO is enforced as follows: - -| Project/Group visibility | Enforce SSO setting | Member with identity | Member without identity | Non-member or not signed in | -|--------------------------|---------------------|--------------------| ------ |------------------------------| -| Private | Off | Enforced | Not enforced | No access | -| Private | On | Enforced | Enforced | No access | -| Public | Off | Enforced | Not enforced | Not enforced | -| Public | On | Enforced | Enforced | Not enforced | - -An [issue exists](https://gitlab.com/gitlab-org/gitlab/-/issues/297389) to add a similar SSO requirement for API activity. - -#### SSO-only for web activity enforcement - -When the **Enforce SSO-only authentication for web activity for this group** option is enabled: - -- All users must access GitLab by using their GitLab group's single sign-on URL - to access group resources, regardless of whether they have an existing SAML - identity. -- SSO is enforced when users access groups and projects in the organization's - group hierarchy. Users can view other groups and projects without SSO sign in. -- Users cannot be added as new members manually. -- Users with the Owner role can use the standard sign in process to make - necessary changes to top-level group settings. - -SSO enforcement for web activity has the following effects when enabled: - -- For groups, users cannot share a project in the group outside the top-level - group, even if the project is forked. -- For Git activity over SSH and HTTPS, users must have at least one active - session signed-in through SSO before they can push to or - pull from a GitLab repository. -- Git activity originating from CI/CD jobs do not have the SSO check enforced. -- Credentials that are not tied to regular users (for example, project and group - access tokens, and deploy keys) do not have the SSO check enforced. -- Users must be signed-in through SSO before they can pull images using the - [Dependency Proxy](../../packages/dependency_proxy/index.md). -- When the **Enforce SSO-only authentication for Git and Dependency Proxy - activity for this group** option is enabled, any API endpoint that involves - Git activity is under SSO enforcement. For example, creating or deleting a - branch, commit, or tag. - -When SSO for web activity is enforced, non-SSO group members do not lose access -immediately. If the user: - -- Has an active session, they can continue accessing the group for up to 24 - hours until the identity provider session times out. -- Is signed out, they cannot access the group after being removed from the - identity provider. - -## Providers - -The SAML standard means that you can use a wide range of identity providers with GitLab. Your identity provider might have relevant documentation. It can be generic SAML documentation or specifically targeted for GitLab. - -When [configuring your identity provider](#configure-your-identity-provider), consider the notes below for specific providers to help avoid common issues and as a guide for terminology used. - -For providers not listed below, you can refer to the [instance SAML notes on configuring an identity provider](../../../integration/saml.md#configure-saml-on-your-idp) -for additional guidance on information your identity provider may require. - -GitLab provides the following information for guidance only. -If you have any questions on configuring the SAML app, contact your provider's support. - -### Set up Azure - -Follow the Azure documentation on [configuring single sign-on to applications](https://learn.microsoft.com/en-us/azure/active-directory/manage-apps/add-application-portal-setup-sso), and use the following notes when needed. +1. Optional. If you use [Group Sync](group_sync.md), customize the name of the + group claim to match the required attribute. -For a demo of the Azure SAML setup including SCIM, see [SCIM Provisioning on Azure Using SAML SSO for Groups Demo](https://youtu.be/24-ZxmTeEBU). -The video is outdated in regard to objectID mapping and you should follow the [SCIM documentation](scim_setup.md#configure-azure-active-directory). - -| GitLab Setting | Azure Field | -| ------------------------------------ | ------------------------------------------ | -| Identifier | Identifier (Entity ID) | -| Assertion consumer service URL | Reply URL (Assertion Consumer Service URL) | -| GitLab single sign-on URL | Sign on URL | -| Identity provider single sign-on URL | Login URL | -| Certificate fingerprint | Thumbprint | - -You should set the following attributes: - -- **Unique User Identifier (Name identifier)** to `user.objectID`. -- **nameid-format** to persistent. -- Additional claims to [supported attributes](#user-attributes). +View a demo of [SCIM provisioning on Azure using SAML SSO for groups](https://youtu.be/24-ZxmTeEBU). The `objectID` mapping is outdated in this video. Follow the [SCIM documentation](scim_setup.md#configure-azure-active-directory) instead. -If using [Group Sync](#group-sync), customize the name of the group claim to match the required attribute. +For more information, see an [example configuration page](example_saml_config.md#azure-active-directory). -See our [example configuration page](example_saml_config.md#azure-active-directory). +### Google Workspace -### Set up Google Workspace +To set up Google Workspace as your identity provider: -1. [Set up SSO with Google as your identity provider](https://support.google.com/a/answer/6087519?hl=en). - The following GitLab settings correspond to the Google Workspace fields. +1. In GitLab, on the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Settings > SAML SSO**. +1. Note the information on this page. +1. Follow the instructions for [setting up SSO with Google as your identity provider](https://support.google.com/a/answer/6087519?hl=en). The following GitLab settings correspond to the Google Workspace fields. - | GitLab setting | Google Workspace field | - |:-------------------------------------|:-----------------------| - | Identifier | **Entity ID** | - | Assertion consumer service URL | **ACS URL** | - | GitLab single sign-on URL | **Start URL** | - | Identity provider single sign-on URL | **SSO URL** | + | GitLab setting | Google Workspace field | + |:-----------------------------------------|:-----------------------| + | **Identifier** | **Entity ID** | + | **Assertion consumer service URL** | **ACS URL** | + | **GitLab single sign-on URL** | **Start URL** | + | **Identity provider single sign-on URL** | **SSO URL** | 1. Google Workspace displays a SHA256 fingerprint. To retrieve the SHA1 fingerprint required by GitLab to [configure SAML](#configure-gitlab): @@ -262,89 +93,151 @@ See our [example configuration page](example_saml_config.md#azure-active-directo ``` 1. Set these values: - - For **Primary email**: `email` - - For **First name**: `first_name` - - For **Last name**: `last_name` - - For **Name ID format**: `EMAIL` - - For **NameID**: `Basic Information > Primary email` + - For **Primary email**: `email`. + - For **First name**: `first_name`. + - For **Last name**: `last_name`. + - For **Name ID format**: `EMAIL`. + - For **NameID**: `Basic Information > Primary email`. + For more information, see [manage user SAML identity](#manage-user-saml-identity). + +1. Make sure the identity provider is set to have provider-initiated calls + to link existing GitLab accounts. On the GitLab SAML SSO page, when you select **Verify SAML Configuration**, disregard the warning that recommends setting the **NameID** format to `persistent`. -For details, see the [example configuration page](example_saml_config.md#google-workspace). - -### Set up Okta +For more information, see an [example configuration page](example_saml_config.md#google-workspace). -For a demo of the Okta SAML setup including SCIM, see [Demo: Okta Group SAML & SCIM setup](https://youtu.be/0ES9HsZq0AQ). +View a demo of [how to configure SAML with Google Workspaces and set up Group Sync](https://youtu.be/NKs0FSQVfCY). + +### Okta + +To set up SSO with Okta as your identity provider: + +1. In GitLab, on the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Settings > SAML SSO**. +1. Note the information on this page. +1. Follow the instructions for [setting up a SAML application in Okta](https://developer.okta.com/docs/guides/build-sso-integration/saml2/main/). -1. [Set up a SAML application in Okta](https://developer.okta.com/docs/guides/build-sso-integration/saml2/main/). The following GitLab settings correspond to the Okta fields. - | GitLab setting | Okta field | - | ------------------------------------ | ---------------------------------------------------------- | - | Identifier | **Audience URI** | - | Assertion consumer service URL | **Single sign-on URL** | - | GitLab single sign-on URL | **Login page URL** (under **Application Login Page** settings) | - | Identity provider single sign-on URL | **Identity Provider Single Sign-On URL** | + | GitLab setting | Okta field | + | ---------------------------------------- | -------------------------------------------------------------- | + | **Identifier** | **Audience URI** | + | **Assertion consumer service URL** | **Single sign-on URL** | + | **GitLab single sign-on URL** | **Login page URL** (under **Application Login Page** settings) | + | **Identity provider single sign-on URL** | **Identity Provider Single Sign-On URL** | 1. Under the Okta **Single sign-on URL** field, select the **Use this for Recipient URL and Destination URL** checkbox. 1. Set these values: - - For **Application username (NameID)**: **Custom** `user.getInternalProperty("id")` - - For **Name ID Format**: `Persistent` + - For **Application username (NameID)**: **Custom** `user.getInternalProperty("id")`. + - For **Name ID Format**: `Persistent`. For more information, see [manage user SAML identity](#manage-user-saml-identity). + +1. Make sure the identity provider is set to have provider-initiated calls + to link existing GitLab accounts. The Okta GitLab application available in the App Catalog only supports [SCIM](scim_setup.md). Support for SAML is proposed in [issue 216173](https://gitlab.com/gitlab-org/gitlab/-/issues/216173). -### Set up OneLogin + +For a demo of the Okta SAML setup including SCIM, see [Demo: Okta Group SAML & SCIM setup](https://youtu.be/0ES9HsZq0AQ). + +For more information, see an [example configuration page](example_saml_config.md#okta) + +### OneLogin OneLogin supports its own [GitLab (SaaS) application](https://onelogin.service-now.com/support?id=kb_article&sys_id=92e4160adbf16cd0ca1c400e0b961923&kb_category=50984e84db738300d5505eea4b961913). +To set up OneLogin as your identity provider: + +1. In GitLab, on the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Settings > SAML SSO**. +1. Note the information on this page. 1. If you use the OneLogin generic [SAML Test Connector (Advanced)](https://onelogin.service-now.com/support?id=kb_article&sys_id=b2c19353dbde7b8024c780c74b9619fb&kb_category=93e869b0db185340d5505eea4b961934), you should [use the OneLogin SAML Test Connector](https://onelogin.service-now.com/support?id=kb_article&sys_id=93f95543db109700d5505eea4b96198f). The following GitLab settings correspond to the OneLogin fields: - | GitLab setting | OneLogin field | - | ------------------------------------------------ | -------------------------------- | - | Identifier | **Audience** | - | Assertion consumer service URL | **Recipient** | - | Assertion consumer service URL | **ACS (Consumer) URL** | - | Assertion consumer service URL (escaped version) | **ACS (Consumer) URL Validator** | - | GitLab single sign-on URL | **Login URL** | - | Identity provider single sign-on URL | **SAML 2.0 Endpoint** | + | GitLab setting | OneLogin field | + | ---------------------------------------------------- | -------------------------------- | + | **Identifier** | **Audience** | + | **Assertion consumer service URL** | **Recipient** | + | **Assertion consumer service URL** | **ACS (Consumer) URL** | + | **Assertion consumer service URL (escaped version)** | **ACS (Consumer) URL Validator** | + | **GitLab single sign-on URL** | **Login URL** | + | **Identity provider single sign-on URL** | **SAML 2.0 Endpoint** | -1. For **NameID**, use `OneLogin ID`. +1. For **NameID**, use `OneLogin ID`. For more information, see [manage user SAML identity](#manage-user-saml-identity). -### Change the SAML app +1. Make sure the identity provider is set to have provider-initiated calls + to link existing GitLab accounts. -To change the SAML app used for sign in: +### Use metadata -- If the NameID is not identical in both the existing and new SAML apps, users must: - 1. [Unlink the current SAML identity](#unlinking-accounts). - 1. [Link their identity](#user-access-and-management) to the new SAML app. -- If the NameID is identical, no change is required. +To configure some identity providers, you need a GitLab metadata URL. +To find this URL: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Settings > SAML SSO**. +1. Copy the provided **GitLab metadata URL**. +1. Follow your identity provider's documentation and paste the metadata URL when it's requested. -### Migrate to a different SAML provider +Check your identity provider's documentation to see if it supports the GitLab metadata URL. -You can migrate to a different SAML provider. During the migration process users will not be able to access any of the SAML groups. -To mitigate this, you can disable [SSO enforcement](#sso-enforcement). +### Manage the identity provider -To migrate SAML providers: +After you have set up your identity provider, you can: -1. [Configure](#configure-your-identity-provider) the group with the new identity provider SAML app. -1. Ask users to [unlink their account from the group](#unlinking-accounts). -1. Ask users to [link their account to the new SAML app](#linking-saml-to-your-existing-gitlabcom-account). +- Change the identity provider. +- Change email domains. -### Change email domains +#### Change the identity provider -To migrate users to a new email domain, users must: +You can change to a different identity provider. During the change process, +users cannot access any of the SAML groups. To mitigate this, you can disable +[SSO enforcement](#sso-enforcement). + +To change identity providers: + +1. [Configure](#set-up-your-identity-provider) the group with the new identity provider. +1. Optional. If the **NameID** is not identical, [change the **NameID** for users](#manage-user-saml-identity). + +#### Change email domains + +To migrate users to a new email domain, tell users to: 1. Add their new email as the primary email to their accounts and verify it. -1. [Unlink their account from the group](#unlinking-accounts). -1. [Link their account to the group](#linking-saml-to-your-existing-gitlabcom-account). -1. (Optional) Remove their old email from the account. +1. Optional. Remove their old email from the account. + +If the **NameID** is configured with the email address, [change the **NameID** for users](#manage-user-saml-identity). + +## Configure GitLab + +After you set up your identity provider to work with GitLab, you must configure GitLab to use it for authentication: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Settings > SAML SSO**. +1. Complete the fields: + - In the **Identity provider single sign-on URL** field, enter the SSO URL from your identity provider. + - In the **Certificate fingerprint** field, enter the fingerprint for the SAML token signing certificate. +1. In the **Default membership role** field, select the role to assign to new users. + The default role is **Guest**. In [GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/issues/214523) + and later, group owners can set a default membership role other than **Guest**. + To do so, [configure the SAML SSO for the group](#configure-gitlab). That role + becomes the starting role of all users added to the group. +1. Select the **Enable SAML authentication for this group** checkbox. +1. Optional. Select: + - **Enforce SSO-only authentication for web activity for this group**. + - **Enforce SSO-only authentication for Git activity for this group**. + For more information, see the [SSO enforcement documentation](#sso-enforcement). +1. Select **Save changes**. + +NOTE: +The certificate [fingerprint algorithm](../../../integration/saml.md#configure-saml-on-your-idp) must be in SHA1. When configuring the identity provider (such as [Google Workspace](#google-workspace)), use a secure signature algorithm. + +If you are having issues configuring GitLab, see the [troubleshooting documentation](#troubleshooting). ## User access and management @@ -362,53 +255,108 @@ When a user tries to sign in with Group SSO, GitLab attempts to find or create a - Create a new account with another email address. - Sign-in to their existing account to link the SAML identity. -### Linking SAML to your existing GitLab.com account +### User attributes + +You can pass user information to GitLab as attributes in the SAML assertion. + +- The user's email address can be an **email** or **mail** attribute. +- The username can be either a **username** or **nickname** attribute. You should specify only + one of these. + +For more information, see the [attributes available for self-managed GitLab instances](../../../integration/saml.md#configure-assertions). + +### Link SAML to your existing GitLab.com account > **Remember me** checkbox [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/121569) in GitLab 15.7. To link SAML to your existing GitLab.com account: -1. Sign in to your GitLab.com account. [Reset your password](https://gitlab.com/users/password/new) if necessary. -1. Locate and visit the **GitLab single sign-on URL** for the group you're signing in to. A group owner can find this on the group's **Settings > SAML SSO** page. If the sign-in URL is configured, users can connect to the GitLab app from the identity provider. -1. Optional. Select the **Remember me** checkbox to stay signed in to GitLab for 2 weeks. You may still be asked to re-authenticate with your SAML provider - more frequently. +1. Sign in to your GitLab.com account. [Reset your password](https://gitlab.com/users/password/new) + if necessary. +1. Locate and visit the **GitLab single sign-on URL** for the group you're signing + in to. A group owner can find this on the group's **Settings > SAML SSO** page. + If the sign-in URL is configured, users can connect to the GitLab app from the identity provider. +1. Optional. Select the **Remember me** checkbox to stay signed in to GitLab for 2 weeks. + You may still be asked to re-authenticate with your SAML provider more frequently. 1. Select **Authorize**. 1. Enter your credentials on the identity provider if prompted. -1. You are then redirected back to GitLab.com and should now have access to the group. In the future, you can use SAML to sign in to GitLab.com. +1. You are then redirected back to GitLab.com and should now have access to the group. + In the future, you can use SAML to sign in to GitLab.com. + +If a user is already a member of the group, linking the SAML identity does not +change their role. -On subsequent visits, you should be able to go [sign in to GitLab.com with SAML](#signing-in-to-gitlabcom-with-saml) or by visiting links directly. If the **enforce SSO** option is turned on, you are then redirected to sign in through the identity provider. +On subsequent visits, you should be able to [sign in to GitLab.com with SAML](#sign-in-to-gitlabcom-with-saml) +or by visiting links directly. If the **enforce SSO** option is turned on, you +are then redirected to sign in through the identity provider. -### Signing in to GitLab.com with SAML +### Sign in to GitLab.com with SAML 1. Sign in to your identity provider. 1. From the list of apps, select the "GitLab.com" app. (The name is set by the administrator of the identity provider.) 1. You are then signed in to GitLab.com and redirected to the group. -### Change NameID for one or more users +### Manage user SAML identity > Update of SAML identities using the SAML API [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227841) in GitLab 15.5. -Group owners can update the SAML identities for their group members using the [SAML API](../../../api/saml.md). +GitLab.com uses the SAML **NameID** to identify users. The **NameID** is: + +- A required field in the SAML response. +- Case sensitive. + +The **NameID** must: + +- Be unique to each user. +- Be a persistent value that never changes, such as a randomly generated unique user ID. +- Match exactly on subsequent sign-in attempts, so it should not rely on user input + that could change between upper and lower case. + +The **NameID** should not be an email address or username because: + +- Email addresses and usernames are more likely to change over time. For example, + when a person's name changes. +- Email addresses are case-insensitive, which can result in users being unable to + sign in. + +The **NameID** format must be `Persistent`, unless you are using a field, like email, that +requires a different format. You can use any format except `Transient`. + +#### Change user **NameID** + +Group owners can use the [SAML API](../../../api/saml.md#update-extern_uid-field-for-a-saml-identity) to change their group members' **NameID** and update their SAML identities . + +If [SCIM](scim_setup.md) is configured, group owners can update the SCIM identities using the [SCIM API](../../../api/scim.md#update-extern_uid-field-for-a-scim-identity). Alternatively, ask the users to reconnect their SAML account. -1. Ask relevant users to [unlink their account from the group](#unlinking-accounts). -1. Ask relevant users to [link their account to the new SAML app](#linking-saml-to-your-existing-gitlabcom-account). +1. Ask relevant users to [unlink their account from the group](#unlink-accounts). +1. Ask relevant users to [link their account to the new SAML app](#link-saml-to-your-existing-gitlabcom-account). + +WARNING: +After users have signed into GitLab using SSO SAML, changing the **NameID** value +breaks the configuration and could lock users out of the GitLab group. + +For more information on the recommended value and format for specific identity +providers, see [set up your identity provider](#set-up-your-identity-provider). ### Configure user settings from SAML response > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/263661) in GitLab 13.7. GitLab allows setting certain user attributes based on values from the SAML response. -Existing users will have these attributes updated if the user was originally -provisioned by the group. Users are provisioned by the group when the account was -created via [SCIM](scim_setup.md) or by first sign-in with SAML SSO for GitLab.com groups. +An existing user's attributes are updated from the SAML response values if that +user was originally provisioned by the group. Users are provisioned by the group +when the account was created either: + +- Through [SCIM](scim_setup.md). +- By first sign-in with SAML SSO for GitLab.com groups. #### Supported user attributes -- `can_create_group` - 'true' or 'false' to indicate whether the user can create +- **can_create_group** - `true` or `false` to indicate whether the user can create new groups. Default is `true`. -- `projects_limit` - The total number of personal projects a user can create. +- **projects_limit** - The total number of personal projects a user can create. A value of `0` means the user cannot create new projects in their personal namespace. Default is `10000`. @@ -446,7 +394,7 @@ convert the information to XML. An example SAML response is shown here. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/238461) in GitLab 15.4. By default, users provisioned with SAML or SCIM are sent a verification email to verify their identity. Instead, you can -[configure GitLab with a custom domain](../../project/pages/custom_domains_ssl_tls_certification/index.md) and GitLab +[configure GitLab with a custom domain](../../enterprise_user/index.md#set-up-a-verified-domain) and GitLab automatically confirms user accounts. Users still receive an [enterprise user](../../enterprise_user/index.md) welcome email. Confirmation is bypassed for users: @@ -454,35 +402,25 @@ bypassed for users: - That are provisioned with SAML or SCIM. - That have an email address that belongs to the verified domain. -### Role - -Starting from [GitLab 13.3](https://gitlab.com/gitlab-org/gitlab/-/issues/214523), group owners can set a -"Default membership role" other than Guest. To do so, [configure the SAML SSO for the group](#configure-gitlab). -That role becomes the starting access level of all users added to the group. - -Existing members with appropriate privileges can promote or demote users, as needed. - -If a user is already a member of the group, linking the SAML identity does not change their role. - -Users given a "minimal access" role have [specific restrictions](../../permissions.md#users-with-minimal-access). - -### Blocking access +### Block user access To rescind a user's access to the group when only SAML SSO is configured, either: - Remove (in order) the user from: 1. The user data store on the identity provider or the list of users on the specific app. 1. The GitLab.com group. -- Use Group Sync at the top-level of your group to [automatically remove the user](group_sync.md#automatic-member-removal). +- Use [Group Sync](group_sync.md#automatic-member-removal) at the top-level of + your group with the default role set to [minimal access](../../permissions.md#users-with-minimal-access) + to automatically block access to all resources in the group. To rescind a user's access to the group when also using SCIM, refer to [Remove access](scim_setup.md#remove-access). -### Unlinking accounts +### Unlink accounts Users can unlink SAML for a group from their profile page. This can be helpful if: - You no longer want a group to be able to sign you in to GitLab.com. -- Your SAML NameID has changed and so GitLab can no longer find your user. +- Your SAML **NameID** has changed and so GitLab can no longer find your user. WARNING: Unlinking an account removes all roles assigned to that user in the group. @@ -499,14 +437,106 @@ For example, to unlink the `MyOrg` account: 1. On the left sidebar, select **Account**. 1. In the **Service sign-in** section, select **Disconnect** next to the connected account. -## Group Sync +## SSO enforcement -For information on automatically managing GitLab group membership, see [SAML Group Sync](group_sync.md). +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/5291) in GitLab 11.8. +> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/9255) in GitLab 11.11 with ongoing enforcement in the GitLab UI. +> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/292811) in GitLab 13.8, with an updated timeout experience. +> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/211962) in GitLab 13.8 with allowing group owners to not go through SSO. +> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/9152) in GitLab 13.11 with enforcing open SSO session to use Git if this setting is switched on. +> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/339888) in GitLab 14.7 to not enforce SSO checks for Git activity originating from CI/CD jobs. +> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/215155) in GitLab 15.5 [with a flag](../../../administration/feature_flags.md) named `transparent_sso_enforcement` to include transparent enforcement even when SSO enforcement is not enabled. Disabled on GitLab.com. +> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/375788) in GitLab 15.8 by enabling transparent SSO by default on GitLab.com. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/389562) in GitLab 15.10. Feature flag `transparent_sso_enforcement` removed. + +On GitLab.com, SSO is enforced: + +- When SAML SSO is enabled. +- For users with an existing SAML identity when accessing groups and projects in the organization's + group hierarchy. Users can view other groups and projects as well as their user settings without SSO sign in by using their GitLab.com credentials. + +A user has a SAML identity if one or both of the following are true: + +- They have signed in to GitLab by using their GitLab group's single sign-on URL. +- They were provisioned by SCIM. + +Users are not prompted to sign in through SSO on each visit. GitLab checks +whether a user has authenticated through SSO. If the user last signed in more +than 24 hours ago, GitLab prompts the user to sign in again through SSO. + +SSO is enforced as follows: + +| Project/Group visibility | Enforce SSO setting | Member with identity | Member without identity | Non-member or not signed in | +|--------------------------|---------------------|----------------------|-------------------------|-----------------------------| +| Private | Off | Enforced | Not enforced | Not enforced | +| Private | On | Enforced | Enforced | Enforced | +| Public | Off | Enforced | Not enforced | Not enforced | +| Public | On | Enforced | Enforced | Not enforced | + +An [issue exists](https://gitlab.com/gitlab-org/gitlab/-/issues/297389) to add a similar SSO requirement for API activity. + +### SSO-only for web activity enforcement + +When the **Enforce SSO-only authentication for web activity for this group** option is enabled: + +- All members must access GitLab by using their GitLab group's single sign-on URL + to access group resources, regardless of whether they have an existing SAML + identity. +- SSO is enforced when users access groups and projects in the organization's + group hierarchy. Users can view other groups and projects without SSO sign in. +- Users cannot be added as new members manually. +- Users with the Owner role can use the standard sign in process to make + necessary changes to top-level group settings. +- For non-members or users who are not signed in: + - SSO is not enforced when they access public group resources. + - SSO is enforced when they access private group resources. + +SSO enforcement for web activity has the following effects when enabled: + +- For groups, users cannot share a project in the group outside the top-level + group, even if the project is forked. +- Git activity originating from CI/CD jobs do not have the SSO check enforced. +- Credentials that are not tied to regular users (for example, project and group + access tokens, and deploy keys) do not have the SSO check enforced. +- Users must be signed-in through SSO before they can pull images using the + [Dependency Proxy](../../packages/dependency_proxy/index.md). +- When the **Enforce SSO-only authentication for Git and Dependency Proxy + activity for this group** option is enabled, any API endpoint that involves + Git activity is under SSO enforcement. For example, creating or deleting a + branch, commit, or tag. For Git activity over SSH and HTTPS, users must + have at least one active session signed-in through SSO before they can push to or + pull from a GitLab repository. + +When SSO for web activity is enforced, non-SSO group members do not lose access +immediately. If the user: + +- Has an active session, they can continue accessing the group for up to 24 + hours until the identity provider session times out. +- Is signed out, they cannot access the group after being removed from the + identity provider. -## Passwords for users created via SAML SSO for Groups +## Related topics -The [Generated passwords for users created through integrated authentication](../../../security/passwords_for_integrated_authentication_methods.md) guide provides an overview of how GitLab generates and sets passwords for users created via SAML SSO for Groups. +- [SAML SSO for self-managed GitLab instances](../../../integration/saml.md) +- [Glossary](../../../integration/saml.md#glossary) +- [Authentication comparison between SaaS and self-managed](../../../administration/auth/index.md#saas-vs-self-managed-comparison) +- [Passwords for users created through integrated authentication](../../../security/passwords_for_integrated_authentication_methods.md) +- [SAML Group Sync](group_sync.md) ## Troubleshooting -See our [troubleshooting SAML guide](troubleshooting.md). +If you find it difficult to match the different SAML terms between GitLab and the +identity provider: + +1. Check your identity provider's documentation. Look at their example SAML + configurations for information on the terms they use. +1. Check the [SAML SSO for self-managed GitLab instances documentation](../../../integration/saml.md). + The self-managed GitLab instance SAML configuration file supports more options + than the GitLab.com file. You can find information on the self-managed instance + file in the: + - External [OmniAuth SAML documentation](https://github.com/omniauth/omniauth-saml/). + - [`ruby-saml` library](https://github.com/onelogin/ruby-saml). +1. Compare the XML response from your provider with our + [example XML used for internal testing](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/spec/fixtures/saml/response.xml). + +For other troubleshooting information, see the [troubleshooting SAML guide](troubleshooting.md). diff --git a/doc/user/group/saml_sso/scim_setup.md b/doc/user/group/saml_sso/scim_setup.md index c6ff5dc63c3..38a1c4125aa 100644 --- a/doc/user/group/saml_sso/scim_setup.md +++ b/doc/user/group/saml_sso/scim_setup.md @@ -116,7 +116,7 @@ For each attribute: 1. Select the required settings. 1. Select **Ok**. -If your SAML configuration differs from [the recommended SAML settings](index.md#set-up-azure), select the mapping +If your SAML configuration differs from [the recommended SAML settings](index.md#azure), select the mapping attributes and modify them accordingly. In particular, the `objectId` source attribute must map to the `externalId` target attribute. @@ -133,13 +133,13 @@ Prerequisites: product tier is required to use SCIM on Okta. - [GitLab is configured](#configure-gitlab). - SAML application for [Okta](https://developer.okta.com/docs/guides/build-sso-integration/saml2/main/) set up as - described in the [Okta setup notes](index.md#set-up-okta). + described in the [Okta setup notes](index.md#okta). - Your Okta SAML setup matches the [configuration steps exactly](index.md), especially the NameID configuration. To configure Okta for SCIM: 1. Sign in to Okta. -1. Ensure you are in the Admin Area by selecting the **Admin** button located in the upper right. The button is not visible from the Admin Area. +1. In the upper-right corner, select **Admin**. The button is not visible from the Admin Area. 1. In the **Application** tab, select **Browse App Catalog**. 1. Search for **GitLab**, find and select the **GitLab** application. 1. On the GitLab application overview page, select **Add**. @@ -200,6 +200,10 @@ On subsequent visits, new and existing users can access groups either: For role information, see the [Group SAML](index.md#user-access-and-management) page. +### Passwords for users created through SCIM for GitLab groups + +GitLab requires passwords for all user accounts. For more information on how GitLab generates passwords for users created through SCIM for GitLab groups, see [generated passwords for users created through integrated authentication](../../../security/passwords_for_integrated_authentication_methods.md). + ### Link SCIM and SAML identities If [group SAML](index.md) is configured and you have an existing GitLab.com account, users can link their SCIM and SAML @@ -210,7 +214,7 @@ To link your SCIM and SAML identities: 1. Update the [primary email](../../profile/index.md#change-your-primary-email) address in your GitLab.com user account to match the user profile email address in your identity provider. -1. [Link your SAML identity](index.md#linking-saml-to-your-existing-gitlabcom-account). +1. [Link your SAML identity](index.md#link-saml-to-your-existing-gitlabcom-account). ### Remove access @@ -222,6 +226,8 @@ Remove or deactivate a user on the identity provider to remove their access to: After the identity provider performs a sync based on its configured schedule, the user's membership is revoked and they lose access. +When you enable SCIM, this does not automatically remove existing users who do not have a SAML identity. + NOTE: Deprovisioning does not delete the GitLab user account. diff --git a/doc/user/group/saml_sso/troubleshooting.md b/doc/user/group/saml_sso/troubleshooting.md index eadf385feb3..62c13e8909b 100644 --- a/doc/user/group/saml_sso/troubleshooting.md +++ b/doc/user/group/saml_sso/troubleshooting.md @@ -167,7 +167,7 @@ you must set `attribute_statements` in the SAML configuration to This error suggests you are signed in as a GitLab user but have already linked your SAML identity to a different GitLab user. Sign out and then try to sign in again using SAML, which should log you into GitLab with the linked user account. -If you do not wish to use that GitLab user with the SAML login, you can [unlink the GitLab account from the SAML app](index.md#unlinking-accounts). +If you do not wish to use that GitLab user with the SAML login, you can [unlink the GitLab account from the SAML app](index.md#unlink-accounts). ### Message: "SAML authentication failed: User has already been taken" @@ -176,7 +176,7 @@ Here are possible causes and solutions: | Cause | Solution | | ---------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| You've tried to link multiple SAML identities to the same user, for a given identity provider. | Change the identity that you sign in with. To do so, [unlink the previous SAML identity](index.md#unlinking-accounts) from this GitLab account before attempting to sign in again. | +| You've tried to link multiple SAML identities to the same user, for a given identity provider. | Change the identity that you sign in with. To do so, [unlink the previous SAML identity](index.md#unlink-accounts) from this GitLab account before attempting to sign in again. | | The `NameID` changes every time the user requests SSO identification | [Check the `NameID`](#verify-nameid) is not set with `Transient` format, or the `NameID` is not changing on subsequent requests.| ### Message: "SAML authentication failed: Email has already been taken" @@ -196,7 +196,7 @@ User accounts are created in one of the following ways: Getting both of these errors at the same time suggests the `NameID` capitalization provided by the identity provider didn't exactly match the previous value for that user. -This can be prevented by configuring the `NameID` to return a consistent value. Fixing this for an individual user involves changing the identifier for the user. For GitLab.com, the user needs to [unlink their SAML from the GitLab account](index.md#unlinking-accounts). +This can be prevented by configuring the `NameID` to return a consistent value. Fixing this for an individual user involves changing the identifier for the user. For GitLab.com, the user needs to [unlink their SAML from the GitLab account](index.md#unlink-accounts). ### Message: "Request to link SAML account must be authorized" @@ -209,7 +209,7 @@ initiated by the service provider and not only the identity provider. ### Message: "There is already a GitLab account associated with this email address. Sign in with your existing credentials to connect your organization's account" **(PREMIUM SAAS)** -A user can see this message when they are trying to [manually link SAML to their existing GitLab.com account](index.md#linking-saml-to-your-existing-gitlabcom-account). +A user can see this message when they are trying to [manually link SAML to their existing GitLab.com account](index.md#link-saml-to-your-existing-gitlabcom-account). To resolve this problem, the user should check they are using the correct GitLab password to sign in. The user first needs to [reset their password](https://gitlab.com/users/password/new) if both: @@ -233,7 +233,7 @@ This can then be compared to the `NameID` being sent by the identity provider by Ensure that the **GitLab single sign-on URL** (for GitLab.com) or the instance URL (for self-managed) has been configured as "Login URL" (or similarly named field) in the identity provider's SAML app. -For GitLab.com, alternatively, when users need to [link SAML to their existing GitLab.com account](index.md#linking-saml-to-your-existing-gitlabcom-account), provide the **GitLab single sign-on URL** and instruct users not to use the SAML app on first sign in. +For GitLab.com, alternatively, when users need to [link SAML to their existing GitLab.com account](index.md#link-saml-to-your-existing-gitlabcom-account), provide the **GitLab single sign-on URL** and instruct users not to use the SAML app on first sign in. ### Users receive a 404 **(PREMIUM SAAS)** @@ -244,8 +244,8 @@ If all users are receiving a `404` when attempting to sign in using SAML, confir If you receive a `404` during setup when using "verify configuration", make sure you have used the correct [SHA-1 generated fingerprint](../../../integration/saml.md#configure-saml-on-your-idp). -If a user is trying to sign in for the first time and the GitLab single sign-on URL has not [been configured](index.md#configure-your-identity-provider), they may see a 404. -As outlined in the [user access section](index.md#linking-saml-to-your-existing-gitlabcom-account), a group Owner needs to provide the URL to users. +If a user is trying to sign in for the first time and the GitLab single sign-on URL has not [been configured](index.md#set-up-your-identity-provider), they may see a 404. +As outlined in the [user access section](index.md#link-saml-to-your-existing-gitlabcom-account), a group Owner needs to provide the URL to users. If all users are receiving a `404` after signing in to the identity provider (IdP): @@ -317,4 +317,4 @@ This error appears when you try to invite a user to a GitLab.com group (or subgr If you see this message after trying to invite a user to a group: 1. Ensure the user has been [added to the SAML identity provider](index.md#user-access-and-management). -1. Ask the user to [link SAML to their existing GitLab.com account](index.md#linking-saml-to-your-existing-gitlabcom-account), if they have one. Otherwise, ask the user to create a GitLab.com account by [accessing GitLab.com through the identity provider's dashboard](index.md#user-access-and-management), or by [signing up manually](https://gitlab.com/users/sign_up) and linking SAML to their new account. +1. Ask the user to [link SAML to their existing GitLab.com account](index.md#link-saml-to-your-existing-gitlabcom-account), if they have one. Otherwise, ask the user to create a GitLab.com account by [accessing GitLab.com through the identity provider's dashboard](index.md#user-access-and-management), or by [signing up manually](https://gitlab.com/users/sign_up) and linking SAML to their new account. diff --git a/doc/user/group/saml_sso/troubleshooting_scim.md b/doc/user/group/saml_sso/troubleshooting_scim.md index 939ed804a99..33343c9b339 100644 --- a/doc/user/group/saml_sso/troubleshooting_scim.md +++ b/doc/user/group/saml_sso/troubleshooting_scim.md @@ -79,7 +79,7 @@ You must not: When the SCIM app changes: -- Users can follow the instructions in the [Change the SAML app](index.md#change-the-saml-app) section. +- Users can follow the instructions in the [Change the SAML app](index.md#change-the-identity-provider) section. - Administrators of the identity provider can: 1. Remove users from the SCIM app, which unlinks all removed users. 1. Turn on sync for the new SCIM app to [link existing users](scim_setup.md#link-scim-and-saml-identities). @@ -163,3 +163,27 @@ error. The error response can include a HTML result of the GitLab URL `https://g This error is harmless and occurs because group provisioning was turned on but GitLab SCIM integration does not support it nor require it. To remove the error, follow the instructions in the Azure configuration guide to disable the option to [synchronize Azure Active Directory groups to AppName](scim_setup.md#configure-azure-active-directory). + +## Okta + +The following troubleshooting information is specifically for SCIM provisioned through Okta. + +### `Error authenticating: null` message when testing API SCIM credentials + +When testing the API credentials in your Okta SCIM application, you may encounter an error: + +```plaintext +Error authenticating: null +``` + +Okta needs to be able to connect to your GitLab instance to provision or deprovision users. + +In your Okta SCIM application, check that the SCIM **Base URL** is correct and pointing to a valid GitLab +SCIM API endpoint URL. Check the following documentation to find information on this URL for: + +- [GitLab.com groups](scim_setup.md#configure-gitlab). +- [Self-managed GitLab instances](../../admin_area/settings/scim_setup.md#configure-gitlab). + +For self-managed GitLab instances, ensure that GitLab is publicly available so Okta can connect to it. If needed, +you can [allow access to Okta IP addresses](https://help.okta.com/en-us/Content/Topics/Security/ip-address-allow-listing.htm) +on your firewall. diff --git a/doc/user/group/settings/group_access_tokens.md b/doc/user/group/settings/group_access_tokens.md index cd50c209b0d..4a2bfd4c4b4 100644 --- a/doc/user/group/settings/group_access_tokens.md +++ b/doc/user/group/settings/group_access_tokens.md @@ -28,16 +28,12 @@ associated with a group rather than a project or user. In self-managed instances, group access tokens are subject to the same [maximum lifetime limits](../../admin_area/settings/account_and_limit_settings.md#limit-the-lifetime-of-access-tokens) as personal access tokens if the limit is set. WARNING: -The ability to create group access tokens without expiry was -[deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/369122) in GitLab 15.4 and is planned for removal in GitLab -16.0. When this ability is removed, existing group access tokens without an expiry are planned to have an expiry added. -The automatic adding of an expiry occurs on GitLab.com during the 16.0 milestone. The automatic adding of an expiry -occurs on self-managed instances when they are upgraded to GitLab 16.0. This change is a breaking change. +The ability to create group access tokens without an expiry date was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/369122) in GitLab 15.4 and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/392855) in GitLab 16.0. In GitLab 16.0 and later, existing group access tokens without an expiry date are automatically given an expiry date 365 days later than the current date. The automatic adding of an expiry date occurs on GitLab.com during the 16.0 milestone. The automatic adding of an expiry date occurs on self-managed instances when they are upgraded to GitLab 16.0. This change is a breaking change. You can use group access tokens: -- On GitLab SaaS if you have the Premium license tier or higher. Group access tokens are not available with a [trial license](https://about.gitlab.com/free-trial/). -- On self-managed instances of GitLab, with any license tier. If you have the Free tier: +- On GitLab SaaS: If you have the Premium or Ultimate license tier. Group access tokens are not available with a [trial license](https://about.gitlab.com/free-trial/). +- On self-managed instances: With any license tier. If you have the Free tier: - Review your security and compliance policies around [user self-enrollment](../../admin_area/settings/sign_up_restrictions.md#disable-new-sign-ups). - Consider [disabling group access tokens](#enable-or-disable-group-access-token-creation) to @@ -48,31 +44,33 @@ You cannot use group access tokens to create other group, project, or personal a Group access tokens inherit the [default prefix setting](../../admin_area/settings/account_and_limit_settings.md#personal-access-token-prefix) configured for personal access tokens. -NOTE: -Group access tokens are not FIPS compliant and creation and use are disabled when [FIPS mode](../../../development/fips_compliance.md) is enabled. - ## Create a group access token using UI > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214045) in GitLab 14.7. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/348660) in GitLab 15.3, default expiration of 30 days and default role of Guest is populated in the UI. +> - Ability to create non-expiring group access tokens [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/392855) in GitLab 16.0. + +WARNING: +Project access tokens are treated as [internal users](../../../development/internal_users.md). +If an internal user creates a project access token, that token is able to access +all projects that have visibility level set to [Internal](../../public_access.md). To create a group access token: 1. On the top bar, select **Main menu > Groups** and find your group. 1. On the left sidebar, select **Settings > Access Tokens**. 1. Enter a name. The token name is visible to any user with permissions to view the group. -1. Optional. Enter an expiry date for the token. The token will expire on that date at midnight UTC. An instance-wide [maximum lifetime](../../admin_area/settings/account_and_limit_settings.md#limit-the-lifetime-of-access-tokens) setting can limit the maximum allowable lifetime in self-managed instances. +1. Enter an expiry date for the token: + - The token expires on that date at midnight UTC. + - If you do not enter an expiry date, the expiry date is automatically set to 365 days later than the current date. + - By default, this date can be a maximum of 365 days later than the current date. + - An instance-wide [maximum lifetime](../../admin_area/settings/account_and_limit_settings.md#limit-the-lifetime-of-access-tokens) setting can limit the maximum allowable lifetime in self-managed instances. 1. Select a role for the token. 1. Select the [desired scopes](#scopes-for-a-group-access-token). 1. Select **Create group access token**. A group access token is displayed. Save the group access token somewhere safe. After you leave or refresh the page, you can't view it again. -WARNING: -Group access tokens are treated as [internal users](../../../development/internal_users.md). -If an internal user creates a group access token, that token is able to access all -groups that have visibility level set to [Internal](../../public_access.md). - ## Create a group access token using Rails console GitLab 14.6 and earlier doesn't support creating group access tokens using the UI @@ -87,8 +85,9 @@ or API. However, administrators can use a workaround: # Set the group group you want to create a token for. For example, group with ID 109. group = Group.find(109) - # Create the group bot user. For further group access tokens, the username should be group_#{group.id}_bot#{bot_count}. For example, group_109_bot2 and email address group_109_bot2@example.com. - bot = Users::CreateService.new(admin, { name: 'group_token', username: "group_#{group.id}_bot", email: "group_#{group.id}_bot@example.com", user_type: :project_bot }).execute + # Create the group bot user. For further group access tokens, the username should be `group_{group_id}_bot_{random_string}` and email address `group_{group_id}_bot_{random_string}@noreply.{Gitlab.config.gitlab.host}`. + random_string = SecureRandom.hex(16) + bot = Users::CreateService.new(admin, { name: 'group_token', username: "group_#{group.id}_bot_#{random_string}", email: "group_#{group.id}_bot_#{random_string}@noreply.#{Gitlab.config.gitlab.host}", user_type: :project_bot }).execute # Confirm the group bot. bot.confirm @@ -172,7 +171,11 @@ to groups instead of projects. Bot users for groups: - Do not count as licensed seats. - Can have a maximum role of Owner for a group. For more information, see [Create a group access token](../../../api/group_access_tokens.md#create-a-group-access-token). -- Have a username set to `group_{group_id}_bot` for the first access token. For example, `group_123_bot`. -- Have an email set to `group{group_id}_bot@noreply.{Gitlab.config.gitlab.host}`. For example, `group123_bot@noreply.example.com`. +- Have a username set to `group_{group_id}_bot_{random_string}`. For example, `group_123_bot_4ffca233d8298ea1`. +- Have an email set to `group_{group_id}_bot_{random_string}@noreply.{Gitlab.config.gitlab.host}`. For example, `group_123_bot_4ffca233d8298ea1@noreply.example.com`. All other properties are similar to [bot users for projects](../../project/settings/project_access_tokens.md#bot-users-for-projects). + +## Token availability + +Group access tokens are only available in paid subscriptions, and not available in trial subscriptions. For more information, see the ["What is included" section of the GitLab Trial FAQ](https://about.gitlab.com/free-trial/#what-is-included-in-my-free-trial-what-is-excluded). diff --git a/doc/user/group/settings/import_export.md b/doc/user/group/settings/import_export.md deleted file mode 100644 index ff64a7dcd54..00000000000 --- a/doc/user/group/settings/import_export.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: '../import/index.md' -remove_date: '2023-03-08' ---- - -This document was moved to [another location](../import/index.md). - - - - - diff --git a/doc/user/group/subgroups/index.md b/doc/user/group/subgroups/index.md index 6c7baa848e7..63ffbf62981 100644 --- a/doc/user/group/subgroups/index.md +++ b/doc/user/group/subgroups/index.md @@ -1,6 +1,6 @@ --- -stage: Manage -group: Organization +stage: Data Stores +group: Tenant Scale info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -85,7 +85,7 @@ You cannot host a GitLab Pages subgroup website with a top-level domain name. Fo To create a subgroup: 1. On the top bar, select **Main menu > Groups** and find and select the parent group to add a subgroup to. -1. On the parent group's overview page, in the upper right, select **New subgroup**. +1. On the parent group's overview page, in the upper-right corner, select **New subgroup**. 1. Select **Create group**. 1. Fill in the fields. View a list of [reserved names](../../reserved_names.md) that cannot be used as group names. 1. Select **Create group**. diff --git a/doc/user/group/value_stream_analytics/img/object_hierarchy_example_V14_10.png b/doc/user/group/value_stream_analytics/img/object_hierarchy_example_V14_10.png new file mode 100644 index 00000000000..9c4d4ae7718 Binary files /dev/null and b/doc/user/group/value_stream_analytics/img/object_hierarchy_example_V14_10.png differ diff --git a/doc/user/group/value_stream_analytics/index.md b/doc/user/group/value_stream_analytics/index.md index c5a95779087..bc48c1050fb 100644 --- a/doc/user/group/value_stream_analytics/index.md +++ b/doc/user/group/value_stream_analytics/index.md @@ -5,9 +5,9 @@ group: Optimize info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Value stream analytics for groups **(PREMIUM)** +# Value stream analytics **(FREE)** -Value stream analytics provides metrics about each stage of your software development process. +Value stream analytics measures the time it takes to go from an idea to production. A **value stream** is the entire work process that delivers value to customers. For example, the [DevOps lifecycle](https://about.gitlab.com/stages-devops-lifecycle/) is a value stream that starts @@ -21,111 +21,83 @@ Use value stream analytics to identify: - Long-running issues or merge requests. - Factors that cause your software development lifecycle to slow down. -Value stream analytics is also available for [projects](../../analytics/value_stream_analytics.md). +Value stream analytics helps businesses: -## View value stream analytics +- Visualize their end-to-end DevSecOps workstreams. +- Identify and solve inefficiencies. +- Optimize their workstreams to deliver more value, faster. -> - Filtering [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13216) in GitLab 13.3 -> - Horizontal stage path [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12196) in 13.0 and [feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/323982) in 13.12 +Value stream analytics is available for projects and groups. -Prerequisites: +## Feature availability -- You must have at least the Reporter role to view value stream analytics for groups. -- You must create a [custom value stream](#create-a-value-stream-with-gitlab-default-stages). Value stream analytics only shows custom value streams created for your group. +Value stream analytics offers different features at the project and group level for FOSS and licensed versions. -To view value stream analytics for your group: +- On GitLab Free, value stream analytics does not aggregate data. It queries the database directly where the date range filter is applied to the creation date of issues and merge request. You can view value stream analytics with pre-defined default stages. +- On GitLab Premium, value stream analytics aggregates data and applies the date range filter on the end event. You can also create, edit, and delete value streams. -1. On the top bar, select **Main menu > Groups** and find your group. -1. On the left sidebar, select **Analytics > Value stream**. -1. To view metrics for a particular stage, select a stage below the **Filter results** text box. -1. Optional. Filter the results: - 1. Select the **Filter results** text box. - 1. Select a parameter. - 1. Select a value or enter text to refine the results. - 1. To adjust the date range: - - In the **From** field, select a start date. - - In the **To** field, select an end date. The charts and list show workflow items created - during the date range. -1. Optional. Sort results by ascending or descending: - - To sort by most recent or oldest workflow item, select the **Last event** header. - - To sort by most or least amount of time spent in each stage, select the **Duration** header. +|Feature|Group level (licensed)|Project level (licensed)|Project level (FOSS)| +|-|-|-|-| +|Create custom value streams|Yes|Yes|no, only one value stream (default) is present with the default stages| +|Create custom stages|Yes|Yes|No| +|Filtering (for example, by author, label, milestone)|Yes|Yes|Yes| +|Stage time chart|Yes|Yes|No| +|Total time chart|Yes|Yes|No| +|Task by type chart|Yes|No|No| +|DORA Metrics|Yes|Yes|No| +|Cycle time and lead time summary (Key metrics)|Yes|Yes|No| +|New issues, commits, and deploys (Key metrics)|Yes, excluding commits|Yes|Yes| +|Uses aggregated backend|Yes|Yes|No| +|Date filter behavior|Filters items [finished within the date range](https://gitlab.com/groups/gitlab-org/-/epics/6046)|Filters items by creation date.|Filters items by creation date.| +|Authorization|At least reporter|At least reporter|Can be public| -A badge next to the workflow items table header shows the number of workflow items that -completed during the selected stage. +## How value stream analytics works -The table shows a list of related workflow items for the selected stage. Based on the stage you select, this can be: +Value stream analytics calculates the duration of every stage of your software development process. -- CI/CD jobs -- Issues -- Merge requests -- Pipelines +Value stream analytics is made of three core objects: -## View DORA metrics and key metrics for a group +- A **value stream** contains a value stream stage list. +- Each value stream stage list contains one or more **stages**. +- Each stage has two **events**: start and stop. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/210315) in GitLab 13.0. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/323982) in GitLab 13.12. +### Value stream stages -The **Overview** dashboard in value stream analytics shows key metrics and DORA metrics of group performance. Based on the filter you select, -the dashboard automatically aggregates DORA metrics and displays the current status of the value stream. Select a DORA metric to view its chart. +A stage represents an event pair (start and end events) with additional metadata, such as the name of the stage. You can configure the stages in the pairing rules defined in the backend. -Prerequisite: +### Value streams -- To view deployment metrics, you must have a -[production environment configured](../../../ci/environments/index.md#deployment-tier-of-environments). +Value streams are container objects for the stages. You can have multiple value streams per group, to focus on different aspects of the DevOps lifecycle. -To view the DORA metrics and key metrics: +### Value stream stage events -1. On the top bar, select **Main menu > Groups** and find your group. -1. On the left sidebar, select **Analytics > Value stream**. -1. Optional. Filter the results: - 1. Select the **Filter results** text box. - 1. Select a parameter. - 1. Select a value or enter text to refine the results. - 1. To adjust the date range: - - In the **From** field, select a start date. - - In the **To** field, select an end date. -Key metrics and DORA metrics display below the **Filter results** text box. - -### Key metrics in the value stream - -The **Overview** dashboard shows the following key metrics that measure team performance: - -- Lead time: Median time from when the issue was created to when it was closed. -- Cycle time: Median time from first commit to issue closed. GitLab measures cycle time from the earliest commit of a - [linked issue's merge request](../../project/issues/crosslinking_issues.md#from-commit-messages) to when that issue is closed. - The cycle time approach underestimates the lead time because merge request creation is always later than commit time. -- New issues: Number of new issues created. -- Deploys: Total number of deployments to production. - -### DORA metrics **(ULTIMATE)** +Events are the smallest building blocks of the value stream analytics feature. A stage consists of a start event and an end event. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/340150) lead time for changes DORA metric in GitLab 14.5. -> - DORA API-based deployment metrics for value stream analytics for groups were [moved](https://gitlab.com/gitlab-org/gitlab/-/issues/337256) from GitLab Ultimate to GitLab Premium in GitLab 14.3. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355304) time to restore service tile in GitLab 15.0. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/357071) change failure rate tile in GitLab 15.0. +The following stage events are available: -The value stream analytics **Overview** dashboard displays the following [DORA](../../../user/analytics/dora_metrics.md) metrics: - -- Deployment Frequency. -- Lead time for changes. -- Time to restore service. -- Change failure rate. - -DORA metrics are calculated based on data from the -[DORA API](../../../api/dora/metrics.md#devops-research-and-assessment-dora-key-metrics-api). +- Issue closed +- Issue created +- Issue first added to board +- Issue first assigned +- Issue first associated with milestone +- Issue first mentioned +- Issue label added +- Issue label removed +- MR closed +- MR merged +- MR created +- MR first commit time +- MR first assigned +- MR first deployed +- MR label added +- MR label removed +- MR last pipeline duration -NOTE: -In GitLab 13.9 and later, deployment frequency metrics are calculated based on when the deployment was finished. -In GitLab 13.8 and earlier, deployment frequency metrics are calculated based on when the deployment was created. +These events play a key role in the duration calculation, which is calculated by the formula: duration = end event time - start event time. -
    - See the video: DORA metrics and value stream analytics. -
    -
    - -
    +To learn what start and end events can be paired, see [Validating start and end events](../../../development/value_stream_analytics.md#validating-start-and-end-events). -### How value stream analytics aggregates data +### How value stream analytics aggregates data **(PREMIUM)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335391) in GitLab 14.5. > - Filter by stop date toggle [added](https://gitlab.com/gitlab-org/gitlab/-/issues/352428) in GitLab 14.9 @@ -147,31 +119,7 @@ longer than 10 minutes in the following cases: To view when the data was most recently updated, in the right corner next to **Edit**, hover over the **Last updated** badge. -## View metrics for each development stage - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/210315) in GitLab 13.0. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/323982) in GitLab 13.12. - -Value stream analytics shows the median time spent by issues or merge requests in each development stage. - -To view the median time spent in each stage by a group: - -1. On the top bar, select **Main menu > Groups** and find your group. -1. On the left sidebar, select **Analytics > Value stream**. -1. Optional. Filter the results: - 1. Select the **Filter results** text box. - 1. Select a parameter. - 1. Select a value or enter text to refine the results. - 1. To adjust the date range: - - In the **From** field, select a start date. - - In the **To** field, select an end date. -1. To view the metrics for each stage, above the **Filter results** text box, hover over a stage. - -NOTE: -The date range selector filters items by the event time. The event time is when the -selected stage finished for the given item. - -## How value stream analytics measures stages +### How value stream analytics measures stages Value stream analytics measures each stage from its start event to its end event. @@ -194,7 +142,7 @@ Each pre-defined stages of value stream analytics is further described in the ta For information about how value stream analytics calculates each stage, see the [Value stream analytics development guide](../../../development/value_stream_analytics.md). -### Example workflow +#### Example workflow This example shows a workflow through all seven stages in one day. @@ -234,9 +182,9 @@ Keep in mind the following observations related to this example: - This example illustrates only **one cycle** of the seven stages. The value stream analytics dashboard shows the median time for multiple cycles. -## How value stream analytics identifies the production environment +### How value stream analytics identifies the production environment -Value stream analytics identifies production environments by looking for project +Value stream analytics identifies [production environments](../../../ci/environments/index.md#deployment-tier-of-environments) by looking for project [environments](../../../ci/yaml/index.md#environment) with a name matching any of these patterns: - `prod` or `prod/*` @@ -246,14 +194,189 @@ These patterns are not case-sensitive. You can change the name of a project environment in your GitLab CI/CD configuration. -## Create a value stream with GitLab default stages +## View value stream analytics + +> - Filtering [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13216) in GitLab 13.3 +> - Horizontal stage path [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12196) in 13.0 and [feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/323982) in 13.12 + +Prerequisites: + +- You must have at least the Reporter role. +- You must create a [custom value stream](#create-a-value-stream-with-gitlab-default-stages). Value stream analytics only shows custom value streams created for your group or project. + +To view value stream analytics for your group or project: + +1. On the top bar, select **Main menu**, and: + - For a project, select **Projects** and find your project. + - For a group, select **Groups** and find your group. +1. On the left sidebar, select **Analytics > Value stream**. +1. To view metrics for a particular stage, select a stage below the **Filter results** text box. +1. Optional. Filter the results: + 1. Select the **Filter results** text box. + 1. Select a parameter. + 1. Select a value or enter text to refine the results. + 1. To adjust the date range: + - In the **From** field, select a start date. + - In the **To** field, select an end date. The charts and list show workflow items created + during the date range. +1. Optional. Sort results by ascending or descending: + - To sort by most recent or oldest workflow item, select the **Last event** header. + - To sort by most or least amount of time spent in each stage, select the **Duration** header. + +A badge next to the workflow items table header shows the number of workflow items that +completed during the selected stage. + +The table shows a list of related workflow items for the selected stage. Based on the stage you select, this can be: + +- Issues +- Merge requests + +### Data filters + +You can filter value stream analytics to view data that matches specific criteria. The following filters are supported: + +- Date range +- Project +- Assignee +- Author +- Milestone +- Label + +NOTE: +For the "Tasks by type" chart, only the Date range and Project selector filters are available. Labels and other filters are not applied, and you need to select labels separately from the dropdown list next to the chart. + +## Value stream analytics metrics + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/210315) in GitLab 13.0. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/323982) in GitLab 13.12. + +The **Overview** page in value stream analytics displays key metrics of the DevSecOps lifecycle performance for projects and groups. + +### Key metrics + +Value stream analytics includes the following key metrics: + +- **Lead time**: Median time from when the issue was created to when it was closed. +- **Cycle time**: Median time from first commit to issue closed. GitLab measures cycle time from the earliest commit of a + [linked issue's merge request](../../project/issues/crosslinking_issues.md#from-commit-messages) to when that issue is closed. + The cycle time approach underestimates the lead time because merge request creation is always later than commit time. +- **New issues**: Number of new issues created. +- **Deploys**: Total number of deployments to production. + +### DORA metrics **(ULTIMATE)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/340150) lead time for changes DORA metric in GitLab 14.5. +> - DORA API-based deployment metrics for value stream analytics for groups were [moved](https://gitlab.com/gitlab-org/gitlab/-/issues/337256) from GitLab Ultimate to GitLab Premium in GitLab 14.3. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355304) time to restore service tile in GitLab 15.0. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/357071) change failure rate tile in GitLab 15.0. + +Value stream analytics includes the following [DORA](../../../user/analytics/dora_metrics.md) metrics: + +- Deployment frequency +- Lead time for changes +- Time to restore service +- Change failure rate + +DORA metrics are calculated based on data from the +[DORA API](../../../api/dora/metrics.md#devops-research-and-assessment-dora-key-metrics-api). + +If you have a GitLab Premium or Ultimate subscription: + +- The number of successful deployments is calculated with DORA data. +- The data is filtered based on environment and environment tier. + +NOTE: +In GitLab 13.9 and later, deployment frequency metrics are calculated based on when the deployment was finished. +In GitLab 13.8 and earlier, deployment frequency metrics are calculated based on when the deployment was created. + +## View key and DORA metrics + +Prerequisite: + +- To view deployment metrics, you must have a +[production environment configured](#how-value-stream-analytics-identifies-the-production-environment). + +To view key lifecycle metrics: + +1. On the top bar, select **Main menu**, and: + - For a project, select **Projects** and find your project. + - For a group, select **Groups** and find your group. +1. On the left sidebar, select **Analytics > Value stream**. + Key metrics display below the **Filter results** text box. +1. Optional. Filter the results: + 1. Select the **Filter results** text box. + Based on the filter you select, the dashboard automatically aggregates key metrics and displays the status of the value stream. + 1. Select a parameter. + 1. Select a value or enter text to refine the results. + 1. To adjust the date range: + - In the **From** field, select a start date. + - In the **To** field, select an end date. + +To view the [Value Streams Dashboard](../../analytics/value_streams_dashboard.md) and [DORA metrics](../../analytics/dora_metrics.md): + +1. On the top bar, select **Main menu**, and: + - For a project, select **Projects** and find your project. + - For a group, select **Groups** and find your group. +1. On the left sidebar, select **Analytics > Value stream**. +1. Below the **Filter results** text box, in the **Key metrics** row, select **Value Streams Dashboard / DORA**. +1. Optional. To open the new page, append this path `/analytics/dashboards/value_streams_dashboard` to the group URL + (for example, `https://gitlab.com/groups/gitlab-org/-/analytics/dashboards/value_streams_dashboard`). + +## View metrics for each development stage + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/210315) in GitLab 13.0. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/323982) in GitLab 13.12. + +Value stream analytics shows the median time spent by issues or merge requests in each development stage. + +To view the median time spent in each stage by a group: + +1. On the top bar, select **Main menu**, and: + - For a project, select **Projects** and find your project. + - For a group, select **Groups** and find your group. +1. On the left sidebar, select **Analytics > Value stream**. +1. Optional. Filter the results: + 1. Select the **Filter results** text box. + 1. Select a parameter. + 1. Select a value or enter text to refine the results. + 1. To adjust the date range: + - In the **From** field, select a start date. + - In the **To** field, select an end date. +1. To view the metrics for each stage, above the **Filter results** text box, hover over a stage. + +NOTE: +The date range selector filters items by the event time. The event time is when the +selected stage finished for the given item. + +## View tasks by type **(PREMIUM)** + +The **Tasks by type** chart displays the cumulative number of issues and merge requests per day for your group. + +The chart uses the global page filters to display data based on the selected +group and time frame. + +To view tasks by type: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Analytics > Value stream**. +1. Below the **Filter results** text box, select **Overview**. The **Tasks by type** chart displays below the **Total time** chart. +1. To switch between the task type, select the **Settings** (**{settings}**) dropdown list + and select **Issues** or **Merge Requests**. +1. To add or remove labels, select the **Settings** (**{settings}**) dropdown list + and select or search for a label. By default the top group-level labels (maximum 10) are selected. You can select a maximum of 15 labels. + +## Create a value stream **(PREMIUM)** + +### Create a value stream with GitLab default stages > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221202) in GitLab 13.3 When you create a value stream, you can use GitLab default stages and hide or re-order them. You can also create custom stages in addition to those provided in the default template. -1. On the top bar, select **Main menu > Groups** and find your group. +1. On the top bar, select **Main menu**, and: + - For a project, select **Projects** and find your project. + - For a group, select **Groups** and find your group. 1. On the left sidebar, select **Analytics > Value Stream**. 1. Select **Create new Value Stream**. 1. Enter a name for the value stream. @@ -269,7 +392,7 @@ create custom stages in addition to those provided in the default template. NOTE: If you have recently upgraded to GitLab Premium, it can take up to 30 minutes for data to collect and display. -## Create a value stream with custom stages +### Create a value stream with custom stages > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50229) in GitLab 13.7. > - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55572) in GitLab 13.10. @@ -277,7 +400,9 @@ If you have recently upgraded to GitLab Premium, it can take up to 30 minutes fo When you create a value stream, you can create and add custom stages that align with your own development workflows. -1. On the top bar, select **Main menu > Groups** and find your group. +1. On the top bar, select **Main menu**, and: + - For a project, select **Projects** and find your project. + - For a group, select **Groups** and find your group. 1. On the left sidebar, select **Analytics > Value Stream**. 1. Select **Create value stream**. 1. For each stage: @@ -287,7 +412,7 @@ When you create a value stream, you can create and add custom stages that align 1. To re-order the stages, select the up or down arrows. 1. Select **Create value stream**. -### Label-based stages for custom value streams +#### Label-based stages for custom value streams To measure complex workflows, you can use [scoped labels](../../project/labels.md#scoped-labels). For example, to measure deployment time from a staging environment to production, you could use the following labels: @@ -297,15 +422,25 @@ time from a staging environment to production, you could use the following label ![Label-based value stream analytics stage](img/vsa_label_based_stage_v14_0.png "Creating a label-based value stream analytics stage") -## Edit a value stream +#### Example for custom value stream configuration + +![Example configuration](img/object_hierarchy_example_V14_10.png "Example custom value stream configuration") + +In the example above, two independent value streams are set up for two teams that are using different development workflows in the **Test Group** (top-level namespace). + +The first value stream uses standard timestamp-based events for defining the stages. The second value stream uses label events. + +## Edit a value stream **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267537) in GitLab 13.10. After you create a value stream, you can customize it to suit your purposes. To edit a value stream: -1. On the top bar, select **Main menu > Groups** and find your group. +1. On the top bar, select **Main menu**, and: + - For a project, select **Projects** and find your project. + - For a group, select **Groups** and find your group. 1. On the left sidebar, select **Analytics > Value Stream**. -1. In the upper right, select the dropdown list, and select a value stream. +1. In the upper-right corner, select the dropdown list, then select a value stream. 1. Next to the value stream dropdown list, select **Edit**. 1. Optional: - Rename the value stream. @@ -316,21 +451,22 @@ After you create a value stream, you can customize it to suit your purposes. To 1. Optional. To undo any modifications, select **Restore value stream defaults**. 1. Select **Save Value Stream**. -## Delete a value stream +## Delete a value stream **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/221205) in GitLab 13.4. To delete a custom value stream: -1. On the top bar, select **Main menu > Groups** and find your group. -1. On the left sidebar, select **Analytics > Value Stream**. -1. In the upper right, select the dropdown list and then select the value stream you would like to delete. +1. On the top bar, select **Main menu**, and: + - For a project, select **Projects** and find your project. + - For a group, select **Groups** and find your group. +1. In the upper-right corner, select the dropdown list, then select the value stream you would like to delete. 1. Select **Delete (name of value stream)**. 1. To confirm, select **Delete**. ![Delete value stream](img/delete_value_stream_v13_12.png "Deleting a custom value stream") -## View number of days for a cycle to complete +## View number of days for a cycle to complete **(PREMIUM)** > - Chart median line [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/235455) in GitLab 13.4. > - Totals [replaced](https://gitlab.com/gitlab-org/gitlab/-/issues/262070) with averages in GitLab 13.12. @@ -338,7 +474,9 @@ To delete a custom value stream: The **Total time chart** shows the average number of days it takes for development cycles to complete. The chart shows data for the last 500 workflow items. -1. On the top bar, select **Main menu > Groups** and find your group. +1. On the top bar, select **Main menu**, and: + - For a project, select **Projects** and find your project. + - For a group, select **Groups** and find your group. 1. On the left sidebar, select **Analytics > Value stream**. 1. Above the **Filter results** box, select a stage: - To view a summary of the cycle time for all stages, select **Overview**. @@ -351,23 +489,42 @@ The chart shows data for the last 500 workflow items. - In the **From** field, select a start date. - In the **To** field, select an end date. -## View tasks by type +## Access permissions for value stream analytics -The **Tasks by type** chart displays the cumulative number of issues and merge requests per day for your group. +Access permissions for value stream analytics depend on the project type. -The chart uses the global page filters to display data based on the selected -group, projects, and time frame. +| Project type | Permissions | +|--------------|----------------------------------------| +| Public | Anyone can access. | +| Internal | Any authenticated user can access. | +| Private | Any member Guest and above can access. | -To view tasks by type: +## Troubleshooting -1. On the top bar, select **Main menu > Groups** and find your group. -1. On the left sidebar, select **Analytics > Value stream**. -1. Below the **Filter results** text box, select **Overview**. The **Tasks by type** chart displays below the **Total time** chart. -1. To switch between the task type, select the **Settings** (**{settings}**) dropdown list - and select **Issues** or **Merge Requests**. -1. To add or remove labels, select the **Settings** (**{settings}**) dropdown list - and select or search for a label. By default the top group-level labels (maximum 10) are selected. You can select a maximum of 15 labels. +### 100% CPU utilization by Sidekiq `cronjob:analytics_cycle_analytics` -## Troubleshooting +It is possible that value stream analytics background jobs +strongly impact performance by monopolizing CPU resources. + +To recover from this situation: + +1. Disable the feature for all projects in [the Rails console](../../../administration/operations/rails_console.md), + and remove existing jobs: + + ```ruby + Project.find_each do |p| + p.analytics_access_level='disabled'; + p.save! + end + + Analytics::CycleAnalytics::GroupStage.delete_all + Analytics::CycleAnalytics::Aggregation.delete_all + ``` -See [Value stream analytics for projects](../../analytics/value_stream_analytics.md#troubleshooting). +1. Configure a [Sidekiq routing](../../../administration/sidekiq/processing_specific_job_classes.md) + with for example a single `feature_category=value_stream_management` + and multiple `feature_category!=value_stream_management` entries. + Find other relevant queue metadata in the + [Enterprise Edition list](../../../administration/sidekiq/processing_specific_job_classes.md#list-of-available-job-classes). +1. Enable value stream analytics for one project after another. + You might need to tweak the Sidekiq routing further according to your performance requirements. diff --git a/doc/user/img/explain_code_experiment.png b/doc/user/img/explain_code_experiment.png new file mode 100644 index 00000000000..1b3b9e3eef3 Binary files /dev/null and b/doc/user/img/explain_code_experiment.png differ diff --git a/doc/user/img/explain_this_vulnerability.png b/doc/user/img/explain_this_vulnerability.png new file mode 100644 index 00000000000..0880ad5f875 Binary files /dev/null and b/doc/user/img/explain_this_vulnerability.png differ diff --git a/doc/user/img/get_domain_verification_code_v16_0.png b/doc/user/img/get_domain_verification_code_v16_0.png new file mode 100644 index 00000000000..34c77866865 Binary files /dev/null and b/doc/user/img/get_domain_verification_code_v16_0.png differ diff --git a/doc/user/img/retry_domain_verification_v16_0.png b/doc/user/img/retry_domain_verification_v16_0.png new file mode 100644 index 00000000000..4183fd5a342 Binary files /dev/null and b/doc/user/img/retry_domain_verification_v16_0.png differ diff --git a/doc/user/img/todos_add_okrs_v16_0.png b/doc/user/img/todos_add_okrs_v16_0.png new file mode 100644 index 00000000000..45c04e647e2 Binary files /dev/null and b/doc/user/img/todos_add_okrs_v16_0.png differ diff --git a/doc/user/img/todos_mark_done_okrs_v16_0.png b/doc/user/img/todos_mark_done_okrs_v16_0.png new file mode 100644 index 00000000000..545b3569ed5 Binary files /dev/null and b/doc/user/img/todos_mark_done_okrs_v16_0.png differ diff --git a/doc/user/infrastructure/clusters/connect/index.md b/doc/user/infrastructure/clusters/connect/index.md index 9325f11b0c7..b571a65b50a 100644 --- a/doc/user/infrastructure/clusters/connect/index.md +++ b/doc/user/infrastructure/clusters/connect/index.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -10,7 +10,7 @@ The [certificate-based Kubernetes integration with GitLab](../index.md) was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. To connect your clusters, use the [GitLab agent](../../../clusters/agent/index.md). -## Cluster levels (DEPRECATED) +## Cluster levels (deprecated) > [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. diff --git a/doc/user/infrastructure/clusters/connect/new_civo_cluster.md b/doc/user/infrastructure/clusters/connect/new_civo_cluster.md index df785cce406..45445c0a0f5 100644 --- a/doc/user/infrastructure/clusters/connect/new_civo_cluster.md +++ b/doc/user/infrastructure/clusters/connect/new_civo_cluster.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -117,18 +117,18 @@ To remove all resources: 1. Add the following to your `.gitlab-ci.yml` file: - ```yaml - stages: - - init - - validate - - build - - deploy - - cleanup - - destroy: - extends: .destroy - needs: [] - ``` + ```yaml + stages: + - init + - validate + - build + - deploy + - cleanup + + destroy: + extends: .destroy + needs: [] + ``` 1. On the left sidebar, select **CI/CD > Pipelines** and select the most recent pipeline. 1. For the `destroy` job, select **Play** (**{play}**). diff --git a/doc/user/infrastructure/clusters/connect/new_eks_cluster.md b/doc/user/infrastructure/clusters/connect/new_eks_cluster.md index cefa8885bfe..4516ea538a9 100644 --- a/doc/user/infrastructure/clusters/connect/new_eks_cluster.md +++ b/doc/user/infrastructure/clusters/connect/new_eks_cluster.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -67,42 +67,42 @@ Set up your AWS credentials when you want to authenticate AWS with GitLab. 1. Create an [IAM User](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users.html) or [IAM Role](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html). 1. Make sure that your IAM user or role has the appropriate permissions for your project. For this example project, you must have the permissions shown below. You can expand this when you set up your own project. - ```json - // IAM custom Policy definition - { - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "VisualEditor0", - "Effect": "Allow", - "Action": [ - "ec2:*", - "eks:*", - "elasticloadbalancing:*", - "autoscaling:*", - "cloudwatch:*", - "logs:*", - "kms:DescribeKey", - "iam:AddRoleToInstanceProfile", - "iam:AttachRolePolicy", - "iam:CreateInstanceProfile", - "iam:CreateRole", - "iam:CreateServiceLinkedRole", - "iam:GetRole", - "iam:ListAttachedRolePolicies", - "iam:ListRolePolicies", - "iam:ListRoles", - "iam:PassRole", - // required for destroy step - "iam:DetachRolePolicy", - "iam:ListInstanceProfilesForRole", - "iam:DeleteRole" - ], - "Resource": "*" - } - ] - } - ``` + ```json + // IAM custom Policy definition + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "ec2:*", + "eks:*", + "elasticloadbalancing:*", + "autoscaling:*", + "cloudwatch:*", + "logs:*", + "kms:DescribeKey", + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreateRole", + "iam:CreateServiceLinkedRole", + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:ListRolePolicies", + "iam:ListRoles", + "iam:PassRole", + // required for destroy step + "iam:DetachRolePolicy", + "iam:ListInstanceProfilesForRole", + "iam:DeleteRole" + ], + "Resource": "*" + } + ] + } + ``` 1. [Create an access key for the user or role](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html). 1. Save your access key and secret. You need these to authenticate AWS with GitLab. @@ -165,19 +165,19 @@ To remove all resources: 1. Add the following to your `.gitlab-ci.yml` file: - ```yaml - stages: - - init - - validate - - test - - build - - deploy - - cleanup - - destroy: - extends: .terraform:destroy - needs: [] - ``` + ```yaml + stages: + - init + - validate + - test + - build + - deploy + - cleanup + + destroy: + extends: .terraform:destroy + needs: [] + ``` 1. On the left sidebar, select **CI/CD > Pipelines** and select the most recent pipeline. 1. For the `destroy` job, select **Play** (**{play}**). diff --git a/doc/user/infrastructure/clusters/connect/new_gke_cluster.md b/doc/user/infrastructure/clusters/connect/new_gke_cluster.md index 7e6a0495d90..c8d2fb674b2 100644 --- a/doc/user/infrastructure/clusters/connect/new_gke_cluster.md +++ b/doc/user/infrastructure/clusters/connect/new_gke_cluster.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -143,19 +143,19 @@ To remove all resources: 1. Add the following to your `.gitlab-ci.yml` file: - ```yaml - stages: - - init - - validate - - build - - test - - deploy - - cleanup - - destroy: - extends: .terraform:destroy - needs: [] - ``` + ```yaml + stages: + - init + - validate + - build + - test + - deploy + - cleanup + + destroy: + extends: .terraform:destroy + needs: [] + ``` 1. On the left sidebar, select **CI/CD > Pipelines** and select the most recent pipeline. 1. For the `destroy` job, select **Play** (**{play}**). diff --git a/doc/user/infrastructure/clusters/deploy/inventory_object.md b/doc/user/infrastructure/clusters/deploy/inventory_object.md index 6bd44c7a0f7..50185966267 100644 --- a/doc/user/infrastructure/clusters/deploy/inventory_object.md +++ b/doc/user/infrastructure/clusters/deploy/inventory_object.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/infrastructure/clusters/index.md b/doc/user/infrastructure/clusters/index.md index 14fd4917141..d0419e08f95 100644 --- a/doc/user/infrastructure/clusters/index.md +++ b/doc/user/infrastructure/clusters/index.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w To connect clusters to GitLab, use the [GitLab agent](../../clusters/agent/index.md). -## Certificate-based Kubernetes integration (DEPRECATED) +## Certificate-based Kubernetes integration (deprecated) WARNING: In GitLab 14.5, the certificate-based method to connect Kubernetes clusters diff --git a/doc/user/infrastructure/clusters/manage/clusters_health.md b/doc/user/infrastructure/clusters/manage/clusters_health.md index 4f63f3e4e2a..d9b849ffccc 100644 --- a/doc/user/infrastructure/clusters/manage/clusters_health.md +++ b/doc/user/infrastructure/clusters/manage/clusters_health.md @@ -1,19 +1,12 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../index.md' --- -# Clusters health (DEPRECATED) **(FREE)** +# Clusters health (removed) **(FREE)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/4701) in GitLab 10.6. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/208224) from GitLab Ultimate to GitLab Free in 13.2. -> - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. - -WARNING: -This feature was [deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. However, you can **still use** Prometheus -for Kubernetes clusters connected to GitLab by [enabling Prometheus manually](../../../project/integrations/prometheus.md#manual-configuration-of-prometheus). - -When [the Prometheus cluster integration is enabled](../../../clusters/integrations.md#prometheus-cluster-integration), GitLab monitors the cluster's health. At the top of the cluster settings page, CPU and Memory utilization is displayed, along with the total amount available. Keeping an eye on cluster resources can be important, if the cluster runs out of memory pods may be shutdown or fail to start. - -![Cluster Monitoring](img/k8s_cluster_monitoring.png) +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/user/infrastructure/clusters/manage/img/k8s_cluster_monitoring.png b/doc/user/infrastructure/clusters/manage/img/k8s_cluster_monitoring.png deleted file mode 100644 index 0a8c5043c65..00000000000 Binary files a/doc/user/infrastructure/clusters/manage/img/k8s_cluster_monitoring.png and /dev/null differ diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/certmanager.md b/doc/user/infrastructure/clusters/manage/management_project_applications/certmanager.md index 8b75d54d352..f07b70dd8e0 100644 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/certmanager.md +++ b/doc/user/infrastructure/clusters/manage/management_project_applications/certmanager.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/ingress.md b/doc/user/infrastructure/clusters/manage/management_project_applications/ingress.md index 14d3a7996e0..1d56e7c1ba1 100644 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/ingress.md +++ b/doc/user/infrastructure/clusters/manage/management_project_applications/ingress.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/infrastructure/clusters/manage/management_project_applications/vault.md b/doc/user/infrastructure/clusters/manage/management_project_applications/vault.md index c2190ad7cfa..bbf8391e8a0 100644 --- a/doc/user/infrastructure/clusters/manage/management_project_applications/vault.md +++ b/doc/user/infrastructure/clusters/manage/management_project_applications/vault.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/infrastructure/clusters/migrate_to_gitlab_agent.md b/doc/user/infrastructure/clusters/migrate_to_gitlab_agent.md index b2b5da61a81..a21c7271e7a 100644 --- a/doc/user/infrastructure/clusters/migrate_to_gitlab_agent.md +++ b/doc/user/infrastructure/clusters/migrate_to_gitlab_agent.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/infrastructure/iac/gitlab_terraform_helpers.md b/doc/user/infrastructure/iac/gitlab_terraform_helpers.md index 8be949c40cd..d5376bfbcb1 100644 --- a/doc/user/infrastructure/iac/gitlab_terraform_helpers.md +++ b/doc/user/infrastructure/iac/gitlab_terraform_helpers.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/infrastructure/iac/index.md b/doc/user/infrastructure/iac/index.md index f9891934067..d4fc345f494 100644 --- a/doc/user/infrastructure/iac/index.md +++ b/doc/user/infrastructure/iac/index.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -100,7 +100,7 @@ For GitLab-curated template recipes, see [Terraform template recipes](terraform_ ## Related topics - View [the images that contain the `gitlab-terraform` shell script](https://gitlab.com/gitlab-org/terraform-images). -- Use GitLab as a [Terraform module registry](../../packages/terraform_module_registry/index.md). +- Use GitLab as a [Terraform Module Registry](../../packages/terraform_module_registry/index.md). - To store state files in local storage or in a remote store, use the [GitLab-managed Terraform state](terraform_state.md). - To collaborate on Terraform code changes and Infrastructure-as-Code workflows, use the [Terraform integration in merge requests](mr_integration.md). diff --git a/doc/user/infrastructure/iac/mr_integration.md b/doc/user/infrastructure/iac/mr_integration.md index 7b4a7cd6b95..b2e02cd7375 100644 --- a/doc/user/infrastructure/iac/mr_integration.md +++ b/doc/user/infrastructure/iac/mr_integration.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/infrastructure/iac/terraform_state.md b/doc/user/infrastructure/iac/terraform_state.md index 93a82023480..1b0065fd165 100644 --- a/doc/user/infrastructure/iac/terraform_state.md +++ b/doc/user/infrastructure/iac/terraform_state.md @@ -1,16 +1,14 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # GitLab-managed Terraform state **(FREE)** > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2673) in GitLab 13.0. -> - Support for state names that contain periods introduced in GitLab 15.7 [with a flag](../../../administration/feature_flags.md) named `allow_dots_on_tf_state_names`. Disabled by default. [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106861) in GitLab 15.7. - -FLAG: -On self-managed GitLab, by default support for state names that contain periods is not available. To make it available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `allow_dots_on_tf_state_names`. On GitLab.com, support for state names that contain periods is available. Requests for state files might generate HTTP 404 errors after enabling this feature. For more information, see [Troubleshooting the Terraform integration with GitLab](troubleshooting.md#state-not-found-if-the-state-name-contains-a-period). +> - Support for state names that contain periods introduced in GitLab 15.7 [with a flag](../../../administration/feature_flags.md) named `allow_dots_on_tf_state_names`. Disabled by default. +> - Support for state names that contain periods [generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/385597) in GitLab 16.0. Feature flag `allow_dots_on_tf_state_names` removed. Terraform uses state files to store details about your infrastructure configuration. With Terraform remote [backends](https://www.terraform.io/language/settings/backends/configuration), @@ -42,7 +40,7 @@ For self-managed GitLab, before you can use GitLab for your Terraform state file - An administrator must [set up Terraform state storage](../../../administration/terraform_state.md). - You must enable the **Infrastructure** menu for your project. Go to **Settings > General**, - expand **Visibility, project features, permissions**, and under **Operations**, turn on the toggle. + expand **Visibility, project features, permissions**, and under **Infrastructure**, turn on the toggle. ## Initialize a Terraform state as a backend by using GitLab CI/CD @@ -58,7 +56,7 @@ WARNING: Like any other job artifact, Terraform plan data is viewable by anyone with the Guest role on the repository. Neither Terraform nor GitLab encrypts the plan file by default. If your Terraform plan includes sensitive data, like passwords, access tokens, or certificates, you should -encrypt plan output or modify the project visibility settings. We also strongly recommend that you **disable** +encrypt plan output or modify the project visibility settings. We also strongly recommend that you **disable** [public pipelines](../../../ci/pipelines/settings.md#change-pipeline-visibility-for-non-project-members-in-public-projects) by setting the artifact's public flag to false (`public: false`). This setting ensures artifacts are accessible only to GitLab Administrators and project members with the Reporter role and above. @@ -119,7 +117,7 @@ inconsistent. Instead, use a remote storage resource. 1. Copy a pre-populated Terraform `init` command: 1. On the top bar, select **Main menu > Projects** and find your project. - 1. On the left sidebar, select **Infrastructure > Terraform**. + 1. On the left sidebar, select **Infrastructure > Terraform states**. 1. Next to the environment you want to use, select **Actions** (**{ellipsis_v}**) and select **Copy Terraform init command**. @@ -297,7 +295,7 @@ To read the Terraform state in the target project, you need at least the Develop To view Terraform state files: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Infrastructure > Terraform**. +1. On the left sidebar, select **Infrastructure > Terraform states**. [An epic exists](https://gitlab.com/groups/gitlab-org/-/epics/4563) to track improvements to this UI. @@ -323,7 +321,7 @@ curl --header "Private-Token: " --request DELETE "https://git If you have at least the Maintainer role, you can remove a state file. -1. On the left sidebar, select **Infrastructure > Terraform**. +1. On the left sidebar, select **Infrastructure > Terraform states**. 1. In the **Actions** column, select **Actions** (**{ellipsis_v}**) and then **Remove state file and versions**. ### Remove a state file by using the API @@ -338,6 +336,5 @@ You can also use [the GraphQL API](../../../api/graphql/reference/index.md#mutat ## Related topics -- [Troubleshooting GitLab-managed Terraform state](troubleshooting.md). -- To use GitLab and Terraform to deploy an AWS EC2 instance in a custom VPC, - see [this sample project](https://gitlab.com/gitlab-org/configure/examples/gitlab-terraform-aws). +- [Troubleshooting GitLab-managed Terraform state](troubleshooting.md) +- [Sample project: Terraform deployment of AWS EC2 instance in a custom VPC](https://gitlab.com/gitlab-org/configure/examples/gitlab-terraform-aws) diff --git a/doc/user/infrastructure/iac/terraform_template_recipes.md b/doc/user/infrastructure/iac/terraform_template_recipes.md index 89a97f305e4..0011e7c9242 100644 --- a/doc/user/infrastructure/iac/terraform_template_recipes.md +++ b/doc/user/infrastructure/iac/terraform_template_recipes.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -36,7 +36,7 @@ include: - template: Terraform.latest.gitlab-ci.yml deploy: - envrionment: + environment: name: $TF_STATE_NAME action: start on_stop: destroy @@ -65,7 +65,7 @@ build: - when: on_success deploy: - envrionment: + environment: name: $TF_STATE_NAME action: start on_stop: destroy diff --git a/doc/user/infrastructure/iac/troubleshooting.md b/doc/user/infrastructure/iac/troubleshooting.md index 2aa1e5d3284..d770c0111d0 100644 --- a/doc/user/infrastructure/iac/troubleshooting.md +++ b/doc/user/infrastructure/iac/troubleshooting.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -160,12 +160,3 @@ If your `TF_HTTP_ADDRESS`, `TF_HTTP_LOCK_ADDRESS` and `TF_HTTP_UNLOCK_ADDRESS` a to update the state names there. Alternatively, you can [migrate your terraform state](terraform_state.md#migrate-to-a-gitlab-managed-terraform-state). - -#### Self-managed GitLab instances - -By default, support for state names with periods is not enabled on self-managed GitLab. -You can enable it from the Rails console: - -```ruby -Feature.enable(:allow_dots_on_tf_state_names) -``` diff --git a/doc/user/infrastructure/index.md b/doc/user/infrastructure/index.md index 87633932e0d..d60f20a3713 100644 --- a/doc/user/infrastructure/index.md +++ b/doc/user/infrastructure/index.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/instance/clusters/index.md b/doc/user/instance/clusters/index.md index 88430df0d67..dc5d402d04b 100644 --- a/doc/user/instance/clusters/index.md +++ b/doc/user/instance/clusters/index.md @@ -4,7 +4,7 @@ group: unassigned info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Instance-level Kubernetes clusters (certificate-based) (DEPRECATED) **(FREE SELF)** +# Instance-level Kubernetes clusters (certificate-based) (deprecated) **(FREE SELF)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/39840) in GitLab 11.11. > - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. diff --git a/doc/user/markdown.md b/doc/user/markdown.md index eaceeccc148..b8ed1c06324 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -124,8 +124,13 @@ to see the color chips next to the color code: ### Diagrams and flowcharts -You can generate diagrams and flowcharts from text by using [Mermaid](https://mermaidjs.github.io/) or [PlantUML](https://plantuml.com). -You can also use [Kroki](https://kroki.io) to create a wide variety of diagrams. +You can generate diagrams from text by using: + +- [Mermaid](https://mermaidjs.github.io/) +- [PlantUML](https://plantuml.com) +- [Kroki](https://kroki.io) to create a wide variety of diagrams. + +In wikis, you can also add and edit diagrams created with the [diagrams.net editor](#diagramsnet-editor). #### Mermaid @@ -206,27 +211,44 @@ For more information, see the [Kroki integration](../administration/integration/ [View this topic in GitLab](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/user/markdown.md#emojis). -```markdown -Sometimes you want to :monkey: around a bit and add some :star2: to your :speech_balloon:. Well we have a gift for you: +::Tabs -:zap: You can use emoji anywhere GitLab Flavored Markdown is supported. :v: +:::TabTitle Rendered Markdown -You can use it to point out a :bug: or warn about :speak_no_evil: patches. And if someone improves your really :snail: code, send them some :birthday:. People :heart: you for that. + -If you're new to this, don't be :fearful:. You can join the emoji :family:. Just look up one of the supported codes. +Sometimes you want to :monkey: around a bit and add some :star2: to your :speech_balloon:. Well we have a gift for you: -Consult the [Emoji Cheat Sheet](https://www.webfx.com/tools/emoji-cheat-sheet/) for a list of all supported emoji codes. :thumbsup: -``` +:zap:You can use emoji anywhere GitLab Flavored Markdown is supported. :v: + +You can use it to point out a :bug: or warn about :speak_no_evil: patches. If someone improves your really :snail: code, send them some :birthday:. People :heart: you for that. -Sometimes you want to around a bit and add some to your . Well we have a gift for you: +If you're new to this, don't be :fearful:. You can join the emoji :family:. Just look up one of the supported codes. -You can use emoji anywhere GitLab Flavored Markdown is supported. +Consult the [Emoji Cheat Sheet](https://www.webfx.com/tools/emoji-cheat-sheet/) for a list of all supported emoji codes. :thumbsup: -You can use it to point out a or warn about patches. If someone improves your really code, send them some . People you for that. + -If you're new to this, don't be . You can join the emoji . Just look up one of the supported codes. +:::TabTitle Code -Consult the [Emoji Cheat Sheet](https://www.webfx.com/tools/emoji-cheat-sheet/) for a list of all supported emoji codes. +```markdown +Sometimes you want to :monkey: around a bit and add some :star2: to your +:speech_balloon:. Well we have a gift for you: + +:zap: You can use emoji anywhere GitLab Flavored Markdown is supported. :v: + +You can use it to point out a :bug: or warn about :speak_no_evil: patches. +And if someone improves your really :snail: code, send them some :birthday:. +People :heart: you for that. + +If you're new to this, don't be :fearful:. You can join the emoji :family:. +Just look up one of the supported codes. + +Consult the [Emoji Cheat Sheet](https://www.webfx.com/tools/emoji-cheat-sheet/) +for a list of all supported emoji codes. :thumbsup: +``` + +::EndTabs #### Emojis and your operating system @@ -244,6 +266,8 @@ this font installed by default. +To learn more about adding custom emojis, see [Custom emojis](award_emojis.md#custom-emojis). + ### Front matter Front matter is metadata included at the beginning of a Markdown document, preceding @@ -357,6 +381,10 @@ _KaTeX only supports a [subset](https://katex.org/docs/supported.html) of LaTeX. This syntax also works for the Asciidoctor `:stem: latexmath`. For details, see the [Asciidoctor user manual](https://asciidoctor.org/docs/user-manual/#activating-stem-support). +To prevent malicious activity, GitLab renders only the first 50 inline math instances. +The number of math blocks is also limited based on render time. If the limit is exceeded, +GitLab renders the excess math instances as text. + Math written between dollar signs with backticks (``$`...`$``) or single dollar signs (`$...$`) is rendered inline with the text. @@ -366,15 +394,15 @@ the language declared as `math` is rendered on a separate line: ````markdown This math is inline: $`a^2+b^2=c^2`$. -This math is on a separate line: +This math is on a separate line using a ```` ```math ```` block: ```math a^2+b^2=c^2 ``` -This math is on a separate line: $$a^2+b^2=c^2$$ +This math is on a separate line using inline `$$`: $$a^2+b^2=c^2$$ -This math is on a separate line: +This math is on a separate line using a `$$...$$` block: $$ a^2+b^2=c^2 @@ -383,23 +411,15 @@ $$ This math is inline: $`a^2+b^2=c^2`$. -This math is on a separate line: +This math is on a separate line using a ```` ```math ```` block: ```math a^2+b^2=c^2 ``` -This math is on a separate line: $$a^2+b^2=c^2$$ - -This math is on a separate line: - -$$ -a^2+b^2=c^2 -$$ - -This math is on a separate line: $$a^2+b^2=c^2$$ +This math is on a separate line using inline `$$`: $$a^2+b^2=c^2$$ -This math is on a separate line: +This math is on a separate line using a `$$...$$` block: $$ a^2+b^2=c^2 @@ -543,6 +563,58 @@ This example links to `/miscellaneous.md`: [Link to Related Page](/miscellaneous.md) ``` +#### diagrams.net editor + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/322174) in GitLab 15.10. + +NOTE: +Use of the diagrams.net editor is not available on offline environments. + +In wikis, you can use the [diagrams.net](https://www.diagrams.net/) editor to create diagrams. You +can also edit diagrams created with the diagrams.net editor. The diagram editor is available in both +the Markdown editor and the content editor. + +##### Markdown editor + +To create a diagram in the Markdown editor: + +1. In the editor's toolbar, select **Insert or edit diagram** (**{diagram}**). +1. Use the diagrams.net editor to create the diagram. +1. Select **Save & exit**. + +A Markdown image reference to the diagram is inserted in the wiki content. + +To edit a diagram in the Markdown editor: + +1. Place the Markdown editor's text field cursor in a Markdown image reference +that contains the diagram. +1. Select **Insert or edit diagram** (**{diagram}**) in the Markdown editor. +1. Use the diagrams.net editor to edit the diagram. +1. Select **Save & exit**. + +A Markdown image reference to the diagram is inserted in the wiki content, +replacing the previous diagram. + +##### Content editor + +To create a diagram in the content editor: + +1. In the editor's toolbar, select **More options** (**{plus}**). +1. In the dropdown list, select **Create or edit diagram**. +1. Use the diagrams.net editor to create the diagram. +1. Select **Save & exit**. + +The diagram as visualized in the diagrams.net editor is inserted in the wiki content. + +To edit a diagram in the content editor: + +1. Select the diagram that you want to edit. +1. In the floating toolbar, select **Edit diagram** (**{diagram}**). +1. Use the diagrams.net editor to edit the diagram. +1. Select **Save & exit**. + +The selected diagram is replaced with an updated version. + ## GitLab-specific references GitLab Flavored Markdown renders GitLab-specific references. For example, you can reference @@ -597,19 +669,44 @@ In addition to this, links to some objects are also recognized and formatted. So ### Show the issue, merge request, or epic title in the reference -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15694) in GitLab 14.6. +> - Support for issues, merge requests, and epics [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15694) in GitLab 14.6. +> - Support for work items [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/390854) in GitLab 16.0. -To include the title in the rendered link of an issue, merge request, or epic, add a plus (`+`) +To include the title in the rendered link of an issue, work item, merge request, or epic, add a plus (`+`) at the end of the reference. For example, a reference like `#123+` is rendered as `The issue title (#123)`. URL references like `https://gitlab.com/gitlab-org/gitlab/-/issues/1234+` are also expanded. -### Embedding metrics in GitLab Flavored Markdown +### Show the issue, work item or merge request summary in the reference + +> - Support for issues and merge requests [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/386937) in GitLab 15.10. +> - Support for work items [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/390854) in GitLab 16.0. + +To include an extended summary in the rendered link of an issue, work item, or merge request, add a `+s` +at the end of the reference. Summary includes information about **assignees**, **milestone** +and **health status** of referenced item. + +For example, a reference like `#123+s` is rendered as +`The issue title (#123) • First Assignee, Second Assignee+ • v15.10 • Needs attention`. + +URL references like `https://gitlab.com/gitlab-org/gitlab/-/issues/1234+s` are also expanded. + +### Embedding metrics Metric charts can be embedded in GitLab Flavored Markdown. Read [Embedding Metrics in GitLab flavored Markdown](../operations/metrics/embed.md) for more details. +### Embedding Observability dashboards + +You can embed GitLab Observability UI dashboards descriptions and comments, for example in epics, issues, and MRs. + +To embed an Observability dashboard URL: + +1. In GitLab Observability UI, copy the URL in the address bar. + +1. Paste your link wherever you want to embed your dashboard. GitLab Flavored Markdown recognizes the URL and displays the source. + ## Features extended from standard Markdown All standard Markdown formatting should work as expected in GitLab. Some standard @@ -656,13 +753,17 @@ you can quote that without having to manually prepend `>` to every line! ``` ->>> -If you paste a message from somewhere else - -that spans multiple lines, + -you can quote that without having to manually prepend `>` to every line! ->>> +> If you paste a message from somewhere else +> +> that spans multiple lines, +> +> you can quote that without having to manually prepend `>` to every line! ### Code spans and blocks @@ -727,7 +828,7 @@ Tildes are OK too. If this section isn't rendered correctly, [view it in GitLab](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/user/markdown.md#colored-code-and-syntax-highlighting). -GitLab uses the [Rouge Ruby library](http://rouge.jneen.net/) for more colorful syntax +GitLab uses the [Rouge Ruby library](https://github.com/rouge-ruby/rouge) for more colorful syntax highlighting in code blocks. For a list of supported languages visit the [Rouge project wiki](https://github.com/rouge-ruby/rouge/wiki/List-of-supported-languages-and-lexers). Syntax highlighting is supported only in code blocks, so you can't highlight inline code. @@ -1594,7 +1695,7 @@ use `
    ` tags to force a cell to have multiple lines: | Item2 | This item has:
    - Multiple items
    - That we want listed separately | You can use HTML formatting in GitLab itself to add [task lists](#task-lists) with checkboxes, -but they do not render properly on `docs.gitlab.com`. Note that these tasks will not save their +but they do not render properly on `docs.gitlab.com`. These tasks will not save their state when selected, like regular GitLab task lists. ```markdown diff --git a/doc/user/namespace/index.md b/doc/user/namespace/index.md index 7463b8f1099..00eb63da3ff 100644 --- a/doc/user/namespace/index.md +++ b/doc/user/namespace/index.md @@ -1,6 +1,6 @@ --- -stage: Manage -group: Organization +stage: Data Stores +group: Tenant Scale info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/okrs.md b/doc/user/okrs.md index 0d3be8474fe..030f6eb82ef 100644 --- a/doc/user/okrs.md +++ b/doc/user/okrs.md @@ -8,8 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/103355) in GitLab 15.6 [with a flag](../administration/feature_flags.md) named `okrs_mvc`. Disabled by default. -WARNING: -OKRs are in [**Alpha**](../policy/alpha-beta-support.md#alpha-features). +OKRs are an [Experiment](../policy/alpha-beta-support.md#experiment). For the OKR feature roadmap, see [epic 7864](https://gitlab.com/groups/gitlab-org/-/epics/7864). FLAG: @@ -17,12 +16,11 @@ On self-managed GitLab, by default this feature is not available. To make it ava On GitLab.com, this feature is not available. The feature is not ready for production use. -Use objectives and key results to align your workforce towards common goals and track the progress. -Set a big goal with an objective and use [child objectives and key results](#child-objectives-and-key-results) -to measure the big goal's completion. +[Objectives and key results](https://en.wikipedia.org/wiki/OKR) (OKRs) are a framework for setting +and tracking goals that are aligned with your organization's overall strategy and vision. The objective and the key result in GitLab share many features. In the documentation, the term -**OKR** refers to either an objective or a key result. +**OKRs** refers to both objectives and key results. OKRs are a type of work item, a step towards [default issue types](https://gitlab.com/gitlab-org/gitlab/-/issues/323404) in GitLab. @@ -31,6 +29,29 @@ to work items and adding custom work item types, see [epic 6033](https://gitlab.com/groups/gitlab-org/-/epics/6033) or the [Plan direction page](https://about.gitlab.com/direction/plan/). +## Designing effective OKRs + +Use objectives and key results to align your workforce towards common goals and track the progress. +Set a big goal with an objective and use [child objectives and key results](#child-objectives-and-key-results) +to measure the big goal's completion. + +**Objectives** are aspirational goals to be achieved and define **what you're aiming to do**. +They show how an individual's, team's, or department's work impacts overall direction of the +organization by connecting their work to overall company strategy. + +**Key results** are measures of progress against aligned objectives. They express +**how you know if you have reached your goal** (objective). +By achieving a specific outcome (key result), you create progress for the linked objective. + +To know if your OKR makes sense, you can use this sentence: + + +> I/we will accomplish (objective) by (date) through attaining and achieving the following metrics (key results). + + +To learn how to create better OKRs and how we use them at GitLab, see the +[Objectives and Key Results handbook page](https://about.gitlab.com/company/okrs/). + ## Create an objective Prerequisites: @@ -93,12 +114,31 @@ To edit an OKR: 1. Optional. To edit the description, select the edit icon (**{pencil}**), make your changes, and select **Save**. +## View OKR system notes + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/378949) in GitLab 15.7 [with a flag](../administration/feature_flags.md) named `work_items_mvc_2`. Disabled by default. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/378949) to feature flag named `work_items_mvc` in GitLab 15.8. Disabled by default. +> - Changing activity sort order [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/378949) in GitLab 15.8. +> - Filtering activity [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/389971) in GitLab 15.10. +> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/334812) in GitLab 15.10. + +Prerequisites: + +- You must have at least the Reporter role for the project. + +You can view all the system notes related to the task. By default they are sorted by **Oldest first**. +You can always change the sorting order to **Newest first**, which is remembered across sessions. + +## Comments and threads + +You can add [comments](discussions/index.md) and reply to threads in tasks. + ## Assign users To show who is responsible for an OKR, you can assign users to it. Users on GitLab Free can assign one user per OKR. -Users on GitLab Premium and higher can assign multiple users to a single OKR. +Users on GitLab Premium and Ultimate can assign multiple users to a single OKR. See also [multiple assignees for issues](project/issues/multiple_assignees_for_issues.md). Prerequisites: @@ -180,6 +220,20 @@ To set health status of an OKR: 1. [Open the key result](okrs.md#view-a-key-result) that you want to edit. 1. Next to **Health status**, select the dropdown list and select the desired health status. +## Promote a key result to an objective + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/386877) in GitLab 16.0. + +Prerequisites: + +- You must have at least the Reporter role for the project. + +To promote a key result: + +1. [Open the key result](#view-a-key-result). +1. In the top right corner, select the vertical ellipsis (**{ellipsis_v}**).. +1. Select **Promote to objective**. + ## Close an OKR When an OKR is achieved, you can close it. @@ -252,3 +306,14 @@ To add an existing key result to an objective: To add multiple objectives, repeat this step. 1. Select **Add key result**. + +### Reorder objective and key result children + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/385887) in GitLab 16.0. + +Prerequisites: + +- You must have at least the Reporter role for the project. + +By default, child OKRs are ordered by creation date. +To reorder them, drag them around. diff --git a/doc/user/operations_dashboard/index.md b/doc/user/operations_dashboard/index.md index 2911a700e14..d013e2813f7 100644 --- a/doc/user/operations_dashboard/index.md +++ b/doc/user/operations_dashboard/index.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/organization/index.md b/doc/user/organization/index.md new file mode 100644 index 00000000000..f5af26250f6 --- /dev/null +++ b/doc/user/organization/index.md @@ -0,0 +1,42 @@ +--- +stage: Data Stores +group: Tenant Scale +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Organization + +DISCLAIMER: +This page contains information related to upcoming products, features, and functionality. +It is important to note that the information presented is for informational purposes only. +Please do not rely on this information for purchasing or planning purposes. +As with all projects, the items mentioned on this page are subject to change or delay. +The development, release, and timing of any products, features, or functionality remain at the +sole discretion of GitLab Inc. + +NOTE: +Organization is in development. + +Organization will be above the [top-level namespaces](../namespace/index.md) for you to manage +everything you do as a GitLab administrator, including: + +- Defining and applying settings to all of your groups, subgroups, and projects. +- Aggregating data from all your groups, subgroups, and projects. + +Our goal is to reach feature parity between SaaS and self-managed installations, with all +[Admin Area settings](/ee/user/admin_area/settings/index.md) moving to either: + +- Groups. Available in the Organization, and subgroups. +- Hardware Controls. For functionality that does not apply to groups, Hardware Controls are only + applicable to self-managed installations. There is one Hardware Controls section per installation. + +For more information about the state of organization development, +see [epic 9265](https://gitlab.com/groups/gitlab-org/-/epics/9265). + + +For a video introduction to the new hierarchy concept for groups and projects for epics, see +[Consolidating groups and projects update (August 2021)](https://www.youtube.com/watch?v=fE74lsG_8yM). + +## Related topics + +- [Organization developer documentation](../../development/organization/index.md) diff --git a/doc/user/packages/conan_repository/index.md b/doc/user/packages/conan_repository/index.md index 05daa525893..f06a7d83f92 100644 --- a/doc/user/packages/conan_repository/index.md +++ b/doc/user/packages/conan_repository/index.md @@ -315,8 +315,17 @@ To search by full or partial package name, or by exact recipe, run the conan search He* --remote=gitlab ``` -The scope of your search includes all projects you have permission to access. -This includes your private projects as well as all public projects. +The scope of your search depends on your Conan remote configuration: + +- If you have a remote configured for your [instance](#add-a-remote-for-your-instance), your search includes +all projects you have permission to access. This includes your private projects + as well as all public projects. + +- If you have a remote configured for a [project](#add-a-remote-for-your-project), your search includes all +packages in the target project, as long as you have permission to access it. + +NOTE: +The limit of the search results is 500 packages, and the results are sorted by the most recently published packages. ## Fetch Conan package information from the Package Registry diff --git a/doc/user/packages/container_registry/authenticate_with_container_registry.md b/doc/user/packages/container_registry/authenticate_with_container_registry.md index f48ba7bbf52..31337d19d59 100644 --- a/doc/user/packages/container_registry/authenticate_with_container_registry.md +++ b/doc/user/packages/container_registry/authenticate_with_container_registry.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w WARNING: -[External authorization](../../admin_area/settings/external_authorization.md) will be enabled by default in GitLab 16.0. External authorization prevents personal access tokens and deploy tokens from accessing container and package registries and affects all users who use these tokens to access the registries. You can disable external authorization if you want to use personal access tokens and deploy tokens with the container or package registries. +In GitLab 16.0 and later, [external authorization](../../admin_area/settings/external_authorization.md) prevents personal access tokens and deploy tokens from accessing container and package registries and affects all users who use these tokens to access the registries. You can disable external authorization if you want to use personal access tokens and deploy tokens with the container or package registries. To authenticate with the Container Registry, you can use a: diff --git a/doc/user/packages/container_registry/index.md b/doc/user/packages/container_registry/index.md index 4d7b25e2d77..c27265ccc3f 100644 --- a/doc/user/packages/container_registry/index.md +++ b/doc/user/packages/container_registry/index.md @@ -143,7 +143,7 @@ see [Container Registry visibility permissions](#container-registry-visibility-p is internal or private, the Container Registry is also internal or private. - **Only Project Members**: The Container Registry is visible only to project members with - Reporter role or higher. This visibility is similar to the behavior of a private project with Container + at least the Reporter role. This visibility is similar to the behavior of a private project with Container Registry visibility set to **Everyone With Access**. 1. Select **Save changes**. diff --git a/doc/user/packages/container_registry/reduce_container_registry_storage.md b/doc/user/packages/container_registry/reduce_container_registry_storage.md index 9229ea61821..f873453e049 100644 --- a/doc/user/packages/container_registry/reduce_container_registry_storage.md +++ b/doc/user/packages/container_registry/reduce_container_registry_storage.md @@ -21,7 +21,7 @@ The Usage Quotas page (**Settings > Usage Quotas > Storage**) displays storage u This page includes the Container Registry usage, which is only available on GitLab.com. Measuring usage is only possible on the new version of the GitLab Container Registry backed by a metadata database. Support for improvements is proposed in epic [5523](https://gitlab.com/groups/gitlab-org/-/epics/5523). -You cannot use the Container Registry in self-managed instances, but epic [5521](https://gitlab.com/groups/gitlab-org/-/epics/5521) proposes to change this behavior. +On self-managed instances, you cannot use the Container Registry with a metadata database. For the plan to change this behavior, see [epic 5521](https://gitlab.com/groups/gitlab-org/-/epics/5521). Image layers stored in the Container Registry are deduplicated at the root namespace level. @@ -112,6 +112,43 @@ to work with all third-party registries in the same predictable way. If you use Registry, this workaround is not required because we implemented a special tag delete operation. In this case, you can expect cleanup policies to be consistent and predictable. +#### Example cleanup policy workflow + +The interaction between the keep and remove rules for the cleanup policy can be complex. +For example, with a project with this cleanup policy configuration: + +- **Keep the most recent**: 1 tag per image name. +- **Keep tags matching**: `production-.*` +- **Remove tags older than**: 7 days. +- **Remove tags matching**: `.*`. + +And a container repository with these tags: + +- `latest`, published 2 hours ago. +- `production-v44`, published 3 days ago. +- `production-v43`, published 6 days ago. +- `production-v42`, published 11 days ago. +- `dev-v44`, published 2 days ago. +- `dev-v43`, published 5 day ago. +- `dev-v42`, published 10 days ago. +- `v44`, published yesterday. +- `v43`, published 12 days ago. +- `v42`, published 20 days ago. + +In this example, the tags that would be deleted in the next cleanup run are `dev-v42`, `v43`, and `v42`. +You can interpret the rules as applying with this precedence: + +1. The keep rules have highest precedence. Tags must be kept when they match **any** rule. + - The `latest` tag must be kept, because `latest` tags are always kept. + - The `production-v44`, `production-v43`, and `production-v42` tags must be kept, + because they match the **Keep tags matching** rule. + - The `v44` tag must be kept because it's the most recent, matching the **Keep the most recent** rule. +1. The remove rules have lower precedence, and tags are only deleted if **all** rules match. + For the tags not matching any keep rules (`dev-44`, `dev-v43`, `dev-v42`, `v43`, and `v42`): + - `dev-44` and `dev-43` do **not** match the **Remove tags older than**, and are kept. + - `dev-v42`, `v43`, and `v42` match both **Remove tags older than** and **Remove tags matching** + rules, so these three tags can be deleted. + ### Create a cleanup policy You can create a cleanup policy in [the API](#use-the-cleanup-policy-api) or the UI. @@ -119,8 +156,8 @@ You can create a cleanup policy in [the API](#use-the-cleanup-policy-api) or the To create a cleanup policy in the UI: 1. For your project, go to **Settings > Packages and registries**. -1. Expand the **Clean up image tags** section. -1. Complete the fields. +1. In the **Cleanup policies** section, select **Set cleanup rules**. +1. Complete the fields: | Field | Description | |----------------------------|-------------------------------------------------| @@ -281,7 +318,7 @@ Here are some other options you can use to reduce the Container Registry storage If you see this error message, check the regex patterns to ensure they are valid. -GitLab uses [RE2 syntax](https://github.com/google/re2/wiki/Syntax) for regular expressions in the cleanup policy. You can test them with the [regex101 regex tester](https://regex101.com/). +GitLab uses [RE2 syntax](https://github.com/google/re2/wiki/Syntax) for regular expressions in the cleanup policy. You can test them with the [regex101 regex tester](https://regex101.com/) using the `Golang` flavor. View some common [regex pattern examples](#regex-pattern-examples). ### The cleanup policy doesn't delete any tags diff --git a/doc/user/packages/debian_repository/index.md b/doc/user/packages/debian_repository/index.md index 7ec20e3d036..220e2085637 100644 --- a/doc/user/packages/debian_repository/index.md +++ b/doc/user/packages/debian_repository/index.md @@ -11,8 +11,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > - [Deployed behind a feature flag](../../feature_flags.md), disabled by default. WARNING: -The Debian package registry for GitLab is under development and isn't ready for production use due to -limited functionality. This [epic](https://gitlab.com/groups/gitlab-org/-/epics/6057) details the remaining +The Debian package registry for GitLab is under development and isn't ready for production use. This [epic](https://gitlab.com/groups/gitlab-org/-/epics/6057) details the remaining work and timelines to make it production ready. Publish Debian packages in your project's Package Registry. Then install the @@ -23,6 +22,12 @@ Project and Group packages are supported. For documentation of the specific API endpoints that Debian package manager clients use, see the [Debian API documentation](../../../api/packages/debian.md). +Prerequisites: + +- The `dpkg-deb` binary must be installed on the GitLab instance. + This binary is usually provided by the [`dpkg` package](https://wiki.debian.org/Teams/Dpkg/Downstream), + installed by default on Debian and derivatives. + ## Enable the Debian API **(FREE SELF)** Debian repository support is still a work in progress. It's gated behind a feature flag that's @@ -30,6 +35,9 @@ Debian repository support is still a work in progress. It's gated behind a featu [GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) can opt to enable it. +WARNING: +Understand the [stability and security risks of enabling features still in development](../../../administration/feature_flags.md#risks-when-enabling-features-still-in-development). + To enable it: ```ruby @@ -46,6 +54,9 @@ Feature.disable(:debian_packages) The Debian group repository is also behind a second feature flag that is disabled by default. +WARNING: +Understand the [stability and security risks of enabling features still in development](../../../administration/feature_flags.md#risks-when-enabling-features-still-in-development). + To enable it: ```ruby @@ -92,8 +103,11 @@ with one of the following: ## Create a Distribution -On the project-level, Debian packages are published using *Debian Distributions*. To publish -packages on the group level, create a distribution with the same `codename`. +At the project level, Debian packages are published with **Debian distributions**. At the +group level, Debian packages are aggregated from the projects in the group provided that: + +- The project visibility is set to `public`. +- The Debian `codename` for the group matches the Debian `codename` for the project. To create a project-level distribution using a personal access token: @@ -135,6 +149,7 @@ Once built, several files are created: - `.deb` files: the binary packages - `.udeb` files: lightened .deb files, used for Debian-Installer (if needed) +- `.ddeb` files: Ubuntu debug .deb files (if needed) - `.tar.{gz,bz2,xz,...}` files: Source files - `.dsc` file: Source metadata, and list of source files (with hashes) - `.buildinfo` file: Used for Reproducible builds (optional) @@ -155,9 +170,9 @@ EOF dput --config=dput.cf --unchecked --no-upload-log gitlab .changes ``` -## Directly upload a package +## Upload a package with explicit distribution and component -> Direct upload [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/101838) in GitLab 15.9. +> Upload with explicit distribution and component [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/101838) in GitLab 15.9. When you don't have access to `.changes` file, you can directly upload a `.deb` by passing distribution `codename` and target `component` as parameters with @@ -166,7 +181,8 @@ For example, to upload to component `main` of distribution `sid` using a persona ```shell curl --request PUT --user ":" \ - "https://gitlab.example.com/api/v4/projects//packages/debian/?distribution=sid&component=main" \ + --get --data "distribution=sid" --data "component=main" \ + "https://gitlab.example.com/api/v4/projects//packages/debian/" \ --upload-file /path/to/your.deb ``` @@ -176,39 +192,39 @@ To install a package: 1. Configure the repository: - If you are using a private project, add your [credentials](#authenticate-to-the-debian-package-repositories) to your apt configuration: + If you are using a private project, add your [credentials](#authenticate-to-the-debian-package-repositories) to your apt configuration: - ```shell - echo 'machine gitlab.example.com login password ' \ - | sudo tee /etc/apt/auth.conf.d/gitlab_project.conf - ``` + ```shell + echo 'machine gitlab.example.com login password ' \ + | sudo tee /etc/apt/auth.conf.d/gitlab_project.conf + ``` - Download your distribution key using your [credentials](#authenticate-to-the-debian-distributions-apis): + Download your distribution key using your [credentials](#authenticate-to-the-debian-distributions-apis): - ```shell - sudo mkdir -p /usr/local/share/keyrings - curl --header "PRIVATE-TOKEN: " \ - "https://gitlab.example.com/api/v4/projects//debian_distributions//key.asc" \ - | \ - gpg --dearmor \ - | \ - sudo tee /usr/local/share/keyrings/-archive-keyring.gpg \ - > /dev/null - ``` + ```shell + sudo mkdir -p /usr/local/share/keyrings + curl --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects//debian_distributions//key.asc" \ + | \ + gpg --dearmor \ + | \ + sudo tee /usr/local/share/keyrings/-archive-keyring.gpg \ + > /dev/null + ``` - Add your project as a source: + Add your project as a source: - ```shell - echo 'deb [ signed-by=/usr/local/share/keyrings/-archive-keyring.gpg ] https://gitlab.example.com/api/v4/projects//packages/debian ' \ - | sudo tee /etc/apt/sources.list.d/gitlab_project.list - sudo apt-get update - ``` + ```shell + echo 'deb [ signed-by=/usr/local/share/keyrings/-archive-keyring.gpg ] https://gitlab.example.com/api/v4/projects//packages/debian ' \ + | sudo tee /etc/apt/sources.list.d/gitlab_project.list + sudo apt-get update + ``` 1. Install the package: - ```shell - sudo apt-get -y install -t - ``` + ```shell + sudo apt-get -y install -t + ``` ## Download a source package @@ -216,36 +232,36 @@ To download a source package: 1. Configure the repository: - If you are using a private project, add your [credentials](#authenticate-to-the-debian-package-repositories) to your apt configuration: + If you are using a private project, add your [credentials](#authenticate-to-the-debian-package-repositories) to your apt configuration: - ```shell - echo 'machine gitlab.example.com login password ' \ - | sudo tee /etc/apt/auth.conf.d/gitlab_project.conf - ``` + ```shell + echo 'machine gitlab.example.com login password ' \ + | sudo tee /etc/apt/auth.conf.d/gitlab_project.conf + ``` - Download your distribution key using your [credentials](#authenticate-to-the-debian-distributions-apis): + Download your distribution key using your [credentials](#authenticate-to-the-debian-distributions-apis): - ```shell - sudo mkdir -p /usr/local/share/keyrings - curl --header "PRIVATE-TOKEN: " \ - "https://gitlab.example.com/api/v4/projects//debian_distributions//key.asc" \ - | \ - gpg --dearmor \ - | \ - sudo tee /usr/local/share/keyrings/-archive-keyring.gpg \ - > /dev/null - ``` + ```shell + sudo mkdir -p /usr/local/share/keyrings + curl --header "PRIVATE-TOKEN: " \ + "https://gitlab.example.com/api/v4/projects//debian_distributions//key.asc" \ + | \ + gpg --dearmor \ + | \ + sudo tee /usr/local/share/keyrings/-archive-keyring.gpg \ + > /dev/null + ``` - Add your project as a source: + Add your project as a source: - ```shell - echo 'deb-src [ signed-by=/usr/local/share/keyrings/-archive-keyring.gpg ] https://gitlab.example.com/api/v4/projects//packages/debian ' \ - | sudo tee /etc/apt/sources.list.d/gitlab_project-sources.list - sudo apt-get update - ``` + ```shell + echo 'deb-src [ signed-by=/usr/local/share/keyrings/-archive-keyring.gpg ] https://gitlab.example.com/api/v4/projects//packages/debian ' \ + | sudo tee /etc/apt/sources.list.d/gitlab_project-sources.list + sudo apt-get update + ``` 1. Download the source package: - ```shell - sudo apt-get source -t - ``` + ```shell + sudo apt-get source -t + ``` diff --git a/doc/user/packages/dependency_proxy/index.md b/doc/user/packages/dependency_proxy/index.md index 8dc3c98795b..d7cf33cc2a4 100644 --- a/doc/user/packages/dependency_proxy/index.md +++ b/doc/user/packages/dependency_proxy/index.md @@ -342,13 +342,21 @@ Authenticating with credentials from job payload (GitLab Registry) Make sure you are using the expected authentication mechanism. -### "Not Found" error when pulling image +### `Not Found` or `404` error when pulling image -Docker errors similar to the following may indicate that the user running the build job doesn't have -a minimum of the Guest role in the specified Dependency Proxy group: +Errors like these might indicate that the user running the job doesn't have +a minimum of the Guest role in the Dependency Proxy group: -```plaintext -ERROR: gitlab.example.com:443/group1/dependency_proxy/containers/alpine:latest: not found +- ```plaintext + ERROR: gitlab.example.com:443/group1/dependency_proxy/containers/alpine:latest: not found -failed to solve with frontend dockerfile.v0: failed to create LLB definition: gitlab.example.com:443/group1/dependency_proxy/containers/alpine:latest: not found -``` + failed to solve with frontend dockerfile.v0: failed to create LLB definition: gitlab.example.com:443/group1/dependency_proxy/containers/alpine:latest: not found + ``` + +- ```plaintext + ERROR: Job failed: failed to pull image "gitlab.example.com:443/group1/dependency_proxy/containers/alpine:latest" with specified policies [always]: + Error response from daemon: error parsing HTTP 404 response body: unexpected end of JSON input: "" (manager.go:237:1s) + ``` + +For more information about the work to improve the error messages in similar cases to `Access denied`, +see [issue 354826](https://gitlab.com/gitlab-org/gitlab/-/issues/354826). diff --git a/doc/user/packages/generic_packages/index.md b/doc/user/packages/generic_packages/index.md index 932de0bcde6..bad099e6733 100644 --- a/doc/user/packages/generic_packages/index.md +++ b/doc/user/packages/generic_packages/index.md @@ -22,6 +22,8 @@ do not support the other available mechanisms. The `user-id` is not checked and may be any value, and the `password` must be either a [personal access token](../../../api/rest/index.md#personalprojectgroup-access-tokens), a [CI/CD job token](../../../ci/jobs/ci_job_token.md), or a [deploy token](../../project/deploy_tokens/index.md). +Do not use authentication methods other than the methods documented here. Undocumented authentication methods might be removed in the future. + ## Publish a package file When you publish a package file, if the package does not exist, it is created. diff --git a/doc/user/packages/go_proxy/index.md b/doc/user/packages/go_proxy/index.md index 1a089cd82be..f0e10d73d08 100644 --- a/doc/user/packages/go_proxy/index.md +++ b/doc/user/packages/go_proxy/index.md @@ -74,7 +74,7 @@ go env -w GOPROXY='https://gitlab.example.com/api/v4/projects/1234/packages/go,h With this configuration, Go fetches dependencies in this order: 1. Go attempts to fetch from the project-specific Go proxy. -1. Go attempts to fetch from [proxy.golang.org](https://proxy.golang.org). +1. Go attempts to fetch from [`proxy.golang.org`](https://proxy.golang.org). 1. Go fetches directly with version control system operations (like `git clone`, `svn checkout`, and so on). diff --git a/doc/user/packages/gradle_repository/index.md b/doc/user/packages/gradle_repository/index.md index 4247c13297d..456acc0da59 100644 --- a/doc/user/packages/gradle_repository/index.md +++ b/doc/user/packages/gradle_repository/index.md @@ -1,372 +1,11 @@ --- -stage: Package -group: Package Registry -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../maven_repository/index.md' +remove_date: '2023-08-09' --- -# Maven packages in the Package Registry **(FREE)** +This document was moved to [another location](../maven_repository/index.md). -Publish [Maven](https://maven.apache.org) artifacts in your project's Package Registry using Gradle. -Then, install the packages whenever you need to use them as a dependency. - -For documentation of the specific API endpoints that the Maven package manager -client uses, see the [Maven API documentation](../../../api/packages/maven.md). - -Learn how to build a [Gradle](../workflows/build_packages.md#gradle) package. - -## Publish to the GitLab Package Registry - -### Tokens - -You need a token to publish a package. Different tokens are available depending on what you're trying to -achieve. For more information, review the [guidance on tokens](../package_registry/index.md#authenticate-with-the-registry). - -- If your organization uses two-factor authentication (2FA), you must use a personal access token with the scope set to `api`. -- If you publish a package via CI/CD pipelines, you must use a CI job token. - -Create a token and save it to use later in the process. - -## Authenticate to the Package Registry with Gradle - -### Authenticate with a personal access token or deploy token in Gradle - -In [your `GRADLE_USER_HOME` directory](https://docs.gradle.org/current/userguide/directory_layout.html#dir:gradle_user_home), -create a file `gradle.properties` with the following content: - -```properties -gitLabPrivateToken=REPLACE_WITH_YOUR_TOKEN -``` - -Your token name depends on which token you use. - -| Token type | Token name | -| --------------------- | --------------- | -| Personal access token | `Private-Token` | -| Deploy token | `Deploy-Token` | - -Add a `repositories` section to your -[`build.gradle`](https://docs.gradle.org/current/userguide/tutorial_using_tasks.html) -file: - -```groovy -repositories { - maven { - url "https://gitlab.example.com/api/v4/groups//-/packages/maven" - name "GitLab" - credentials(HttpHeaderCredentials) { - name = 'REPLACE_WITH_TOKEN_NAME' - value = gitLabPrivateToken - } - authentication { - header(HttpHeaderAuthentication) - } - } -} -``` - -Or add it to your `build.gradle.kts` file if you are using Kotlin DSL: - -```kotlin -repositories { - maven { - url = uri("https://gitlab.example.com/api/v4/groups//-/packages/maven") - name = "GitLab" - credentials(HttpHeaderCredentials::class) { - name = "REPLACE_WITH_TOKEN_NAME" - value = findProperty("gitLabPrivateToken") as String? - } - authentication { - create("header", HttpHeaderAuthentication::class) - } - } -} -``` - -### Authenticate with a CI job token in Gradle - -To authenticate with a CI job token, add a `repositories` section to your -[`build.gradle`](https://docs.gradle.org/current/userguide/tutorial_using_tasks.html) -file: - -```groovy -repositories { - maven { - url "${CI_API_V4_URL}/groups//-/packages/maven" - name "GitLab" - credentials(HttpHeaderCredentials) { - name = 'Job-Token' - value = System.getenv("CI_JOB_TOKEN") - } - authentication { - header(HttpHeaderAuthentication) - } - } -} -``` - -Or add it to your `build.gradle.kts` file if you are using Kotlin DSL: - -```kotlin -repositories { - maven { - url = uri("$CI_API_V4_URL/groups//-/packages/maven") - name = "GitLab" - credentials(HttpHeaderCredentials::class) { - name = "Job-Token" - value = System.getenv("CI_JOB_TOKEN") - } - authentication { - create("header", HttpHeaderAuthentication::class) - } - } -} -``` - -### Naming convention - -You can use one of three API endpoints to install a Maven package. You must publish a package to a project, but note which endpoint -you use to install the package. The option you choose determines the settings you add to your `pom.xml` file for publishing. - -The three endpoints are: - -- **Project-level**: Use when you have a few Maven packages that are not in the same GitLab group. -- **Group-level**: Use when installing packages from many different projects in the same GitLab group. GitLab does not guarantee the uniqueness of package names in the group. You can have two projects with the same package name and package version. As a result, GitLab serves whichever one is more recent. -- **Instance-level**: Use when installing many packages from different GitLab groups or in their own namespace. - -**Only packages with the same path as the project** are exposed by the instance-level endpoint. - -| Project | Package | Instance-level endpoint available | -| ------------------- | -------------------------------- | --------------------------------- | -| `foo/bar` | `foo/bar/1.0-SNAPSHOT` | Yes | -| `gitlab-org/gitlab` | `foo/bar/1.0-SNAPSHOT` | No | -| `gitlab-org/gitlab` | `gitlab-org/gitlab/1.0-SNAPSHOT` | Yes | - -#### Endpoint URLs - -| Endpoint | Endpoint URL | Additional information | -| -------- | ------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------- | -| Project | `https://gitlab.example.com/api/v4/projects//packages/maven` | Replace `gitlab.example.com` with your domain name. Replace `` with your project ID found on your project's homepage. | -| Group | `https://gitlab.example.com/api/v4/groups//-/packages/maven` | Replace `gitlab.example.com` with your domain name. Replace `` with your group ID found on your group's homepage. | -| Instance | `https:///gitlab.example.com/api/v4/packages/maven` | Replace `gitlab.example.com` with your domain name. | - -In all cases, to publish a package, you need: - -- A project-specific URL in the `distributionManagement` section. -- A `repository` and `distributionManagement` section. - -### Edit the Groovy DSL or Kotlin DSL - -The Gradle Groovy DSL `repositories` section should look like this: - -```groovy -repositories { - maven { - url "" - name "GitLab" - } -} -``` - -In Kotlin DSL: - -```kotlin -repositories { - maven { - url = uri("") - name = "GitLab" - } -} -``` - -- Replace `` with the [endpoint](#endpoint-urls) you chose. - -## Publish using Gradle - -Your token name depends on which token you use. - -| Token type | Token name | -| --------------------- | --------------- | -| Personal access token | `Private-Token` | -| Deploy token | `Deploy-Token` | - -To publish a package by using Gradle: - -1. Add the Gradle plugin [`maven-publish`](https://docs.gradle.org/current/userguide/publishing_maven.html) to the plugins section: - - In Groovy DSL: - - ```groovy - plugins { - id 'java' - id 'maven-publish' - } - ``` - - In Kotlin DSL: - - ```kotlin - plugins { - java - `maven-publish` - } - ``` - -1. Add a `publishing` section: - - In Groovy DSL: - - ```groovy - publishing { - publications { - library(MavenPublication) { - from components.java - } - } - repositories { - maven { - url "https://gitlab.example.com/api/v4/projects//packages/maven" - credentials(HttpHeaderCredentials) { - name = "REPLACE_WITH_TOKEN_NAME" - value = gitLabPrivateToken // the variable resides in $GRADLE_USER_HOME/gradle.properties - } - authentication { - header(HttpHeaderAuthentication) - } - } - } - } - ``` - - In Kotlin DSL: - - ```kotlin - publishing { - publications { - create("library") { - from(components["java"]) - } - } - repositories { - maven { - url = uri("https://gitlab.example.com/api/v4/projects//packages/maven") - credentials(HttpHeaderCredentials::class) { - name = "REPLACE_WITH_TOKEN_NAME" - value = - findProperty("gitLabPrivateToken") as String? // the variable resides in $GRADLE_USER_HOME/gradle.properties - } - authentication { - create("header", HttpHeaderAuthentication::class) - } - } - } - } - ``` - -1. Replace `PROJECT_ID` with your project ID, which you can find on your project's home page. - -1. Run the publish task: - - ```shell - gradle publish - ``` - -Go to your project's **Packages and registries** page and view the published packages. - -## Install a package - -To install a package from the GitLab Package Registry, you must configure -the [remote and authenticate](#authenticate-to-the-package-registry-with-gradle). -After configuring the remote and authenticate, you can install a package from a project, group, or namespace. - -If multiple packages have the same name and version, when you install -a package, the most recently-published package is retrieved. - -Add a [dependency](https://docs.gradle.org/current/userguide/declaring_dependencies.html) to `build.gradle` in the dependencies section: - -```groovy -dependencies { - implementation 'com.mycompany.mydepartment:my-project:1.0-SNAPSHOT' -} -``` - -Or to `build.gradle.kts` if you are using Kotlin DSL: - -```kotlin -dependencies { - implementation("com.mycompany.mydepartment:my-project:1.0-SNAPSHOT") -} -``` - -## Helpful hints - -For the complete list of helpful hints, see the [Maven documentation](../maven_repository/index.md#helpful-hints). - -### Create Maven packages with GitLab CI/CD by using Gradle - -You can create a package each time the `main` branch -is updated. - -1. Authenticate with [a CI job token in Gradle](#authenticate-with-a-ci-job-token-in-gradle). - -1. Add a `deploy` job to your `.gitlab-ci.yml` file: - - ```yaml - deploy: - image: gradle:6.5-jdk11 - script: - - 'gradle publish' - only: - - main - ``` - -1. Commit files to your repository. - -When the pipeline is successful, the Maven package is created. - -### Publishing a package with the same name or version - -When you publish a package with the same name and version as an existing package, the new package -files are added to the existing package. You can still use the UI or API to access and view the -existing package's older assets. - -Consider using the Packages API or the UI to delete older package versions. - -### Do not allow duplicate Maven packages - -To prevent users from publishing duplicate Maven packages, you can use the [GraphQl API](../../../api/graphql/reference/index.md#packagesettings) or the UI. - -In the UI: - -1. For your group, go to **Settings > Packages and registries**. -1. Expand the **Package Registry** section. -1. Turn on the **Do not allow duplicates** toggle. -1. Optional. To allow some duplicate packages, in the **Exceptions** box, enter a regex pattern that matches the names and/or versions of packages you want to allow. - -Your changes are automatically saved. - -### Request forwarding to Maven Central - -FLAG: -By default, this feature is not available for self-managed. To make it available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `maven_central_request_forwarding`. -This feature is not available for SaaS users. - -When a Maven package is not found in the Package Registry, the request is forwarded -to [Maven Central](https://search.maven.org/). - -When the feature flag is enabled, administrators can disable this behavior in the -[Continuous Integration settings](../../admin_area/settings/continuous_integration.md). - -There are many ways to configure your Maven project to request packages -in Maven Central from GitLab. Maven repositories are queried in a -[specific order](https://maven.apache.org/guides/mini/guide-multiple-repositories.html#repository-order). -By default, maven-central is usually checked first through the -[Super POM](https://maven.apache.org/guides/introduction/introduction-to-the-pom.html#Super_POM), so -GitLab needs to be configured to be queried before maven-central. - -[Using GitLab as a mirror of the central proxy](../maven_repository/index.md#setting-gitlab-as-a-mirror-for-the-central-proxy) is one -way to force GitLab to be queried in place of maven-central. - -Maven forwarding is restricted to only the project level and -group level [endpoints](#naming-convention). The instance-level endpoint -has naming restrictions that prevent it from being used for packages that don't follow that convention and also -introduces too much security risk for supply-chain style attacks. + + + + diff --git a/doc/user/packages/harbor_container_registry/index.md b/doc/user/packages/harbor_container_registry/index.md index 1ad5e2c2f05..6cea541a55d 100644 --- a/doc/user/packages/harbor_container_registry/index.md +++ b/doc/user/packages/harbor_container_registry/index.md @@ -18,7 +18,7 @@ You can view the Harbor Registry for a project or group. You can search, sort, and filter images on this page. You can share a filtered view by copying the URL from your browser. -At the project level, you can see **CLI Commands** in the upper right corner, where you can copy +At the project level, in the upper-right corner, you can see **CLI Commands** where you can copy corresponding commands to sign in, build images, and push images. **CLI Commands** is not shown at the group level. diff --git a/doc/user/packages/index.md b/doc/user/packages/index.md index 7418977e0d1..2084de58bdb 100644 --- a/doc/user/packages/index.md +++ b/doc/user/packages/index.md @@ -14,15 +14,9 @@ packages, which can be easily consumed as a dependency in downstream projects. The GitLab [Container Registry](container_registry/index.md) is a secure and private registry for container images. It's built on open source software and completely integrated within GitLab. Use GitLab CI/CD to create and publish images. Use the GitLab [API](../../api/container_registry.md) to manage the registry across groups and projects. -## Infrastructure Registry +## Terraform Module Registry -The GitLab [Infrastructure Registry](infrastructure_registry/index.md) is a secure and private registry for infrastructure packages. You can use GitLab CI/CD to create and publish infrastructure packages. - -The Infrastructure Registry supports the following formats: - -| Package type | GitLab version | -| ------------ | -------------- | -| [Terraform Module](terraform_module_registry/index.md) | 14.0+ | +The GitLab [Terraform Module Registry](terraform_module_registry/index.md) is a secure and private registry for Terraform modules. You can use GitLab CI/CD to create and publish modules. ## Dependency Proxy diff --git a/doc/user/packages/infrastructure_registry/index.md b/doc/user/packages/infrastructure_registry/index.md index d06c5e7fb1e..8c1e8a2ad8a 100644 --- a/doc/user/packages/infrastructure_registry/index.md +++ b/doc/user/packages/infrastructure_registry/index.md @@ -1,102 +1,11 @@ --- -stage: Package -group: Package Registry -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: '../terraform_module_registry/index.md' +remove_date: '2023-06-13' --- -# Infrastructure Registry **(FREE)** +This document was moved to [another location](../terraform_module_registry/index.md). -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3221) in GitLab 14.0. - -With the GitLab Infrastructure Registry, you can use GitLab projects as a -private registry for infrastructure packages. You can create and publish -packages with GitLab CI/CD, which can then be consumed from other private -projects. - -## View packages - -To view packages within your project: - -1. Go to the project. -1. Go to **Packages and registries > Infrastructure Registry**. - -You can search, sort, and filter packages on this page. - -For information on how to create and upload a package, view the GitLab -documentation for your package type: - -- [Terraform modules](../terraform_module_registry/index.md) - -## Use GitLab CI/CD to build packages - -To use [GitLab CI/CD](../../../ci/index.md) to build packages, you can -authenticate with the [`CI_JOB_TOKEN` predefined variable](../../../ci/variables/predefined_variables.md). - -CI/CD templates, which you can use to get started, are in [this repository](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates). - -For more information about using CI/CD to build Terraform modules, -see [Publish a Terraform module by using CI/CD](../terraform_module_registry/index.md#publish-a-terraform-module-by-using-cicd). - -If you use CI/CD to build a package, you can find extended activity information -when you view the package details: - -![Package CI/CD activity](../package_registry/img/package_activity_v12_10.png) - -You can see the pipeline that published the package as well as the commit and the user who triggered it. However, the history is limited to five updates per package. - -## Download a package - -To download a package: - -1. Go to **Packages and registries > Infrastructure Registry**. -1. Select the name of the package you want to download. -1. In the **Activity** section, select the name of the package you want to download. - -## Delete a package - -You cannot edit a package after you publish it in the Infrastructure Registry. Instead, you -must delete and recreate it. - -To delete a package, you must have suitable [permissions](../../permissions.md). - -You can delete packages by using [the API](../../../api/packages.md#delete-a-project-package) or the UI. - -To delete a package in the UI, from your project: - -1. Go to **Packages and registries > Infrastructure Registry**. -1. Find the name of the package you want to delete. -1. Select **Delete**. - -The package is permanently deleted. - -## Disable the Infrastructure Registry - -The Infrastructure Registry is automatically enabled. - -For self-managed instances, a GitLab administrator can -[disable](../../../administration/packages/index.md) **Packages and registries**, -which removes this menu item from the sidebar. - -You can also remove the Infrastructure Registry for a specific project: - -1. In your project, go to **Settings > General**. -1. Expand the **Visibility, project features, permissions** section and toggle **Packages** off (in gray). -1. Select **Save changes**. - -To enable it back, follow the same steps above and toggle it on (in blue). - -## How module resolution works - -When you upload a new module, GitLab generates a path for the module, for example, `https://gitlab.example.com/parent-group/my-infra-package`. - -- This path conforms with [the Terraform spec](https://www.terraform.io/internals/module-registry-protocol). -- The name of the path must be unique within the namespace. - -For projects in subgroups, GitLab checks that the module name does not already exist anywhere in the namespace, including all subgroups and the parent group. - -For example, if: - -- The project is `gitlab.example.com/parent-group/sub-group/my-project`. -- The infrastructure package is `my-infra-package`. - -The project name must be unique in all projects in all groups under `parent-group`. + + + + diff --git a/doc/user/packages/maven_repository/index.md b/doc/user/packages/maven_repository/index.md index 1899cdc213f..fd8049d9b75 100644 --- a/doc/user/packages/maven_repository/index.md +++ b/doc/user/packages/maven_repository/index.md @@ -12,7 +12,13 @@ Then, install the packages whenever you need to use them as a dependency. For documentation of the specific API endpoints that the Maven package manager client uses, see the [Maven API documentation](../../../api/packages/maven.md). -Learn how to build a [Maven](../workflows/build_packages.md#maven) package. +Supported clients: + +- `mvn`. Learn how to build a [Maven](../workflows/build_packages.md#maven) package. +- `gradle`. Learn how to build a [Gradle](../workflows/build_packages.md#gradle) package. +- `sbt`. + - `sbt` can only be used to [pull dependencies](#install-a-package). + See this [issue 408479](https://gitlab.com/gitlab-org/gitlab/-/issues/408479) for more details. ## Publish to the GitLab Package Registry @@ -22,13 +28,16 @@ You need an token to publish a package. There are different tokens available dep Create a token and save it to use later in the process. -### Edit the `settings.xml` +Do not use authentication methods other than the methods documented here. Undocumented authentication methods might be removed in the future. -Add the following section to your -[`settings.xml`](https://maven.apache.org/settings.html) file. +#### Edit the client configuration -NOTE: -The `` field must be named to match the token you chose. +You must add the authentication details to the configuration file +for your client. + +::Tabs + +:::TabTitle `mvn` | Token type | Name must be | Token | | --------------------- | --------------- | ---------------------------------------------------------------------- | @@ -36,6 +45,12 @@ The `` field must be named to match the token you chose. | Deploy token | `Deploy-Token` | Paste token as-is, or define an environment variable to hold the token | | CI Job token | `Job-Token` | `${CI_JOB_TOKEN}` | +NOTE: +The `` field must be named to match the token you chose. + +Add the following section to your +[`settings.xml`](https://maven.apache.org/settings.html) file. + ```xml @@ -54,6 +69,104 @@ The `` field must be named to match the token you chose. ``` +:::TabTitle `gradle` + +| Token type | Name must be | Token | +| --------------------- | --------------- | ---------------------------------------------------------------------- | +| Personal access token | `Private-Token` | Paste token as-is, or define an environment variable to hold the token | +| Deploy token | `Deploy-Token` | Paste token as-is, or define an environment variable to hold the token | +| CI Job token | `Job-Token` | `System.getenv("CI_JOB_TOKEN")` | + +NOTE: +The `` field must be named to match the token you chose. + +In [your `GRADLE_USER_HOME` directory](https://docs.gradle.org/current/userguide/directory_layout.html#dir:gradle_user_home), +create a file `gradle.properties` with the following content: + +```properties +gitLabPrivateToken=REPLACE_WITH_YOUR_TOKEN +``` + +Add a `repositories` section to your +[`build.gradle`](https://docs.gradle.org/current/userguide/tutorial_using_tasks.html) +file: + +- In Groovy DSL: + + ```groovy + repositories { + maven { + url "https://gitlab.example.com/api/v4/groups//-/packages/maven" + name "GitLab" + credentials(HttpHeaderCredentials) { + name = 'REPLACE_WITH_NAME' + value = gitLabPrivateToken + } + authentication { + header(HttpHeaderAuthentication) + } + } + } + ``` + +- In Kotlin DSL: + + ```kotlin + repositories { + maven { + url = uri("https://gitlab.example.com/api/v4/groups//-/packages/maven") + name = "GitLab" + credentials(HttpHeaderCredentials::class) { + name = "REPLACE_WITH_NAME" + value = findProperty("gitLabPrivateToken") as String? + } + authentication { + create("header", HttpHeaderAuthentication::class) + } + } + } + ``` + +:::TabTitle `sbt` + +| Token type | Name must be | Token | +|-----------------------|------------------------------|------------------------------------------------------------------------| +| Personal access token | The username of the user | Paste token as-is, or define an environment variable to hold the token | +| Deploy token | The username of deploy token | Paste token as-is, or define an environment variable to hold the token | +| CI Job token | `gitlab-ci-token` | `sys.env.get("CI_JOB_TOKEN").get` | + +Authentication for [SBT](https://www.scala-sbt.org/index.html) is based on +[basic HTTP Authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication). +You must to provide a name and a password. + +NOTE: +The name field must be named to match the token you chose. + +To install a package from the Maven GitLab Package Registry by using `sbt`, you must configure +a [Maven resolver](https://www.scala-sbt.org/1.x/docs/Resolvers.html#Maven+resolvers). +If you're accessing a private or an internal project or group, you need to set up +[credentials](https://www.scala-sbt.org/1.x/docs/Publishing.html#Credentials). +After configuring the resolver and authentication, you can install a package +from a project, group, or namespace. + +In your [`build.sbt`](https://www.scala-sbt.org/1.x/docs/Directories.html#sbt+build+definition+files), add the following lines: + +```scala +resolvers += ("gitlab" at "") + +credentials += Credentials("GitLab Packages Registry", "", "", "") +``` + +In this example: + +- `` is the [endpoint URL](#endpoint-urls). +Example: `https://gitlab.example.com/api/v4/projects//packages/maven`. +- `` is the host present in the `` without the protocol +scheme or the port. Example: `gitlab.example.com`. +- `` and `` are explained in the table above. + +::EndTabs + ### Naming convention You can use one of three endpoints to install a Maven package. You must publish a package to a project, but the endpoint you choose determines the settings you add to your `pom.xml` file for publishing. @@ -64,6 +177,13 @@ The three endpoints are: - **Group-level**: Use when you want to install packages from many different projects in the same GitLab group. GitLab does not guarantee the uniqueness of package names within the group. You can have two projects with the same package name and package version. As a result, GitLab serves whichever one is more recent. - **Instance-level**: Use when you have many packages in different GitLab groups or in their own namespace. +For the instance-level endpoint, ensure the relevant section of your `pom.xml` in Maven looks like this: + +```xml + group-slug.subgroup-slug + project-slug +``` + **Only packages that have the same path as the project** are exposed by the instance-level endpoint. | Project | Package | Instance-level endpoint available | @@ -80,7 +200,13 @@ The three endpoints are: | Group | `https://gitlab.example.com/api/v4/groups//-/packages/maven` | Replace `gitlab.example.com` with your domain name. Replace `` with your group ID, found on your group's homepage. | | Instance | `https://gitlab.example.com/api/v4/packages/maven` | Replace `gitlab.example.com` with your domain name. | -### Edit the `pom.xml` for publishing +### Edit the configuration file for publishing + +You must add publishing details to the configuration file for your client. + +::Tabs + +:::TabTitle `mvn` No matter which endpoint you choose, you must have: @@ -108,16 +234,97 @@ The relevant `repository` section of your `pom.xml` in Maven should look like th ``` -- The `id` is what you [defined in `settings.xml`](#edit-the-settingsxml). +- The `id` is what you [defined in `settings.xml`](#edit-the-client-configuration). - The `` depends on which [endpoint](#endpoint-urls) you choose. - Replace `gitlab.example.com` with your domain name. +:::TabTitle `gradle` + +To publish a package by using Gradle: + +1. Add the Gradle plugin [`maven-publish`](https://docs.gradle.org/current/userguide/publishing_maven.html) to the plugins section: + + - In Groovy DSL: + + ```groovy + plugins { + id 'java' + id 'maven-publish' + } + ``` + + - In Kotlin DSL: + + ```kotlin + plugins { + java + `maven-publish` + } + ``` + +1. Add a `publishing` section: + + - In Groovy DSL: + + ```groovy + publishing { + publications { + library(MavenPublication) { + from components.java + } + } + repositories { + maven { + url "https://gitlab.example.com/api/v4/projects//packages/maven" + credentials(HttpHeaderCredentials) { + name = "REPLACE_WITH_TOKEN_NAME" + value = gitLabPrivateToken // the variable resides in $GRADLE_USER_HOME/gradle.properties + } + authentication { + header(HttpHeaderAuthentication) + } + } + } + } + ``` + + - In Kotlin DSL: + + ```kotlin + publishing { + publications { + create("library") { + from(components["java"]) + } + } + repositories { + maven { + url = uri("https://gitlab.example.com/api/v4/projects//packages/maven") + credentials(HttpHeaderCredentials::class) { + name = "REPLACE_WITH_TOKEN_NAME" + value = + findProperty("gitLabPrivateToken") as String? // the variable resides in $GRADLE_USER_HOME/gradle.properties + } + authentication { + create("header", HttpHeaderAuthentication::class) + } + } + } + } + ``` + +::EndTabs + ## Publish a package After you have set up the [authentication](#authenticate-to-the-package-registry) and [chosen an endpoint for publishing](#naming-convention), publish a Maven package to your project. +::Tabs + +:::TabTitle `mvn` + To publish a package by using Maven: ```shell @@ -138,6 +345,18 @@ The message should also show that the package was published to the correct locat Uploading to gitlab-maven: https://example.com/api/v4/projects/PROJECT_ID/packages/maven/com/mycompany/mydepartment/my-project/1.0-SNAPSHOT/my-project-1.0-20200128.120857-1.jar ``` +:::TabTitle `gradle` + +Run the publish task: + +```shell +gradle publish +``` + +Go to your project's **Packages and registries** page and view the published packages. + +::EndTabs + ## Install a package To install a package from the GitLab Package Registry, you must configure @@ -148,7 +367,9 @@ group, or namespace. If multiple packages have the same name and version, when you install a package, the most recently-published package is retrieved. -### Use Maven with `mvn install` +::Tabs + +:::TabTitle `mvn` To install a package by using `mvn install`: @@ -175,9 +396,7 @@ The message should show that the package is downloading from the Package Registr Downloading from gitlab-maven: http://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven/com/mycompany/mydepartment/my-project/1.0-SNAPSHOT/my-project-1.0-20200128.120857-1.pom ``` -### Use Maven with `mvn dependency:get` - -You can install packages by using the Maven `dependency:get` [command](https://maven.apache.org/plugins/maven-dependency-plugin/get-mojo.html) directly. +You can also install packages by using the Maven [`dependency:get` command](https://maven.apache.org/plugins/maven-dependency-plugin/get-mojo.html) directly. 1. In your project directory, run: @@ -186,7 +405,7 @@ You can install packages by using the Maven `dependency:get` [command](https://m ``` - `` is the URL of the GitLab [endpoint](#endpoint-urls). - - `` is the path to the `settings.xml` file that contains the [authentication details](#edit-the-settingsxml). + - `` is the path to the `settings.xml` file that contains the [authentication details](#edit-the-client-configuration). NOTE: The repository IDs in the command(`gitlab-maven`) and the `settings.xml` file must match. @@ -197,6 +416,52 @@ The message should show that the package is downloading from the Package Registr Downloading from gitlab-maven: http://gitlab.example.com/api/v4/projects/PROJECT_ID/packages/maven/com/mycompany/mydepartment/my-project/1.0-SNAPSHOT/my-project-1.0-20200128.120857-1.pom ``` +:::TabTitle `gradle` + +To install a package by using `gradle`: + +1. Add a [dependency](https://docs.gradle.org/current/userguide/declaring_dependencies.html) to `build.gradle` in the dependencies section: + + - In Groovy DSL: + + ```groovy + dependencies { + implementation 'com.mycompany.mydepartment:my-project:1.0-SNAPSHOT' + } + ``` + + - In Kotlin DSL: + + ```kotlin + dependencies { + implementation("com.mycompany.mydepartment:my-project:1.0-SNAPSHOT") + } + ``` + +1. In your project, run the following: + + ```shell + gradle install + ``` + +:::TabTitle `sbt` + +To install a package by using `sbt`: + +1. Add an [inline dependency](https://www.scala-sbt.org/1.x/docs/Library-Management.html#Dependencies) to `build.sbt`: + + ```scala + libraryDependencies += "com.mycompany.mydepartment" % "my-project" % "8.4" + ``` + +1. In your project, run the following: + + ```shell + sbt update + ``` + +::EndTabs + ## Helpful hints ### Publishing a package with the same name or version @@ -232,22 +497,19 @@ to [Maven Central](https://search.maven.org/). When the feature flag is enabled, administrators can disable this behavior in the [Continuous Integration settings](../../admin_area/settings/continuous_integration.md). -There are many ways to configure your Maven project so that it requests packages -in Maven Central from GitLab. Maven repositories are queried in a -[specific order](https://maven.apache.org/guides/mini/guide-multiple-repositories.html#repository-order). -By default, maven-central is usually checked first through the -[Super POM](https://maven.apache.org/guides/introduction/introduction-to-the-pom.html#Super_POM), so -GitLab needs to be configured to be queried before maven-central. - -[Using GitLab as a mirror of the central proxy](#setting-gitlab-as-a-mirror-for-the-central-proxy) is one -way to force GitLab to be queried in place of maven-central. - Maven forwarding is restricted to only the project level and group level [endpoints](#naming-convention). The instance level endpoint has naming restrictions that prevent it from being used for packages that don't follow that convention and also introduces too much security risk for supply-chain style attacks. -#### Setting GitLab as a mirror for the central proxy +#### Additional configuration for `mvn` + +When using `mvn`, there are many ways to configure your Maven project so that it requests packages +in Maven Central from GitLab. Maven repositories are queried in a +[specific order](https://maven.apache.org/guides/mini/guide-multiple-repositories.html#repository-order). +By default, Maven Central is usually checked first through the +[Super POM](https://maven.apache.org/guides/introduction/introduction-to-the-pom.html#Super_POM), so +GitLab needs to be configured to be queried before maven-central. To ensure all package requests are sent to GitLab instead of Maven Central, you can override Maven Central as the central repository by adding a `` @@ -284,9 +546,11 @@ section to your `settings.xml`: After you have configured your repository to use the Package Repository for Maven, you can configure GitLab CI/CD to build new packages automatically. -### Create Maven packages with GitLab CI/CD using Maven +::Tabs + +:::TabTitle `mvn` -You can create a new package each time the `main` branch is updated. +You can create a new package each time the default branch is updated. 1. Create a `ci_settings.xml` file that serves as Maven's `settings.xml` file. @@ -342,8 +606,8 @@ You can create a new package each time the `main` branch is updated. image: maven:3.6-jdk-11 script: - 'mvn deploy -s ci_settings.xml' - only: - - main + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH ``` 1. Push those files to your repository. @@ -354,6 +618,30 @@ user's home location. In this example: - The user is `root`, because the job runs in a Docker container. - Maven uses the configured CI/CD variables. +:::TabTitle `gradle` + +You can create a package each time the default branch +is updated. + +1. Authenticate with [a CI job token in Gradle](#edit-the-client-configuration). + +1. Add a `deploy` job to your `.gitlab-ci.yml` file: + + ```yaml + deploy: + image: gradle:6.5-jdk11 + script: + - 'gradle publish' + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + ``` + +1. Commit files to your repository. + +When the pipeline is successful, the Maven package is created. + +::EndTabs + ### Version validation The version string is validated by using the following regex. @@ -394,27 +682,44 @@ that you can use when performing tasks with GitLab CI/CD. ### Supported CLI commands -The GitLab Maven repository supports the following Maven CLI commands: +The GitLab Maven repository supports the following CLI commands: + +::Tabs + +:::TabTitle `mvn` - `mvn deploy`: Publish your package to the Package Registry. - `mvn install`: Install packages specified in your Maven project. - `mvn dependency:get`: Install a specific package. +:::TabTitle `gradle` + +- `gradle publish`: Publish your package to the Package Registry. +- `gradle install`: Install packages specified in your Gradle project. + +::EndTabs + ## Troubleshooting -To improve performance, Maven caches files related to a package. If you encounter issues, clear +To improve performance, clients cache files related to a package. If you encounter issues, clear the cache with these commands: +::Tabs + +:::TabTitle `mvn` + ```shell rm -rf ~/.m2/repository ``` -If you're using Gradle, run this command to clear the cache: +:::TabTitle `gradle` ```shell rm -rf ~/.gradle/caches # Or replace ~/.gradle with your custom GRADLE_USER_HOME ``` +::EndTabs + ### Review network trace logs If you are having issues with the Maven Repository, you may want to review network trace logs. diff --git a/doc/user/packages/npm_registry/index.md b/doc/user/packages/npm_registry/index.md index 11e3d0e5131..52fdda0d523 100644 --- a/doc/user/packages/npm_registry/index.md +++ b/doc/user/packages/npm_registry/index.md @@ -23,6 +23,8 @@ You need an token to publish a package. There are different tokens available dep Create a token and save it to use later in the process. +Do not use authentication methods other than the methods documented here. Undocumented authentication methods might be removed in the future. + ### Naming convention Depending on how the package is installed, you may need to adhere to the naming convention. @@ -122,21 +124,50 @@ You can install a package from a GitLab project or instance: - **Instance-level**: Use when you have many npm packages in different GitLab groups or in their own namespace. - **Project-level**: Use when you have few npm packages and they are not in the same GitLab group. -### Install from the instance level +### Authenticate to the Package Registry -WARNING: -To install a package from the instance level, the package must have been published following the scoped [naming convention](#naming-convention). +You must authenticate to the Package Registry to install a package from a private project. +No authentication is needed if the project is public. -1. Authenticate to the Package Registry +To authenticate with `npm`: - If you would like to install a package from a private project, you would have to authenticate to the Package Registry. Skip this step if the project is not private. +```shell +npm config set -- //your_domain_name/:_authToken=your_token +``` - ```shell - npm config set -- //your_domain_name/api/v4/packages/npm/:_authToken=your_token - ``` +With npm version 7 or earlier, use the full URL to the endpoint. - - Replace `your_domain_name` with your domain name, for example, `gitlab.com`. - - Replace `your_token` with a deploy token, group access token, project access token, or personal access token. +If you're installing: + +- From the instance level: + + ```shell + npm config set -- //your_domain_name/api/v4/packages/npm/:_authToken=your_token + ``` + + From the project level: + + ```shell + npm config set -- //your_domain_name/api/v4/projects/your_project_id/packages/npm/:_authToken=your_token + ``` + +In these examples: + +- Replace `your_domain_name` with your domain name, for example, `gitlab.com`. +- Replace `your_project_id` is your project ID, found on the project's home page. +- Replace `your_token` with a deploy token, group access token, project access token, or personal access token. + +NOTE: +Starting with npm version 8, you can [use a URI fragment instead of a full URL](https://docs.npmjs.com/cli/v8/configuring-npm/npmrc?v=true#auth-related-configuration) +in the `_authToken` parameter. However, [group-level endpoints](https://gitlab.com/gitlab-org/gitlab/-/issues/299834) +are not supported. + +### Install from the instance level + +WARNING: +To install a package from the instance level, the package must have been published following the scoped [naming convention](#naming-convention). + +1. [Authenticate to the Package Registry](#authenticate-to-the-package-registry). 1. Set the registry @@ -156,17 +187,7 @@ To install a package from the instance level, the package must have been publish ### Install from the project level -1. Authenticate to the Package Registry - - If you would like to install a package from a private project, you would have to authenticate to the Package Registry. Skip this step if the project is not private. - - ```shell - npm config set -- //your_domain_name/api/v4/projects/your_project_id/packages/npm/:_authToken=your_token - ``` - - - Replace `your_domain_name` with your domain name, for example, `gitlab.com`. - - Replace `your_project_id` is your project ID, found on the project's home page. - - Replace `your_token` with a deploy token, group access token, project access token, or personal access token. +1. [Authenticate to the Package Registry](#authenticate-to-the-package-registry). 1. Set the registry @@ -184,6 +205,39 @@ To install a package from the instance level, the package must have been publish npm install @scope/my-package ``` +## Deprecate a package + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/396763) in GitLab 16.0. + +You can deprecate a package so that a deprecation warning displays when the package is fetched. + +Pre-requisites: + +- The same [permissions](../../permissions.md) as deleting a package. +- [Authenticated to the package registry](#authentication-to-the-package-registry). + +From the command line, run: + +```shell +npm deprecate @scope/package "Deprecation message" +``` + +The CLI also accepts version ranges for `@scope/package`. For example: + +```shell +npm deprecate @scope/package "All package versions are deprecated" +npm deprecate @scope/package@1.0.1 "Only version 1.0.1 is deprecated" +npm deprecate @scope/package@"< 1.0.5" "All package versions less than 1.0.5 are deprecated" +``` + +### Remove deprecation warning + +To remove a package's deprecation warning, specify `""` (an empty string) for the message. For example: + +```shell +npm deprecate @scope/package "" +``` + ## Helpful hints ### Package forwarding to npmjs.com @@ -244,8 +298,19 @@ npm dist-tag rm @scope/package@version my-tag # Delete a tag from the package npm install @scope/package@my-tag # Install a specific tag ``` -You cannot use your `CI_JOB_TOKEN` or deploy token with the `npm dist-tag` commands. -View [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/258835) for details. +#### From CI/CD + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/258835) in GitLab 15.10. + +You can use a [`CI_JOB_TOKEN`](../../../ci/jobs/ci_job_token.md) or [deploy token](../../project/deploy_tokens/index.md) +to run `npm dist-tag` commands in a GitLab CI/CD job. For example: + +```yaml +npm-deploy-job: + script: + - echo "//${CI_SERVER_HOST}/api/v4/projects/${CI_PROJECT_ID}/packages/npm/:_authToken=${CI_JOB_TOKEN}">.npmrc + - npm dist-tag add @scope/package@version my-tag +``` Due to a bug in npm 6.9.0, deleting distribution tags fails. Make sure your npm version is 6.9.1 or later. @@ -261,24 +326,35 @@ The GitLab npm repository supports the following commands for the npm CLI (`npm` - `npm dist-tag rm`: Delete a dist-tag. - `npm ci`: Install npm packages directly from your `package-lock.json` file. - `npm view`: Show package metadata. +- `npm pack`: Create a tarball from a package. +- `npm deprecate`: Deprecate a version of a package. ## Troubleshooting ### `404 Not Found` errors are happening on `npm install` or `yarn` -Using `CI_JOB_TOKEN` to install npm packages with dependencies in another project gives you 404 Not Found errors. A fix for this problem is proposed in [issue 352962](https://gitlab.com/gitlab-org/gitlab/-/issues/352962). +Using `CI_JOB_TOKEN` to install npm packages with dependencies in another project gives you 404 Not Found errors. You need to authenticate with a token that has access to the package and all its dependencies. -As a workaround, you can: +If the package and its dependencies are in separate projects but in the same group, you can use a +[group deploy token](../../project/deploy_tokens/index.md#create-a-deploy-token): -1. Create a [personal access token](../../profile/personal_access_tokens.md). -1. Authenticate at both the instance level and project level for each package: +```ini +//gitlab.example.com/api/v4/packages/npm/:_authToken= +@group-scope:registry=https://gitlab.example.com/api/v4/packages/npm/ +``` - ```ini - @foo:registry=https://gitlab.example.com/api/v4/packages/npm/ - //gitlab.example.com/api/v4/packages/npm/:_authToken=${MY_TOKEN} - //gitlab.example.com/api/v4/projects//packages/npm/:_authToken=${MY_TOKEN} - //gitlab.example.com/api/v4/projects//packages/npm/:_authToken=${MY_TOKEN} - ``` +If the package and its dependencies are spread across multiple groups, you can use a [personal access token](../../profile/personal_access_tokens.md) +from a user that has access to all the groups or individual projects: + +```ini +//gitlab.example.com/api/v4/packages/npm/:_authToken= +@group-1:registry=https://gitlab.example.com/api/v4/packages/npm/ +@group-2:registry=https://gitlab.example.com/api/v4/packages/npm/ +``` + +WARNING: +Personal access tokens must be treated carefully. Read our [token security considerations](../../../security/token_overview.md#security-considerations) +for guidance on managing personal access tokens (for example, setting a short expiry and using minimal scopes). ### `npm publish` targets default npm registry (`registry.npmjs.org`) diff --git a/doc/user/packages/nuget_repository/index.md b/doc/user/packages/nuget_repository/index.md index 6710c147c87..c97ce9ec593 100644 --- a/doc/user/packages/nuget_repository/index.md +++ b/doc/user/packages/nuget_repository/index.md @@ -39,6 +39,8 @@ Some features such as [publishing](#publish-a-nuget-package) a package are only When asking for versions of a given NuGet package name, the GitLab Package Registry returns a maximum of 300 most recent versions. +Do not use authentication methods other than the methods documented here. Undocumented authentication methods might be removed in the future. + WARNING: Because of how NuGet handles credentials, the Package Registry rejects anonymous requests on the group-level endpoint. To work around this limitation, set up [authentication](#add-the-package-registry-as-a-source-for-nuget-packages). @@ -472,4 +474,4 @@ nuget locals all -clear ### `Error publishing` or `Invalid Package: Failed metadata extraction error` messages when trying to publish NuGet packages in a Docker-based GitLab installation -Webhook requests to local network addresses are blocked to prevent the exploitation of internal web services. If you get `Error publishing` or `Invalid Package` messages when you try to publish NuGet packages, change your network settings to [allow webhook and service requests to the local network](../../../security/webhooks.md#allow-webhook-and-service-requests-to-local-network). +Webhook requests to local network addresses are blocked to prevent exploitation of internal web services. If you get `Error publishing` or `Invalid Package` messages when you try to publish NuGet packages, change your network settings to [allow webhook and integration requests to the local network](../../../security/webhooks.md#allow-requests-to-the-local-network-from-webhooks-and-integrations). diff --git a/doc/user/packages/package_registry/index.md b/doc/user/packages/package_registry/index.md index cd60a229479..91186e6c159 100644 --- a/doc/user/packages/package_registry/index.md +++ b/doc/user/packages/package_registry/index.md @@ -71,9 +71,13 @@ NOTE: If you have not activated the "Package registry" feature for your project at **Settings > General > Visibility, project features, permissions**, you receive a 403 Forbidden response. Accessing package registry via deploy token is not available when external authorization is enabled. -## Use GitLab CI/CD to build packages +## Use GitLab CI/CD + +You can use [GitLab CI/CD](../../../ci/index.md) to build or import packages into +a package registry. + +### To build packages -You can use [GitLab CI/CD](../../../ci/index.md) to build packages. For Maven, NuGet, npm, Conan, Helm, and PyPI packages, and Composer dependencies, you can authenticate with GitLab by using the `CI_JOB_TOKEN`. @@ -97,6 +101,16 @@ when you view the package details: You can view which pipeline published the package, and the commit and user who triggered it. However, the history is limited to five updates of a given package. +### To import packages + +If you already have packages built in a different registry, you can import them +into your GitLab package registry with the [Packages Importer](https://gitlab.com/gitlab-org/ci-cd/package-stage/pkgs_importer) + +The Packages Importer runs a CI/CD pipeline that [can import these package types](https://gitlab.com/gitlab-org/ci-cd/package-stage/pkgs_importer#formats-supported): + +- NPM +- NuGet + ## Reduce storage usage For information on reducing your storage use for the Package Registry, see @@ -129,17 +143,17 @@ your project's settings. For example, if you have a public project and set the r to **Only Project Members**, the Package Registry is then public. Disabling the Package Registry disables all Package Registry operations. -| Project visibility | Action | [Role](../../permissions.md#roles) required | +| Project visibility | Action | Minimum [role](../../permissions.md#roles) required | |--------------------|-----------------------|---------------------------------------------------------| | Public | View Package Registry | `n/a`, everyone on the internet can perform this action | -| Public | Publish a package | Developer or higher | +| Public | Publish a package | Developer | | Public | Pull a package | `n/a`, everyone on the internet can perform this action | -| Internal | View Package Registry | Guest or higher | -| Internal | Publish a package | Developer or higher | -| Internal | Pull a package | Guest or higher(1) | -| Private | View Package Registry | Reporter or higher | -| Private | Publish a package | Developer or higher | -| Private | Pull a package | Reporter or higher(1) | +| Internal | View Package Registry | Guest | +| Internal | Publish a package | Developer | +| Internal | Pull a package | Guest (1) | +| Private | View Package Registry | Reporter | +| Private | Publish a package | Developer | +| Private | Pull a package | Reporter (1) | ### Allow anyone to pull from Package Registry diff --git a/doc/user/packages/package_registry/reduce_package_registry_storage.md b/doc/user/packages/package_registry/reduce_package_registry_storage.md index 673196ebad5..020cad5ac14 100644 --- a/doc/user/packages/package_registry/reduce_package_registry_storage.md +++ b/doc/user/packages/package_registry/reduce_package_registry_storage.md @@ -35,6 +35,9 @@ To delete a package in the UI, from your group or project: The package is permanently deleted. +If [request forwarding](supported_functionality.md#forwarding-requests) is enabled, +deleting a package can introduce a [dependency confusion risk](supported_functionality.md#deleting-packages). + ## Delete assets associated with a package To delete package assets, you must have suitable [permissions](../../permissions.md). diff --git a/doc/user/packages/package_registry/supported_functionality.md b/doc/user/packages/package_registry/supported_functionality.md index e56ae88872a..ca174c43565 100644 --- a/doc/user/packages/package_registry/supported_functionality.md +++ b/doc/user/packages/package_registry/supported_functionality.md @@ -13,77 +13,102 @@ and pulling packages, request forwarding, managing duplicates, and authenticatio Packages can be published to your project, group, or instance. -| Package type | Project | Group | Instance | -|-----------------------------------------------------|---------|-------|----------| -| [Maven](../maven_repository/index.md) | Y | N | N | -| [npm](../npm_registry/index.md) | Y | N | N | -| [NuGet](../nuget_repository/index.md) | Y | N | N | -| [PyPI](../pypi_repository/index.md) | Y | N | N | -| [Generic packages](../generic_packages/index.md) | Y | N | N | -| [Terraform](../terraform_module_registry/index.md) | Y | N | N | -| [Composer](../composer_repository/index.md) | N | Y | N | -| [Conan](../conan_repository/index.md) | Y | N | N | -| [Helm](../helm_repository/index.md) | Y | N | N | -| [Debian](../debian_repository/index.md) | Y | N | N | -| [Go](../go_proxy/index.md) | Y | N | Y | -| [Ruby gems](../rubygems_registry/index.md) | Y | N | N | +| Package type | Project | Group | Instance | +|-------------------------------------------------------|---------|-------|----------| +| [Maven (with `mvn`)](../maven_repository/index.md) | Y | N | N | +| [Maven (with `gradle`)](../maven_repository/index.md) | Y | N | N | +| [Maven (with `sbt`)](../maven_repository/index.md) | N | N | N | +| [npm](../npm_registry/index.md) | Y | N | N | +| [NuGet](../nuget_repository/index.md) | Y | N | N | +| [PyPI](../pypi_repository/index.md) | Y | N | N | +| [Generic packages](../generic_packages/index.md) | Y | N | N | +| [Terraform](../terraform_module_registry/index.md) | Y | N | N | +| [Composer](../composer_repository/index.md) | N | Y | N | +| [Conan](../conan_repository/index.md) | Y | N | Y | +| [Helm](../helm_repository/index.md) | Y | N | N | +| [Debian](../debian_repository/index.md) | Y | N | N | +| [Go](../go_proxy/index.md) | Y | N | N | +| [Ruby gems](../rubygems_registry/index.md) | Y | N | N | ## Pulling packages **(FREE)** Packages can be pulled from your project, group, or instance. -| Package type | Project | Group | Instance | -|-----------------------------------------------------|---------|-------|----------| -| [Maven](../maven_repository/index.md) | Y | Y | Y | -| [npm](../npm_registry/index.md) | Y | Y | Y | -| [NuGet](../nuget_repository/index.md) | Y | Y | N | -| [PyPI](../pypi_repository/index.md) | Y | Y | N | -| [Generic packages](../generic_packages/index.md) | Y | N | N | -| [Terraform](../terraform_module_registry/index.md) | N | Y | N | -| [Composer](../composer_repository/index.md) | Y | Y | N | -| [Conan](../conan_repository/index.md) | Y | N | Y | -| [Helm](../helm_repository/index.md) | Y | N | N | -| [Debian](../debian_repository/index.md) | Y | N | N | -| [Go](../go_proxy/index.md) | Y | N | Y | -| [Ruby gems](../rubygems_registry/index.md) | Y | N | N | +| Package type | Project | Group | Instance | +|-------------------------------------------------------|---------|-------|----------| +| [Maven (with `mvn`)](../maven_repository/index.md) | Y | Y | Y | +| [Maven (with `gradle`)](../maven_repository/index.md) | Y | Y | Y | +| [Maven (with `sbt`)](../maven_repository/index.md) | Y | Y | Y | +| [npm](../npm_registry/index.md) | Y | N | Y | +| [NuGet](../nuget_repository/index.md) | Y | Y | N | +| [PyPI](../pypi_repository/index.md) | Y | Y | N | +| [Generic packages](../generic_packages/index.md) | Y | N | N | +| [Terraform](../terraform_module_registry/index.md) | N | Y | N | +| [Composer](../composer_repository/index.md) | Y | Y | N | +| [Conan](../conan_repository/index.md) | Y | N | Y | +| [Helm](../helm_repository/index.md) | Y | N | N | +| [Debian](../debian_repository/index.md) | Y | N | N | +| [Go](../go_proxy/index.md) | Y | N | Y | +| [Ruby gems](../rubygems_registry/index.md) | Y | N | N | ## Forwarding requests **(PREMIUM)** Requests for packages not found in your GitLab project are forwarded to the public registry. For example, Maven Central, npmjs, or PyPI. -| Package type | Supports request forwarding | -|-----------------------------------------------------|-----------------------------| -| [Maven](../maven_repository/index.md) | Y | -| [npm](../npm_registry/index.md) | Y | -| [NuGet](../nuget_repository/index.md) | N | -| [PyPI](../pypi_repository/index.md) | Y | -| [Generic packages](../generic_packages/index.md) | N | -| [Terraform](../terraform_module_registry/index.md) | N | -| [Composer](../composer_repository/index.md) | N | -| [Conan](../conan_repository/index.md) | N | -| [Helm](../helm_repository/index.md) | N | -| [Debian](../debian_repository/index.md) | N | -| [Go](../go_proxy/index.md) | N | -| [Ruby gems](../rubygems_registry/index.md) | N | +| Package type | Supports request forwarding | +|-------------------------------------------------------|-----------------------------| +| [Maven (with `mvn`)](../maven_repository/index.md) | [Yes (disabled by default)](../../admin_area/settings/continuous_integration.md#maven-forwarding) | +| [Maven (with `gradle`)](../maven_repository/index.md) | [Yes (disabled by default)](../../admin_area/settings/continuous_integration.md#maven-forwarding) | +| [Maven (with `sbt`)](../maven_repository/index.md) | [Yes (disabled by default)](../../admin_area/settings/continuous_integration.md#maven-forwarding) | +| [npm](../npm_registry/index.md) | [Yes](../../admin_area/settings/continuous_integration.md#npm-forwarding) | +| [NuGet](../nuget_repository/index.md) | N | +| [PyPI](../pypi_repository/index.md) | [Yes](../../admin_area/settings/continuous_integration.md#pypi-forwarding) | +| [Generic packages](../generic_packages/index.md) | N | +| [Terraform](../terraform_module_registry/index.md) | N | +| [Composer](../composer_repository/index.md) | N | +| [Conan](../conan_repository/index.md) | N | +| [Helm](../helm_repository/index.md) | N | +| [Debian](../debian_repository/index.md) | N | +| [Go](../go_proxy/index.md) | N | +| [Ruby gems](../rubygems_registry/index.md) | N | + +### Deleting packages + +When package requests are forwarded to a public registry, deleting packages can +be a [dependency confusion vulnerability](https://medium.com/@alex.birsan/dependency-confusion-4a5d60fec610). + +If a system tries to pull a deleted package, the request is forwarded to the public +registry. If a package with the same name and version is found in the public registry, that package +is pulled instead. There is a risk that the package pulled from the registry might not be +what is expected, and could even be malicious. + +To reduce the associated security risks, before deleting a package you can: + +- Verify the package is not being actively used. +- Disable request forwarding: + - Instance administrators can disable forwarding in the [**Continuous Integration** section](../../admin_area/settings/continuous_integration.md#package-registry-configuration) of the Admin Area. + - Group owners can disable forwarding in the **Packages and Registries** section of the group settings. ## Allow or prevent duplicates **(FREE)** By default, the GitLab package registry either allows or prevents duplicates based on the default of that specific package manager format. -| Package type | Duplicates allowed? | -|-----------------------------------------------------|---------------------| -| [Maven](../maven_repository/index.md) | Y (configurable) | -| [npm](../npm_registry/index.md) | N | -| [NuGet](../nuget_repository/index.md) | Y | -| [PyPI](../pypi_repository/index.md) | N | -| [Generic packages](../generic_packages/index.md) | Y (configurable) | -| [Terraform](../terraform_module_registry/index.md) | N | -| [Composer](../composer_repository/index.md) | N | -| [Conan](../conan_repository/index.md) | N | -| [Helm](../helm_repository/index.md) | Y | -| [Debian](../debian_repository/index.md) | Y | -| [Go](../go_proxy/index.md) | N | -| [Ruby gems](../rubygems_registry/index.md) | Y | +| Package type | Duplicates allowed? | +|-------------------------------------------------------|---------------------| +| [Maven (with `mvn`)](../maven_repository/index.md) | Y (configurable) | +| [Maven (with `gradle`)](../maven_repository/index.md) | Y (configurable) | +| [Maven (with `sbt`)](../maven_repository/index.md) | Y (configurable) | +| [npm](../npm_registry/index.md) | N | +| [NuGet](../nuget_repository/index.md) | Y | +| [PyPI](../pypi_repository/index.md) | N | +| [Generic packages](../generic_packages/index.md) | Y (configurable) | +| [Terraform](../terraform_module_registry/index.md) | N | +| [Composer](../composer_repository/index.md) | N | +| [Conan](../conan_repository/index.md) | N | +| [Helm](../helm_repository/index.md) | Y | +| [Debian](../debian_repository/index.md) | Y | +| [Go](../go_proxy/index.md) | N | +| [Ruby gems](../rubygems_registry/index.md) | Y | ## Authentication tokens **(FREE)** @@ -91,39 +116,45 @@ GitLab tokens are used to authenticate with the GitLab Package Registry. The following tokens are supported: -| Package type | Supported tokens | -|-----------------------------------------------------|------------------------------------------------------------------------| -| [Maven](../maven_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | -| [npm](../npm_registry/index.md) | Personal access, job tokens, deploy (project or group), project access | -| [NuGet](../nuget_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | -| [PyPI](../pypi_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | -| [Generic packages](../generic_packages/index.md) | Personal access, job tokens, deploy (project or group), project access | -| [Terraform](../terraform_module_registry/index.md) | Personal access, job tokens, deploy (project or group), project access | -| [Composer](../composer_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | -| [Conan](../conan_repository/index.md) | Personal access, job tokens, project access | -| [Helm](../helm_repository/index.md) | Personal access, job tokens, deploy (project or group) | -| [Debian](../debian_repository/index.md) | Personal access, job tokens, deploy (project or group) | -| [Go](../go_proxy/index.md) | Personal access, job tokens, project access | -| [Ruby gems](../rubygems_registry/index.md) | Personal access, job tokens, deploy (project or group) | +| Package type | Supported tokens | +|-------------------------------------------------------|------------------------------------------------------------------------| +| [Maven (with `mvn`)](../maven_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | +| [Maven (with `gradle`)](../maven_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | +| [Maven (with `sbt`)](../maven_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | +| [npm](../npm_registry/index.md) | Personal access, job tokens, deploy (project or group), project access | +| [NuGet](../nuget_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | +| [PyPI](../pypi_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | +| [Generic packages](../generic_packages/index.md) | Personal access, job tokens, deploy (project or group), project access | +| [Terraform](../terraform_module_registry/index.md) | Personal access, job tokens, deploy (project or group), project access | +| [Composer](../composer_repository/index.md) | Personal access, job tokens, deploy (project or group), project access | +| [Conan](../conan_repository/index.md) | Personal access, job tokens, project access | +| [Helm](../helm_repository/index.md) | Personal access, job tokens, deploy (project or group) | +| [Debian](../debian_repository/index.md) | Personal access, job tokens, deploy (project or group) | +| [Go](../go_proxy/index.md) | Personal access, job tokens, project access | +| [Ruby gems](../rubygems_registry/index.md) | Personal access, job tokens, deploy (project or group) | ## Authentication protocols **(FREE)** The following authentication protocols are supported: -| Package type | Supported auth protocols | -|-----------------------------------------------------|--------------------------| -| [Maven](../maven_repository/index.md) | Headers | -| [npm](../npm_registry/index.md) | OAuth | -| [NuGet](../nuget_repository/index.md) | Basic auth | -| [PyPI](../pypi_repository/index.md) | Basic auth | -| [Generic packages](../generic_packages/index.md) | Basic auth | -| [Terraform](../terraform_module_registry/index.md) | Token | -| [Composer](../composer_repository/index.md) | OAuth | -| [Conan](../conan_repository/index.md) | OAuth, Basic auth | -| [Helm](../helm_repository/index.md) | Basic auth | -| [Debian](../debian_repository/index.md) | Basic auth | -| [Go](../go_proxy/index.md) | Basic auth | -| [Ruby gems](../rubygems_registry/index.md) | Token | +| Package type | Supported auth protocols | +|-------------------------------------------------------|-------------------------------------------------------------| +| [Maven (with `mvn`)](../maven_repository/index.md) | Headers, Basic auth ([pulling](#pulling-packages) only) (1) | +| [Maven (with `gradle`)](../maven_repository/index.md) | Headers, Basic auth ([pulling](#pulling-packages) only) (1) | +| [Maven (with `sbt`)](../maven_repository/index.md) | Basic auth (1) | +| [npm](../npm_registry/index.md) | OAuth | +| [NuGet](../nuget_repository/index.md) | Basic auth | +| [PyPI](../pypi_repository/index.md) | Basic auth | +| [Generic packages](../generic_packages/index.md) | Basic auth | +| [Terraform](../terraform_module_registry/index.md) | Token | +| [Composer](../composer_repository/index.md) | OAuth | +| [Conan](../conan_repository/index.md) | OAuth, Basic auth | +| [Helm](../helm_repository/index.md) | Basic auth | +| [Debian](../debian_repository/index.md) | Basic auth | +| [Go](../go_proxy/index.md) | Basic auth | +| [Ruby gems](../rubygems_registry/index.md) | Token | + +1. Basic authentication for Maven packages [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/212854) in GitLab 16.0. ## Supported hash types **(FREE)** @@ -131,16 +162,18 @@ Hash values are used to ensure you are using the correct package. You can view t The Package Registry supports the following hash types: -| Package type | Supported hashes | -|--------------------------------------------------|----------------------------------| -| [Maven](../maven_repository/index.md) | MD5, SHA1 | -| [npm](../npm_registry/index.md) | SHA1 | -| [NuGet](../nuget_repository/index.md) | not applicable | -| [PyPI](../pypi_repository/index.md) | MD5, SHA256 | -| [Generic packages](../generic_packages/index.md) | SHA256 | -| [Composer](../composer_repository/index.md) | not applicable | -| [Conan](../conan_repository/index.md) | MD5, SHA1 | -| [Helm](../helm_repository/index.md) | not applicable | -| [Debian](../debian_repository/index.md) | MD5, SHA1, SHA256 | -| [Go](../go_proxy/index.md) | MD5, SHA1, SHA256 | -| [Ruby gems](../rubygems_registry/index.md) | MD5, SHA1, SHA256 (gemspec only) | +| Package type | Supported hashes | +|-------------------------------------------------------|----------------------------------| +| [Maven (with `mvn`)](../maven_repository/index.md) | MD5, SHA1 | +| [Maven (with `gradle`)](../maven_repository/index.md) | MD5, SHA1 | +| [Maven (with `sbt`)](../maven_repository/index.md) | MD5, SHA1 | +| [npm](../npm_registry/index.md) | SHA1 | +| [NuGet](../nuget_repository/index.md) | not applicable | +| [PyPI](../pypi_repository/index.md) | MD5, SHA256 | +| [Generic packages](../generic_packages/index.md) | SHA256 | +| [Composer](../composer_repository/index.md) | not applicable | +| [Conan](../conan_repository/index.md) | MD5, SHA1 | +| [Helm](../helm_repository/index.md) | not applicable | +| [Debian](../debian_repository/index.md) | MD5, SHA1, SHA256 | +| [Go](../go_proxy/index.md) | MD5, SHA1, SHA256 | +| [Ruby gems](../rubygems_registry/index.md) | MD5, SHA1, SHA256 (gemspec only) | diff --git a/doc/user/packages/package_registry/supported_hash_types.md b/doc/user/packages/package_registry/supported_hash_types.md deleted file mode 100644 index d39b8d60c93..00000000000 --- a/doc/user/packages/package_registry/supported_hash_types.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'supported_functionality.md' -remove_date: '2023-04-22' ---- - -This document was moved to [another location](supported_functionality.md). - - - - - diff --git a/doc/user/packages/package_registry/supported_package_managers.md b/doc/user/packages/package_registry/supported_package_managers.md index 75b8c95a0fa..cd2b288094b 100644 --- a/doc/user/packages/package_registry/supported_package_managers.md +++ b/doc/user/packages/package_registry/supported_package_managers.md @@ -11,24 +11,20 @@ Not all package manager formats are ready for production use. The Package Registry supports the following package manager types: -| Package type | GitLab version | Status | -| ------------------------------------------------ | -------------- | ---------------------------------------------------------- | -| [Maven](../maven_repository/index.md) | 11.3+ | GA | -| [npm](../npm_registry/index.md) | 11.7+ | GA | -| [NuGet](../nuget_repository/index.md) | 12.8+ | GA | -| [PyPI](../pypi_repository/index.md) | 12.10+ | GA | -| [Generic packages](../generic_packages/index.md) | 13.5+ | GA | -| [Composer](../composer_repository/index.md) | 13.2+ | [Beta](https://gitlab.com/groups/gitlab-org/-/epics/6817) | -| [Conan](../conan_repository/index.md) | 12.6+ | [Beta](https://gitlab.com/groups/gitlab-org/-/epics/6816) | -| [Helm](../helm_repository/index.md) | 14.1+ | [Beta](https://gitlab.com/groups/gitlab-org/-/epics/6366) | -| [Debian](../debian_repository/index.md) | 14.2+ | [Alpha](https://gitlab.com/groups/gitlab-org/-/epics/6057) | -| [Go](../go_proxy/index.md) | 13.1+ | [Alpha](https://gitlab.com/groups/gitlab-org/-/epics/3043) | -| [Ruby gems](../rubygems_registry/index.md) | 13.10+ | [Alpha](https://gitlab.com/groups/gitlab-org/-/epics/3200) | +| Package type | GitLab version | Status | +| ------------------------------------------------ | -------------- | --------------------------------------------------------------- | +| [Maven](../maven_repository/index.md) | 11.3+ | GA | +| [npm](../npm_registry/index.md) | 11.7+ | GA | +| [NuGet](../nuget_repository/index.md) | 12.8+ | GA | +| [PyPI](../pypi_repository/index.md) | 12.10+ | GA | +| [Generic packages](../generic_packages/index.md) | 13.5+ | GA | +| [Composer](../composer_repository/index.md) | 13.2+ | [Beta](https://gitlab.com/groups/gitlab-org/-/epics/6817) | +| [Helm](../helm_repository/index.md) | 14.1+ | [Beta](https://gitlab.com/groups/gitlab-org/-/epics/6366) | +| [Conan](../conan_repository/index.md) | 12.6+ | [Experiment](https://gitlab.com/groups/gitlab-org/-/epics/6816) | +| [Debian](../debian_repository/index.md) | 14.2+ | [Experiment](https://gitlab.com/groups/gitlab-org/-/epics/6057) | +| [Go](../go_proxy/index.md) | 13.1+ | [Experiment](https://gitlab.com/groups/gitlab-org/-/epics/3043) | +| [Ruby gems](../rubygems_registry/index.md) | 13.10+ | [Experiment](https://gitlab.com/groups/gitlab-org/-/epics/3200) | -[Status](../../../policy/alpha-beta-support.md): - -- Alpha: behind a feature flag and not officially supported. -- Beta: several known issues that may prevent expected use. -- GA (Generally Available): ready for production use at any scale. +[View what each status means](../../../policy/alpha-beta-support.md). You can also use the [API](../../../api/packages.md) to administer the Package Registry. diff --git a/doc/user/packages/pypi_repository/index.md b/doc/user/packages/pypi_repository/index.md index e5b7f06f6ca..d23966f26fe 100644 --- a/doc/user/packages/pypi_repository/index.md +++ b/doc/user/packages/pypi_repository/index.md @@ -34,6 +34,8 @@ To do this, you can use: `read_package_registry`, `write_package_registry`, or both. - A [CI job token](#authenticate-with-a-ci-job-token). +Do not use authentication methods other than the methods documented here. Undocumented authentication methods might be removed in the future. + ### Authenticate with a personal access token To authenticate with a personal access token, edit the `~/.pypirc` file and add: diff --git a/doc/user/packages/terraform_module_registry/index.md b/doc/user/packages/terraform_module_registry/index.md index 66a42f398b0..d7f33dd9e79 100644 --- a/doc/user/packages/terraform_module_registry/index.md +++ b/doc/user/packages/terraform_module_registry/index.md @@ -4,30 +4,53 @@ group: Package Registry info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Terraform module registry **(FREE)** +# Terraform Module Registry **(FREE)** -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3221) in GitLab 14.0. +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3221) in GitLab 14.0. +> - Infrastructure registry and Terraform Module Registry [merged](https://gitlab.com/gitlab-org/gitlab/-/issues/404075) into a single Terraform Module Registry feature in GitLab 15.11. -Publish Terraform modules in your project's Infrastructure Registry, then reference them using GitLab -as a Terraform module registry. +With the Terraform Module Registry, you can use GitLab projects as a +private registry for terraform modules. You can create and publish +modules with GitLab CI/CD, which can then be consumed from other private +projects. -## Authenticate to the Terraform module registry +## View Terraform modules -To authenticate to the Terraform module registry, you need either: +To view Terraform modules in your project: + +1. Go to the project. +1. On the left sidebar, select **Packages and registries > Terraform modules**. + +You can search, sort, and filter modules on this page. + +For information on how to create and upload a package, view the GitLab +documentation for your package type: + +- [Terraform modules](../terraform_module_registry/index.md) + +## Authenticate to the Terraform Module Registry + +To authenticate to the Terraform Module Registry, you need either: - A [personal access token](../../../api/rest/index.md#personalprojectgroup-access-tokens) with at least `read_api` rights. - A [CI/CD job token](../../../ci/jobs/ci_job_token.md). -## Publish a Terraform Module +Do not use authentication methods other than the methods documented here. Undocumented authentication methods might be removed in the future. -When you publish a Terraform Module, if it does not exist, it is created. +## Publish a Terraform module + +When you publish a Terraform module, if it does not exist, it is created. + +### Using the API + +You can publish Terraform modules by using the [Terraform Module Registry API](../../../api/packages/terraform-modules.md). Prerequisites: -- The package name and version [must be unique in the top-level namespace](../infrastructure_registry/index.md#how-module-resolution-works). +- The package name and version [must be unique in the top-level namespace](#how-module-resolution-works). - Your project and group names must not include a dot (`.`). For example, `source = "gitlab.example.com/my.group/project.name"`. - You must [authenticate with the API](../../../api/rest/index.md#authentication). If authenticating with a deploy token, it must be configured with the `write_package_registry` scope. -- The name of a module [must be unique within the scope of its group](../infrastructure_registry/index.md#how-module-resolution-works), otherwise an +- The name of a module [must be unique in the scope of its group](#how-module-resolution-works), otherwise an [error occurs](#troubleshooting). ```plaintext @@ -37,9 +60,9 @@ PUT /projects/:id/packages/terraform/modules/:module-name/:module-system/:module | Attribute | Type | Required | Description | | -------------------| --------------- | ---------| -------------------------------------------------------------------------------------------------------------------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](../../../api/rest/index.md#namespaced-path-encoding). | -| `module-name` | string | yes | The package name. **Supported syntax**: One to 64 ASCII characters, including lowercase letters (a-z) and digits (0-9). The package name can't exceed 64 characters. -| `module-system` | string | yes | The package system. **Supported syntax**: One to 64 ASCII characters, including lowercase letters (a-z) and digits (0-9). The package system can't exceed 64 characters. More information can be found in the [Terraform Module Registry Protocol documentation](https://www.terraform.io/internals/module-registry-protocol). -| `module-version` | string | yes | The package version. It must be valid according to the [Semantic Versioning Specification](https://semver.org/). +| `module-name` | string | yes | The module name. **Supported syntax**: One to 64 ASCII characters, including lowercase letters (a-z) and digits (0-9). The module name can't exceed 64 characters. +| `module-system` | string | yes | The module system. **Supported syntax**: One to 64 ASCII characters, including lowercase letters (a-z) and digits (0-9). The module system can't exceed 64 characters. More information can be found in the [Terraform Module Registry protocol documentation](https://www.terraform.io/internals/module-registry-protocol). +| `module-version` | string | yes | The module version. It must be valid according to the [semantic versioning specification](https://semver.org/). Provide the file content in the request body. @@ -55,14 +78,6 @@ curl --fail-with-body --header "PRIVATE-TOKEN: " \ "https://gitlab.example.com/api/v4/projects//packages/terraform/modules/my-module/my-system/0.0.1/file" ``` -Example response: - -```json -{ - "message":"201 Created" -} -``` - Example request using a deploy token: ```shell @@ -79,41 +94,13 @@ Example response: } ``` -## Reference a Terraform Module +### Using a CI/CD template (recommended) -Prerequisites: - -- You need to [authenticate with the API](../../../api/rest/index.md#authentication). If authenticating with a personal access token, it must be configured with the `read_api` scope. - -Authentication tokens (Job Token or Personal Access Token) can be provided for `terraform` in your `~/.terraformrc` file: - -```plaintext -credentials "gitlab.com" { - token = "" -} -``` - -Where `gitlab.com` can be replaced with the hostname of your self-managed GitLab instance. - -You can then refer to your Terraform Module from a downstream Terraform project: - -```plaintext -module "" { - source = "gitlab.com///" -} -``` - -Where `` is the [namespace](../../../user/namespace/index.md) of the Terraform module registry. - -## Publish a Terraform module by using CI/CD - -> CI/CD template [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/110493) in GitLab 15.9. - -### Use a CI/CD template (recommended) +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/110493) in GitLab 15.9. You can use the [`Terraform-Module.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform-Module.gitlab-ci.yml) or the advanced [`Terraform/Module-Base.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform/Module-Base.gitlab-ci.yml) -CI/CD template to publish a Terraform module to the GitLab Terraform Registry: +CI/CD template to publish a Terraform module to the GitLab terraform registry: ```yaml include: @@ -124,7 +111,7 @@ The pipeline contains the following jobs: - `fmt` - Validate the formatting of the Terraform module. - `kics-iac-sast` - Test the Terraform module for security issues. -- `deploy` - For tag pipelines only. Deploy the Terraform module to the GitLab Terraform Registry. +- `deploy` - For tag pipelines only. Deploy the Terraform module to the Terraform Module Registry. #### Pipeline variables @@ -137,7 +124,7 @@ You can configure the pipeline with the following variables: | `TERRAFORM_MODULE_SYSTEM` | `local` | The system or provider of your Terraform module targets. For example, `local`, `aws`, `google`. | | `TERRAFORM_MODULE_VERSION` | `${CI_COMMIT_TAG}` | The Terraform module version. You should follow the semantic versioning specification. | -### Deploy manually via CI/CD +### Using CI/CD manually To work with Terraform modules in [GitLab CI/CD](../../../ci/index.md), you can use `CI_JOB_TOKEN` in place of the personal access token in your commands. @@ -166,14 +153,97 @@ upload: - if: $CI_COMMIT_TAG ``` -To trigger this upload job, add a Git tag to your commit. Ensure the tag follows the [Semantic Versioning Specification](https://semver.org/) that Terraform requires. The `rules:if: $CI_COMMIT_TAG` ensures that only tagged commits to your repository trigger the module upload job. +To trigger this upload job, add a Git tag to your commit. Ensure the tag follows the [Semantic versioning specification](https://semver.org/) that Terraform requires. The `rules:if: $CI_COMMIT_TAG` ensures that only tagged commits to your repository trigger the module upload job. For other ways to control jobs in your CI/CD pipeline, refer to the [`.gitlab-ci.yml`](../../../ci/yaml/index.md) keyword reference. +## Reference a Terraform module + +Prerequisites: + +- You need to [authenticate with the API](../../../api/rest/index.md#authentication). If authenticating with a personal access token, it must be configured with the `read_api` scope. + +Authentication tokens (Job Token or Personal Access Token) can be provided for `terraform` in your `~/.terraformrc` file: + +```terraform +credentials "gitlab.com" { + token = "" +} +``` + +Where `gitlab.com` can be replaced with the hostname of your self-managed GitLab instance. + +You can then refer to your Terraform module from a downstream Terraform project: + +```terraform +module "" { + source = "gitlab.com///" +} +``` + +Where `` is the [namespace](../../../user/namespace/index.md) of the Terraform Module Registry. + +## Download a Terraform module + +To download a Terraform module: + +1. On the left sidebar, select **Packages and registries > Terraform modules**. +1. Select the name of the module you want to download. +1. In the **Activity** section, select the name of the module you want to download. + +## How module resolution works + +When you upload a new module, GitLab generates a path for the module, for example, `https://gitlab.example.com/parent-group/my-infra-package`. + +- This path conforms with [the Terraform spec](https://www.terraform.io/internals/module-registry-protocol). +- The name of the path must be unique in the namespace. + +For projects in subgroups, GitLab checks that the module name does not already exist anywhere in the namespace, including all subgroups and the parent group. + +For example, if: + +- The project is `gitlab.example.com/parent-group/sub-group/my-project`. +- The Terraform module is `my-infra-package`. + +The project name must be unique in all projects in all groups under `parent-group`. + +## Delete a Terraform module + +You cannot edit a Terraform module after you publish it in the Terraform Module Registry. Instead, you +must delete and recreate it. + +To delete a module, you must have suitable [permissions](../../permissions.md). + +You can delete modules by using [the packages API](../../../api/packages.md#delete-a-project-package) or the UI. + +To delete a module in the UI, from your project: + +1. On the left sidebar, select **Packages and registries > Terraform modules**. +1. Find the name of the package you want to delete. +1. Select **Delete**. + +The package is permanently deleted. + +## Disable the Terraform Module Registry + +The Terraform Module Registry is automatically enabled. + +For self-managed instances, a GitLab administrator can +[disable](../../../administration/packages/index.md) **Packages and registries**, +which removes this menu item from the sidebar. + +You can also remove the Terraform Module Registry for a specific project: + +1. In your project, go to **Settings > General**. +1. Expand the **Visibility, project features, permissions** section and toggle **Packages** off (in gray). +1. Select **Save changes**. + +To enable it back, follow the same steps above and toggle it on (in blue). + ## Example projects -For examples of the Terraform module registry, check the projects below: +For examples of the Terraform Module Registry, check the projects below: -- The [_GitLab local file_ project](https://gitlab.com/mattkasa/gitlab-local-file) creates a minimal Terraform module and uploads it into the Terraform module registry using GitLab CI/CD. +- The [_GitLab local file_ project](https://gitlab.com/mattkasa/gitlab-local-file) creates a minimal Terraform module and uploads it into the Terraform Module Registry using GitLab CI/CD. - The [_Terraform module test_ project](https://gitlab.com/mattkasa/terraform-module-test) uses the module from the previous example. ## Troubleshooting diff --git a/doc/user/packages/yarn_repository/index.md b/doc/user/packages/yarn_repository/index.md index 7e2f45019cd..c8f4985a518 100644 --- a/doc/user/packages/yarn_repository/index.md +++ b/doc/user/packages/yarn_repository/index.md @@ -6,220 +6,310 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Publish packages with Yarn -Publish npm packages in your project's Package Registry using Yarn. Then install the -packages whenever you need to use them as a dependency. +You can publish packages with [Yarn 1 (Classic)](https://classic.yarnpkg.com) and [Yarn 2+](https://yarnpkg.com). -Learn how to build a [yarn](../workflows/build_packages.md#yarn) package. +To find the Yarn version used in the deployment container, run `yarn --version` in the `script` block of the CI +script job block that is responsible for calling `yarn publish`**`. The Yarn version is shown in the pipeline output. -You can get started with Yarn 2 by following the [Yarn documentation](https://yarnpkg.com/getting-started/install/). +Learn how to build a [Yarn](../workflows/build_packages.md#yarn) package. + +You can use the Yarn documentation to get started with +[Yarn Classic](https://classic.yarnpkg.com/en/docs/getting-started) and +[Yarn 2+](https://yarnpkg.com/getting-started/). ## Publish to GitLab Package Registry +You can use Yarn to publish to the GitLab Package Registry. + ### Authentication to the Package Registry You need a token to publish a package. Different tokens are available depending on what you're trying to achieve. For more information, review the [guidance on tokens](../../../user/packages/package_registry/index.md#authenticate-with-the-registry). -- If your organization uses two-factor authentication (2FA), you must use a personal access token with the scope set to `api`. -- If you publish a package via CI/CD pipelines, you must use a CI job token. +- If your organization uses two-factor authentication (2FA), you must use a + personal access token with the scope set to `api`. +- If you publish a package via CI/CD pipelines, you can use a CI job token in + private runners or you can register a variable for shared runners. -Create a token and save it to use later in the process. +### Publish configuration -### Naming convention +To publish, set the following configuration in `.yarnrc.yml`. This file should be +located in the root directory of your package project source where `package.json` is found. -Depending on how you install the package, you may need to adhere to the naming convention. +```yaml +npmScopes: + : + npmPublishRegistry: 'https:///api/v4/projects//packages/npm/' + npmAlwaysAuth: true + npmAuthToken: '' +``` -You can use one of two API endpoints to install packages: +In this configuration: -- **Instance-level**: Use when you have many npm packages in different GitLab groups or in their own namespace. -- **Project-level**: Use when you have a few npm packages, and they are not in the same GitLab group. +- Replace `` with your organization scope, exclude the `@` symbol. +- Replace `` with your domain name. +- Replace `` with your project's ID, which you can find on the project's home page. +- Replace `` with a deployment token, group access token, project access token, or personal access token. -If you plan to install a package through the [project level](#install-from-the-project-level), you do not have to -adhere to the naming convention. +Scoped registry does not work in Yarn Classic in `package.json` file, based on +this [issue](https://github.com/yarnpkg/yarn/pull/7829). +Therefore, under `publishConfig` there should be `registry` and not `@scope:registry` for Yarn Classic. +You can publish using your command line or a CI/CD pipeline to the GitLab Package Registry. -If you plan to install a package through the [instance level](#install-from-the-instance-level), then you must name -your package with a [scope](https://docs.npmjs.com/misc/scope/). Scoped packages begin with a `@` and have the -`@owner/package-name` format. You can set up the scope for your package in the `.yarnrc.yml` file and by using the -`publishConfig` option in the `package.json`. +### Publishing via the command line - Manual Publish -- The value used for the `@scope` is the root of the project that hosts the packages and not the root - of the project with the package's source code. The scope should be lowercase. -- The package name can be anything you want +```shell +# Yarn 1 (Classic) +yarn publish -| Project URL | Package Registry in | Scope | Full package name | -| ------------------------------------------------------- | ------------------- | --------- | ---------------------- | -| `https://gitlab.com/my-org/engineering-group/analytics` | Analytics | `@my-org` | `@my-org/package-name` | +# Yarn 2+ +yarn npm publish +``` -### Configuring `.yarnrc.yml` to publish from the project level +Your package should now publish to the Package Registry. -To publish with the project-level npm endpoint, set the following configuration in -`.yarnrc.yml`: +### Publishing via a CI/CD pipeline - Automated Publish -```yaml -npmScopes: - foo: - npmRegistryServer: 'https:///api/v4/projects//packages/npm/' - npmPublishRegistry: 'https:///api/v4/projects//packages/npm/' +You can use pipeline variables when you use this method. -npmRegistries: - //gitlab.example.com/api/v4/projects//packages/npm/: - npmAlwaysAuth: true - npmAuthToken: '' -``` +You can use **Shared Runners** *(Default)* or **Private Runners** (Advanced). -In this configuration: +#### Shared runners -- Replace `` with your domain name. -- Replace `` with your project's ID, which you can find on the project's home page. -- Replace `` with a deploy token, group access token, project access token, or personal access token. +Third party images such as `node:latest` or `node:current` do not have direct access +to the `CI_JOB_TOKEN` when operating in a shared runner. You must configure an +authentication token or use a private runner. + +To create a authentication token: -### Configuring `.yarnrc.yml` to publish from the instance level +1. On the top bar, select **Main menu**, and: + - For a project, select **Projects** and find your project. + - For a group, select **Groups** and find your group. +1. On the left sidebar, select **Settings > Repository > Deploy Tokens**. +1. Create a deployment token with `read_package_registry` and `write_package_registry` scopes and copy the generated token. +1. On the left sidebar, select **Settings > CI/CD > Variables**. +1. Select `Add variable` and use the following settings: -For the instance-level npm endpoint, use this Yarn 2 configuration in `.yarnrc.yml`: +| Field | Value | +|--------------------|------------------------------| +| key | `NPM_AUTH_TOKEN` | +| value | `` | +| type | Variable | +| Protected variable | `CHECKED` | +| Mask variable | `CHECKED` | +| Expand variable | `CHECKED` | + +To use any **Protected variable**: + + 1. Go to the repository that contains the Yarn package source code. + 1. On the left sidebar, select **Settings > Repository**. + - If you are building from branches with tags, select **Protected Tags** and add `v*` (wildcard) for semantic versioning. + - If you are building from branches without tags, select **Protected Branches**. + +Then add the `NPM_AUTH_TOKEN` created above, to the `.yarnrc.yml` configuration +in your package project root directory where `package.json` is found: ```yaml npmScopes: - : - npmRegistryServer: 'https:///api/v4/packages/npm/' - -npmRegistries: - //gitlab.example.com/api/v4/packages/npm/: + esp-code: + npmPublishRegistry: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/npm/" npmAlwaysAuth: true - npmAuthToken: '' + npmAuthToken: "${NPM_AUTH_TOKEN}" ``` -In this configuration: +#### Private runners -- Replace `` with your domain name. -- Your scope is ``, without `@`. -- Replace `` with a deploy token, group access token, project access token, or personal access token. +Add the `CI_JOB_TOKEN` to the `.yarnrc.yml` configuration in your package project +root directory where `package.json` is found: -### Publishing a package via the command line +```yaml +npmScopes: + esp-code: + npmPublishRegistry: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/npm/" + npmAlwaysAuth: true + npmAuthToken: "${CI_JOB_TOKEN}" +``` -Publish a package: +To publish the package using CI/CD pipeline, In the GitLab project that houses +your `yarnrc.yml`, edit or create a `.gitlab-ci.yml` file. For example to trigger +only on any tag push: -```shell -npm publish -``` +```yaml +# Yarn 1 +image: node:lts -Your package should now publish to the Package Registry. +stages: + - deploy -### Publishing via a CI/CD pipeline +rules: +- if: $CI_COMMIT_TAG -In the GitLab project that houses your `yarnrc.yml`, edit or create a `.gitlab-ci.yml` file. For example: +deploy: + stage: deploy + script: + - yarn publish +``` ```yaml -image: node:latest +# Yarn 2+ +image: node:lts stages: - deploy +rules: + - if: $CI_COMMIT_TAG + deploy: stage: deploy + before_script: + - corepack enable + - yarn set version stable script: - - npm publish + - yarn npm publish ``` Your package should now publish to the Package Registry when the pipeline runs. ## Install a package -If multiple packages have the same name and version, the most recently-published package is retrieved when you install a package. +NOTE: +If multiple packages have the same name and version, the most recently-published +package is retrieved when you install a package. -You can install a package from a GitLab project or instance: +You can use one of two API endpoints to install packages: -- **Instance-level**: Use when you have many npm packages in different GitLab groups or in their own namespace. -- **Project-level**: Use when you have a few npm packages, and they are not in the same GitLab group. +- **Instance-level**: Best used when working with many packages in an organization scope. -### Install from the instance level +- If you plan to install a package through the [instance level](#install-from-the-instance-level), + then you must name your package with a [scope](https://docs.npmjs.com/misc/scope/). + Scoped packages begin with a `@` and have the `@owner/package-name` format. You can set up + the scope for your package in the `.yarnrc.yml` file and by using the `publishConfig` + option in the `package.json`. -WARNING: -You must use packages published with the scoped [naming convention](#naming-convention) when you install a package from the instance level. +- The value used for the `@scope` is the organization root (top-level project) `...com/my-org` + *(@my-org)* that hosts the packages, not the root of the project with the package's source code. +- The scope is always lowercase. +- The package name can be anything you want `@my-org/any-name`. -1. Authenticate to the Package Registry +- **Project-level**: For when you have a one-off package. - If you install a package from a private project, you must authenticate to the Package Registry. Skip this step if the project is not private. +If you plan to install a package through the [project level](#install-from-the-project-level), +you do not have to adhere to the naming convention. - ```shell - npm config set -- //your_domain_name/api/v4/packages/npm/:_authToken=your_token - ``` +| Project URL | Package Registry | Organization Scope | Full package name | +|-------------------------------------------------------------------|----------------------|--------------------|-----------------------------| +| `https://gitlab.com///` | Package Name Example | `@my-org` | `@my-org/package-name` | +| `https://gitlab.com///` | Project Name | `@example-org` | `@example-org/project-name` | - - Replace `your_domain_name` with your domain name, for example, `gitlab.com`. - - Replace `your_token` with a deploy token, group access token, project access token, or personal access token. +You can install from the instance level or from the project level. -1. Set the registry +The configurations for `.yarnrc.yml` can be added per package consuming project +root where `package.json` is located, or you can use a global +configuration located in your system user home directory. - ```shell - npm config set @scope:registry https://your_domain_name.com/api/v4/packages/npm/ - ``` +### Install from the instance level - - Replace `@scope` with the [root level group](#naming-convention) of the project you're installing to the package from. - - Replace `your_domain_name` with your domain name, for example, `gitlab.com`. - - Replace `your_token` with a deploy token, group access token, project access token, or personal access token. +Use these steps for global configuration in the `.yarnrc.yml` file: -1. Install the package +1. [Configure organization scope](#configure-organization-scope). +1. [Set the registry](#set-the-registry). - ```shell - yarn add @scope/my-package - ``` +#### Configure organization scope + +```yaml +npmScopes: + : + npmRegistryServer: "https:///api/v4/packages/npm" +``` + +- Replace `` with the root level group of the project you're installing to the package from excluding the `@` symbol. +- Replace `` with your domain name, for example, `gitlab.com`. + +#### Set the registry + +Skip this step if your package is public not private. + +```yaml + npmRegistries: + ///api/v4/packages/npm: + npmAlwaysAuth: true + npmAuthToken: "" +``` + +- Replace `` with your domain name, for example, `gitlab.com`. +- Replace `` with a deployment token (recommended), group access token, project access token, or personal access token. ### Install from the project level -1. Authenticate to the Package Registry +Use these steps for each project in the `.yarnrc.yml` file: + +1. [Configure project scope](#configure-project-scope). +1. [Set the registry](#set-the-registry-project-level). - If you install a package from a private project, you must authenticate to the Package Registry. Skip this step if the project is not private. +#### Configure project scope - ```shell - npm config set -- //your_domain_name/api/v4/projects/your_project_id/packages/npm/:_authToken=your_token - ``` + ```yaml + npmScopes: + : + npmRegistryServer: "https:///api/v4/projects//packages/npm" +``` - - Replace `your_domain_name` with your domain name, for example, `gitlab.com`. - - Replace `your_project_id` is your project ID, found on the project's home page. - - Replace `your_token` with a deploy token, group access token, project access token, or personal access token. +- Replace `` with the root level group of the project you're installing to the package from excluding the `@` symbol. +- Replace `` with your domain name, for example, `gitlab.com`. +- Replace `` with your project ID, found on the project's home page. -1. Set the registry +#### Set the registry (project level) - ```shell - npm config set @scope:registry=https://your_domain_name/api/v4/projects/your_project_id/packages/npm/ - ``` +Skip this step if your package is public not private. - - Replace `@scope` with the [root level group](#naming-convention) of the project you're installing to the package from. - - Replace `your_domain_name` with your domain name, for example, `gitlab.com`. - - Replace `your_project_id` is your project ID, found on the project's home page. +```yaml +npmRegistries: + ///api/v4/projects//packages/npm: + npmAlwaysAuth: true + npmAuthToken: "" +``` -1. Install the package +- Replace `` with your domain name, for example, `gitlab.com`. +- Replace `` with a deployment token (recommended), group access token, project access token, or personal access token. +- Replace `` with your project ID, found on the project's home page. - ```shell - yarn add @scope/my-package - ``` +### Install the package -## Helpful hints +For Yarn 2+, use `yarn add` either in the command line or in the CI/CD pipelines to install your packages: -For full helpful hints information, refer to the [npm documentation](../npm_registry/index.md#helpful-hints). +```shell +yarn add @scope/my-package +``` -### Supported CLI commands +#### For Yarn Classic -The GitLab npm repository supports the following commands for the npm CLI (`npm`) and yarn CLI -(`yarn`): +The Yarn Classic setup, requires both `.npmrc` and `.yarnrc` files as +[mentioned in issue](https://github.com/yarnpkg/yarn/issues/4451#issuecomment-753670295): -- `npm install`: Install npm packages. -- `npm publish`: Publish an npm package to the registry. -- `npm dist-tag add`: Add a dist-tag to an npm package. -- `npm dist-tag ls`: List dist-tags for a package. -- `npm dist-tag rm`: Delete a dist-tag. -- `npm ci`: Install npm packages directly from your `package-lock.json` file. -- `npm view`: Show package metadata. -- `yarn add`: Install an npm package. -- `yarn update`: Update your dependencies. +- Place credentials in the `.npmrc` file. +- Place the scoped registry in the `.yarnrc` file. -## Troubleshooting +```shell +# .npmrc +///api/v4/projects//packages/npm/:_authToken="" + +# .yarnrc +"@scope:registry" "https:///api/v4/projects//packages/npm/" +``` + +Then you can use `yarn add` to install your packages. + +## Related topics -For full troubleshooting information, refer to the [npm documentation](../npm_registry/index.md#troubleshooting). +- [npm documentation](../npm_registry/index.md#helpful-hints) +- [Yarn Migration Guide](https://yarnpkg.com/getting-started/migration) + +## Troubleshooting ### Error running Yarn with the Package Registry for the npm registry -If you are using [Yarn](https://classic.yarnpkg.com/en/) with the npm registry, you may get -an error message like: +If you are using [Yarn](https://classic.yarnpkg.com/en/) with the npm registry, you may get an error message like: ```shell yarn install v1.15.2 @@ -233,14 +323,7 @@ info If you think this is a bug, please open a bug report with the information p info Visit https://classic.yarnpkg.com/en/docs/cli/install for documentation about this command ``` -In this case, try adding this to your `.npmrc` file (and replace `` -with your personal access token or deploy token): - -```plaintext -//gitlab.example.com/api/v4/projects/:_authToken= -``` - -You can also use `yarn config` instead of `npm config` when setting your auth-token dynamically: +In this case, the following commands creates a file called `.yarnrc` in the current directory. Make sure to be in either your user home directory for global configuration or your project root for per-project configuration: ```shell yarn config set '//gitlab.example.com/api/v4/projects//packages/npm/:_authToken' "" diff --git a/doc/user/permissions.md b/doc/user/permissions.md index 8e736b6d83e..418c01cd851 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -23,6 +23,7 @@ The available roles are: - Developer - Maintainer - Owner +- Minimal Access (available for the top-level group only) A user assigned the Guest role has the least permissions, and the Owner has the most. @@ -58,7 +59,7 @@ The following table lists project permissions available for each role: |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|----------|-----------|------------|----------| | [Analytics](analytics/index.md):
    View [issue analytics](analytics/issue_analytics.md) | ✓ | ✓ | ✓ | ✓ | ✓ | | [Analytics](analytics/index.md):
    View [merge request analytics](analytics/merge_request_analytics.md) | ✓ | ✓ | ✓ | ✓ | ✓ | -| [Analytics](analytics/index.md):
    View [value stream analytics](analytics/value_stream_analytics.md) | ✓ | ✓ | ✓ | ✓ | ✓ | +| [Analytics](analytics/index.md):
    View [value stream analytics](group/value_stream_analytics/index.md) | ✓ | ✓ | ✓ | ✓ | ✓ | | [Analytics](analytics/index.md):
    View [DORA metrics](analytics/ci_cd_analytics.md) | | ✓ | ✓ | ✓ | ✓ | | [Analytics](analytics/index.md):
    View [CI/CD analytics](analytics/ci_cd_analytics.md) | | ✓ | ✓ | ✓ | ✓ | | [Analytics](analytics/index.md):
    View [code review analytics](analytics/code_review_analytics.md) | | ✓ | ✓ | ✓ | ✓ | @@ -69,8 +70,8 @@ The following table lists project permissions available for each role: | [Application security](application_security/index.md):
    View [dependency list](application_security/dependency_list/index.md) | | | ✓ | ✓ | ✓ | | [Application security](application_security/index.md):
    Create a [CVE ID Request](application_security/cve_id_request.md) | | | | ✓ | ✓ | | [Application security](application_security/index.md):
    Create or assign [security policy project](application_security/policies/index.md) | | | | | ✓ | -| [Clusters](infrastructure/clusters/index.md):
    View clusters | | | ✓ | ✓ | ✓ | -| [Clusters](infrastructure/clusters/index.md):
    Manage clusters | | | | ✓ | ✓ | +| [GitLab Agent for Kubernetes](clusters/agent/index.md):
    View agents | | | ✓ | ✓ | ✓ | +| [GitLab Agent for Kubernetes](clusters/agent/index.md):
    Manage agents | | | | ✓ | ✓ | | [Container Registry](packages/container_registry/index.md):
    Create, edit, delete [cleanup policies](packages/container_registry/delete_container_registry_images.md#use-a-cleanup-policy) | | | | ✓ | ✓ | | [Container Registry](packages/container_registry/index.md):
    Push an image to the Container Registry | | | ✓ | ✓ | ✓ | | [Container Registry](packages/container_registry/index.md):
    Pull an image from the Container Registry | ✓ (19) | ✓ (19) | ✓ | ✓ | ✓ | @@ -102,6 +103,8 @@ The following table lists project permissions available for each role: | [Issues](project/issues/index.md):
    View [Design Management](project/issues/design_management.md) pages | ✓ | ✓ | ✓ | ✓ | ✓ | | [Issues](project/issues/index.md):
    View [related issues](project/issues/related_issues.md) | ✓ | ✓ | ✓ | ✓ | ✓ | | [Issues](project/issues/index.md):
    Set [weight](project/issues/issue_weight.md) | ✓ (15) | ✓ | ✓ | ✓ | ✓ | +| [Issues](project/issues/index.md):
    Set metadata such as labels, milestones, or assignees when creating an issue | ✓ (15) | ✓ | ✓ | ✓ | ✓ | +| [Issues](project/issues/index.md):
    Edit metadata such labels, milestones, or assignees for an existing issue | (15) | ✓ | ✓ | ✓ | ✓ | | [Issues](project/issues/index.md):
    Set [parent epic](group/epics/manage_epics.md#add-an-existing-issue-to-an-epic) | | ✓ | ✓ | ✓ | ✓ | | [Issues](project/issues/index.md):
    View [confidential issues](project/issues/confidential_issues.md) | (2) | ✓ | ✓ | ✓ | ✓ | | [Issues](project/issues/index.md):
    Close / reopen (18) | | ✓ | ✓ | ✓ | ✓ | @@ -137,7 +140,7 @@ The following table lists project permissions available for each role: | [Package registry](packages/index.md):
    Delete a package | | | | ✓ | ✓ | | [Package registry](packages/index.md):
    Delete a file associated with a package | | | | ✓ | ✓ | | [Project operations](../operations/index.md):
    View [Error Tracking](../operations/error_tracking.md) list | | ✓ | ✓ | ✓ | ✓ | -| [Project operations](../operations/index.md):
    Manage [Feature Flags](../operations/feature_flags.md) | | | ✓ | ✓ | ✓ | +| [Project operations](../operations/index.md):
    Manage [Feature flags](../operations/feature_flags.md) | | | ✓ | ✓ | ✓ | | [Project operations](../operations/index.md):
    Manage [Error Tracking](../operations/error_tracking.md) | | | | ✓ | ✓ | | [Projects](project/index.md):
    Download project | ✓ (1) | ✓ | ✓ | ✓ | ✓ | | [Projects](project/index.md):
    Leave comments | ✓ | ✓ | ✓ | ✓ | ✓ | @@ -220,7 +223,7 @@ The following table lists project permissions available for each role: -1. On self-managed GitLab instances, users with the Guest role are able to perform this action only on public and internal projects (not on private projects). [External users](admin_area/external_users.md) must be given explicit access even if the project is internal. Users with the Guest role on GitLab.com are only able to perform this action on public projects because internal visibility is not available. In GitLab 15.9 and later, this restriction only applies to users with the non-custom Guest role on self-managed GitLab instances and GitLab.com. +1. On self-managed GitLab instances, users with the Guest role are able to perform this action only on public and internal projects (not on private projects). [External users](admin_area/external_users.md) must be given explicit access even if the project is internal. Users with the Guest role on GitLab.com are only able to perform this action on public projects because internal visibility is not available. 2. Guest users can only view the [confidential issues](project/issues/confidential_issues.md) they created themselves or are assigned to. 3. Not allowed for Guest, Reporter, Developer, Maintainer, or Owner. See [protected branches](project/protected_branches.md). 4. If the [branch is protected](project/protected_branches.md), this depends on the access given to Developers and Maintainers. @@ -271,8 +274,7 @@ More details about the permissions for some project-level features follow. | View and download artifacts | ✓ (1) | ✓ (2) | ✓ | ✓ | ✓ | ✓ | | View [environments](../ci/environments/index.md) | ✓ (3) | ✓ (3) | ✓ | ✓ | ✓ | ✓ | | View job logs and job details page | ✓ (1) | ✓ (2) | ✓ | ✓ | ✓ | ✓ | -| View pipeline details page | ✓ (1) | ✓ (2) | ✓ | ✓ | ✓ | ✓ | -| View pipelines page | ✓ (1) | ✓ (2) | ✓ | ✓ | ✓ | ✓ | +| View pipelines and pipeline details pages | ✓ (1) | ✓ (2) | ✓ | ✓ | ✓ | ✓ | | View pipelines tab in MR | ✓ (3) | ✓ (3) | ✓ | ✓ | ✓ | ✓ | | [View vulnerabilities in a pipeline](application_security/vulnerability_report/pipeline.md#view-vulnerabilities-in-a-pipeline) | | ✓ (2) | ✓ | ✓ | ✓ | ✓ | | View and download project-level [Secure Files](../api/secure_files.md) | | | | ✓ | ✓ | ✓ | @@ -328,7 +330,7 @@ This table shows granted privileges for jobs triggered by specific types of user | Push source and LFS | | | | | 1. Only if the triggering user is not an external one. -1. Only if the triggering user is a member of the project. See also [Usage of private Docker images with `if-not-present` pull policy](http://docs.gitlab.com/runner/security/index.html#usage-of-private-docker-images-with-if-not-present-pull-policy). +1. Only if the triggering user is a member of the project. See also [Usage of private Docker images with `if-not-present` pull policy](https://docs.gitlab.com/runner/security/index.html#usage-of-private-docker-images-with-if-not-present-pull-policy). ## Group members permissions @@ -424,39 +426,38 @@ nested groups if you have membership in one of its parents. For more information, see [subgroup memberships](group/subgroups/index.md#subgroup-membership). -## Users with minimal access **(PREMIUM)** +## Users with Minimal Access **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/40942) in GitLab 13.4. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/40942) in GitLab 13.4. +> - Support for inviting users with Minimal Access role [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106438) in GitLab 15.9. -Owners can add members with a "minimal access" role to a root group. Such users don't automatically have access to -projects and subgroups underneath. Owners must explicitly add these "minimal access" users to the specific subgroups and +Users with the Minimal Access role do not: + +- Count as licensed seats on self-managed Ultimate subscriptions or any GitLab.com subscriptions. +- Automatically have access to projects and subgroups in that root group. + +Owners must explicitly add these users to the specific subgroups and projects. -You can use minimal access to give the same member more than one role in a group: +You can use the Minimal Access role to give the same member more than one role in a group: -1. Add the member to the root group with a minimal access role. +1. Add the member to the root group with a Minimal Access role. 1. Invite the member as a direct member with a specific role in any subgroup or project in that group. -Because of an [outstanding issue](https://gitlab.com/gitlab-org/gitlab/-/issues/267996), when minimal access users: +Because of an [outstanding issue](https://gitlab.com/gitlab-org/gitlab/-/issues/267996), when a user with the Minimal Access role: -- Sign in with standard web authentication, they receive a `404` error when accessing the parent group. -- Sign in with Group SSO, they receive a `404` error immediately because they are redirected to the parent group page. +- Signs in with standard web authentication, they receive a `404` error when accessing the parent group. +- Signs in with Group SSO, they receive a `404` error immediately because they are redirected to the parent group page. To work around the issue, give these users the Guest role or higher to any project or subgroup within the parent group. -### Minimal access users take license seats - -Users with even a "minimal access" role are counted against your number of license seats. This -requirement does not apply for [GitLab Ultimate](https://about.gitlab.com/pricing/) -subscriptions. - ## Related topics - [The GitLab principles behind permissions](https://about.gitlab.com/handbook/product/gitlab-the-product/#permissions-in-gitlab) - [Members](project/members/index.md) - Customize permissions on [protected branches](project/protected_branches.md) - [LDAP user permissions](group/access_and_permissions.md#manage-group-memberships-via-ldap) -- [Value stream analytics permissions](analytics/value_stream_analytics.md#access-permissions-for-value-stream-analytics) +- [Value stream analytics permissions](group/value_stream_analytics/index.md#access-permissions-for-value-stream-analytics) - [Project aliases](../user/project/import/index.md#project-aliases) - [Auditor users](../administration/auditor_users.md) - [Confidential issues](project/issues/confidential_issues.md) @@ -467,9 +468,10 @@ subscriptions. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106256) in GitLab 15.7 [with a flag](../administration/feature_flags.md) named `customizable_roles`. > - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/110810) in GitLab 15.9. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/114524) in GitLab 15.10. Custom roles allow group members who are assigned the Owner role to create roles -specific to the needs of their organization. +specific to the needs of their organization. For a demo of the custom roles feature, see [[Demo] Ultimate Guest can view code on private repositories via custom role](https://www.youtube.com/watch?v=46cp_-Rtxps). @@ -481,29 +483,36 @@ To enable custom roles for your group, a group member with the Owner role: 1. Makes sure that there is at least one private project in this group or one of its subgroups, so that you can see the effect of giving a Guest a custom role. 1. Creates a personal access token with the API scope. -1. Uses [the API](../api/member_roles.md#add-a-member-role-to-a-group) to create the Guest+1 role for the group. +1. Uses [the API](../api/member_roles.md#add-a-member-role-to-a-group) to create the Guest+1 role for the root group. ### Associate a custom role with an existing group member To associate a custom role with an existing group member, a group member with the Owner role: -1. Invites a test user account to the root group as a Guest. - At this point, this Guest user cannot see any code on the projects in the group. +1. Invites a user to the root group or any subgroup or project in the root + group's hierarchy as a Guest. At this point, this Guest user cannot see any + code on the projects in the group or subgroup. 1. Optional. If the Owner does not know the `ID` of the Guest user receiving a custom role, finds that `ID` by making an [API request](../api/member_roles.md#list-all-member-roles-of-a-group). -1. Associates the group member with the Guest+1 role using the [Group and Project Members API endpoint](../api/members.md#edit-a-member-of-a-group-or-project) +1. Associates the member with the Guest+1 role using the [Group and Project Members API endpoint](../api/members.md#edit-a-member-of-a-group-or-project) + + ```shell + # to update a project membership + curl --request PUT --header "Content-Type: application/json" --header "Authorization: Bearer $YOUR_ACCESS_TOKEN" --data '{"member_role_id": '$MEMBER_ROLE_ID', "access_level": 10}' "https://example.gitlab.com/api/v4/projects/$ID/members/$GUEST_USER_ID" - ```shell - curl --request PUT --header "Content-Type: application/json" --header "Authorization: Bearer $YOUR_ACCESS_TOKEN" --data '{"member_role_id": '$MEMBER_ROLE_ID', "access_level": 10}' "https://example.gitlab.com/api/v4/groups/$GROUP_PATH/members/$GUEST_USER_ID" - ``` + # to update a group membership + curl --request PUT --header "Content-Type: application/json" --header "Authorization: Bearer $YOUR_ACCESS_TOKEN" --data '{"member_role_id": '$MEMBER_ROLE_ID', "access_level": 10}' "https://example.gitlab.com/api/v4/groups/$ID/members/$GUEST_USER_ID" + ``` Where: + + - `$ID`: The `ID` or [URL-encoded path of the project or group](../api/rest/index.md#namespaced-path-encoding) associated with the membership receiving the custom role. - `$MEMBER_ROLE_ID`: The `ID` of the member role created in the previous section. - `$GUEST_USER_ID`: The `ID` of the Guest user receiving a custom role. - Now the Guest+1 user can view code on all projects in the root group. + Now the Guest+1 user can view code on all projects associated with this membership. ### Remove a custom role from a group member @@ -532,8 +541,5 @@ the Owner role: ### Known issues - Additional permissions can only be applied to users with the Guest role. -- There is no visual distinction in the UI between the Guest role and the Guest role with additional permission. For more information, see [issue 384099](https://gitlab.com/gitlab-org/gitlab/-/issues/384099). - If a user with a custom role is shared with a group or project, their custom role is not transferred over with them. The user has the regular Guest role in the new group or project. -- If a custom role is deleted, the users associated with that custom role are also removed from the group. For more information, see [issue 370352](https://gitlab.com/gitlab-org/gitlab/-/issues/370352). -- The API endpoint for associating a custom role with a user only works for users with the Guest role in a group. A project member can be associated with a custom role, but not through the API yet. For more information, see [issue 385495](https://gitlab.com/gitlab-org/gitlab/-/issues/385495). -- The only way to remove a custom role from a user's membership to a Group is to delete the custom role, which deletes the user membership entirely. See [issue 387769](https://gitlab.com/gitlab-org/gitlab/-/issues/387769). +- You cannot use an [Auditor user](../administration/auditor_users.md) as a template for a custom role. diff --git a/doc/user/product_analytics/index.md b/doc/user/product_analytics/index.md index 6d6a609618b..b5f936e7848 100644 --- a/doc/user/product_analytics/index.md +++ b/doc/user/product_analytics/index.md @@ -4,20 +4,30 @@ group: Product Analytics info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Product analytics **(ULTIMATE)** +# Product analytics (Experiment) **(ULTIMATE)** -> - Introduced in GitLab 15.4 as an [Alpha](../../policy/alpha-beta-support.md#alpha-features) feature [with a flag](../../administration/feature_flags.md) named `cube_api_proxy`. Disabled by default. +> - Introduced in GitLab 15.4 as an [Experiment](../../policy/alpha-beta-support.md#experiment) feature [with a flag](../../administration/feature_flags.md) named `cube_api_proxy`. Disabled by default. > - `cube_api_proxy` revised to only reference the [Product Analytics API](../../api/product_analytics.md) in GitLab 15.6. +> - `cube_api_proxy` removed and replaced with `product_analytics_internal_preview` in GitLab 15.10. +> - `product_analytics_internal_preview` replaced with `product_analytics_dashboards` in GitLab 15.11. FLAG: -On self-managed GitLab, by default this feature is not available. To make it available per project or for your entire instance, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `cube_api_proxy`. +On self-managed GitLab, by default this feature is not available. To make it available per project or for your entire instance, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `product_analytics_dashboards`. On GitLab.com, this feature is not available. This feature is not ready for production use. This page is a work in progress, and we're updating the information as we add more features. For more information, see the [group direction page](https://about.gitlab.com/direction/analytics/product-analytics/). -## How Product Analytics works +## How product analytics works + +Product analytics uses several tools: + +- [**Jitsu**](https://jitsu.com/docs) - A web and app event collection platform that provides a consistent API to collect user data and pass it through to ClickHouse. +- [**ClickHouse**](https://clickhouse.com/docs) - A database suited to store, query, and retrieve analytical data. +- [**Cube.js**](https://cube.dev/docs/) - An analytical graphing library that provides an API to run queries against the data stored in Clickhouse. + +The following diagram illustrates the product analytics flow: ```mermaid --- @@ -26,7 +36,7 @@ title: Product Analytics flow flowchart TB subgraph Adding data A([SDK]) --Send user data--> B[Analytics Proxy] - B --Transform data and pass it through--> C[Jitsu] + B --Transform data and pass it through--> C[Snowplow] C --Pass the data to the associated database--> D([Clickhouse]) end subgraph Showing dashboards @@ -40,24 +50,20 @@ flowchart TB end ``` -Product Analytics uses several tools: - -- [**Jitsu**](https://jitsu.com/docs) - A web and app event collection platform that provides a consistent API to collect user data and pass it through to Clickhouse. -- [**Clickhouse**](https://clickhouse.com/docs) - A database suited to store, query, and retrieve analytical data. -- [**Cube.js**](https://cube.dev/docs/) - An analytical graphing library that provides an API to run queries against the data stored in Clickhouse. - ## Enable product analytics > - Introduced in GitLab 15.6 behind the [feature flag](../../administration/feature_flags.md) named `cube_api_proxy`. Disabled by default. > - Moved to be behind the [feature flag](../../administration/feature_flags.md) named `product_analytics_admin_settings` in GitLab 15.7. Disabled by default. +> - `cube_api_proxy` removed and replaced with `product_analytics_internal_preview` in GitLab 15.10. +> - `product_analytics_internal_preview` replaced with `product_analytics_dashboards` in GitLab 15.11. FLAG: -On self-managed GitLab, by default this feature is not available. To make it available per project or for your entire instance, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `product_analytics_admin_settings`. +On self-managed GitLab, by default this feature is not available. To make it available per project or for your entire instance, ask an administrator to [enable the feature flags](../../administration/feature_flags.md) named `product_analytics_dashboards` and `product_analytics_admin_settings`. On GitLab.com, this feature is not available. This feature is not ready for production use. -You can enable and configure product analytics to track events -within your project applications on a self-managed instance. +To track events in your project applications on a self-managed instance, +you must enable and configure product analytics. Prerequisite: @@ -65,35 +71,41 @@ Prerequisite: 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Settings > General**. -1. Expand the **Product analytics** section. +1. Expand the **Analytics** tab and find the **Product analytics** section. 1. Select **Enable product analytics** and enter the configuration values. The following table shows the required configuration parameters and example values: - | Name | Value | - |------------------------------|------------------------------------------------------------| - | Jitsu host | `https://jitsu.gitlab.com` | - | Jitsu project ID | `g0maofw84gx5sjxgse2k` | - | Jitsu administrator email | `jitsu.admin@gitlab.com` | - | Jitsu administrator password | `` | - | Clickhouse URL | `https://:@clickhouse.gitlab.com:8123` | - | Cube API URL | `https://cube.gitlab.com` | - | Cube API key | `25718201b3e9...ae6bbdc62dbb` | + | Name | Value | + |--------------------------------|------------------------------------------------------------| + | Configurator connection string | `https://test:test@configurator.gitlab.com` | + | Jitsu host | `https://jitsu.gitlab.com` | + | Jitsu project ID | `g0maofw84gx5sjxgse2k` | + | Jitsu administrator email | `jitsu.admin@gitlab.com` | + | Jitsu administrator password | `` | + | Collector host | `https://collector.gitlab.com` | + | ClickHouse URL | `https://:@clickhouse.gitlab.com:8123` | + | Cube API URL | `https://cube.gitlab.com` | + | Cube API key | `25718201b3e9...ae6bbdc62dbb` | 1. Select **Save changes**. ## Product analytics dashboards -> Introduced in GitLab 15.5 behind the [feature flag](../../administration/feature_flags.md) named `product_analytics_internal_preview`. Disabled by default. +> - Introduced in GitLab 15.5 behind the [feature flag](../../administration/feature_flags.md) named `product_analytics_internal_preview`. Disabled by default. +> - `product_analytics_internal_preview` replaced with `product_analytics_dashboards` in GitLab 15.11. FLAG: -On self-managed GitLab, by default this feature is not available. To make it available per project or for your entire instance, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `product_analytics_internal_preview`. +On self-managed GitLab, by default this feature is not available. To make it available per project or for your entire instance, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `product_analytics_dashboards`. On GitLab.com, this feature is not available. This feature is not ready for production use. -Each project can define an unlimited number of dashboards. These dashboards are defined using our YAML schema and stored -in the `.gitlab/product_analytics/dashboards/` directory of a project repository. The name of the file is the name of the dashboard, and visualizations are shared across dashboards. +Each project can have an unlimited number of dashboards. +These dashboards are defined using the GitLab YAML schema, and stored in the `.gitlab/analytics/dashboards/` directory of a project repository. +The name of the file is the name of the dashboard. +Each dashboard can contain one or more visualizations (charts), which are shared across dashboards. -Project maintainers can enforce approval rules on dashboard changes using features such as code owners and approval rules. Dashboards are versioned in source control with the rest of a project's code. +Project maintainers can enforce approval rules on dashboard changes using features such as code owners and approval rules. +Dashboards are versioned in source control with the rest of a project's code. ### View project dashboards @@ -108,20 +120,28 @@ To view a list of product analytics dashboards for a project: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Analytics > Dashboards**. +1. From the list of available dashboards, select the dashboard you want to view. ### Define a dashboard To define a dashboard: -1. In `.gitlab/product_analytics/dashboards/`, create a directory named like the dashboard. Each dashboard should have its own directory. -1. In the new directory, create a `.yaml` file with the same name as the directory. This file contains the dashboard definition, and must conform to the JSON schema defined in `ee/app/validators/json_schemas/product_analytics_dashboard.json`. -1. In the `.gitlab/product_analytics/dashboards/visualizations/` directory, create a `yaml` file. This file defines the visualization type for the dashboard, and must conform to the schema in +1. In `.gitlab/analytics/dashboards/`, create a directory named like the dashboard. + + Each dashboard should have its own directory. +1. In the new directory, create a `.yaml` file with the same name as the directory. + + This file contains the dashboard definition. It must conform to the JSON schema defined in `ee/app/validators/json_schemas/product_analytics_dashboard.json`. +1. In the `.gitlab/analytics/dashboards/visualizations/` directory, create a `.yaml` file. + + This file defines the visualization type for the dashboard. It must conform to the schema in `ee/app/validators/json_schemas/product_analytics_visualization.json`. -The example below includes three dashboards and one visualization that applies to all dashboards. +For example, if you want to create three dashboards (Conversion funnels, Demographic breakdown, and North star metrics) +and one visualization (line chart) that applies to all dashboards, the file structure would be: ```plaintext -.gitlab/product_analytics/dashboards +.gitlab/analytics/dashboards ├── conversion_funnels │ └── conversion_funnels.yaml ├── demographic_breakdown @@ -132,15 +152,40 @@ The example below includes three dashboards and one visualization that applies t │ └── example_line_chart.yaml ``` +### Define a chart visualization + +You can define different charts, and add visualization options to some of them: + +- Line chart, with the options listed in the [ECharts documentation](https://echarts.apache.org/en/option.html). +- Column chart, with the options listed in the [ECharts documentation](https://echarts.apache.org/en/option.html). +- Data table, with the only option to render `links` (array of objects, each with `text` and `href` properties to specify the dimensions to be used in links). See [example](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/validators/json_schemas/analytics_visualization.json?ref_type=heads#L112)). +- Single stat, with the only option to set `decimalPlaces` (number, default value is 0). + +To define a chart for your dashboards: + +1. In the `.gitlab/product_analytics/dashboards/visualizations/` directory, create a `.yaml` file. +The filename should be descriptive of the visualization it defines. +1. In the `.yaml` file, define the visualization options, according to the schema in +`ee/app/validators/json_schemas/analytics_visualization.json`. + +For [example](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/assets/javascripts/analytics/analytics_dashboards/gl_dashboards/product_analytics/visualizations/events_over_time.json), to create a line chart that illustrates event count over time, in the `visualizations` folder +create a `line_chart.yaml` file with the following required fields: + +- version +- title +- type +- data +- options + ## Funnel analysis -Funnel analysis can be used to understand the flow of users through your application and where +Use funnel analysis to understand the flow of users through your application, and where users drop out of a predefined flow (for example, a checkout process or ticket purchase). Each product can also define an unlimited number of funnels. -These funnels are defined using our YAML schema and stored in the `.gitlab/product_analytics/funnels/` directory of a project repository. +Like dashboards, funnels are defined using the GitLab YAML schema, and stored in the `.gitlab/analytics/funnels/` directory of a project repository. -Funnel definitions must include the keys `name`, `seconds_to_convert`, and an array of `steps`. +Funnel definitions must include the keys `name` and `seconds_to_convert`, and an array of `steps`. | Key | Description | |----------------------|----------------------------------------------------------| @@ -158,6 +203,8 @@ Each step must include the keys `name`, `target`, and `action`. ### Example funnel definition +The following example defines a funnel that tracks users who completed a purchase within one hour by going through three target pages: + ```yaml name: completed_purchase seconds_to_convert: 3600 @@ -205,3 +252,49 @@ The `afterDate` filter is not supported. Please use `beforeDate` or `inDateRange } } ``` + +## Raw data export + +Exporting the raw event data from the underlying storage engine can help you debug and create datasets for data analysis. + +Because Cube acts as an abstraction layer between the raw data and the API, the exported raw data has some caveats: + +- Data is grouped by the selected dimensions. Therefore, the exported data might be incomplete, unless including both `utcTime` and `userAnonymousId`. +- Data is by default limited to 10,000 rows, but you can increase the limit to maximum 50,000 rows. If your dataset has more than 50,000 rows, you must paginate through the results by using the `limit` and `offset` parameters. +- Data is always returned in JSON format. If you need it in a different format, you need to convert the JSON to the required format using a scripting language of your choice. + +[Issue 391683](https://gitlab.com/gitlab-org/gitlab/-/issues/391683) tracks efforts to implement a more scalable export solution. + +### Export raw data with Cube queries + +You can [query the raw data with the REST API](../../api/product_analytics.md#send-query-request-to-cube), +and convert the JSON output to any required format. + +To export the raw data for a specific dimension, pass a list of dimensions to the `dimensions` key. +For example, the following query outputs the raw data for the attributes listed: + +```json +POST /api/v4/projects/PROJECT_ID/product_analytics/request/load?queryType=multi + +{ + "query":{ + "dimensions": [ + "TrackedEvents.docEncoding", + "TrackedEvents.docHost", + "TrackedEvents.docPath", + "TrackedEvents.docSearch", + "TrackedEvents.eventType", + "TrackedEvents.localTzOffset", + "TrackedEvents.pageTitle", + "TrackedEvents.src", + "TrackedEvents.utcTime", + "TrackedEvents.vpSize" + ], + "order": { + "TrackedEvents.apiKey": "asc" + } + } +} +``` + +If the request is successful, the returned JSON includes an array of rows of results. diff --git a/doc/user/profile/account/create_accounts.md b/doc/user/profile/account/create_accounts.md index 60b8fe8ab88..f405dc42f46 100644 --- a/doc/user/profile/account/create_accounts.md +++ b/doc/user/profile/account/create_accounts.md @@ -16,9 +16,9 @@ You can create users: ## Create users on sign-in page -Prerequisites: +Prerequisite: -- [Sign-up enabled](../../admin_area/settings/sign_up_restrictions.md) +- [Sign-up must be enabled](../../admin_area/settings/sign_up_restrictions.md). Users can create their own accounts by either: @@ -27,18 +27,36 @@ Users can create their own accounts by either: ## Create users in Admin Area -Prerequisites: +Prerequisite: -- You must have administrator access for the instance. +- You must have administrator access to the instance. To create a user manually: 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Overview > Users** (`/admin/users`). 1. Select **New user**. -1. Complete the fields. +1. Complete the required fields, such as name, username, and email. 1. Select **Create user**. +A reset link is sent to the user's email and they are forced to set their +password on first sign in. + +To set a user's password without relying on the email confirmation, after you +create a user following the previous steps: + +1. Select the user. +1. Select **Edit**. +1. Complete the password and password confirmation fields. +1. Select **Save changes**. + +The user can now sign in with the new username and password, and they are asked +to change the password you set up for them. + +NOTE: +If you wanted to create a test user, you could follow the previous steps +by providing a fake email and using the same password in the final confirmation. + ## Create users through authentication integrations Users are: diff --git a/doc/user/profile/account/delete_account.md b/doc/user/profile/account/delete_account.md index a74fd8d5215..a2c82fdeadf 100644 --- a/doc/user/profile/account/delete_account.md +++ b/doc/user/profile/account/delete_account.md @@ -48,9 +48,10 @@ When deleting users, you can either: - Delete just the user. Not all associated records are deleted with the user. Instead of being deleted, these records are moved to a system-wide user with the username Ghost User. The Ghost User's purpose is to act as a container for such records. Any commits made by a deleted user still display the username of the original user. + The user's personal projects are deleted, not moved to the Ghost User. - Delete the user and their contributions, including: - Abuse reports. - - Award emojis. + - Emoji reactions. - Epics. - Groups of which the user is the only user with the Owner role. - Issues. diff --git a/doc/user/profile/account/two_factor_authentication.md b/doc/user/profile/account/two_factor_authentication.md index b76b99b5242..8827e1e27c5 100644 --- a/doc/user/profile/account/two_factor_authentication.md +++ b/doc/user/profile/account/two_factor_authentication.md @@ -14,7 +14,7 @@ GitLab supports as a second factor of authentication: - Time-based one-time passwords ([TOTP](https://datatracker.ietf.org/doc/html/rfc6238)). When enabled, GitLab prompts you for a code when you sign in. Codes are generated by your one-time password authenticator (for example, a password manager on one of your devices). -- U2F or WebAuthn devices. You're prompted to activate your U2F or WebAuthn device (usually by pressing a button on it) when +- WebAuthn devices. You're prompted to activate your WebAuthn device (usually by pressing a button on it) when you supply your username and password to sign in. This performs secure authentication on your behalf. If you set up a device, also set up a TOTP so you can still access your account if you lose the device. @@ -24,28 +24,37 @@ If you set up a device, also set up a TOTP so you can still access your account When 2FA is enabled, you can't use your password to authenticate with Git over HTTPS or the [GitLab API](../../../api/rest/index.md). You can use a [personal access token](../personal_access_tokens.md) instead. -## Git Credential Manager +## OAuth credential helpers -For Git over HTTPS, [Git Credential Manager](https://github.com/GitCredentialManager/git-credential-manager) (GCM) offers an alternative to personal access tokens. By default, GCM -authenticates using OAuth, opening GitLab in your web browser. The first time you authenticate, GitLab asks you to authorize the app. If you remain signed in to GitLab, subsequent -authentication requires no interaction. +The following Git credential helpers authenticate to GitLab using OAuth. This is compatible with two-factor authentication. The first time you authenticate, the helper opens the web browser and GitLab asks you to authorize the app. Subsequent authentication requires no interaction. -So you don't need to reauthenticate on every push, GCM supports caching as well as a variety of platform-specific credential stores that persist between sessions. This feature is useful whether you use personal access tokens or OAuth. +### Git Credential Manager -GCM supports GitLab.com out the box. To use with self-managed GitLab, see [GitLab support](https://github.com/GitCredentialManager/git-credential-manager/blob/main/docs/gitlab.md) -documentation. +[Git Credential Manager](https://github.com/GitCredentialManager/git-credential-manager) (GCM) authenticates by default using OAuth. GCM supports GitLab.com without any manual configuration. To use GCM with self-managed GitLab, see [GitLab support](https://github.com/GitCredentialManager/git-credential-manager/blob/main/docs/gitlab.md). + +So you do not need to re-authenticate on every push, GCM supports caching as well as a variety of platform-specific credential stores that persist between sessions. This feature is useful whether you use personal access tokens or OAuth. + +Git for Windows includes Git Credential Manager. Git Credential Manager is developed primarily by GitHub, Inc. It is an open-source project and is supported by the community. +### git-credential-oauth + +[git-credential-oauth](https://github.com/hickford/git-credential-oauth) supports GitLab.com and several popular public hosts without any manual configuration needed. To use with self-managed GitLab, see the [git-credential-oauth custom hosts documentation](https://github.com/hickford/git-credential-oauth#custom-hosts). + +Many Linux distributions include git-credential-oauth as a package. + +git-credential-oauth is an open-source project supported by the community. + ## Enable two-factor authentication > - Account email confirmation requirement [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35102) in GitLab 14.3. [Deployed behind the `ensure_verified_primary_email_for_2fa` flag](../../../administration/feature_flags.md), enabled by default. > - Account email confirmation requirement generally available and [feature flag `ensure_verified_primary_email_for_2fa` removed](https://gitlab.com/gitlab-org/gitlab/-/issues/340151) in GitLab 14.4. -You can enable 2FA: +You can enable 2FA using a: -- Using a one-time password authenticator. After you enable 2FA, back up your [recovery codes](#recovery-codes). -- Using a U2F or WebAuthn device. +- One-time password authenticator. After you enable 2FA, back up your [recovery codes](#recovery-codes). +- WebAuthn device. In GitLab 14.3 and later, your account email must be confirmed to enable 2FA. @@ -60,10 +69,11 @@ To enable 2FA with a one-time password: 1. **On your device (usually your phone):** 1. Install a compatible application. For example: - Cloud-based (recommended because you can restore access if you lose the hardware device): - - [Authy](https://authy.com/) + - [Authy](https://authy.com/). + - [Duo](https://duo.com/). - Other: - - [Google Authenticator](https://support.google.com/accounts/answer/1066447?hl=en) - - [Microsoft Authenticator](https://www.microsoft.com/en-us/security/mobile-authenticator-app) + - [Google Authenticator](https://support.google.com/accounts/answer/1066447?hl=en). + - [Microsoft Authenticator](https://www.microsoft.com/en-us/security/mobile-authenticator-app). 1. In the application, add a new entry in one of two ways: - Scan the code displayed by GitLab with your device's camera to add the entry automatically. - Enter the details provided to add the entry manually. @@ -72,9 +82,6 @@ To enable 2FA with a one-time password: 1. Enter your current password. 1. Select **Submit**. -NOTE: -DUO [cannot be used for 2FA](https://gitlab.com/gitlab-org/gitlab/-/issues/15760). - If you entered the correct pin, GitLab displays a list of [recovery codes](#recovery-codes). Download them and keep them in a safe place. @@ -141,6 +148,70 @@ Configure FortiAuthenticator in GitLab. On your GitLab server: 1. [Reconfigure](../../../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) (Omnibus GitLab) or [restart](../../../administration/restart_gitlab.md#installations-from-source) (GitLab installed from source). +### Enable one-time password using Duo + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15760) in GitLab 15.10. + +FLAG: +On self-managed GitLab, by default this feature is available. On GitLab.com this feature is not available. + +You can use Duo as an OTP provider in GitLab. + +#### Prerequisites + +To use Duo as an OTP provider: + +- Your account must exist in both Duo and GitLab, with the same username in both applications. +- You must have [configured Duo](https://admin.duosecurity.com/) and have an integration key, secret key, and API hostname. + +For more information, see the [Duo API documentation](https://duo.com/docs/authapi). + +GitLab 15.10 has been tested with Duo version D261.14 + +#### Configure Duo in GitLab + +On your GitLab server: + +1. Open the configuration file. + + For Omnibus GitLab: + + ```shell + sudo editor /etc/gitlab/gitlab.rb + ``` + + For installations from source: + + ```shell + cd /home/git/gitlab + sudo -u git -H editor config/gitlab.yml + ``` + +1. Add the provider configuration: + + For Omnibus package: + + ```ruby + gitlab_rails['duo_auth_enabled'] = false + gitlab_rails['duo_auth_integration_key'] = '' + gitlab_rails['duo_auth_secret_key'] = '' + gitlab_rails['duo_auth_hostname'] = '' + ``` + + For installations from source: + + ```yaml + duo_auth: + enabled: true + hostname: + integration_key: + secret_key: + ``` + +1. Save the configuration file. +1. For Omnibus GitLab, [reconfigure GitLab](../../../administration/restart_gitlab.md#omnibus-gitlab-reconfigure). + For installations from source, [restart GitLab](../../../administration/restart_gitlab.md#installations-from-source). + ### Enable one-time password using FortiToken Cloud > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/212313) in GitLab 13.7 [with a flag](../../../administration/feature_flags.md) named `forti_token_cloud`. Disabled by default. @@ -198,68 +269,61 @@ Configure FortiToken Cloud in GitLab. On your GitLab server: 1. [Reconfigure](../../../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) (Omnibus GitLab) or [restart](../../../administration/restart_gitlab.md#installations-from-source) (GitLab installed from source). -### Set up a U2F device - -GitLab officially supports [YubiKey](https://www.yubico.com/products/) U2F devices, but users have successfully used -[SoloKeys](https://solokeys.com/) and [Google Titan Security Key](https://cloud.google.com/titan-security-key). - -U2F is [supported by](https://caniuse.com/#search=U2F) the following desktop browsers: - -- Chrome -- Edge -- Opera -- Firefox 67+. For Firefox 47-66: - - 1. Enable the FIDO U2F API in [`about:config`](https://support.mozilla.org/en-US/kb/about-config-editor-firefox). - 1. Search for `security.webauth.u2f` and select it to toggle to `true`. - -To set up 2FA with a U2F device: - -1. Access your [**User settings**](../index.md#access-your-user-settings). -1. Select **Account**. -1. Select **Enable Two-Factor Authentication**. -1. Connect your U2F device. -1. Select **Set up New U2F Device**. -1. A light begins blinking on your device. Activate it by pressing its button. - -A message displays indicating that your device was successfully set up. Select **Register U2F Device** to complete the -process. Recovery codes are not generated for U2F devices. - ### Set up a WebAuthn device -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22506) in GitLab 13.4 [with a flag](../../../administration/feature_flags.md) named `webauthn`. Disabled by default. -> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/232671) in GitLab 14.6. +> - WebAuthn devices [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22506) in GitLab 13.4 [with a flag](../../../administration/feature_flags.md) named `webauthn`. Disabled by default. +> - WebAuthn devices [enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/232671) in GitLab 14.6. +> - Optional one-time password authentication for WebAuthn devices [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/378844) in GitLab 15.10 [with a flag](../../../administration/feature_flags.md) named `webauthn_without_topt`. [Enabled on GitLab.com and self-managed by default](https://gitlab.com/gitlab-org/gitlab/-/issues/232671). FLAG: -On self-managed GitLab, by default this feature is available. To disable the feature, ask an administrator to +On self-managed GitLab, by default, WebAuthn devices are available. To disable the feature, ask an administrator to [disable the feature flag](../../../administration/feature_flags.md) named `webauthn`. If you disable the WebAuthn feature flag after WebAuthn devices have been registered, these devices are not usable until you re-enable this feature. +On GitLab.com, WebAuthn devices are available. + +FLAG: +On self-managed GitLab, by default, optional one-time password authentication for WebAuthn devices is not available. To enable the feature, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `webauthn_without_totp`. On GitLab.com, this feature is available. -WebAuthn [supported by](https://caniuse.com/#search=webauthn): +WebAuthn is [supported by](https://caniuse.com/#search=webauthn) the following: -- The following desktop browsers: +- Desktop browsers: - Chrome - Edge - Firefox - Opera - Safari -- The following mobile browsers: +- Mobile browsers: - Chrome for Android - Firefox for Android - iOS Safari (since iOS 13.3) To set up 2FA with a WebAuthn-compatible device: +1. Optional. [Set up a one-time password](#enable-one-time-password). 1. Access your [**User settings**](../index.md#access-your-user-settings). 1. Select **Account**. 1. Select **Enable Two-Factor Authentication**. 1. Plug in your WebAuthn device. +1. Enter a device name and in GitLab 15.10 and later, your GitLab account password. + You might not need to enter this password if you have signed in through your + identity provider. 1. Select **Set up New WebAuthn Device**. 1. Depending on your device, you might have to press a button or touch a sensor. -A message displays indicating that your device was successfully set up. Recovery codes are not generated for WebAuthn -devices. +You should receive a message indicating that you successfully set up your device. + +When you set up 2FA with a WebAuthn-compatible device, that device is linked to +a specific browser on a specific computer. Depending on the browser and WebAuthn +device, you might be able to configure settings to use the WebAuthn device on a +different browser or computer. + +If this is the first time you have set up 2FA, you +must [download recovery codes](#recovery-codes) so you can recover access to your +account if you lose access. + +WARNING: +You can lose access to your account if you clear your browser data. ## Recovery codes @@ -272,11 +336,11 @@ these recovery codes to sign in to your account. WARNING: Each code can be used only once to sign in to your account. -We recommend copying and printing them, or downloading them using the **Download codes** button for storage in a safe +You should copy and print the codes, or use **Download codes** to download them for storage in a safe place. If you choose to download them, the file is called `gitlab-recovery-codes.txt`. NOTE: -Recovery codes are not generated for U2F or WebAuthn devices. +Recovery codes are not generated for WebAuthn devices. If you lose the recovery codes, or want to generate new ones, you can use either: @@ -302,18 +366,7 @@ and you're presented with a second prompt, depending on which type of 2FA you've ### Sign in using a one-time password -When asked, enter the pin from your one time password authenticator's application or a recovery code to sign in. - -### Sign in using a U2F device - -To sign in by using a U2F device: - -1. Select **Login via U2F Device**. -1. A light begins blinking on your device. Activate it by touching/pressing - its button. - -A message displays indicating that your device responded to the authentication request, and you're automatically signed -in. +When asked, enter the pin from your one-time password authenticator's application or a recovery code to sign in. ### Sign in using a WebAuthn device @@ -333,7 +386,7 @@ To disable 2FA: 1. Under **Register Two-Factor Authenticator**, enter your current password and select **Disable two-factor authentication**. -This clears all your 2FA registrations, including mobile applications and U2F or WebAuthn devices. +This clears all your 2FA registrations, including mobile applications and WebAuthn devices. ## Recovery options @@ -412,18 +465,18 @@ a GitLab global administrator disable 2FA for your account: ## Information for GitLab administrators **(FREE SELF)** - Take care that 2FA keeps working after [restoring a GitLab backup](../../../raketasks/backup_restore.md). -- To ensure 2FA authorizes correctly with a time-based one time passwords (TOTP) server, synchronize your GitLab +- To ensure 2FA authorizes correctly with a time-based one-time password (TOTP) server, synchronize your GitLab server's time using a service like NTP. Otherwise, authorization can always fail because of time differences. -- The GitLab U2F and WebAuthn implementation does _not_ work when the GitLab instance is accessed from multiple hostnames - or FQDNs. Each U2F or WebAuthn registration is linked to the _current hostname_ at the time of registration, and +- The GitLab WebAuthn implementation does _not_ work when the GitLab instance is accessed from multiple hostnames + or FQDNs. Each WebAuthn registration is linked to the _current hostname_ at the time of registration, and cannot be used for other hostnames or FQDNs. For example, if a user is trying to access a GitLab instance from `first.host.xyz` and `second.host.xyz`: - - The user signs in by using `first.host.xyz` and registers their U2F key. - - The user signs out and attempts to sign in by using `first.host.xyz` - U2F authentication succeeds. - - The user signs out and attempts to sign in by using `second.host.xyz` - U2F authentication fails, because - the U2F key has only been registered on `first.host.xyz`. + - The user signs in by using `first.host.xyz` and registers their WebAuthn key. + - The user signs out and attempts to sign in by using `first.host.xyz` - WebAuthn authentication succeeds. + - The user signs out and attempts to sign in by using `second.host.xyz` - WebAuthn authentication fails, because + the WebAuthn key has only been registered on `first.host.xyz`. - To enforce 2FA at the system or group levels see, [Enforce two-factor authentication](../../../security/two_factor_authentication.md). @@ -441,23 +494,25 @@ access token instead of a password. This error occurs in the following scenarios: - You have 2FA enabled and have attempted to authenticate with a username and - password. For 2FA-enabled users, a [personal access token](../personal_access_tokens.md) (PAT) - must be used instead of a password. To authenticate: - - Git requests over HTTP(S), a PAT with `read_repository` or `write_repository` scope is required. - - [GitLab Container Registry](../../packages/container_registry/authenticate_with_container_registry.md) requests, a PAT - with `read_registry` or `write_registry` scope is required. - - [Dependency Proxy](../../packages/dependency_proxy/index.md#authenticate-with-the-dependency-proxy) requests, a PAT with - `read_registry` and `write_registry` scopes is required. + password. - You do not have 2FA enabled and have sent an incorrect username or password with your request. - You do not have 2FA enabled but an administrator has enabled the [enforce 2FA for all users](../../../security/two_factor_authentication.md#enforce-2fa-for-all-users) setting. - You do not have 2FA enabled, but an administrator has disabled the [password authentication enabled for Git over HTTP(S)](../../admin_area/settings/sign_in_restrictions.md#password-authentication-enabled) - setting. You can authenticate Git requests: - - Over HTTP(S) using a [personal access token](../personal_access_tokens.md). - - In your browser using [Git Credential Manager](#git-credential-manager). - - If you have configured LDAP, over HTTP(S) using an [LDAP password](../../../administration/auth/ldap/index.md). + setting. + +Instead you can authenticate: + +- Using a [personal access token](../personal_access_tokens.md) (PAT): + - For Git requests over HTTP(S), a PAT with `read_repository` or `write_repository` scope is required. + - For [GitLab Container Registry](../../packages/container_registry/authenticate_with_container_registry.md) requests, a PAT + with `read_registry` or `write_registry` scope is required. + - For [Dependency Proxy](../../packages/dependency_proxy/index.md#authenticate-with-the-dependency-proxy) requests, a PAT with + `read_registry` and `write_registry` scopes is required. +- If you have configured LDAP, using an [LDAP password](../../../administration/auth/ldap/index.md) +- Using an [OAuth credential helper](#oauth-credential-helpers). ### Error: "invalid pin code" diff --git a/doc/user/profile/achievements.md b/doc/user/profile/achievements.md new file mode 100644 index 00000000000..1313c714dd0 --- /dev/null +++ b/doc/user/profile/achievements.md @@ -0,0 +1,236 @@ +--- +stage: Data Stores +group: Tenant Scale +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Achievements (Experiment) **(FREE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/113156) in GitLab 15.10 [with a flag](../../administration/feature_flags.md) named `achievements`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, +ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `achievements`. +The feature is not ready for production use. + +Achievements are a way to reward users for their activity on GitLab. +As a namespace maintainer or owner, you can create custom achievements for specific contributions, which you can award to or revoke from users based on your criteria. + +As a user, you can collect achievements to highlight your contributions to different projects or groups on your profile. +An achievement consists of a name, a description, and an avatar. + +![Achievements on user profile page](img/user_profile_achievements_v15_11.png) + +Achievements are considered to be owned by the user. They are visible regardless of the visibility setting of the namespace that created the Achievement. + +This feature is an Experiment. +For more information about planned work, see [epic 9429](https://gitlab.com/groups/gitlab-org/-/epics/9429). +Tell us about your use cases by leaving comments in the epic. + +## Types of achievement + +Programmatically, there is only one way to create, award, revoke, or delete achievements. + +Practically, you can differentiate between achievements that are awarded: + +- Once and irrevocable. For example, a "First contribution merged" achievement. +- Once and revocable. For example, a "Core team member" achievement. +- Multiple times. For example, a "Contributor of the month" achievement. + +## View a user's achievements + +You can view a user's achievements on their profile page. + +Prerequisites: + +- The user profile must be public. + +To view a user's achievements: + +1. Go to the user's profile page. +1. Below the user's avatar, see their achievements. +1. To view details about an achievement, hover over it. + It displays the following information: + + - Name of the achievement + - Description of the achievement + - Date when the achievement was awarded to the user + - Namespace that awarded the achievement if the user is a member of the namespace or the namespace is public + +To retrieve a list of a user's achievements, query the [`user` GraphQL type](../../api/graphql/reference/index.md#user). + +```graphql +query { + user(username: "") { + userAchievements { + nodes { + achievement { + name + description + avatarUrl + namespace { + fullPath + name + } + } + } + } + } +} +``` + +## Create an achievement + +You can create custom achievements to award for specific contributions. + +Prerequisites: + +- You must have the Maintainer or Owner role for the namespace. + +To create an achievement, call the [`achievementsCreate` GraphQL mutation](../../api/graphql/reference/index.md#mutationachievementscreate). + +```graphql +mutation achievementsCreate($file: Upload!) { + achievementsCreate( + input: { + namespaceId: "gid://gitlab/Namespace/", + name: "", + description: "", + avatar: $file} + ) { + errors + achievement { + id + name + description + avatarUrl + } + } +} +``` + +## Update an achievement + +You can change the name, description, and avatar of an achievement at any time. + +Prerequisites: + +- You must have the Maintainer or Owner role for the namespace. + +To update an achievement, call the [`achievementsUpdate` GraphQL mutation](../../api/graphql/reference/index.md#mutationachievementsupdate). + +```graphql +mutation achievementsUpdate($file: Upload!) { + achievementsUpdate( + input: { + achievementId: "gid://gitlab/Achievements::Achievement/", + name: "", + description: "", + avatar: $file} + ) { + errors + achievement { + id + name + description + avatarUrl + } + } +} +``` + +## Award an achievement + +You can award an achievement to a user to recognize their contributions. +The user receives an email notification when they are awarded an achievement. + +Prerequisites: + +- You must have the Maintainer or Owner role for the namespace. + +To award an achievement to a user, call the [`achievementsAward` GraphQL mutation](../../api/graphql/reference/index.md#mutationachievementsaward). + +```graphql +mutation { + achievementsAward(input: { + achievementId: "gid://gitlab/Achievements::Achievement/", + userId: "gid://gitlab/User/" }) { + userAchievement { + id + achievement { + id + name + } + user { + id + username + } + } + errors + } +} +``` + +## Revoke an achievement + +You can revoke a user's achievement if you consider the user no longer meets the awarding criteria. + +Prerequisites: + +- You must have the Maintainer or Owner role for the namespace. + +To revoke an achievement, call the [`achievementsRevoke` GraphQL mutation](../../api/graphql/reference/index.md#mutationachievementsrevoke). + +```graphql +mutation { + achievementsRevoke(input: { + userAchievementId: "gid://gitlab/Achievements::UserAchievement/" }) { + userAchievement { + id + achievement { + id + name + } + user { + id + username + } + revokedAt + } + errors + } +} +``` + +## Delete an achievement + +If you consider you no longer need an achievement, you can delete it. +This will delete all related awarded and revoked instances of the achievement. + +Prerequisites: + +- You must have the Maintainer or Owner role for the namespace. + +To delete an achievement, call the [`achievementsDelete` GraphQL mutation](../../api/graphql/reference/index.md#mutationachievementsdelete). + +```graphql +mutation { + achievementsDelete(input: { + achievementId: "gid://gitlab/Achievements::Achievement/" }) { + achievement { + id + name + } + errors + } +} +``` + +## Hide achievements + +If you don't want to display achievements on your profile, you can opt out. To do this: + +1. On the top bar, in the upper-right corner, select your avatar. +1. Select **Edit profile**. +1. In the **Main settings** section, clear the **Display achievements on your profile** checkbox. +1. Select **Update profile settings**. diff --git a/doc/user/profile/active_sessions.md b/doc/user/profile/active_sessions.md index 1f7264d740e..f22accf2ff5 100644 --- a/doc/user/profile/active_sessions.md +++ b/doc/user/profile/active_sessions.md @@ -40,8 +40,8 @@ To revoke an active session: NOTE: When any session is revoked all **Remember me** tokens for all -devices are revoked. See [Why do you keep getting signed out?](index.md#why-do-you-keep-getting-signed-out) -for more information about the **Remember me** feature. +devices are revoked. For details about **Remember me**, see +[cookies used for sign-in](index.md#cookies-used-for-sign-in). -If you don't see this action on the right sidebar, your project or instance may have +If you don't see this action on the right sidebar, your project or instance might have enabled a feature flag for [moved sidebar actions](../project/merge_requests/index.md#move-sidebar-actions). ### Notification events on issues, merge requests, and epics @@ -286,7 +285,8 @@ To always receive notifications on your own issues, merge requests, and so on, t ## Notifications for unknown sign-ins -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27211) in GitLab 13.0. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27211) in GitLab 13.0. +> - Listing the full name and username of the signed-in user [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225183) in GitLab 15.10. NOTE: This feature is enabled by default for self-managed instances. Administrators may disable this feature @@ -295,7 +295,12 @@ The feature is always enabled on GitLab.com. When a user successfully signs in from a previously unknown IP address or device, GitLab notifies the user by email. In this way, GitLab proactively alerts users of potentially -malicious or unauthorized sign-ins. +malicious or unauthorized sign-ins. This notification email includes the: + +- Hostname. +- User's name and username. +- IP address. +- Date and time of sign-in. GitLab uses several methods to identify a known sign-in. All methods must fail for a notification email to be sent. @@ -306,7 +311,7 @@ GitLab uses several methods to identify a known sign-in. All methods must fail f - Cookie: After successful sign in, an encrypted cookie is stored in the browser. This cookie is set to expire 14 days after the last successful sign in. -## Notifications for attempted sign-in using wrong two-factor authentication codes +## Notifications for attempted sign-ins using incorrect verification codes > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/374740) in GitLab 15.5. @@ -357,6 +362,7 @@ The following table lists all GitLab-specific email headers: | ------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | | `List-Id` | The path of the project in an RFC 2919 mailing list identifier. You can use it for email organization with filters. | | `X-GitLab-(Resource)-ID` | The ID of the resource the notification is for. The resource, for example, can be `Issue`, `MergeRequest`, `Commit`, or another such resource. | +| `X-GitLab-ConfidentialIssue` | The boolean value indicating issue confidentiality for notifications. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/222908) in GitLab 16.0. | | `X-GitLab-Discussion-ID` | The ID of the thread the comment belongs to, in notification emails for comments. | | `X-GitLab-Group-Id` | The group's ID. Only present on notification emails for [epics](../group/epics/index.md). | | `X-GitLab-Group-Path` | The group's path. Only present on notification emails for [epics](../group/epics/index.md) | diff --git a/doc/user/profile/personal_access_tokens.md b/doc/user/profile/personal_access_tokens.md index 3826c602fb4..e59d7313281 100644 --- a/doc/user/profile/personal_access_tokens.md +++ b/doc/user/profile/personal_access_tokens.md @@ -20,11 +20,7 @@ Personal access tokens can be an alternative to [OAuth2](../../api/oauth2.md) an In both cases, you authenticate with a personal access token in place of your password. WARNING: -The ability to create personal access tokens without expiry was -[deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/369122) in GitLab 15.4 and is planned for removal in GitLab -16.0. When this ability is removed, existing personal access tokens without an expiry are planned to have an expiry added. -The automatic adding of an expiry occurs on GitLab.com during the 16.0 milestone. The automatic adding of an expiry -occurs on self-managed instances when they are upgraded to GitLab 16.0. This change is a breaking change. +The ability to create personal access tokens without expiry was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/369122) in GitLab 15.4 and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/392855) in GitLab 16.0. In GitLab 16.0 and later, existing personal access tokens without an expiry date are automatically given an expiry date of 365 days later than the current date. The automatic adding of an expiry date occurs on GitLab.com during the 16.0 milestone. The automatic adding of an expiry date occurs on self-managed instances when they are upgraded to GitLab 16.0. This change is a breaking change. Personal access tokens are: @@ -45,19 +41,20 @@ For examples of how you can use a personal access token to authenticate with the Alternately, GitLab administrators can use the API to create [impersonation tokens](../../api/rest/index.md#impersonation-tokens). Use impersonation tokens to automate authentication as a specific user. -NOTE: -Personal access tokens are not FIPS compliant and creation and use are disabled when [FIPS mode](../../development/fips_compliance.md) is enabled. - ## Create a personal access token -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/348660) in GitLab 15.3, default expiration of 30 days is populated in the UI. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/348660) in GitLab 15.3, default expiration of 30 days is populated in the UI. +> - Ability to create non-expiring personal access tokens [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/392855) in GitLab 16.0. You can create as many personal access tokens as you like. 1. In the upper-right corner, select your avatar. 1. Select **Edit profile**. 1. On the left sidebar, select **Access Tokens**. -1. Enter a name and optional expiry date for the token. +1. Enter a name and expiry date for the token. + - The token expires on that date at midnight UTC. + - If you do not enter an expiry date, the expiry date is automatically set to 365 days later than the current date. + - By default, this date can be a maximum of 365 days later than the current date. 1. Select the [desired scopes](#personal-access-token-scopes). 1. Select **Create personal access token**. @@ -103,6 +100,8 @@ To view the last time a token was used: ## Personal access token scopes +> Personal access tokens no longer being able to access container or package registries [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/387721) in GitLab 16.0. + A personal access token can perform actions based on the assigned scopes. | Scope | Access | @@ -117,6 +116,9 @@ A personal access token can perform actions based on the assigned scopes. | `sudo` | Grants permission to perform API actions as any user in the system, when authenticated as an administrator. | | `admin_mode` | Grants permission to perform API actions as an administrator, when Admin Mode is enabled. ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/107875) in GitLab 15.8.) | +WARNING: +If you enabled [external authorization](../admin_area/settings/external_authorization.md), personal access tokens cannot access container or package registries. If you use personal access tokens to access these registries, this measure breaks this use of these tokens. Disable external authorization to use personal access tokens with container or package registries. + ## When personal access tokens expire Personal access tokens expire on the date you define, at midnight UTC. @@ -225,7 +227,7 @@ Remember this if you set up an automation pipeline that depends on authenticatio ### Unrevoke a personal access token **(FREE SELF)** -If a personal access token is revoked accidentally by any method, administrators can unrevoke that token. +If a personal access token is revoked accidentally by any method, administrators can unrevoke that token. By default, a daily job deletes revoked tokens at 1:00 AM system time. WARNING: Running the following commands changes data directly. This could be damaging if not done correctly, or under the right conditions. You should first run these commands in a test environment with a backup of the instance ready to be restored, just in case. @@ -247,5 +249,4 @@ Running the following commands changes data directly. This could be damaging if ## Alternatives to personal access tokens -For Git over HTTPS, an alternative to personal access tokens is [Git Credential Manager](account/two_factor_authentication.md#git-credential-manager), -which securely authenticates using OAuth. +For Git over HTTPS, an alternative to personal access tokens is to use an [OAuth credential helper](account/two_factor_authentication.md#oauth-credential-helpers). diff --git a/doc/user/profile/preferences.md b/doc/user/profile/preferences.md index 7e581bb7419..da4d2da70fe 100644 --- a/doc/user/profile/preferences.md +++ b/doc/user/profile/preferences.md @@ -39,11 +39,11 @@ The default theme is Indigo. You can choose between 10 themes: ## Dark mode -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28252) in GitLab 13.1 as an [Alpha](../../policy/alpha-beta-support.md#alpha-features) release. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28252) in GitLab 13.1 as an [Experiment](../../policy/alpha-beta-support.md#experiment) release. -GitLab has started work on dark mode! The dark mode Alpha release is available in the +GitLab has started work on dark mode! The dark mode Experiment release is available in the spirit of iteration and the lower expectations of -[Alpha versions](../../policy/alpha-beta-support.md#alpha-features). +[Experiment features](../../policy/alpha-beta-support.md#experiment). Progress on dark mode is tracked in the [Dark theme epic](https://gitlab.com/groups/gitlab-org/-/epics/2902). See the epic for: @@ -63,7 +63,9 @@ Dark theme only works with the **Dark** syntax highlighting theme. ## Syntax highlighting theme -GitLab uses the [rouge Ruby library](http://rouge.jneen.net/ "Rouge website") +> Changing the default syntax highlighting theme for new users and users who are not signed in [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/25129) in GitLab 15.10. + +GitLab uses the [Rouge Ruby library](https://github.com/rouge-ruby/rouge) for syntax highlighting outside of any Editor context. The WebIDE (like Snippets) uses [Monaco Editor](https://microsoft.github.io/monaco-editor/) and it's provided [Monarch](https://microsoft.github.io/monaco-editor/monarch.html) library for @@ -73,22 +75,14 @@ the respective libraries. Changing this setting allows you to customize the color theme when viewing any syntax highlighted code on GitLab. -The default syntax theme is White, and you can choose among 5 different themes: - - - -- White -- Dark -- Solarized light -- Solarized dark -- Monokai - - - -![Profile preferences syntax highlighting themes](img/profile-preferences-syntax-themes.png) +![Profile preferences syntax highlighting themes](img/profile-preferences-syntax-themes_v15_11.png) Introduced in GitLab 13.6, the themes [Solarized](https://gitlab.com/gitlab-org/gitlab/-/issues/221034) and [Monokai](https://gitlab.com/gitlab-org/gitlab/-/issues/221034) also apply to the [Web IDE](../project/web_ide/index.md) and [Snippets](../snippets.md). +You can use an API call to change the default syntax highlighting theme for new users and users +who are not signed in. For more information, see the `default_syntax_highlighting_theme` +in the [list of settings that can be accessed through API calls](../../api/settings.md#list-of-settings-that-can-be-accessed-via-api-calls). + ## Diff colors A diff compares the old/removed content with the new/added content (for example, when @@ -116,12 +110,9 @@ between the fixed (max. `1280px`) and the fluid (`100%`) application layout. NOTE: While `1280px` is the standard max width when using fixed layout, some pages still use 100% width, depending on the content. -### Dashboard +### Homepage -For users who have access to a large number of projects but only keep up with a -select few, the amount of activity on the your dashboard can be -overwhelming. From the **Dashboard** dropdown list, select what you'd like displayed on your -personal dashboard. +This setting changes the behavior of the tanuki icon in the upper-left corner of GitLab. ### Group overview content @@ -191,22 +182,6 @@ NOTE: This feature is experimental, and choosing absolute times might break certain layouts. Open an issue if you notice that using absolute times breaks a layout. -## Web IDE - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/370139) in GitLab 15.7 [with a flag](../../administration/feature_flags.md) named `vscode_web_ide`. Disabled by default. - -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) named `vscode_web_ide`. On GitLab.com, this feature is available. - -The [Web IDE Beta](../project/web_ide_beta/index.md) is -the default editing environment when the `vscode_web_ide` feature -flag is enabled. - -To stop using the Web IDE Beta: - -1. In the **Web IDE** section, select the **Opt out of the Web IDE Beta** checkbox. -1. Select **Save changes**. - ## Integrations Configure your preferences with third-party services which provide enhancements to your GitLab experience. diff --git a/doc/user/profile/saved_replies.md b/doc/user/profile/saved_replies.md new file mode 100644 index 00000000000..1f4e4f5fa51 --- /dev/null +++ b/doc/user/profile/saved_replies.md @@ -0,0 +1,11 @@ +--- +redirect_to: 'comment_templates.md' +remove_date: '2023-06-22' +--- + +This document was moved to [another location](comment_templates.md). + + + + + diff --git a/doc/user/project/badges.md b/doc/user/project/badges.md index 0ea1a80bc54..26708dece50 100644 --- a/doc/user/project/badges.md +++ b/doc/user/project/badges.md @@ -1,6 +1,6 @@ --- -stage: Manage -group: Organization +stage: Data Stores +group: Tenant Scale info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -62,7 +62,7 @@ You can access a test coverage report badge image by using the following link: https://gitlab.example.com///badges//coverage.svg ``` -You can define the regular expression for the [coverage report](../../ci/pipelines/settings.md#merge-request-test-coverage-results) +You can define the regular expression for the [coverage report](../../ci/testing/code_coverage.md#view-code-coverage-results-in-the-mr) that each job log is matched against. This means that each job in the pipeline can have the test coverage percentage value defined. @@ -107,7 +107,7 @@ If there is no release, it shows `none`. You can access a latest release badge image by using the following link: ```plaintext -https://gitlab.example.com///badges//release.svg +https://gitlab.example.com///-/badges/release.svg ``` By default, the badge fetches the release sorted using the [`released_at`](../../api/releases/index.md#create-a-release) @@ -117,6 +117,10 @@ time with the `?order_by` query parameter. https://gitlab.example.com///-/badges/release.svg?order_by=release_at ``` +You can change the width of the release name field by using the `value_width` parameter ([Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/113615) in GitLab 15.10). +The value must be between 1 and 200, and the default value is 54. +If you set an out of range value, GitLab automatically adjusts it to the default value. + ## Project badges Badges can be added to a project by Maintainers or Owners, and are visible on the project's overview page. @@ -275,7 +279,7 @@ To add a new badge with a custom image to a group or project: 1. Select **Add badge**. To learn how to use custom images generated through a pipeline, see the documentation on -[accessing the latest job artifacts by URL](../../ci/pipelines/job_artifacts.md#access-the-latest-job-artifacts). +[accessing the latest job artifacts by URL](../../ci/jobs/job_artifacts.md#from-a-url). ## Placeholders diff --git a/doc/user/project/canary_deployments.md b/doc/user/project/canary_deployments.md index 95f38c7e354..8ea762777ac 100644 --- a/doc/user/project/canary_deployments.md +++ b/doc/user/project/canary_deployments.md @@ -4,18 +4,16 @@ group: unassigned info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Canary Deployments **(FREE)** +# Canary deployments **(FREE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1659) in GitLab 9.1. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212320) from GitLab Premium to GitLab Free in 13.8. -A popular [Continuous Deployment](https://en.wikipedia.org/wiki/Continuous_deployment) +Canary deployments are a popular [continuous deployment](https://en.wikipedia.org/wiki/Continuous_deployment) strategy, where a small portion of the fleet is updated to the new version of your application. -## Overview - -When embracing [Continuous Delivery](https://about.gitlab.com/blog/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/), a company needs to decide what +When embracing [continuous delivery](https://about.gitlab.com/blog/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/), an organization needs to decide what type of deployment strategy to use. One of the most popular strategies is canary deployments, where a small portion of the fleet is updated to the new version first. This subset, the canaries, then serve as the proverbial diff --git a/doc/user/project/changelogs.md b/doc/user/project/changelogs.md index a64edd053b1..7e622110291 100644 --- a/doc/user/project/changelogs.md +++ b/doc/user/project/changelogs.md @@ -5,7 +5,7 @@ info: "To determine the technical writer assigned to the Stage/Group associated type: reference, api --- -# Changelogs +# Changelogs **(FREE)** Changelogs are generated based on commit titles and Git trailers. To be included in a changelog, a commit must contain a specific Git trailer. Changelogs are generated @@ -313,5 +313,5 @@ an error is produced when generating a changelog. ## Related topics -- [Changelog-related endpoints](../../api/repositories.md) in the Repositories API. -- Developer documentation for [changelog entries](../../development/changelog.md) in GitLab. +- [Changelog-related endpoints](../../api/repositories.md) in the Repositories API +- Developer documentation for [changelog entries](../../development/changelog.md) in GitLab diff --git a/doc/user/project/clusters/add_eks_clusters.md b/doc/user/project/clusters/add_eks_clusters.md index 52288af101a..6c8edb8c3e5 100644 --- a/doc/user/project/clusters/add_eks_clusters.md +++ b/doc/user/project/clusters/add_eks_clusters.md @@ -1,10 +1,10 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Connect EKS clusters through cluster certificates (DEPRECATED) **(FREE)** +# Connect EKS clusters through cluster certificates (deprecated) **(FREE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22392) in GitLab 12.5. > - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. @@ -25,7 +25,7 @@ use the [GitLab agent](../../clusters/agent/index.md). To create a new cluster from GitLab, use [Infrastructure as Code](../../infrastructure/iac/index.md). -### How to create a new cluster on EKS through cluster certificates (DEPRECATED) +### How to create a new cluster on EKS through cluster certificates (deprecated) > [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 14.0. @@ -174,11 +174,11 @@ When you create a new cluster, you have the following settings: | Kubernetes cluster name | Your cluster's name. | | Environment scope | The [associated environment](multiple_kubernetes_clusters.md#setting-the-environment-scope). | | Service role | The **EKS IAM role** (**role A**). | -| Kubernetes version | The [Kubernetes version](../../clusters/agent/index.md#supported-cluster-versions) for your cluster. | +| Kubernetes version | The [Kubernetes version](../../clusters/agent/index.md#gitlab-agent-for-kubernetes-supported-cluster-versions) for your cluster. | | Key pair name | The [key pair](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html) that you can use to connect to your worker nodes. | | VPC | The [VPC](https://docs.aws.amazon.com/vpc/latest/userguide/what-is-amazon-vpc.html) to use for your EKS Cluster resources. | | Subnets | The [subnets](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Subnets.html) in your VPC where your worker nodes run. Two are required. | -| Security group | The [security group](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html) to apply to the EKS-managed Elastic Network Interfaces that are created in your worker node subnets. | +| Security group | The [security group](https://docs.aws.amazon.com/vpc/latest/userguide/vpc-security-groups.html) to apply to the EKS-managed Elastic Network Interfaces that are created in your worker node subnets. | | Instance type | The [instance type](https://aws.amazon.com/ec2/instance-types/) of your worker nodes. | | Node count | The number of worker nodes. | | GitLab-managed cluster | Check if you want GitLab to manage namespaces and service accounts for this cluster. | diff --git a/doc/user/project/clusters/add_existing_cluster.md b/doc/user/project/clusters/add_existing_cluster.md index 0b0555806ce..bb85662d442 100644 --- a/doc/user/project/clusters/add_existing_cluster.md +++ b/doc/user/project/clusters/add_existing_cluster.md @@ -1,10 +1,10 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Connect existing clusters through cluster certificates (DEPRECATED) **(FREE)** +# Connect existing clusters through cluster certificates (deprecated) **(FREE)** > [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. diff --git a/doc/user/project/clusters/add_gke_clusters.md b/doc/user/project/clusters/add_gke_clusters.md index b1c5bbfc4f7..c6561fffa0b 100644 --- a/doc/user/project/clusters/add_gke_clusters.md +++ b/doc/user/project/clusters/add_gke_clusters.md @@ -1,10 +1,10 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Connect GKE clusters through cluster certificates (DEPRECATED) **(FREE)** +# Connect GKE clusters through cluster certificates (deprecated) **(FREE)** > [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. @@ -77,7 +77,7 @@ cluster certificates: - **Zone** - Choose the [region zone](https://cloud.google.com/compute/docs/regions-zones/) under which to create the cluster. - **Number of nodes** - Enter the number of nodes you wish the cluster to have. - - **Machine type** - The [machine type](https://cloud.google.com/compute/docs/machine-types) + - **Machine type** - The [machine type](https://cloud.google.com/compute/docs/machine-resource) of the Virtual Machine instance to base the cluster on. - **Enable Cloud Run for Anthos** - Check this if you want to use Cloud Run for Anthos for this cluster. See the [Cloud Run for Anthos section](#cloud-run-for-anthos) for more information. diff --git a/doc/user/project/clusters/add_remove_clusters.md b/doc/user/project/clusters/add_remove_clusters.md index d3048158958..bba05f1e724 100644 --- a/doc/user/project/clusters/add_remove_clusters.md +++ b/doc/user/project/clusters/add_remove_clusters.md @@ -1,10 +1,10 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Add a cluster using cluster certificates (DEPRECATED) **(FREE)** +# Add a cluster using cluster certificates (deprecated) **(FREE)** > [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/327908) in GitLab 14.0. diff --git a/doc/user/project/clusters/cluster_access.md b/doc/user/project/clusters/cluster_access.md index 529e7a6da12..ff30da2ca98 100644 --- a/doc/user/project/clusters/cluster_access.md +++ b/doc/user/project/clusters/cluster_access.md @@ -1,10 +1,10 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Access controls with cluster certificates (RBAC or ABAC) (DEPRECATED) **(FREE)** +# Access controls with cluster certificates (RBAC or ABAC) (deprecated) **(FREE)** > - Restricted service account for deployment was [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/51716) in GitLab 11.5. > - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. diff --git a/doc/user/project/clusters/deploy_to_cluster.md b/doc/user/project/clusters/deploy_to_cluster.md index 173f1f39a66..31d5a73757a 100644 --- a/doc/user/project/clusters/deploy_to_cluster.md +++ b/doc/user/project/clusters/deploy_to_cluster.md @@ -1,10 +1,10 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Deploy to a Kubernetes cluster with cluster certificates (DEPRECATED) **(FREE)** +# Deploy to a Kubernetes cluster with cluster certificates (deprecated) **(FREE)** > [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. diff --git a/doc/user/project/clusters/gitlab_managed_clusters.md b/doc/user/project/clusters/gitlab_managed_clusters.md index b2a4bd85de4..b321646d8d8 100644 --- a/doc/user/project/clusters/gitlab_managed_clusters.md +++ b/doc/user/project/clusters/gitlab_managed_clusters.md @@ -1,10 +1,10 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# GitLab-managed clusters (DEPRECATED) **(FREE)** +# GitLab-managed clusters (deprecated) **(FREE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/22011) in GitLab 11.5. > - Became [optional](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/26565) in GitLab 11.11. diff --git a/doc/user/project/clusters/index.md b/doc/user/project/clusters/index.md index 94513fc7124..5ce1994ba36 100644 --- a/doc/user/project/clusters/index.md +++ b/doc/user/project/clusters/index.md @@ -1,10 +1,10 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Project-level Kubernetes clusters (certificate-based) (DEPRECATED) **(FREE)** +# Project-level Kubernetes clusters (certificate-based) (deprecated) **(FREE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/35954) in GitLab 10.1. > - [Deprecated](https://gitlab.com/groups/gitlab-org/configure/-/epics/8) in GitLab 14.5. diff --git a/doc/user/project/clusters/multiple_kubernetes_clusters.md b/doc/user/project/clusters/multiple_kubernetes_clusters.md index c79f250da7a..c8f49b1917d 100644 --- a/doc/user/project/clusters/multiple_kubernetes_clusters.md +++ b/doc/user/project/clusters/multiple_kubernetes_clusters.md @@ -1,10 +1,10 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Multiple clusters per project with cluster certificates (DEPRECATED) **(FREE)** +# Multiple clusters per project with cluster certificates (deprecated) **(FREE)** > - Introduced in GitLab 10.3 > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35094) from GitLab Premium to GitLab Free in 13.2. diff --git a/doc/user/project/clusters/runbooks/index.md b/doc/user/project/clusters/runbooks/index.md index df0ffff8561..a985436d67d 100644 --- a/doc/user/project/clusters/runbooks/index.md +++ b/doc/user/project/clusters/runbooks/index.md @@ -1,6 +1,6 @@ --- -stage: Configure -group: Configure +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -107,7 +107,7 @@ the components outlined above and the pre-loaded demo runbook. ''' We set user's id, login and access token on single user image to enable repository integration for JupyterHub. - See: https://gitlab.com/gitlab-org/gitlab-foss/issues/47138#note_154294790 + See: https://gitlab.com/gitlab-org/gitlab-foss/-/issues/47138#note_154294790 ''' auth_state = await spawner.user.get_auth_state() diff --git a/doc/user/project/code_owners.md b/doc/user/project/code_owners.md index 9de9d445965..f9244177aa5 100644 --- a/doc/user/project/code_owners.md +++ b/doc/user/project/code_owners.md @@ -1,373 +1,11 @@ --- -stage: Create -group: Source Code -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: 'codeowners/index.md' +remove_date: '2023-07-07' --- -# Code Owners **(PREMIUM)** +This document was moved to [another location](codeowners/index.md). -> Moved to GitLab Premium in 13.9. - -Code Owners define who develops and maintains a feature, and own the resulting -files or directories in a repository. - -- The users you define as Code Owners are displayed in the UI when you browse directories. -- You can set your merge requests so they must be approved by Code Owners before merge. -- You can protect a branch and allow only Code Owners to approve changes to the branch. - -
    - Video introduction: Code Owners. -
    -
    - -
    - - - -Use Code Owners and approvers together with -[approval rules](merge_requests/approvals/rules.md) to build a flexible approval -workflow: - -- Use **Code Owners** to define the users who have domain expertise for specific paths in your repository. -- Use **Approvers** and **Approval rules** to define domains of expertise (such as a security team) - that are not scoped to specific file paths in your repository. - - **Approvers** define the users. - - **Approval rules** define when these users can approve work, and whether or not their approval is required. - -For example: - -| Type | Name | Scope | Comment | -|------|------|--------|------------| -| Approval rule | UX | All files | A user experience (UX) team member reviews the user experience of all changes made in your project. | -| Approval rule | Security | All files | A security team member reviews all changes for vulnerabilities. | -| Code Owner approval rule | Frontend: Code Style | `*.css` files | A frontend engineer reviews CSS file changes for adherence to project style standards. | -| Code Owner approval rule | Backend: Code Review | `*.rb` files | A backend engineer reviews the logic and code style of Ruby files. | - -## Code Owners file - -A `CODEOWNERS` file (with no extension) can specify users or [shared groups](members/share_project_with_groups.md) -that are responsible for specific files and directories in a repository. Each repository -can have a single `CODEOWNERS` file, and it must be found one of these three locations: - -- In the root directory of the repository. -- In the `.gitlab/` directory. -- In the `docs/` directory. - -A CODEOWNERS file in any other location is ignored. - -## Set up Code Owners - -1. Create a file named `CODEOWNERS` (with no extension) in one of these locations: - -- In the root directory of the repository -- In the `.gitlab/` directory -- In the `docs/` directory - -1. In the file, enter text that follows one of these patterns: - - ```plaintext - # Code Owners for a file - filename @username1 @username2 - - # Code Owners for a directory - directoryname/ @username1 @username2 - - # All group members as Code Owners for a file - filename @groupname - - # All group members as Code Owners for a directory - directoryname/ @groupname - ``` - -The Code Owners are now displayed in the UI. They apply to the current branch only. - -Next steps: - -- [Add Code Owners as merge request approvers](merge_requests/approvals/rules.md#code-owners-as-eligible-approvers). -- Set up [Code Owner approval on a protected branch](protected_branches.md#require-code-owner-approval-on-a-protected-branch). - -## Groups as Code Owners - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/53182) in GitLab 12.1. -> - Group and subgroup hierarchy support was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32432) in GitLab 13.0. - -You can use members of groups and subgroups as Code Owners for projects: - -```mermaid -graph TD - A[Parent group X] -->|owns| B[Project A] - A -->|contains| C[Subgroup Y] - C -->|owns| D[Project B] - A-. inherits ownership .-> D -``` - -In this example: - -- **Parent group X** (`group-x`) owns **Project A**. -- **Parent group X** also contains a subgroup, **Subgroup Y**. (`group-x/subgroup-y`) -- **Subgroup Y** owns **Project B**. - -The eligible Code Owners are: - -- **Project A**: the members of **Group X** only, because **Project A** doesn't belong to **Subgroup Y**. -- **Project B**: the members of both **Group X** and **Subgroup Y**. - -You can [invite](members/share_project_with_groups.md) **Subgroup Y** to **Project A** -so that their members also become eligible Code Owners. - -```mermaid -graph LR - A[Parent group X] -->|owns| B[Project A] - A -->|also contains| C[Subgroup Y] - C -.->D{Invite Subgroup Y
    to Project A?} -.->|yes| F[Approvals can be
    required] -.-> B - D{Invite Subgroup Y
    to Project A?} -.->|no| I[Subgroup Y cannot be
    an approver] -.-> B - C -.->E{Add Subgroup Y
    as Code Owners
    to Project A?} -.->|yes| H[Approvals can only
    be optional] -.-> B -``` - -If you do not invite **Subgroup Y** to **Project A**, but make them Code Owners, their approval -of the merge request becomes optional. - -Inviting **Subgroup Y** to a parent group of **Project A** -[is not supported](https://gitlab.com/gitlab-org/gitlab/-/issues/288851). To set **Subgroup Y** as -Code Owners, add this group directly to the project itself. - -NOTE: -For approval to be required, groups as Code Owners must have a direct membership -(not inherited membership) in the project. Approval can only be optional for groups -that inherit membership. Members in the Code Owners group also must be direct members, -and not inherit membership from any parent groups. - -### Add a group as a Code Owner - -To set a group as a Code Owner: - -In the `CODEOWNERS` file, enter text that follows one of these patterns: - -```plaintext -# All group members as Code Owners for a file -file.md @group-x - -# All subgroup members as Code Owners for a file -file.md @group-x/subgroup-y - -# All group and subgroup members as Code Owners for a file -file.md @group-x @group-x/subgroup-y -``` - -## When a file matches multiple `CODEOWNERS` entries - -When a file matches multiple entries in the `CODEOWNERS` file, -the users from last pattern matching the file are used. - -For example, in the following `CODEOWNERS` file: - -```plaintext -README.md @user1 - -# This line would also match the file README.md -*.md @user2 -``` - -The Code Owner for `README.md` would be `@user2`. - -If you use sections, the last pattern matching the file for each section is used. -For example, in a `CODEOWNERS` file using sections: - -```plaintext -[README Owners] -README.md @user1 @user2 -internal/README.md @user4 - -[README other owners] -README.md @user3 -``` - -The Code Owners for the `README.md` in the root directory are `@user1`, `@user2`, -and `@user3`. The Code Owners for `internal/README.md` are `@user4` and `@user3`. - -Only one CODEOWNERS pattern can match per file path. - -### Organize Code Owners by putting them into sections - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12137) in GitLab 13.2 [with a flag](../../administration/feature_flags.md) named `sectional_codeowners`. Disabled by default. -> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42389) in GitLab 13.4. Feature flag `sectional_codeowners` removed. - -You can organize Code Owners by putting them into named sections. - -You can use sections for shared directories, so that multiple -teams can be reviewers. - -To add a section to the `CODEOWNERS` file, enter a section name in brackets, -followed by the files or directories, and users, groups, or subgroups: - -```plaintext -[README Owners] -README.md @user1 @user2 -internal/README.md @user2 -``` - -Each Code Owner in the merge request widget is listed under a label. -The following image shows a **Groups** and **Documentation** section: - -![MR widget - Sectional Code Owners](img/sectional_code_owners_v13.2.png) - -### Sections with duplicate names - -If multiple sections have the same name, they are combined. -Also, section headings are not case-sensitive. For example: - -```plaintext -[Documentation] -ee/docs/ @docs -docs/ @docs - -[Database] -README.md @database -model/db/ @database - -[DOCUMENTATION] -README.md @docs -``` - -This code results in three entries under the **Documentation** section header, and two -entries under **Database**. The entries defined under the sections **Documentation** and -**DOCUMENTATION** are combined, using the case of the first section. - -### Make a Code Owners section optional - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/232995) in GitLab 13.8. - -You can designate optional sections in your Code Owners file. Prepend the -section name with the caret `^` character to treat the entire section as optional. -Optional sections enable you to designate responsible parties for various parts -of your codebase, but not require approval from them. This approach provides -a more relaxed policy for parts of your project that are frequently updated, -but don't require stringent reviews. - -In this example, the `[Go]` section is optional: - -```plaintext -[Documentation] -*.md @root - -[Ruby] -*.rb @root - -^[Go] -*.go @root -``` - -The optional Code Owners section displays in merge requests under the **Approval Rules** area: - -![MR widget - Optional Code Owners sections](img/optional_code_owners_sections_v13_8.png) - -If a section is duplicated in the file, and one of them is marked as optional and the other isn't, the section is required. - -Optional sections in the `CODEOWNERS` file are treated as optional only -when changes are submitted by using merge requests. If a change is submitted directly -to the protected branch, approval from Code Owners is still required, even if the -section is marked as optional. - -### Allowed to Push - -The Code Owner approval and protected branch features do not apply to users who -are **Allowed to push**. - -## Example `CODEOWNERS` file - -```plaintext -# This is an example of a CODEOWNERS file. -# Lines that start with `#` are ignored. - -# app/ @commented-rule - -# Specify a default Code Owner by using a wildcard: -* @default-codeowner - -# Specify multiple Code Owners by using a tab or space: -* @multiple @code @owners - -# Rules defined later in the file take precedence over the rules -# defined before. -# For example, for all files with a filename ending in `.rb`: -*.rb @ruby-owner - -# Files with a `#` can still be accessed by escaping the pound sign: -\#file_with_pound.rb @owner-file-with-pound - -# Specify multiple Code Owners separated by spaces or tabs. -# In the following case the CODEOWNERS file from the root of the repo -# has 3 Code Owners (@multiple @code @owners): -CODEOWNERS @multiple @code @owners - -# You can use both usernames or email addresses to match -# users. Everything else is ignored. For example, this code -# specifies the `@legal` and a user with email `janedoe@gitlab.com` as the -# owner for the LICENSE file: -LICENSE @legal this_does_not_match janedoe@gitlab.com - -# Use group names to match groups, and nested groups to specify -# them as owners for a file: -README @group @group/with-nested/subgroup - -# End a path in a `/` to specify the Code Owners for every file -# nested in that directory, on any level: -/docs/ @all-docs - -# End a path in `/*` to specify Code Owners for every file in -# a directory, but not nested deeper. This code matches -# `docs/index.md` but not `docs/projects/index.md`: -/docs/* @root-docs - -# Include `/**` to specify Code Owners for all subdirectories -# in a directory. This rule matches `docs/projects/index.md` or -# `docs/development/index.md` -/docs/**/*.md @root-docs - -# This code makes matches a `lib` directory nested anywhere in the repository: -lib/ @lib-owner - -# This code match only a `config` directory in the root of the repository: -/config/ @config-owner - -# If the path contains spaces, escape them like this: -path\ with\ spaces/ @space-owner - -# Code Owners section: -[Documentation] -ee/docs @docs -docs @docs - -[Database] -README.md @database -model/db @database - -# This section is combined with the previously defined [Documentation] section: -[DOCUMENTATION] -README.md @docs -``` - -## Troubleshooting - -### Approvals shown as optional - -A Code Owner approval rule is optional if any of these conditions are true: - -- The user or group are not a member of the project. Code Owners [cannot inherit from parent groups](https://gitlab.com/gitlab-org/gitlab/-/issues/288851/). -- [Code Owner approval on a protected branch](protected_branches.md#require-code-owner-approval-on-a-protected-branch) has not been set up. -- The section is [marked as optional](#make-a-code-owners-section-optional). - -### Approvals do not show - -Code Owner approval rules only update when the merge request is created. -If you update the `CODEOWNERS` file, close the merge request and create a new one. - -### User not shown as possible approver - -A user might not show as an approver on the Code Owner merge request approval rules -if any of these conditions are true: - -- A rule prevents the specific user from approving the merge request. - Check the project [merge request approval](merge_requests/approvals/settings.md#edit-merge-request-approval-settings) settings. -- A Code Owner group has a visibility of **private**, and the current user is not a - member of the Code Owner group. + + + + diff --git a/doc/user/project/codeowners/index.md b/doc/user/project/codeowners/index.md new file mode 100644 index 00000000000..e69dba6f977 --- /dev/null +++ b/doc/user/project/codeowners/index.md @@ -0,0 +1,381 @@ +--- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Code Owners **(PREMIUM)** + +> Moved to GitLab Premium in 13.9. + +Use the Code Owners feature to define who has expertise for specific parts of your project's codebase. +Define the owners of files and directories in a repository to: + +- **Require owners to approve changes.** Combine protected branches with Code Owners to require + experts to approve merge requests before they merge into a protected branch. +- **Identify owners.** Code Owner names are displayed on the files and directories they own: + ![Code Owners displayed in UI](../img/codeowners_in_UI_v15_10.png) + +Use Code Owners in combination with merge request +[approval rules](../merge_requests/approvals/rules.md) (either optional or required) +to build a flexible approval workflow: + +- Use **Code Owners** to ensure quality. Define the users who have domain expertise + for specific paths in your repository. +- Use **Approval rules** to define areas of expertise that don't correspond to specific + file paths in your repository. Approval rules help guide merge request creators to + the correct set of reviewers, such as frontend developers or a security team. + +For example: + +| Type | Name | Scope | Comment | +|------|------|--------|------------| +| Approval rule | UX | All files | A user experience (UX) team member reviews the user experience of all changes made in your project. +| Approval rule | Security | All files | A security team member reviews all changes for vulnerabilities. +| Code Owner approval rule | Frontend: Code Style | `*.css` files | A frontend engineer reviews CSS file changes for adherence to project style standards. +| Code Owner approval rule | Backend: Code Review | `*.rb` files | A backend engineer reviews the logic and code style of Ruby files. + +
    + Video introduction: Code Owners. +
    +
    + +
    + + + +## View Code Owners of a file or directory + +To view the Code Owners of a file or directory: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Repository > Files**. +1. Go to the file or directory you want to see the Code Owners for. +1. Optional. Select a branch or tag. + +GitLab shows the Code Owners at the top of the page. + +## Set up Code Owners + +1. Create a `CODEOWNERS` file in your [preferred location](#code-owners-file). +1. Define some rules in the file following the [Code Owners syntax reference](reference.md). + Some suggestions: + - Configure [All eligible approvers](../merge_requests/approvals/rules.md#code-owners-as-eligible-approvers) approval rule. + - [Require Code Owner approval](../protected_branches.md#require-code-owner-approval-on-a-protected-branch) on a protected branch. +1. Commit your changes, and push them up to GitLab. + +### Code Owners file + +A `CODEOWNERS` file (with no extension) specifies the users or +[shared groups](../members/share_project_with_groups.md) responsible for +specific files and directories in a repository. + +Each repository uses a single `CODEOWNERS` file. GitLab checks these locations +in your repository in this order. The first `CODEOWNERS` file found is used, and +all others are ignored: + +1. In the root directory: `./CODEOWNERS`. +1. In the `docs` directory: `./docs/CODEOWNERS`. +1. In the `.gitlab` directory: `./.gitlab/CODEOWNERS`. + +### Add a group as a Code Owner + +> Group and subgroup hierarchy support was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32432) in GitLab 13.0. + +To set the members of a group or subgroup as a Code Owner: + +In the `CODEOWNERS` file, enter text that follows one of these patterns: + +```plaintext +# All group members as Code Owners for a file +file.md @group-x + +# All subgroup members as Code Owners for a file +file.md @group-x/subgroup-y + +# All group and subgroup members as Code Owners for a file +file.md @group-x @group-x/subgroup-y +``` + +#### Group inheritance and eligibility + +> Group and subgroup hierarchy support was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32432) in GitLab 13.0. + +```mermaid +graph TD + A[Parent group X] -->|owns| B[Project A] + A -->|contains| C[Subgroup Y] + C -->|owns| D[Project B] + A-. inherits ownership .-> D +``` + +In this example: + +- **Parent group X** (`group-x`) owns **Project A**. +- **Parent group X** also contains a subgroup, **Subgroup Y**. (`group-x/subgroup-y`) +- **Subgroup Y** owns **Project B**. + +The eligible Code Owners are: + +- **Project A**: the members of **Group X** only, because **Project A** doesn't belong to **Subgroup Y**. +- **Project B**: the members of both **Group X** and **Subgroup Y**. + +##### Inviting subgroups to projects in parent groups + +You can [invite](../members/share_project_with_groups.md) **Subgroup Y** to **Project A** +so that their members also become eligible Code Owners. + +```mermaid +graph LR + A[Parent group X] -->|owns| B[Project A] + A -->|also contains| C[Subgroup Y] + C -.->D{Invite Subgroup Y
    to Project A?} -.->|yes| E[Members of Subgroup Y
    can submit Approvals] + D{Invite Subgroup Y
    to Project A?} -.->|no| F[Members of Subgroup Y
    cannot submit Approvals] + E -.->|Add Subgroup Y
    as Code Owner to Project A| I[Approvals can be
    required] -.-> B + F -.-> |Add Subgroup Y
    as Code Owners to Project A| J[Approvals can only
    be optional] -.-> B +``` + +If you do not invite **Subgroup Y** to **Project A**, but make them Code Owners, their approval +of the merge request becomes optional. + +##### Inviting subgroups to parent groups + +Inviting **Subgroup Y** to a parent group of **Project A** +[is not supported](https://gitlab.com/gitlab-org/gitlab/-/issues/288851). To set **Subgroup Y** as +Code Owners [invite this group directly to the project](#inviting-subgroups-to-projects-in-parent-groups) itself. + +NOTE: +For approval to be required, groups as Code Owners must have a direct membership +(not inherited membership) in the project. Approval can only be optional for groups +that inherit membership. Members in the Code Owners group also must be direct members, +and not inherit membership from any parent groups. + +### Define more specific owners for more specifically defined files or directories + +When a file or directory matches multiple entries in the `CODEOWNERS` file, +the users from last pattern matching the file or directory are used. This enables you +to define more specific owners for more specifically defined files or directories, when +you order the entries in a sensible way. + +For example, in the following `CODEOWNERS` file: + +```plaintext +# This line would match the file terms.md +*.md @doc-team + +# This line would also match the file terms.md +terms.md @legal-team +``` + +The Code Owner for `terms.md` would be `@legal-team`. + +If you use sections, the last pattern matching the file or directory for each section is used. +For example, in a `CODEOWNERS` file using sections: + +```plaintext +[README Owners] +README.md @user1 @user2 +internal/README.md @user4 + +[README other owners] +README.md @user3 +``` + +The Code Owners for the `README.md` in the root directory are `@user1`, `@user2`, +and `@user3`. The Code Owners for `internal/README.md` are `@user4` and `@user3`. + +Only one CODEOWNERS pattern per section is matched to a file path. + +### Organize Code Owners by putting them into sections + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/12137) in GitLab 13.2 [with a flag](../../../administration/feature_flags.md) named `sectional_codeowners`. Disabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42389) in GitLab 13.4. Feature flag `sectional_codeowners` removed. + +You can organize Code Owners by putting them into named sections. + +You can use sections for shared directories, so that multiple +teams can be reviewers. + +To add a section to the `CODEOWNERS` file, enter a section name in brackets, +followed by the files or directories, and users, groups, or subgroups: + +```plaintext +[README Owners] +README.md @user1 @user2 +internal/README.md @user2 +``` + +Each Code Owner in the merge request widget is listed under a label. +The following image shows a **Groups** and **Documentation** section: + +![MR widget - Sectional Code Owners](../img/sectional_code_owners_v13.2.png) + +#### Set default owner for a section + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/371711) in GitLab 15.11 [with a flag](../../../administration/feature_flags.md) named `codeowners_default_owners`. Disabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/115888) in GitLab 15.11. Feature flag `codeowners_default_owners` removed. + +If multiple file paths inside a section share the same ownership, define a default +Code Owner for the section. All paths in that section inherit this default, unless +you override the section default on a specific line. + +Default owners are applied when specific owners are not specified for file paths. +Specific owners defined beside the file path override default owners: + +```plaintext +[Documentation] @docs-team +docs/ +README.md + +[Database] @database-team +model/db/ +config/db/database-setup.md @docs-team +``` + +In this example: + +- `@docs-team` owns all items in the `Documentation` section. +- `@database-team` owns all items in the `Database` section except + `config/db/database-setup.md`, which has an override assigning it to `@docs-team`. + +To combine the syntax for default owners with [optional sections](#make-a-code-owners-section-optional) +and required approvals, place default owners at the end: + +```plaintext +[Documentation][2] @docs-team +docs/ +README.md + +^[Database] @database-team +model/db/ +config/db/database-setup.md @docs-team +``` + +#### Sections with duplicate names + +If multiple sections have the same name, they are combined. +Also, section headings are not case-sensitive. For example: + +```plaintext +[Documentation] +ee/docs/ @docs +docs/ @docs + +[Database] +README.md @database +model/db/ @database + +[DOCUMENTATION] +README.md @docs +``` + +This code results in three entries under the **Documentation** section header, and two +entries under **Database**. The entries defined under the sections **Documentation** and +**DOCUMENTATION** are combined, using the case of the first section. + +#### Make a Code Owners section optional + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/232995) in GitLab 13.8. + +You can designate optional sections in your Code Owners file. Prepend the +section name with the caret `^` character to treat the entire section as optional. +Optional sections enable you to designate responsible parties for various parts +of your codebase, but not require approval from them. This approach provides +a more relaxed policy for parts of your project that are frequently updated, +but don't require stringent reviews. + +In this example, the `[Go]` section is optional: + +```plaintext +[Documentation] +*.md @root + +[Ruby] +*.rb @root + +^[Go] +*.go @root +``` + +The optional Code Owners section displays in merge requests under the **Approval Rules** area: + +![MR widget - Optional Code Owners sections](../img/optional_code_owners_sections_v13_8.png) + +If a section is duplicated in the file, and one of them is marked as optional and the other isn't, the section is required. + +Optional sections in the `CODEOWNERS` file are treated as optional only +when changes are submitted by using merge requests. If a change is submitted directly +to the protected branch, approval from Code Owners is still required, even if the +section is marked as optional. + +### Require multiple approvals from Code Owners + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335451) in GitLab 15.9. + +You can require multiple approvals for the Code Owners sections under the Approval Rules area in merge requests. +Append the section name with a number `n` in brackets. This requires `n` approvals from the Code Owners in this section. +Please note valid entries for `n` are integers `≥ 1`. `[1]` is optional as it is the default. Invalid values for `n` are treated as `1`. + +WARNING: +[Issue #384881](https://gitlab.com/gitlab-org/gitlab/-/issues/385881) proposes changes +to the behavior of this setting. Do not intentionally set invalid values. They may +become valid in the future, and cause unexpected behavior. + +Please confirm you enabled `Require approval from code owners` in `Settings > Repository > Protected branches`, otherwise the Code Owner approvals will be optional. + +In this example, the `[Documentation]` section requires 2 approvals: + +```plaintext +[Documentation][2] +*.md @tech-writer-team + +[Ruby] +*.rb @dev-team +``` + +The `Documentation` Code Owners section under the **Approval Rules** area displays 2 approvals are required: + +![MR widget - Multiple Approval Code Owners sections](../img/multi_approvals_code_owners_sections_v15_9.png) + +### Allowed to Push + +The Code Owner approval and protected branch features do not apply to users who +are **Allowed to push**. + +## Technical Resources + +[Code Owners development guidelines](../../../development/code_owners/index.md) + +## Troubleshooting + +For more information about how the Code Owners feature handles errors, see the +[Code Owners reference](reference.md). + +### Approvals shown as optional + +A Code Owner approval rule is optional if any of these conditions are true: + +- The user or group are not a member of the project. Code Owners [cannot inherit from parent groups](https://gitlab.com/gitlab-org/gitlab/-/issues/288851/). +- [Code Owner approval on a protected branch](../protected_branches.md#require-code-owner-approval-on-a-protected-branch) has not been set up. +- The section is [marked as optional](#make-a-code-owners-section-optional). + +### Approvals do not show + +Code Owner approval rules only update when the merge request is created. +If you update the `CODEOWNERS` file, close the merge request and create a new one. + +### User not shown as possible approver + +A user might not show as an approver on the Code Owner merge request approval rules +if any of these conditions are true: + +- A rule prevents the specific user from approving the merge request. + Check the project [merge request approval](../merge_requests/approvals/settings.md#edit-merge-request-approval-settings) settings. +- A Code Owner group has a visibility of **private**, and the current user is not a + member of the Code Owner group. +- Current user is an external user who does not have permission to the internal Code Owner group. + +### Approval rule is invalid. GitLab has approved this rule automatically to unblock the merge request + +This message may appear if an approval rule uses a Code Owner that is not a direct member of the project. +Check that the group or user has been invited to the project. diff --git a/doc/user/project/codeowners/reference.md b/doc/user/project/codeowners/reference.md new file mode 100644 index 00000000000..d486b689638 --- /dev/null +++ b/doc/user/project/codeowners/reference.md @@ -0,0 +1,371 @@ +--- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Code Owners syntax and error handling **(PREMIUM)** + +This page describes the syntax and error handling used in Code Owners files, +and provides an example file. + +## Code Owners syntax + +### Comments + +Lines beginning with `#` are ignored: + +```plaintext +# This is a comment +``` + +### Sections + +Sections are groups of entries. A section begins with a section heading in square brackets, followed by the entries. + +```plaintext +[Section name] +/path/of/protected/file.rb @username +/path/of/protected/dir/ @group +``` + +#### Section headings + +Section headings must always have a name. They can also be made optional, or +require a number of approvals. A list of default owners can be added to the section heading line. + +```plaintext +# Required section +[Section name] + +# Optional section +^[Section name] + +# Section requiring 5 approvals +[Section name][5] + +# Section with @username as default owner +[Section name] @username + +# Section with @group and @subgroup as default owners and requiring 2 approvals +[Section name][2] @group @subgroup +``` + +#### Section names + +Sections names are defined between square brackets. Section names are not case-sensitive. +[Sections with duplicate names](index.md#sections-with-duplicate-names) are combined. + +```plaintext +[Section name] +``` + +#### Required sections + +Required sections do not include `^` before the [section name](#section-names). + +```plaintext +[Required section] +``` + +#### Optional sections + +Optional sections include a `^` before the [section name](#section-names). + +```plaintext +^[Optional section] +``` + +#### Sections requiring multiple approvals + +Sections requiring multiple approvals include the number of approvals in square brackets after the [section name](#section-names). + +```plaintext +[Section requiring 5 approvals][5] +``` + +NOTE: +Optional sections ignore the number of approvals required. + +#### Sections with default owners + +You can define a default owner for the entries in a section by appending the owners to the [section heading](#section-headings). + +```plaintext +# Section with @username as default owner +[Section name] @username + +# Section with @group and @subgroup as default owners and requiring 2 approvals +[Section name][2] @group @subgroup +``` + +### Code Owner entries + +Each Code Owner entry includes a path followed by one or more owners. + +```plaintext +README.md @username1 +``` + +NOTE: +If an entry is duplicated in a section, [the last entry is used from each section.](index.md#define-more-specific-owners-for-more-specifically-defined-files-or-directories) + +### Relative paths + +If a path does not start with a `/`, the path is treated as if it starts with +a [globstar](#globstar-paths). `README.md` is treated the same way as `/**/README.md`: + +```plaintext +# This will match /README.md, /internal/README.md, /app/lib/README.md +README.md @username + +# This will match /internal/README.md, /docs/internal/README.md, /docs/api/internal/README.md +internal/README.md +``` + +### Absolute paths + +If a path starts with a `/` it matches the root of the repository. + +```plaintext +# Matches only the file named `README.md` in the root of the repository. +/README.md + +# Matches only the file named `README.md` inside the `/docs` directory. +/docs/README.md +``` + +### Directory paths + +If a path ends with `/`, the path matches any file in the directory. + +```plaintext +# This is the same as `/docs/**/*` +/docs/ +``` + +### Wildcard paths + +Wildcards can be used to match one of more characters of a path. + +```plaintext +# Any markdown files in the docs directory +/docs/*.md @username + +# /docs/index file of any filetype +# For example: /docs/index.md, /docs/index.html, /docs/index.xml +/docs/index.* @username + +# Any file in the docs directory with 'spec' in the name. +# For example: /docs/qa_specs.rb, /docs/spec_helpers.rb, /docs/runtime.spec +/docs/*spec* @username + +# README.md files one level deep within the docs directory +# For example: /docs/api/README.md +/docs/*/README.md @username +``` + +### Globstar paths + +Globstars (`**`) can be used to match zero or more directories and subdirectories. + +```plaintext +# This will match /docs/index.md, /docs/api/index.md, /docs/api/graphql/index.md +/docs/**/index.md +``` + +### Entry owners + +Entries must be followed by one or more owner. These can be groups, subgroups, +and users. Order of owners is not important. + +```plaintext +/path/to/entry.rb @group +/path/to/entry.rb @group/subgroup +/path/to/entry.rb @user +/path/to/entry.rb @group @group/subgroup @user +``` + +#### Groups as entry owners + +Groups and subgroups can be owners of an entry. +Each entry can be owned by [one or more owners](#entry-owners). +For more details see the [Add a group as a Code Owner](index.md#add-a-group-as-a-code-owner). + +```plaintext +/path/to/entry.rb @group +/path/to/entry.rb @group/subgroup +/path/to/entry.rb @group @group/subgroup +``` + +### Users as entry owners + +Users can be owners of an entry. Each entry can be owned by +[one or more owners](#entry-owners). + +```plaintext +/path/to/entry.rb @username1 +/path/to/entry.rb @username1 @username2 +``` + +## Error handling in Code Owners + +### Entries with spaces + +Paths containing whitespace must be escaped with backslashes: `path\ with\ spaces/*.md`. +Without the backslashes, the path after the first whitespace is parsed as an owner. +GitLab the parses `folder with spaces/*.md @group` into +`path: "folder", owners: " with spaces/*.md @group"`. + +### Unparsable sections + +If a section heading cannot be parsed, the section is: + +1. Parsed as an entry. +1. Added to the previous section. +1. If no previous section exists, the section is added to the default section. + +For example, this file is missing a square closing bracket: + +```plaintext +* @group + +[Section name +docs/ @docs_group +``` + +GitLab recognizes the heading `[Section name` as an entry. The default section includes 3 rules: + +- Default section + - `*` owned by `@group` + - `[Section` owned by `name` + - `docs/` owned by `@docs_group` + +This file contains an unescaped space between the words `Section` and `name`. +GitLab recognizes the intended heading as an entry: + +```plaintext +[Docs] +docs/**/* @group + +[Section name]{2} @group +docs/ @docs_group +``` + +The `[Docs]` section then includes 3 rules: + +- `docs/**/*` owned by `@group` +- `[Section` owned by `name]{2} @group` +- `docs/` owned by `@docs_group` + +### Malformed owners + +Each entry must contain 1 or more owners to be valid, malformed owners are ignored. +For example `/path/* @group user_without_at_symbol @user_with_at_symbol` +is owned by `@group` and `@user_with_at_symbol`. + +### Inaccessible or incorrect owners + +Inaccessible or incorrect owners are ignored. For example, if `@group`, `@username`, +and `example@gitlab.com` are accessible on the project and we create an entry: + +```plaintext +* @group @grou @username @i_left @i_dont_exist example@gitlab.com invalid@gitlab.com +``` + +GitLab ignores `@grou`, `@i_left`, `@i_dont_exist`, and `invalid@gitlab.com`. + +For more information on who is accessible, see [Add a group as a Code Owner](index.md#add-a-group-as-a-code-owner). + +### Zero owners + +If an entry includes no owners, or zero [accessible owners](#inaccessible-or-incorrect-owners) +exist, the entry is invalid. Because this rule can never be satisfied, GitLab +auto-approves it in merge requests. + +NOTE: +When a protected branch has `Require code owner approval` enabled, rules with +zero owners are still honored. + +### Less than 1 required approval + +When [defining the number of approvals](index.md#require-multiple-approvals-from-code-owners) for a section, +the minimum number of approvals is `1`. Setting the number of approvals to +`0` results in GitLab requiring one approval. + +## Example `CODEOWNERS` file + +```plaintext +# This is an example of a CODEOWNERS file. +# Lines that start with `#` are ignored. + +# app/ @commented-rule + +# Specify a default Code Owner by using a wildcard: +* @default-codeowner + +# Specify multiple Code Owners by using a tab or space: +* @multiple @code @owners + +# Rules defined later in the file take precedence over the rules +# defined before. +# For example, for all files with a filename ending in `.rb`: +*.rb @ruby-owner + +# Files with a `#` can still be accessed by escaping the pound sign: +\#file_with_pound.rb @owner-file-with-pound + +# Specify multiple Code Owners separated by spaces or tabs. +# In the following case the CODEOWNERS file from the root of the repo +# has 3 Code Owners (@multiple @code @owners): +CODEOWNERS @multiple @code @owners + +# You can use both usernames or email addresses to match +# users. Everything else is ignored. For example, this code +# specifies the `@legal` and a user with email `janedoe@gitlab.com` as the +# owner for the LICENSE file: +LICENSE @legal this_does_not_match janedoe@gitlab.com + +# Use group names to match groups, and nested groups to specify +# them as owners for a file: +README @group @group/with-nested/subgroup + +# End a path in a `/` to specify the Code Owners for every file +# nested in that directory, on any level: +/docs/ @all-docs + +# End a path in `/*` to specify Code Owners for every file in +# a directory, but not nested deeper. This code matches +# `docs/index.md` but not `docs/projects/index.md`: +/docs/* @root-docs + +# Include `/**` to specify Code Owners for all subdirectories +# in a directory. This rule matches `docs/projects/index.md` or +# `docs/development/index.md` +/docs/**/*.md @root-docs + +# This code makes matches a `lib` directory nested anywhere in the repository: +lib/ @lib-owner + +# This code match only a `config` directory in the root of the repository: +/config/ @config-owner + +# If the path contains spaces, escape them like this: +path\ with\ spaces/ @space-owner + +# Code Owners section: +[Documentation] +ee/docs @docs +docs @docs + +# Use of default owners for a section. In this case, all files (*) are owned by +the dev team except the README.md and data-models which are owned by other teams. +[Development] @dev-team +* +README.md @docs-team +data-models/ @data-science-team + +# This section is combined with the previously defined [Documentation] section: +[DOCUMENTATION] +README.md @docs +``` diff --git a/doc/user/project/deploy_boards.md b/doc/user/project/deploy_boards.md index a68f9550ebf..09a443614d0 100644 --- a/doc/user/project/deploy_boards.md +++ b/doc/user/project/deploy_boards.md @@ -1,11 +1,11 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: howto, reference --- -# Deploy boards (DEPRECATED) **(FREE)** +# Deploy boards (deprecated) **(FREE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1589) in GitLab 9.0. > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212320) from GitLab Premium to GitLab Free in 13.8. diff --git a/doc/user/project/deploy_keys/index.md b/doc/user/project/deploy_keys/index.md index fc88535dc77..13ee07097e1 100644 --- a/doc/user/project/deploy_keys/index.md +++ b/doc/user/project/deploy_keys/index.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -15,8 +15,7 @@ Depending on your needs, you might want to use a [deploy token](../deploy_tokens |------------------|-------------|--------------| | Sharing | Shareable between multiple projects, even those in different groups. | Belong to a project or group. | | Source | Public SSH key generated on an external host. | Generated on your GitLab instance, and is provided to users only at creation time. | -| Validity | Valid as long as it's registered and enabled, and the user that created it exists. | Can be given an expiration date. | -| Registry access | Cannot access a package registry. | Can read from and write to a package registry. | +| Accessible resources | Git repository over SSH | Git repository over HTTP, package registry, and container registry. | Deploy keys can't be used for Git operations if [external authorization](../../admin_area/settings/external_authorization.md) is enabled. @@ -41,10 +40,8 @@ A deploy key is given a permission level when it is created: You can change a deploy key's permission level after creating it. Changing a project deploy key's permissions only applies for the current project. -When a read-write deploy key is used to push a commit, GitLab checks if the creator of the -deploy key has permission to access the resource. - -For example: +Although a deploy key is a secret that isn't associated with a specific user, +GitLab authorizes the creator of the deploy key if the Git-command triggers additional processes. For example: - When a deploy key is used to push a commit to a [protected branch](../protected_branches.md), the _creator_ of the deploy key must have access to the branch. @@ -52,6 +49,15 @@ For example: deploy key must have access to the CI/CD resources, including protected environments and secret variables. +## Security implications + +The intended use case for deploy keys is for non-human interaction with GitLab, for example: an automated script running on a server in your organization. +As with all sensitive information, you should ensure only those who need access to the secret can read it. +For human interactions, use credentials tied to users such as Personal Access Tokens. + +To help detect a potential secret leak, you can use the +[Audit Event](../../../administration/audit_event_streaming.md#example-payloads-for-ssh-events-with-deploy-key) feature. + ## View deploy keys To view the deploy keys available to a project: @@ -80,6 +86,7 @@ Prerequisites: 1. Complete the fields. 1. Optional. To grant `read-write` permission, select the **Grant write permissions to this key** checkbox. +1. Optional. Update the **Expiration date**. A project deploy key is enabled when it is created. You can modify only a project deploy key's name and permissions. @@ -88,9 +95,9 @@ name and permissions. Prerequisites: -- You must have administrator access. -- [Generate an SSH key pair](../../ssh.md#generate-an-ssh-key-pair). Put the private SSH - key on the host that requires access to the repository. +- You must have administrator access to the instance. +- You must [generate an SSH key pair](../../ssh.md#generate-an-ssh-key-pair). +- You must put the private SSH key on the host that requires access to the repository. To create a public deploy key: @@ -153,7 +160,7 @@ There are a few scenarios where a deploy key fails to push to a - The owner associated to a deploy key does not have access to the protected branch. - The owner associated to a deploy key does not have [membership](../members/index.md) to the project of the protected branch. -- **No one** is selected in [the **Allowed to push** section](../protected_branches.md#configure-a-protected-branch) of the protected branch. +- **No one** is selected in [the **Allowed to push and merge** section](../protected_branches.md#configure-a-protected-branch) of the protected branch. All deploy keys are associated to an account. Since the permissions for an account can change, this might lead to scenarios where a deploy key that was working is suddenly unable to push to a protected branch. diff --git a/doc/user/project/deploy_tokens/index.md b/doc/user/project/deploy_tokens/index.md index cfb382d73e2..a81ccd043bd 100644 --- a/doc/user/project/deploy_tokens/index.md +++ b/doc/user/project/deploy_tokens/index.md @@ -1,6 +1,6 @@ --- -stage: Release -group: Release +stage: Deploy +group: Environments info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/project/description_templates.md b/doc/user/project/description_templates.md index ffbe7447aa8..f0ced95c8eb 100644 --- a/doc/user/project/description_templates.md +++ b/doc/user/project/description_templates.md @@ -16,7 +16,7 @@ You might want to use these templates: - For different stages of your workflow, for example, feature proposal, feature improvement, or a bug report. - For every issue or merge request for a specific project, so the layout is consistent. -- For a [Service Desk email template](service_desk.md#new-service-desk-issues). +- For a [Service Desk email template](service_desk.md#use-a-custom-template-for-service-desk-issues). For description templates to work, they must be: @@ -46,10 +46,11 @@ and see if you can find your description template in the **Choose a template** d ## Create a merge request template Similarly to issue templates, create a new Markdown (`.md`) file inside the -`.gitlab/merge_request_templates/` directory in your repository. Commit and -push to your default branch. +`.gitlab/merge_request_templates/` directory in your repository. Unlike issue +templates, merge requests have [additional inheritance rules](merge_requests/creating_merge_requests.md) +that depend on the contents of commit messages and branch names. -To create a merge request description template: +To create a merge request description template for a project: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Repository**. @@ -87,6 +88,10 @@ For example: `https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_templat > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89810) in GitLab 15.7. +NOTE: +This feature is available only for +[the default template](#set-a-default-template-for-merge-requests-and-issues). + When you save a merge request for the first time, GitLab replaces these variables in your merge request template with their values: @@ -95,7 +100,7 @@ your merge request template with their values: | `%{all_commits}` | Messages from all commits in the merge request. Limited to 100 most recent commits. Skips commit bodies exceeding 100 KiB and merge commit messages. | `* Feature introduced`

    `This commit implements feature`
    `Changelog:added`

    `* Bug fixed`

    `* Documentation improved`

    `This commit introduced better docs.` | | `%{co_authored_by}` | Names and emails of commit authors in a `Co-authored-by` Git commit trailer format. Limited to authors of 100 most recent commits in merge request. | `Co-authored-by: Zane Doe `
    `Co-authored-by: Blake Smith ` | | `%{first_commit}` | Full message of the first commit in merge request diff. | `Update README.md` | -| `%{first_multiline_commit}` | Full message of the first commit that's not a merge commit and has more than one line in message body. Merge request title if all commits aren't multiline. | `Update README.md`

    `Improved project description in readme file.` | +| `%{first_multiline_commit}` | Full message of the first commit that's not a merge commit and has more than one line in message body. Merge request title if all commits aren't multiline. | `Update README.md`

    `Improved project description in readme file.` | | `%{source_branch}` | The name of the branch being merged. | `my-feature-branch` | | `%{target_branch}` | The name of the branch that the changes are applied to. | `main` | @@ -148,11 +153,11 @@ To set a default description template for merge requests, either: - [In GitLab 14.8 and later](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78302), [create a merge request template](#create-a-merge-request-template) named `Default.md` (case insensitive) and save it in `.gitlab/merge_request_templates/`. This [doesn't overwrite](#priority-of-default-description-templates) the default template if one has been set in the project settings. -- Users on GitLab Premium and higher: set the default template in project settings: +- Users on GitLab Premium and Ultimate: set the default template in project settings: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Settings > Merge requests**. - 1. In the **Merge commit message template** section, fill in **Default description template for merge requests**. + 1. In the **Default description template for merge requests** section, fill in the text area. 1. Select **Save changes**. To set a default description template for issues, either: @@ -160,12 +165,12 @@ To set a default description template for issues, either: - [In GitLab 14.8 and later](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78302), [create an issue template](#create-an-issue-template) named `Default.md` (case insensitive) and save it in `.gitlab/issue_templates/`. This [doesn't overwrite](#priority-of-default-description-templates) the default template if one has been set in the project settings. -- Users on GitLab Premium and higher: set the default template in project settings: +- Users on GitLab Premium and Ultimate: set the default template in project settings: 1. On the top bar, select **Main menu > Projects** and find your project. - 1. On the left sidebar, select **Settings**. - 1. Expand **Default issue template**. - 1. Fill in the **Default description template for issues** text area. + 1. On the left sidebar, select **Settings > General**. + 1. Expand **Default description template for issues**. + 1. Fill in the text area. 1. Select **Save changes**. Because GitLab merge request and issues support [Markdown](../markdown.md), you can use it to format @@ -176,7 +181,7 @@ You can also provide `issues_template` and `merge_requests_template` attributes #### Priority of default description templates -When you set [merge request and issue description templates](#set-a-default-template-for-merge-requests-and-issues) +When you set [issue description templates](#set-a-default-template-for-merge-requests-and-issues) in various places, they have the following priorities in a project. The ones higher up override the ones below: @@ -184,6 +189,9 @@ The ones higher up override the ones below: 1. `Default.md` (case insensitive) from the parent group. 1. `Default.md` (case insensitive) from the project repository. +Merge requests have [additional inheritance rules](merge_requests/creating_merge_requests.md) +that depend on the contents of commit messages and branch names. + ## Example description template We use description templates for issues and merge requests in the diff --git a/doc/user/project/file_lock.md b/doc/user/project/file_lock.md index 8d3446994e8..c81ba4b7dd3 100644 --- a/doc/user/project/file_lock.md +++ b/doc/user/project/file_lock.md @@ -58,7 +58,7 @@ git-lfs --version ``` If it doesn't recognize this command, you must install it. There are -several [installation methods](https://git-lfs.github.com/) that you can +several [installation methods](https://git-lfs.com/) that you can choose according to your OS. To install it with Homebrew: ```shell @@ -195,8 +195,7 @@ Suggested workflow for shared projects: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/440) in GitLab 8.9. This process allows you to lock one file at a time through the GitLab UI and -requires access to [GitLab Premium](https://about.gitlab.com/pricing/) -or higher tiers. +requires access to the [GitLab Premium or Ultimate tier](https://about.gitlab.com/pricing/). Default branch file and directory locks only apply to the [default branch](repository/branches/default.md) set in the project's settings. @@ -209,7 +208,7 @@ requests that modify locked files. Unlock the file to allow changes. To lock a file: 1. Open the file or directory in GitLab. -1. In the upper right, above the file, select **Lock**. +1. In the upper-right corner, above the file, select **Lock**. 1. On the confirmation dialog box, select **OK**. If you do not have permission to lock the file, the button is not enabled. @@ -221,7 +220,7 @@ To view the user who locked the file (if it was not you), hover over the button. To view and remove file locks: 1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Repository > Locked Files**. +1. On the left sidebar, select **Repository > Locked files**. This list shows all the files locked either through LFS or GitLab UI. diff --git a/doc/user/project/git_attributes.md b/doc/user/project/git_attributes.md index 1feb17b19c8..698b888a32a 100644 --- a/doc/user/project/git_attributes.md +++ b/doc/user/project/git_attributes.md @@ -5,7 +5,7 @@ info: "To determine the technical writer assigned to the Stage/Group associated type: reference --- -# Git Attributes **(FREE)** +# Git attributes **(FREE)** GitLab supports defining custom [Git attributes](https://git-scm.com/docs/gitattributes) such as what files to treat as binary, and what language to use for syntax highlighting @@ -14,14 +14,41 @@ diffs. To define these attributes, create a file called `.gitattributes` in the root directory of your repository and push it to the default branch of your project. -## Encoding Requirements +## Encoding requirements The `.gitattributes` file _must_ be encoded in UTF-8 and _must not_ contain a Byte Order Mark. If a different encoding is used, the file's contents are ignored. -## Syntax Highlighting +## Support for mixed file encodings + +GitLab attempts to detect the encoding of files automatically, but defaults to UTF-8 unless +the detector is confident of a different type (such as `ISO-8859-1`). Incorrect encoding +detection can result in some characters not displaying in the text, such as accented characters in a +non-UTF-8 encoding. + +Git has built-in support for handling this eventuality and automatically converts files between +a designated encoding and UTF-8 for the repository itself. Configure support for mixed file encoding in the `.gitattributes` +file using the `working-tree-encoding` attribute. + +Example: + +```plaintext +*.xhtml text working-tree-encoding=ISO-8859-1 +``` + +With this example configuration, Git maintains all `.xhtml` files in the repository in ISO-8859-1 +encoding in the local tree, but converts to and from UTF-8 when committing into the repository. GitLab +renders the files accurately as it only sees correctly encoded UTF-8. + +If applying this configuration to an existing repository, files may need to be touched and recommitted +if the local copy has the correct encoding but the repository does not. This can +be performed for the whole repository by running `git add --renormalize .`. + +For more information, see [working-tree-encoding](https://git-scm.com/docs/gitattributes#_working_tree_encoding). + +## Syntax highlighting The `.gitattributes` file can be used to define which language to use when -syntax highlighting files and diffs. See -["Syntax Highlighting"](highlighting.md) for more information. +syntax highlighting files and diffs. For more information, see +[Syntax highlighting](highlighting.md). diff --git a/doc/user/project/img/codeowners_in_UI_v15_10.png b/doc/user/project/img/codeowners_in_UI_v15_10.png new file mode 100644 index 00000000000..c643d333791 Binary files /dev/null and b/doc/user/project/img/codeowners_in_UI_v15_10.png differ diff --git a/doc/user/project/img/issue_board_system_notes_v13_6.png b/doc/user/project/img/issue_board_system_notes_v13_6.png deleted file mode 100644 index 4958f63541e..00000000000 Binary files a/doc/user/project/img/issue_board_system_notes_v13_6.png and /dev/null differ diff --git a/doc/user/project/img/multi_approvals_code_owners_sections_v15_9.png b/doc/user/project/img/multi_approvals_code_owners_sections_v15_9.png new file mode 100644 index 00000000000..a7fea76d5b1 Binary files /dev/null and b/doc/user/project/img/multi_approvals_code_owners_sections_v15_9.png differ diff --git a/doc/user/project/img/protected_tags_list_v12_3.png b/doc/user/project/img/protected_tags_list_v12_3.png deleted file mode 100644 index 6a30f615f2f..00000000000 Binary files a/doc/user/project/img/protected_tags_list_v12_3.png and /dev/null differ diff --git a/doc/user/project/img/protected_tags_page_v12_3.png b/doc/user/project/img/protected_tags_page_v12_3.png deleted file mode 100644 index 841e19af8a7..00000000000 Binary files a/doc/user/project/img/protected_tags_page_v12_3.png and /dev/null differ diff --git a/doc/user/project/img/protected_tags_permissions_dropdown_v12_3.png b/doc/user/project/img/protected_tags_permissions_dropdown_v12_3.png deleted file mode 100644 index 913d4725d53..00000000000 Binary files a/doc/user/project/img/protected_tags_permissions_dropdown_v12_3.png and /dev/null differ diff --git a/doc/user/project/import/bitbucket.md b/doc/user/project/import/bitbucket.md index 7114974d8db..b523e007f0e 100644 --- a/doc/user/project/import/bitbucket.md +++ b/doc/user/project/import/bitbucket.md @@ -1,7 +1,6 @@ --- -type: reference, howto stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -33,10 +32,13 @@ When importing: ## Prerequisites +> Requirement for Maintainer role instead of Developer role introduced in GitLab 16.0 and backported to GitLab 15.11.1 and GitLab 15.10.5. + - [Bitbucket Cloud integration](../../../integration/bitbucket.md) must be enabled. If that integration is not enabled, ask your GitLab administrator to enable it. The Bitbucket Cloud integration is enabled by default on GitLab.com. -- At least the Maintainer role on the destination group to import to. Using the Developer role for this purpose was - [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387891) in GitLab 15.8 and will be removed in GitLab 16.0. +- [Bitbucket Cloud import source](../../admin_area/settings/visibility_and_access_controls.md#configure-allowed-import-sources) must be enabled. If not enabled, ask your + GitLab administrator to enable it. The Bitbucket Cloud import source is enabled by default on GitLab.com. +- At least the Maintainer role on the destination group to import to. ## How it works diff --git a/doc/user/project/import/bitbucket_server.md b/doc/user/project/import/bitbucket_server.md index e6d2e3e00b6..4d3a6eb87e0 100644 --- a/doc/user/project/import/bitbucket_server.md +++ b/doc/user/project/import/bitbucket_server.md @@ -1,7 +1,6 @@ --- -type: reference, howto stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -26,11 +25,18 @@ created as private in GitLab as well. > Ability to re-import projects [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/23905) in GitLab 15.9. -Prerequisites: +You can import Bitbucket repositories to GitLab. -- An administrator must enable **Bitbucket Server** in **Admin > Settings > General > Visibility and access controls > Import sources**. -- At least the Maintainer role on the destination group to import to. Using the Developer role for this purpose was - [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387891) in GitLab 15.8 and will be removed in GitLab 16.0. +### Prerequisites + +> Requirement for Maintainer role instead of Developer role introduced in GitLab 16.0 and backported to GitLab 15.11.1 and GitLab 15.10.5. + +- [Bitbucket Server import source](../../admin_area/settings/visibility_and_access_controls.md#configure-allowed-import-sources) + must be enabled. If not enabled, ask your GitLab administrator to enable it. The Bitbucket Server import source is enabled + by default on GitLab.com. +- At least the Maintainer role on the destination group to import to. + +### Import repositories To import your Bitbucket repositories: @@ -120,7 +126,16 @@ If the project import completes but LFS objects can't be downloaded or cloned, y password or personal access token containing special characters. For more information, see [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/337769). +### Pull requests are missing + +Importing large projects spawns a process that can consume a lot of memory. Sometimes you can see messages such as `Sidekiq worker RSS out of range` in the +[Sidekiq logs](../../../administration/logs/index.md#sidekiq-logs). This can mean that MemoryKiller is interrupting the `RepositoryImportWorker` because it's using +too much memory. + +To resolve this problem, temporarily increase the `SIDEKIQ_MEMORY_KILLER_MAX_RSS` environment variable using +[custom environment variables](https://docs.gitlab.com/omnibus/settings/environment-variables.html) from the default `2000000` value to a larger value like `3000000`. +Consider memory constraints on the system before deciding to increase `SIDEKIQ_MEMORY_KILLER_MAX_RSS`. + ## Related topics -For information on automating user, group, and project import API calls, see -[Automate group and project import](index.md#automate-group-and-project-import). +- [Automate group and project import](index.md#automate-group-and-project-import) diff --git a/doc/user/project/import/clearcase.md b/doc/user/project/import/clearcase.md index 0878476d59f..35ed7a043d0 100644 --- a/doc/user/project/import/clearcase.md +++ b/doc/user/project/import/clearcase.md @@ -1,7 +1,6 @@ --- -type: reference, howto stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/project/import/cvs.md b/doc/user/project/import/cvs.md index 00aebb75a50..fb3416a3492 100644 --- a/doc/user/project/import/cvs.md +++ b/doc/user/project/import/cvs.md @@ -1,14 +1,13 @@ --- -type: reference, howto stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Migrating from CVS **(FREE)** [CVS](https://savannah.nongnu.org/projects/cvs) is an old centralized version -control system similar to [SVN](svn.md). +control system similar to [SVN](https://subversion.apache.org/). ## CVS vs Git @@ -71,6 +70,5 @@ Here's a few links to get you started with the migration: - [Migrate using the `cvs-fast-export` tool](https://gitlab.com/esr/cvs-fast-export) - [Stack Overflow post on importing the CVS repository](https://stackoverflow.com/a/11490134/974710) -- [Convert a CVS repository to Git](https://www.techrepublic.com/article/convert-cvs-repositories-to-git/) - [Man page of the `git-cvsimport` tool](https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git-cvsimport.html) - [Migrate using `reposurgeon`](http://www.catb.org/~esr/reposurgeon/repository-editing.html#conversion) diff --git a/doc/user/project/import/fogbugz.md b/doc/user/project/import/fogbugz.md index d9e03662434..692ec1390d2 100644 --- a/doc/user/project/import/fogbugz.md +++ b/doc/user/project/import/fogbugz.md @@ -1,7 +1,6 @@ --- -type: reference, howto stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -16,10 +15,16 @@ The importer imports all of your cases and comments with the original case numbers and timestamps. You can also map FogBugz users to GitLab users. -Prerequisite: +## Prerequisites -- At least the Maintainer role on the destination group to import to. Using the Developer role for this purpose was - [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387891) in GitLab 15.8 and will be removed in GitLab 16.0. +> Requirement for Maintainer role instead of Developer role introduced in GitLab 16.0 and backported to GitLab 15.11.1 and GitLab 15.10.5. + +- [FogBugz import source](../../admin_area/settings/visibility_and_access_controls.md#configure-allowed-import-sources) + must be enabled. If not enabled, ask your GitLab administrator to enable it. The FogBugz import source is enabled + by default on GitLab.com. +- At least the Maintainer role on the destination group to import to. + +## Import project from FogBugz To import your project from FogBugz: diff --git a/doc/user/project/import/gitea.md b/doc/user/project/import/gitea.md index 404bb4c8600..62cda62c2fe 100644 --- a/doc/user/project/import/gitea.md +++ b/doc/user/project/import/gitea.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -8,15 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/381902) in GitLab 15.8, GitLab no longer automatically creates namespaces or groups that don't exist. GitLab also no longer falls back to using the user's personal namespace if the namespace or group name is taken. -Import your projects from Gitea to GitLab with minimal effort. - -NOTE: -This requires Gitea `v1.0.0` or later. - -Prerequisite: - -- At least the Maintainer role on the destination group to import to. Using the Developer role for this purpose was - [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387891) in GitLab 15.8 and will be removed in GitLab 16.0. +Import your projects from Gitea to GitLab. The Gitea importer can import: @@ -30,13 +22,21 @@ The Gitea importer can import: When importing, repository public access is retained. If a repository is private in Gitea, it's created as private in GitLab as well. -## How it works - Because Gitea isn't an OAuth provider, author/assignee can't be mapped to users in your GitLab instance. This means the project creator (usually the user that started the import process) is set as the author. A reference, however, is kept on the issue about the original Gitea author. +## Prerequisites + +> Requirement for Maintainer role instead of Developer role introduced in GitLab 16.0 and backported to GitLab 15.11.1 and GitLab 15.10.5. + +- Gitea version 1.0.0 or later. +- [Gitea import source](../../admin_area/settings/visibility_and_access_controls.md#configure-allowed-import-sources) + must be enabled. If not enabled, ask your GitLab administrator to enable it. The Gitea import source is enabled + by default on GitLab.com. +- At least the Maintainer role on the destination group to import to. + ## Import your Gitea repositories The importer page is visible when you create a new project. @@ -75,10 +75,9 @@ From there, you can view the import statuses of your Gitea repositories: You also can: -- Import all of your Gitea projects in one go by selecting **Import all projects** - in the upper left corner. -- Filter projects by name. If filter is applied, selecting **Import all projects** - imports only matched projects. +- In the upper-left corner, select **Import all projects** to import all of your Gitea projects at once. +- Filter projects by name. If a filter is applied, **Import all projects** + imports only selected projects. ![Gitea importer page](img/import_projects_from_gitea_importer_v12_3.png) diff --git a/doc/user/project/import/github.md b/doc/user/project/import/github.md index eeebb5a166c..b2b1ede12d4 100644 --- a/doc/user/project/import/github.md +++ b/doc/user/project/import/github.md @@ -1,13 +1,14 @@ --- -type: reference, howto stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Import your project from GitHub to GitLab **(FREE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/381902) in GitLab 15.8, GitLab no longer automatically creates namespaces or groups that don't exist. GitLab also no longer falls back to using the user's personal namespace if the namespace or group name is taken. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/381902) in GitLab 15.8, GitLab no longer automatically creates namespaces or groups that don't exist. GitLab also no longer falls back to using the user's personal namespace if the namespace or group name is taken. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/388716) in GitLab 15.10, you no longer need to add any users to the parent group in GitLab to successfully import the **Require a pull request before merging - Allow specified actors to bypass required pull requests** branch protection rule. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/378267) in GitLab 15.11, GitLab instances behind proxies no longer require `github.com` and `api.github.com` entries in the [allowlist for local requests](../../../security/webhooks.md#allow-outbound-requests-to-certain-ip-addresses-and-domains). You can import your GitHub projects from either GitHub.com or GitHub Enterprise. Importing projects does not migrate or import any types of groups or organizations from GitHub to GitLab. @@ -30,18 +31,19 @@ When importing projects: imported with a naming scheme similar to `GH-SHA-username/pull-request-number/fork-name/branch`. This may lead to a discrepancy in branches compared to those of the GitHub repository. -For additional technical details, refer to the [GitHub Importer](../../../development/github_importer.md) -developer documentation. - -For an overview of the import process, see the video [Migrating from GitHub to GitLab](https://youtu.be/VYOXuOg9tQI). + +For an overview of the import process, see [Migrating from GitHub to GitLab](https://youtu.be/VYOXuOg9tQI). ## Prerequisites +> Requirement for Maintainer role instead of Developer role introduced in GitLab 16.0 and backported to GitLab 15.11.1 and GitLab 15.10.5. + To import projects from GitHub: -- You must have at least the Maintainer role on the destination group to import to. Using the Developer role for this - purpose was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387891) in GitLab 15.8 and will be removed in - GitLab 16.0. +- [GitHub import source](../../admin_area/settings/visibility_and_access_controls.md#configure-allowed-import-sources) + must be enabled. If not enabled, ask your GitLab administrator to enable it. The GitHub import source is enabled + by default on GitLab.com. +- You must have at least the Maintainer role on the destination group to import to. - Each GitHub author and assignee in the repository must have a [public-facing email address](https://docs.github.com/en/account-and-profile/setting-up-and-managing-your-personal-account-on-github/managing-email-preferences/setting-your-commit-email-address) on GitHub that matches their GitLab email address (regardless of how the account was created). If their email address @@ -54,17 +56,18 @@ To import projects from GitHub: are properly mapped to the same user in GitLab. GitHub Enterprise does not require this field to be populated so you may have to add it on existing accounts. -See also [Branch protection rules and project settings](#branch-protection-rules-and-project-settings) for additional -prerequisites for those imports. +Because of a [known issue](https://gitlab.com/gitlab-org/gitlab/-/issues/383047), if you are using GitHub as an OmniAuth provider, ensure that the URL +perimeter is specified in the [OmniAuth configuration](../../../integration/github.md#enable-github-oauth-in-gitlab). ### Importing from GitHub Enterprise to self-managed GitLab If you are importing from GitHub Enterprise to a self-managed GitLab instance: - You must first enable the [GitHub integration](../../../integration/github.md). -- If GitLab is behind a HTTP/HTTPS proxy, you must populate the [allowlist for local requests](../../../security/webhooks.md#create-an-allowlist-for-local-requests) - with `github.com` and `api.github.com` to solve the hostname. For more information, read the issue - [Importing a GitHub project requires DNS resolution even when behind a proxy](https://gitlab.com/gitlab-org/gitlab/-/issues/37941). +- GitHub must be enabled as an import source in the + [Admin Area](../../admin_area/settings/visibility_and_access_controls.md#configure-allowed-import-sources). +- For GitLab 15.10 and earlier, you must add `github.com` and `api.github.com` entries in the + [allowlist for local requests](../../../security/webhooks.md#allow-outbound-requests-to-certain-ip-addresses-and-domains). ### Importing from GitHub.com to self-managed GitLab @@ -82,10 +85,9 @@ Before you begin, ensure that any GitHub user you want to map to a GitLab user h [publicly visible email address](https://docs.github.com/en/rest/users#get-a-user) on GitHub. If you are importing to GitLab.com, you can alternatively import GitHub repositories using a [personal access token](#use-a-github-token). -We do not recommend this method, as it does not associate all user activity (such as issues and pull requests) with matching GitLab users. -User-matching attempts occur in that order, and if a user is not identified either way, the activity is associated with -the user account that is performing the import. +If a GitHub user's public email address doesn't match any GitLab user email address, the user's activity is associated with the user account that is +performing the import. NOTE: If you are using a self-managed GitLab instance or if you are importing from GitHub Enterprise, this process requires that you have configured @@ -103,9 +105,7 @@ Prerequisite: - Authentication token with administrator access. -Using a personal access token to import projects is not recommended. If you are a GitLab.com user, -you can use a personal access token to import your project from GitHub, but this method cannot -associate all user activity (such as issues and pull requests) with matching GitLab users. +If you are a GitLab.com user, you can use a personal access token to import your project from GitHub. If you are an administrator of a self-managed GitLab instance or if you are importing from GitHub Enterprise, you cannot use a personal access token. The [GitHub integration method (above)](#use-the-github-integration) is recommended for all users. @@ -124,6 +124,21 @@ If you are not using the GitHub integration, you can still perform an authorizat To use a newer personal access token in imports after previously performing these steps, sign out of your GitLab account and sign in again, or revoke the older personal access token in GitHub. +### Filter repositories list + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/385113) in GitLab 16.0. + +After you authorize access to your GitHub repositories, GitLab redirects you to the importer page and +your GitHub repositories are listed. + +Use one of the following tabs to filter the list of repositories: + +- **Owner** (default): Filter the list to the repositories that you are the owner of. +- **Collaborated**: Filter the list to the repositories that you have contributed to. +- **Organization**: Filter the list to the repositories that belong to an organization you are a member of. + +When the **Organization** tab is selected, you can further narrow down your search by selecting an available GitHub organization from a dropdown. + ### Select additional items to import > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/373705) in GitLab 15.5. @@ -140,15 +155,15 @@ You can choose to import these items, but this could significantly increase impo - **Import issue and pull request events**. - **Use alternative comments import method**. - **Import Markdown attachments**. +- **Import collaborators** (selected by default). Leaving it selected might result in new users using a seat in the group or namespace, + and being granted permissions [as high as project owner](#collaborators-members). Only direct collaborators are imported. + Outside collaborators are never imported. ### Select which repositories to import > - Ability to cancel pending or active imports [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/247325) in GitLab 15.7. > - Ability to re-import projects [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/23905) in GitLab 15.9. -After you have authorized access to your GitHub repositories, you are redirected to the GitHub importer page and -your GitHub repositories are listed. - By default, the proposed repository namespaces match the names as they exist in GitHub, but based on your permissions, you can choose to edit these names before you proceed to import any of them. @@ -170,6 +185,18 @@ Completed imports can be re-imported by selecting **Re-import** and specifying n ![GitHub importer page](img/import_projects_from_github_importer_v12_3.png) +### Check status of imports + +> Details of partially completed imports with a list of entities that failed to import [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/386748) in GitLab 16.0. + +After imports are completed, they can be in one of three states: + +- **Completed**: GitLab imported all repository entities. +- **Partially completed**: GitLab failed to import some repository entities. +- **Failed**: GitLab imported no repository entities. + +Expand **Details** to see a list of [repository entities](#imported-data) that failed to import. + ## Mirror a repository and share pipeline status **(PREMIUM)** Depending on your GitLab tier, [repository mirroring](../repository/mirror/index.md) can be set up to keep @@ -209,17 +236,19 @@ The following items of a project are imported: - Repository description. - Git repository data. - Branch protection rules. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22650) in GitLab 15.4. +- Collaborators (members). [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/388716) in GitLab 15.10. From GitLab 16.0, can + be imported [as an additional item](#select-additional-items-to-import). - Issues. - Pull requests. - Wiki pages. - Milestones. - Labels. -- Release note descriptions. +- Release notes content. - Attachments for: - Release notes. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15620) in GitLab 15.4. - - Comments and notes. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18052) in GitLab 15.5. + - Comments. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18052) in GitLab 15.5. - Issue description. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18052) in GitLab 15.5. - - Merge Request description. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18052) in GitLab 15.5. + - Pull Request description. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18052) in GitLab 15.5. All attachment imports are disabled by default behind `github_importer_attachments_import` [feature flag](../../../administration/feature_flags.md). From GitLab 15.5, can @@ -232,7 +261,7 @@ The following items of a project are imported: - Pull request "merged by" information. - Pull request comments replies in discussions. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336596) in GitLab 14.5. -- Diff Notes suggestions. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/340624) in GitLab 14.7. +- Pull request review comments suggestions. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/340624) in GitLab 14.7. - Issue events and pull requests events. [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/7673) in GitLab 15.4 with `github_importer_issue_events_import` [feature flag](../../../administration/feature_flags.md) disabled by default. From GitLab 15.5, can be imported [as an additional item](#select-additional-items-to-import). The feature flag was @@ -252,14 +281,11 @@ When they are imported, supported GitHub branch protection rules are mapped to e | GitHub rule | GitLab rule | Introduced in | | :---------------------------------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------ | | **Require conversation resolution before merging** for the project's default branch | **All threads must be resolved** [project setting](../../discussions/index.md#prevent-merge-unless-all-threads-are-resolved) | [GitLab 15.5](https://gitlab.com/gitlab-org/gitlab/-/issues/371110) | -| **Require a pull request before merging** | **No one** option in the **Allowed to push** list of [branch protection settings](../protected_branches.md#configure-a-protected-branch) | [GitLab 15.5](https://gitlab.com/gitlab-org/gitlab/-/issues/370951) | +| **Require a pull request before merging** | **No one** option in the **Allowed to push and merge** list of [branch protection settings](../protected_branches.md#configure-a-protected-branch) | [GitLab 15.5](https://gitlab.com/gitlab-org/gitlab/-/issues/370951) | | **Require signed commits** for the project's default branch | **Reject unsigned commits** GitLab [push rule](../repository/push_rules.md#prevent-unintended-consequences) **(PREMIUM)** | [GitLab 15.5](https://gitlab.com/gitlab-org/gitlab/-/issues/370949) | | **Allow force pushes - Everyone** | **Allowed to force push** [branch protection setting](../protected_branches.md#allow-force-push-on-a-protected-branch) | [GitLab 15.6](https://gitlab.com/gitlab-org/gitlab/-/issues/370943) | | **Require a pull request before merging - Require review from Code Owners** | **Require approval from code owners** [branch protection setting](../protected_branches.md#require-code-owner-approval-on-a-protected-branch) **(PREMIUM)** | [GitLab 15.6](https://gitlab.com/gitlab-org/gitlab/-/issues/376683) | -| **Require a pull request before merging - Allow specified actors to bypass required pull requests** (1) | List of users in the **Allowed to push** list of [branch protection settings](../protected_branches.md#configure-a-protected-branch) **(PREMIUM)**. Without a Premium license, the list of users that are allowed to push is limited to roles. | [GitLab 15.8](https://gitlab.com/gitlab-org/gitlab/-/issues/384939) | - -1. To successfully import the **Require a pull request before merging - Allow specified actors to bypass required pull requests** rule, you must add to the parent group in GitLab - the users that are allowed to bypass required pull requests before you begin importing. +| **Require a pull request before merging - Allow specified actors to bypass required pull requests** | List of users in the **Allowed to push and merge** list of [branch protection settings](../protected_branches.md#configure-a-protected-branch) **(PREMIUM)**. Without a **Premium** subscription, the list of users that are allowed to push and merge is limited to roles. | [GitLab 15.8](https://gitlab.com/gitlab-org/gitlab/-/issues/384939) | Mapping GitHub rule **Require status checks to pass before merging** to [external status checks](../merge_requests/status_checks.md) was considered in issue @@ -267,39 +293,25 @@ Mapping GitHub rule **Require status checks to pass before merging** to into GitLab due to technical difficulties. You can still create [external status checks](../merge_requests/status_checks.md) manually. -## Alternative way to import notes and diff notes - -When GitHub Importer runs on extremely large projects not all notes & diff notes can be imported due to GitHub API `issues_comments` & `pull_requests_comments` endpoints limitation. -Not all pages can be fetched due to the following error coming from GitHub API: `In order to keep the API fast for everyone, pagination is limited for this resource. Check the rel=last link relation in the Link response header to see how far back you can traverse.`. - -An [alternative approach](#select-additional-items-to-import) for importing comments is available. - -Instead of using `issues_comments` and `pull_requests_comments`, use individual resources `issue_comments` and `pull_request_comments` instead to pull notes from one object at a time. -This allows us to carry over any missing comments, however it increases the number of network requests required to perform the import, which means its execution takes a longer time. +### Collaborators (members) -## Reduce GitHub API request objects per page +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/388716) in GitLab 15.10. -Some GitHub API endpoints may return a 500 or 502 error for project imports from large repositories. -To reduce the chance of such errors, you can enable the feature flag -`github_importer_lower_per_page_limit` in the group project importing the data. This reduces the -page size from 100 to 50. +These GitHub collaborator roles are mapped to these GitLab [member roles](../../permissions.md#roles): -To enable this feature flag, start a [Rails console](../../../administration/operations/rails_console.md#starting-a-rails-console-session) -and run the following `enable` command: +| GitHub role | Mapped GitLab role | +|:------------|:-------------------| +| Read | Guest | +| Triage | Reporter | +| Write | Developer | +| Maintain | Maintainer | +| Admin | Owner | -```ruby -group = Group.find_by_full_path('my/group/fullpath') +GitHub Enterprise Cloud has +[custom repository roles](https://docs.github.com/en/enterprise-cloud@latest/organizations/managing-peoples-access-to-your-organization-with-roles/about-custom-repository-roles). +These roles aren't supported and cause partially completed imports. -# Enable -Feature.enable(:github_importer_lower_per_page_limit, group) -``` - -To disable the feature, run this command: - -```ruby -# Disable -Feature.disable(:github_importer_lower_per_page_limit, group) -``` +To import GitHub collaborators, you must have at least the Write role on the GitHub project. Otherwise collaborators import is skipped. ## Import from GitHub Enterprise on an internal network @@ -443,3 +455,47 @@ repository to be imported manually. Administrators can manually import the repos # Trigger import from second step Gitlab::GithubImport::Stage::ImportRepositoryWorker.perform_async(project.id) ``` + +### Errors when importing large projects + +The GitHub importer might encounter some errors when importing large projects. + +#### Alternative way to import notes and diff notes + +When the GitHub importer runs on extremely large projects, not all notes and diff notes can be imported due to the GitHub API `issues_comments` and `pull_requests_comments` endpoint limitations. + +If it's not possible to fetch all pages, the GitHub API might return the following error: + +```plaintext +In order to keep the API fast for everyone, pagination is limited for this resource. Check the rel=last link relation in the Link response header to see how far back you can traverse. +``` + +An [alternative approach](#select-additional-items-to-import) for importing comments is available. + +Instead of using `issues_comments` and `pull_requests_comments`, use individual resources to pull notes from one object at a time. This way, you can carry over any missing comments. However, execution takes longer because this method increases the number of network requests required to perform the import. + +#### Reduce GitHub API request objects per page + +Some GitHub API endpoints might return a `500` or `502` error for project imports from large repositories. +To reduce the chance of these errors, in the group project importing the data, enable the +`github_importer_lower_per_page_limit` feature flag. When enabled, the flag reduces the +page size from `100` to `50`. + +To enable this feature flag: + +1. Start a [Rails console](../../../administration/operations/rails_console.md#starting-a-rails-console-session). +1. Run the following `enable` command: + + ```ruby + group = Group.find_by_full_path('my/group/fullpath') + + # Enable + Feature.enable(:github_importer_lower_per_page_limit, group) + ``` + +To disable the feature flag, run this command: + +```ruby +# Disable +Feature.disable(:github_importer_lower_per_page_limit, group) +``` diff --git a/doc/user/project/import/gitlab_com.md b/doc/user/project/import/gitlab_com.md index c00709d9697..2b69cd40fc7 100644 --- a/doc/user/project/import/gitlab_com.md +++ b/doc/user/project/import/gitlab_com.md @@ -1,39 +1,17 @@ --- -type: reference, howto stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Import a project from GitLab.com to your self-managed GitLab instance (deprecated) **(FREE)** +# Import a project from GitLab.com to your self-managed GitLab instance (removed) **(FREE)** WARNING: The GitLab.com importer was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/108502) in GitLab 15.8 -and will be removed in GitLab 16.0. To import GitLab projects from GitLab.com to a self-managed GitLab instance use +and removed in GitLab 16.0. To import GitLab projects from GitLab.com to a self-managed GitLab instance use [migrating groups and projects by direct transfer](../../group/import/index.md#migrate-groups-by-direct-transfer-recommended). -You can import your existing GitLab.com projects to your GitLab instance. - -Prerequisite: - -- GitLab.com integration must be enabled on your GitLab instance. - [Read more about GitLab.com integration for self-managed GitLab instances](../../../integration/gitlab.md). - -To import a GitLab.com project to your self-managed GitLab instance: - -1. In GitLab, on the top bar, select **Main menu > Projects > View all projects**. -1. On the right of the page, select **New project**. -1. Select **Import project**. -1. Select **GitLab.com**. -1. Give GitLab.com permission to access your projects. -1. Select **Import**. - -The importer imports your repository and issues. -When the importer is done, a new GitLab project is created with your imported data. - ## Related topics -- To automate user, group, and project import API calls, see - [Automate group and project import](index.md#automate-group-and-project-import). -- To import Wiki and merge request data to your new instance, - see [exporting a project](../settings/import_export.md#export-a-project-and-its-data). +- [Automate group and project import](index.md#automate-group-and-project-import) +- [Export a project](../settings/import_export.md#export-a-project-and-its-data) diff --git a/doc/user/project/import/img/bitbucket_import_select_project_v12_3.png b/doc/user/project/import/img/bitbucket_import_select_project_v12_3.png deleted file mode 100644 index bbc72a0b4b7..00000000000 Binary files a/doc/user/project/import/img/bitbucket_import_select_project_v12_3.png and /dev/null differ diff --git a/doc/user/project/import/img/fogbugz_import_finished.png b/doc/user/project/import/img/fogbugz_import_finished.png deleted file mode 100644 index 62c5c86c9b3..00000000000 Binary files a/doc/user/project/import/img/fogbugz_import_finished.png and /dev/null differ diff --git a/doc/user/project/import/img/manifest_status_v13_3.png b/doc/user/project/import/img/manifest_status_v13_3.png deleted file mode 100644 index c1a55ba1f50..00000000000 Binary files a/doc/user/project/import/img/manifest_status_v13_3.png and /dev/null differ diff --git a/doc/user/project/import/index.md b/doc/user/project/import/index.md index 24748b2e89c..1265b8534d0 100644 --- a/doc/user/project/import/index.md +++ b/doc/user/project/import/index.md @@ -1,7 +1,6 @@ --- -type: reference, howto stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -15,16 +14,11 @@ If you want to bring existing projects to GitLab or copy GitLab projects to a di - Between a self-managed instance and GitLab.com in both directions. - In the same GitLab instance. -Prerequisite: - -- At least the Maintainer role on the destination group to import to. Using the Developer role for this purpose was - [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387891) in GitLab 15.8 and will be removed in GitLab 16.0. - For any type of source and target, you can migrate GitLab projects: - When [migrating groups by direct transfer](../../group/import/index.md#migrate-groups-by-direct-transfer-recommended), which allows you to migrate all projects in a group simultaneously. Migrating projects by direct transfer is in - [Beta](../../../policy/alpha-beta-support.md#beta-features). The feature is not ready for production use. + [Beta](../../../policy/alpha-beta-support.md#beta). The feature is not ready for production use. - Using [file exports](../settings/import_export.md). With this method you can migrate projects one by one. No network connection between instances is required. @@ -32,13 +26,27 @@ If you only need to migrate Git repositories, you can [import each project by UR import issues and merge requests this way. To retain metadata like issues and merge requests, either: - [Migrate projects with groups by direct transfer](../../group/import/index.md#migrate-groups-by-direct-transfer-recommended). - This feature is in [Beta](../../../policy/alpha-beta-support.md#beta-features). It is not ready for production use. + This feature is in [Beta](../../../policy/alpha-beta-support.md#beta). It is not ready for production use. - Use [file exports](../settings/import_export.md) to import projects. Keep in mind the limitations of [migrating using file exports](../settings/import_export.md#items-that-are-exported). When migrating from self-managed to GitLab.com, user associations (such as comment author) are changed to the user who is importing the projects. +## Security + +Only import projects from sources you trust. If you import a project from an untrusted source, +an attacker could steal your sensitive data. For example, an imported project +with a malicious `.gitlab-ci.yml` file could allow an attacker to exfiltrate group CI/CD variables. + +GitLab self-managed administrators can reduce their attack surface by disabling import sources they don't need: + +1. On the top bar, select **Main menu > Admin**. +1. On the left sidebar, select **Settings > General**. +1. Expand **Visibility and access controls**. +1. Scroll to **Import sources**. +1. Clear checkboxes for importers that are not required. + ## Available project importers You can import projects from: @@ -65,7 +73,7 @@ You can then [connect your external repository to get CI/CD benefits](../../../c GitLab can not automatically migrate Subversion repositories to Git. Converting Subversion repositories to Git can be difficult, but several tools exist including: -- [`git svn`](https://git-scm.com/book/en/v2/Git-and-Other-Systems-Migrating-to-Git), for very small and simple repositories. +- [`git svn`](https://git-scm.com/book/en/v2/Git-and-Other-Systems-Migrating-to-Git), for very small and basic repositories. - [`reposurgeon`](http://www.catb.org/~esr/reposurgeon/repository-editing.html), for larger and more complex repositories. ## Migrate using the API @@ -82,10 +90,9 @@ over a series of Docker pulls and pushes. Re-run any CI pipelines to retrieve an ## Migrate between two self-managed GitLab instances -To migrate from an existing self-managed GitLab instance to a new self-managed GitLab instance, it's -best to [back up](../../../raketasks/backup_restore.md) -the existing instance and restore it on the new instance. For example, this is useful when migrating -a self-managed instance from an old server to a new server. +To migrate from an existing self-managed GitLab instance to a new self-managed GitLab instance, +you should [back up](../../../raketasks/backup_restore.md) +the existing instance and restore it on the new instance. For example, you could use this method to migrate a self-managed instance from an old server to a new server. The backups produced don't depend on the operating system running GitLab. You can therefore use the restore method to switch between different operating system distributions or versions, as long @@ -109,7 +116,7 @@ To view project import history: 1. On the top bar, select **Create new...** (**{plus-square}**). 1. Select **New project/repository**. 1. Select **Import project**. -1. Select **History** in the upper right corner. +1. In the upper-right corner, select **History**. 1. If there are any errors for a particular import, you can see them by selecting **Details**. The history also includes projects created from [built-in](../index.md#create-a-project-from-a-built-in-template) @@ -154,8 +161,20 @@ GitLab from: For more information, see: +- Information on paid GitLab [migration services](https://about.gitlab.com/services/migration/). - [Quick Start](https://gitlab.com/gitlab-org/professional-services-automation/tools/migration/congregate/-/blob/master/docs/using-congregate.md#quick-start). - [Frequently Asked Migration Questions](https://gitlab.com/gitlab-org/professional-services-automation/tools/migration/congregate/-/blob/master/customer/famq.md), including settings that need checking afterwards and other limitations. For support, customers must enter into a paid engagement with GitLab Professional Services. + +## Troubleshooting + +### Imported repository is missing branches + +If an imported repository does not contain all branches of the source repository: + +1. Set the [environment variable](../../../administration/logs/index.md#override-default-log-level) `IMPORT_DEBUG=true`. +1. Retry the import with a [different group, subgroup, or project name](https://about.gitlab.com/releases/2023/02/22/gitlab-15-9-released/#re-import-projects-from-external-providers). +1. If some branches are still missing, inspect [`importer.log`](../../../administration/logs/index.md#importerlog) + (for example, with [`jq`](../../../administration/logs/log_parsing.md#parsing-gitlab-railsimporterlog)). diff --git a/doc/user/project/import/jira.md b/doc/user/project/import/jira.md index c8b717d0421..ede9eb244c6 100644 --- a/doc/user/project/import/jira.md +++ b/doc/user/project/import/jira.md @@ -37,16 +37,10 @@ iterations of the GitLab Jira importer. ## Prerequisites -### Permissions - -To be able to import issues from a Jira project you must have read access on Jira -issues and at least the Maintainer role in the GitLab project that you wish to import into. - -### Jira integration - -This feature uses the existing GitLab [Jira integration](../../../integration/jira/index.md). - -Make sure you have the integration set up before trying to import Jira issues. +- To be able to import issues from a Jira project you must have read access on Jira + issues and at least the Maintainer role in the GitLab project that you wish to import into. +- This feature uses the existing GitLab [Jira integration](../../../integration/jira/index.md). + Make sure you have the integration set up before trying to import Jira issues. ## Import Jira issues to GitLab @@ -59,11 +53,11 @@ Importing large projects may take several minutes depending on the size of the i To import Jira issues to a GitLab project: -1. On the **{issues}** **Issues** page, select **Import Issues** (**{import}**) **> Import from Jira**. +1. On the **{issues}** **Issues** page, select **Actions** (**{ellipsis_v}**) **> Import from Jira**. ![Import issues from Jira button](img/jira/import_issues_from_jira_button_v12_10.png) - The **Import from Jira** option is only visible if you have the [correct permissions](#permissions). + The **Import from Jira** option is only visible if you have the [correct permissions](#prerequisites). The following form appears. If you've previously set up the [Jira integration](../../../integration/jira/index.md), you can now see diff --git a/doc/user/project/import/manifest.md b/doc/user/project/import/manifest.md index 514a6a0cb5a..7b3550d2dc9 100644 --- a/doc/user/project/import/manifest.md +++ b/doc/user/project/import/manifest.md @@ -1,7 +1,6 @@ --- -type: howto stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -16,12 +15,16 @@ based on a manifest file like the one used by the Use the manifest to import a project with many repositories like the Android Open Source Project (AOSP). -## Requirements +## Prerequisites +> Requirement for Maintainer role instead of Developer role introduced in GitLab 16.0 and backported to GitLab 15.11.1 and GitLab 15.10.5. + +- [Manifest import source](../../admin_area/settings/visibility_and_access_controls.md#configure-allowed-import-sources) + must be enabled. If not enabled, ask your GitLab administrator to enable it. The Manifest import source is enabled + by default on GitLab.com. - GitLab must use PostgreSQL for its database, because [subgroups](../../group/subgroups/index.md) are needed for the manifest import to work. Read more about the [database requirements](../../../install/requirements.md#database). -- At least the Maintainer role on the destination group to import to. Using the Developer role for this purpose was - [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387891) in GitLab 15.8 and will be removed in GitLab 16.0. +- At least the Maintainer role on the destination group to import to. ## Manifest format diff --git a/doc/user/project/import/perforce.md b/doc/user/project/import/perforce.md index a96297b1a38..5f06daef0aa 100644 --- a/doc/user/project/import/perforce.md +++ b/doc/user/project/import/perforce.md @@ -1,7 +1,6 @@ --- -type: howto stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -53,7 +52,7 @@ submit back from Git to Perforce. Here's a few links to get you started: - [`git-p4` manual page](https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git-p4.html) -- [`git-p4` example usage](https://git.wiki.kernel.org/index.php/Git-p4_Usage) +- [`git-p4` example usage](https://archive.kernel.org/oldwiki/git.wiki.kernel.org/index.php/Git-p4_Usage.html) - [Git book migration guide](https://git-scm.com/book/en/v2/Git-and-Other-Systems-Migrating-to-Git#_perforce_import) `git p4` and `git filter-branch` are not very good at diff --git a/doc/user/project/import/phabricator.md b/doc/user/project/import/phabricator.md index 8a2dbba1343..7b9615b883c 100644 --- a/doc/user/project/import/phabricator.md +++ b/doc/user/project/import/phabricator.md @@ -1,43 +1,11 @@ --- -type: howto stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' --- -# Import Phabricator tasks into a GitLab project (deprecated) **(FREE SELF)** +# Import Phabricator tasks into a GitLab project (removed) **(FREE SELF)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/60562) in GitLab 12.0. - -WARNING: -The Phabricator task importer was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106369) in GitLab 15.7 -and will be removed in GitLab 16.0. - -WARNING: -The Phabricator task importer is in -[beta](../../../policy/alpha-beta-support.md#beta-features) and is -[**not** complete](https://gitlab.com/gitlab-org/gitlab/-/issues/284406). It imports -only an issue's title and description. The GitLab project created during the import -process contains only issues, and the associated repository is disabled. - -GitLab allows you to import all tasks from a Phabricator instance into -GitLab issues. The import creates a single project with the -repository disabled. - -Only the following basic fields are imported: - -- Title -- Description -- State (open or closed) -- Created at -- Closed at - -## Users - -The assignee and author of a user are deducted from a Task's owner and -author: If a user with the same username has access to the namespace -of the project being imported into, then the user is linked. - -## Enable this feature - -Enable Phabricator as an [import source](../../admin_area/settings/visibility_and_access_controls.md#configure-allowed-import-sources) in the Admin Area. +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106369) in GitLab 15.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/117649) in 16.0. diff --git a/doc/user/project/import/repo_by_url.md b/doc/user/project/import/repo_by_url.md index 93e3991ba19..4ad51ccb97f 100644 --- a/doc/user/project/import/repo_by_url.md +++ b/doc/user/project/import/repo_by_url.md @@ -1,18 +1,23 @@ --- -type: howto stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Import project from repository by URL **(FREE)** -Prerequisite: +You can import your existing repositories by providing the Git URL. -- At least the Maintainer role on the destination group to import to. Using the Developer role for this purpose was - [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387891) in GitLab 15.8 and will be removed in GitLab 16.0. +## Prerequisites -You can import your existing repositories by providing the Git URL: +> Requirement for Maintainer role instead of Developer role introduced in GitLab 16.0 and backported to GitLab 15.11.1 and GitLab 15.10.5. + +- [Repository by URL import source](../../admin_area/settings/visibility_and_access_controls.md#configure-allowed-import-sources) + must be enabled. If not enabled, ask your GitLab administrator to enable it. The Repository by URL import source is enabled + by default on GitLab.com. +- At least the Maintainer role on the destination group to import to. + +## Import project by URL 1. In GitLab, on the top bar, select **Main menu > Projects > View all projects**. 1. On the right of the page, select **New project**. diff --git a/doc/user/project/import/svn.md b/doc/user/project/import/svn.md deleted file mode 100644 index c9abc0f459d..00000000000 --- a/doc/user/project/import/svn.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'index.md#import-from-subversion' -remove_date: '2023-03-15' ---- - -This document was moved to [another location](index.md). - - - - - diff --git a/doc/user/project/import/tfvc.md b/doc/user/project/import/tfvc.md index 0a03513467e..da9d31e0ca8 100644 --- a/doc/user/project/import/tfvc.md +++ b/doc/user/project/import/tfvc.md @@ -1,8 +1,7 @@ --- stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments -type: concepts --- # Migrate from TFVC to Git **(FREE)** diff --git a/doc/user/project/index.md b/doc/user/project/index.md index decf3b071e7..1512039fb25 100644 --- a/doc/user/project/index.md +++ b/doc/user/project/index.md @@ -1,6 +1,6 @@ --- -stage: Manage -group: Organization +stage: Data Stores +group: Tenant Scale info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments" --- @@ -139,9 +139,8 @@ Prerequisites: - You must have permission to add new projects to a namespace. To check if you have permission: 1. On the top bar, select **Main menu > Groups** and find your group. - 1. Confirm that **New project** is visible in the upper right - corner. Contact your GitLab - administrator if you require permission. + 1. In the upper-right corner, confirm that **New project** is visible. + Contact your GitLab administrator if you require permission. To push your repository and create a project: @@ -180,8 +179,6 @@ Your project's visibility is set to **Private** by default. To change project vi ## Related topics -- For a list of words that you cannot use as project names, see - [reserved project and group names](../../user/reserved_names.md). -- For a list of characters that you cannot use in project and group names, see - [limitations on project and group names](../../user/reserved_names.md#limitations-on-project-and-group-names). -- [Manage projects](working_with_projects.md). +- [Reserved project and group names](../../user/reserved_names.md) +- [Limitations on project and group names](../../user/reserved_names.md#limitations-on-project-and-group-names) +- [Manage projects](working_with_projects.md) diff --git a/doc/user/project/integrations/apple_app_store.md b/doc/user/project/integrations/apple_app_store.md index 1b41dd0b669..633d565064c 100644 --- a/doc/user/project/integrations/apple_app_store.md +++ b/doc/user/project/integrations/apple_app_store.md @@ -1,19 +1,24 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Apple App Store **(FREE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104888) in GitLab 15.8 [with a flag](../../../administration/feature_flags.md) named `apple_app_store_integration`. Disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/104888) in GitLab 15.8 [with a flag](../../../administration/feature_flags.md) named `apple_app_store_integration`. Disabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/385335) in GitLab 15.10. Feature flag `apple_app_store_integration` removed. -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `apple_app_store_integration`. On GitLab.com, this feature is not available. +This feature is part of [Mobile DevOps](../../../ci/mobile_devops.md) developed by [GitLab Incubation Engineering](https://about.gitlab.com/handbook/engineering/incubation/). +The feature is still in development, but you can: -The Apple App Store integration makes it easy to configure your CI/CD pipelines to connect to [App Store Connect](https://appstoreconnect.apple.com) to build and release apps for iOS, iPadOS, macOS, tvOS, and watchOS. +- [Request a feature](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=feature_request). +- [Report a bug](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=report_bug). +- [Share feedback](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=general_feedback). -The integration is designed to be able to work out of the box with [fastlane](http://fastlane.tools/), but can be used with other build tools as well. +With the Apple App Store integration, you can configure your CI/CD pipelines to connect to [App Store Connect](https://appstoreconnect.apple.com) to build and release apps for iOS, iPadOS, macOS, tvOS, and watchOS. + +The Apple App Store integration works out of the box with [fastlane](https://fastlane.tools/). You can also use this integration with other build tools. ## Prerequisites @@ -29,16 +34,17 @@ GitLab supports enabling the Apple App Store integration at the project level. C 1. Select **Apple App Store**. 1. Turn on the **Active** toggle under **Enable Integration**. 1. Provide the Apple App Store Connect configuration information: - - **Issuer ID**: The Apple App Store Connect Issuer ID can be found in the *Keys* section under *Users and Access* the Apple App Store Connect portal. - - **Key ID**: The Key ID of the new private key that was just generated. - - **Private Key**: The Private Key that was just generated. Note: you are only be able to download this key one time. + - **Issuer ID**: The Apple App Store Connect issuer ID. + - **Key ID**: The key ID of the generated private key. + - **Private Key**: The generated private key. You can download this key only once. 1. Select **Save changes**. After the Apple App Store integration is activated: -- The global variables `$APP_STORE_CONNECT_API_KEY_ISSUER_ID`, `$APP_STORE_CONNECT_API_KEY_KEY_ID`, and `$APP_STORE_CONNECT_API_KEY_KEY` are created for CI/CD use. +- The global variables `$APP_STORE_CONNECT_API_KEY_ISSUER_ID`, `$APP_STORE_CONNECT_API_KEY_KEY_ID`, `$APP_STORE_CONNECT_API_KEY_KEY`, and `$APP_STORE_CONNECT_API_KEY_IS_KEY_CONTENT_BASE64` are created for CI/CD use. - `$APP_STORE_CONNECT_API_KEY_KEY` contains the Base64 encoded Private Key. +- `$APP_STORE_CONNECT_API_KEY_IS_KEY_CONTENT_BASE64` is always `true`. ## Security considerations @@ -48,12 +54,10 @@ Malicious code pushed to your `.gitlab-ci.yml` file could compromise your variab `$APP_STORE_CONNECT_API_KEY_KEY`, and send them to a third-party server. For more details, see [CI/CD variable security](../../../ci/variables/index.md#cicd-variable-security). -## fastlane Example +## Enable the integration in fastlane -Because this integration works out of the box with fastlane adding the code below to an app's `fastlane/Fastfile` activates the integration, and create the connection for any interactions with the Apple App Store uploading a Test Flight or public App Store release. +To enable the integration in fastlane and upload a TestFlight or public App Store release, you can add the following code to your app's `fastlane/Fastfile`: ```ruby -app_store_connect_api_key( - is_key_content_base64: true -) +app_store_connect_api_key ``` diff --git a/doc/user/project/integrations/asana.md b/doc/user/project/integrations/asana.md index 8bc1a984e3d..b1a4d2a2f78 100644 --- a/doc/user/project/integrations/asana.md +++ b/doc/user/project/integrations/asana.md @@ -1,12 +1,12 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Asana **(FREE)** -This integration adds commit messages as comments to Asana tasks. +The Asana integration adds commit messages as comments to Asana tasks. Once enabled, commit messages are checked for Asana task URLs (for example, `https://app.asana.com/0/123456/987654`) or task IDs starting with `#` (for example, `#987654`). Every task ID found gets the commit comment added to it. diff --git a/doc/user/project/integrations/bamboo.md b/doc/user/project/integrations/bamboo.md index 12039a70d0e..23990036f4e 100644 --- a/doc/user/project/integrations/bamboo.md +++ b/doc/user/project/integrations/bamboo.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -125,7 +125,7 @@ For example: ### Builds not triggered If builds are not triggered, ensure you entered the right GitLab IP address in -Bamboo under **Trigger IP addresses**. Also check [service hook logs](index.md#troubleshooting-integrations) for request failures. +Bamboo under **Trigger IP addresses**. Also check [integration webhook logs](index.md#troubleshooting) for request failures. ### Advanced Atlassian Bamboo features not available in GitLab UI diff --git a/doc/user/project/integrations/bugzilla.md b/doc/user/project/integrations/bugzilla.md index 2933203c593..f2af4dc6e4d 100644 --- a/doc/user/project/integrations/bugzilla.md +++ b/doc/user/project/integrations/bugzilla.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -57,4 +57,4 @@ internal issue tracker, the internal issue is linked. ## Troubleshooting -To see recent service hook deliveries, check [service hook logs](index.md#troubleshooting-integrations). +For recent integration webhook deliveries, check [integration webhook logs](index.md#troubleshooting). diff --git a/doc/user/project/integrations/custom_issue_tracker.md b/doc/user/project/integrations/custom_issue_tracker.md index 24a2e3d1ebc..b529d728e3b 100644 --- a/doc/user/project/integrations/custom_issue_tracker.md +++ b/doc/user/project/integrations/custom_issue_tracker.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/project/integrations/discord_notifications.md b/doc/user/project/integrations/discord_notifications.md index 0163bce3496..60b37f07bb5 100644 --- a/doc/user/project/integrations/discord_notifications.md +++ b/doc/user/project/integrations/discord_notifications.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/project/integrations/emails_on_push.md b/doc/user/project/integrations/emails_on_push.md index c3134e986d3..7f8755f1114 100644 --- a/doc/user/project/integrations/emails_on_push.md +++ b/doc/user/project/integrations/emails_on_push.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/project/integrations/ewm.md b/doc/user/project/integrations/ewm.md index c2371d32853..39dd548e7ca 100644 --- a/doc/user/project/integrations/ewm.md +++ b/doc/user/project/integrations/ewm.md @@ -1,12 +1,12 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Engineering Workflow Management (EWM) **(FREE)** -This integration allows you to go from GitLab to EWM work items mentioned in merge request +The EWM integration allows you to go from GitLab to EWM work items mentioned in merge request descriptions and commit messages. Each work item reference is automatically converted to a link to the work item. diff --git a/doc/user/project/integrations/github.md b/doc/user/project/integrations/github.md index 990d0839581..3e75106a9bc 100644 --- a/doc/user/project/integrations/github.md +++ b/doc/user/project/integrations/github.md @@ -1,13 +1,13 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # GitHub **(PREMIUM)** You can update GitHub with pipeline status updates from GitLab. -This integration can help you if you use GitLab for CI/CD. +The GitHub integration can help you if you use GitLab for CI/CD. ![Pipeline status update on GitHub](img/github_status_check_pipeline_update.png) diff --git a/doc/user/project/integrations/gitlab_slack_application.md b/doc/user/project/integrations/gitlab_slack_application.md index 649b0c51818..fabdf07f76f 100644 --- a/doc/user/project/integrations/gitlab_slack_application.md +++ b/doc/user/project/integrations/gitlab_slack_application.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -18,6 +18,10 @@ integrations on GitLab.com. ## Installation +In GitLab 15.0 and later, the GitLab for Slack app uses +[granular permissions](https://medium.com/slack-developer-blog/more-precision-less-restrictions-a3550006f9c3). +Although functionality has not changed, you should [reinstall the app](#update-the-gitlab-for-slack-app). + ### Through the Slack App Directory To enable the GitLab for Slack app for your workspace, @@ -43,8 +47,8 @@ To enable the GitLab integration for your Slack workspace: 1. Select **Install GitLab for Slack app**. 1. Select **Allow** on Slack's confirmation screen. -You can also select **Reinstall GitLab for Slack app** to update the app in your Slack workspace -to the latest version. See [Version history](#version-history) for details. +To update the app in your Slack workspace to the latest version, +you can also select **Reinstall GitLab for Slack app**. ## Slash commands @@ -63,6 +67,7 @@ Replace `` with the project full path, or create a shorter [project ali | `/gitlab issue comment ` | Adds a new comment with the comment body `` to the issue with the ID ``. | | `/gitlab deploy to ` | [Deploys](#the-deploy-slash-command) from the `` environment to the `` environment. | | `/gitlab run ` | Executes the [ChatOps](../../../ci/chatops/index.md) job `` on the default branch. | +| `/gitlab incident declare` | **Beta** Opens modal to [create a new incident](../../../operations/incident_management/slack.md). | ### The deploy slash command @@ -191,8 +196,3 @@ If you're not receiving notifications to a Slack channel, ensure: ### The App Home does not display properly If the [App Home](https://api.slack.com/start/overview#app_home) does not display properly, ensure your [app is up to date](#update-the-gitlab-for-slack-app). - -## Version history - -In GitLab 15.0 and later, the GitLab for Slack app is updated to [Slack's new granular permissions model](https://medium.com/slack-developer-blog/more-precision-less-restrictions-a3550006f9c3). -Although there is no change in functionality, you should [reinstall the app](#update-the-gitlab-for-slack-app). diff --git a/doc/user/project/integrations/google_play.md b/doc/user/project/integrations/google_play.md new file mode 100644 index 00000000000..2ae5a504e06 --- /dev/null +++ b/doc/user/project/integrations/google_play.md @@ -0,0 +1,55 @@ +--- +stage: Manage +group: Import and Integrate +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Google Play **(FREE)** + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/111621) in GitLab 15.10 [with a flag](../../../administration/feature_flags.md) named `google_play_integration`. Disabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/389611) in GitLab 15.11. Feature flag `google_play_integration` removed. + +This feature is part of [Mobile DevOps](../../../ci/mobile_devops.md) developed by [GitLab Incubation Engineering](https://about.gitlab.com/handbook/engineering/incubation/). +The feature is still in development, but you can: + +- [Request a feature](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=feature_request). +- [Report a bug](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=report_bug). +- [Share feedback](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=general_feedback). + +With the Google Play integration, you can configure your CI/CD pipelines to connect to the [Google Play Console](https://play.google.com/console) to build and release apps for Android devices. + +The Google Play integration works out of the box with [fastlane](https://fastlane.tools/). You can also use this integration with other build tools. + +## Enable the integration in GitLab + +Prerequisites: + +- You must have a [Google Play Console](https://play.google.com/console/developers) developer account. +- You must [generate a new service account key for your project](https://developers.google.com/android-publisher/getting_started) from the Google Cloud console. + +To enable the Google Play integration in GitLab: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Integrations**. +1. Select **Google Play**. +1. In **Enable integration**, select the **Active** checkbox. +1. In **Package name**, enter the package name of the app (for example, `com.gitlab.app_name`). +1. In **Service account key (.JSON)**, drag or upload your key file. +1. Select **Save changes**. + +After you enable the integration, the global variables `$SUPPLY_PACKAGE_NAME` and `$SUPPLY_JSON_KEY_DATA` are created for CI/CD use. + +### CI/CD variable security + +Malicious code pushed to your `.gitlab-ci.yml` file could compromise your variables, including `$SUPPLY_JSON_KEY_DATA`, and send them to a third-party server. For more information, see [CI/CD variable security](../../../ci/variables/index.md#cicd-variable-security). + +## Enable the integration in fastlane + +To enable the integration in fastlane and upload the build to the given track in Google Play, you can add the following code to your app's `fastlane/Fastfile`: + +```ruby +upload_to_play_store( + track: 'internal', + aab: '../build/app/outputs/bundle/release/app-release.aab' +) +``` diff --git a/doc/user/project/integrations/hangouts_chat.md b/doc/user/project/integrations/hangouts_chat.md index 3537033902d..3470d11b983 100644 --- a/doc/user/project/integrations/hangouts_chat.md +++ b/doc/user/project/integrations/hangouts_chat.md @@ -1,13 +1,13 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Google Chat **(FREE)** -Integrate your project to send notifications from GitLab to a -room of your choice in [Google Chat](https://chat.google.com/) (former Google +You can configure your project to send notifications from GitLab to a +room of your choice in [Google Chat](https://chat.google.com/) (formerly Google Hangouts). ## Integration workflow @@ -28,7 +28,7 @@ notifications to Google Chat: To enable the integration in Google Chat: 1. Enter the room where you want to receive notifications from GitLab. -1. Open the room dropdown list in the upper left and select **Manage webhooks**. +1. In the upper-left corner, from the room dropdown list, select **Manage webhooks**. 1. Enter the name for your webhook, for example "GitLab integration". 1. Optional. Add an avatar for your bot. 1. Select **Save**. diff --git a/doc/user/project/integrations/harbor.md b/doc/user/project/integrations/harbor.md index 596821ba12b..4a586054aee 100644 --- a/doc/user/project/integrations/harbor.md +++ b/doc/user/project/integrations/harbor.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -8,11 +8,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/80999) in GitLab 14.9. -Use Harbor as the container registry for your GitLab project. +You can use Harbor as the container registry for your GitLab project. -[Harbor](https://goharbor.io/) is an open source registry that can help you manage artifacts across cloud-native compute platforms, like Kubernetes and Docker. +[Harbor](https://goharbor.io/) is an open-source registry that can help you manage artifacts across cloud-native compute platforms like Kubernetes and Docker. -This integration can help you if you need GitLab CI/CD and a container image repository. +The Harbor integration can help you if you need GitLab CI/CD and a container image repository. ## Prerequisites diff --git a/doc/user/project/integrations/index.md b/doc/user/project/integrations/index.md index 57947e21736..f7019b2eeb2 100644 --- a/doc/user/project/integrations/index.md +++ b/doc/user/project/integrations/index.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -72,13 +72,14 @@ You can configure the following integrations. | [Pipelines emails](pipeline_status_emails.md) | Send the pipeline status to a list of recipients by email. | **{dotted-circle}** No | | [Pivotal Tracker](pivotal_tracker.md) | Add commit messages as comments to Pivotal Tracker stories. | **{dotted-circle}** No | | [Prometheus](prometheus.md) | Monitor application metrics. | **{dotted-circle}** No | -| [Pumble](pumble.md) | Send event notifications to a Pumble channel. | **{dotted-circle}** No | +| [Pumble](pumble.md) | Send event notifications to a Pumble channel. | **{dotted-circle}** No | | Pushover | Get real-time notifications on your device. | **{dotted-circle}** No | | [Redmine](redmine.md) | Use Redmine as the issue tracker. | **{dotted-circle}** No | -| [Shimo Workspace](shimo.md) | Use Shimo instead of the GitLab Wiki. | **{dotted-circle}** No | +| [Shimo Workspace](shimo.md) | Use Shimo instead of the GitLab Wiki. | **{dotted-circle}** No | | [GitLab for Slack app](gitlab_slack_application.md) | Use Slack's official GitLab application. | **{dotted-circle}** No | | [Slack notifications](slack.md) | Send notifications about project events to Slack. | **{dotted-circle}** No | | [Slack slash commands](slack_slash_commands.md) | Enable slash commands in a workspace. | **{dotted-circle}** No | +| [Squash TM](squash_tm.md) | Update Squash TM requirements when GitLab issues are modified. | **{check-circle}** Yes | | [Unify Circuit](unify_circuit.md) | Send notifications about project events to Unify Circuit. | **{dotted-circle}** No | | [Webex Teams](webex_teams.md) | Receive events notifications. | **{dotted-circle}** No | | [YouTrack](youtrack.md) | Use YouTrack as the issue tracker. | **{dotted-circle}** No | @@ -102,9 +103,16 @@ supported by `push_hooks` and `tag_push_hooks` events aren't executed. You can change the number of supported branches or tags by changing the [`push_event_hooks_limit` application setting](../../../api/settings.md#list-of-settings-that-can-be-accessed-via-api-calls). -## Troubleshooting integrations +## Contribute to integrations + +If you're interested in developing a new native integration for GitLab, see: + +- [Integrations development guidelines](../../../development/integrations/index.md) +- [GitLab Developer Portal](https://developer.gitlab.com) + +## Troubleshooting -Some integrations use hooks to integrate with external applications. To confirm which ones use integration hooks, see the [available integrations](#available-integrations). For more information, see [Troubleshooting webhooks](webhooks.md#troubleshoot-webhooks). +Some integrations use hooks to integrate with external applications. To confirm which ones use integration hooks, see the [available integrations](#available-integrations). For more information, see [webhook troubleshooting](webhooks.md#troubleshooting). ### `Test Failed. Save Anyway` error @@ -114,7 +122,3 @@ push data to build the test payload, and there are no push events in the project To resolve this error, initialize the repository by pushing a test file to the project and set up the integration again. - -## Contribute to integrations - -To add a new integration, see the [Integrations development guide](../../../development/integrations/index.md). diff --git a/doc/user/project/integrations/irker.md b/doc/user/project/integrations/irker.md index 23525c33e84..df11ed3e57c 100644 --- a/doc/user/project/integrations/irker.md +++ b/doc/user/project/integrations/irker.md @@ -1,13 +1,13 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # irker (IRC gateway) **(FREE)** -GitLab provides a way to push update messages to an irker server. When -configured, pushes to a project trigger the integration to send data directly +GitLab provides a way to push update messages to an irker server. After you configure +the integration, each push to a project triggers the integration to send data directly to the irker server. See also the [irker integration API documentation](../../../api/integrations.md). diff --git a/doc/user/project/integrations/mattermost.md b/doc/user/project/integrations/mattermost.md index fcc364724b7..a50d341c38e 100644 --- a/doc/user/project/integrations/mattermost.md +++ b/doc/user/project/integrations/mattermost.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/project/integrations/mattermost_slash_commands.md b/doc/user/project/integrations/mattermost_slash_commands.md index 192360f5440..3b5e4030487 100644 --- a/doc/user/project/integrations/mattermost_slash_commands.md +++ b/doc/user/project/integrations/mattermost_slash_commands.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/project/integrations/microsoft_teams.md b/doc/user/project/integrations/microsoft_teams.md index fc5d4d3cc80..5d2c27fc2a4 100644 --- a/doc/user/project/integrations/microsoft_teams.md +++ b/doc/user/project/integrations/microsoft_teams.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -58,4 +58,4 @@ GitLab to send the notifications: ## Related topics -- [Setting up an incoming webhook on Microsoft Teams](https://learn.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/connectors-using#setting-up-a-custom-incoming-webhook). +- [Setting up an incoming webhook on Microsoft Teams](https://learn.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/connectors-using#setting-up-a-custom-incoming-webhook) diff --git a/doc/user/project/integrations/mlflow_client.md b/doc/user/project/integrations/mlflow_client.md index bd14021ab1c..75fae4647d0 100644 --- a/doc/user/project/integrations/mlflow_client.md +++ b/doc/user/project/integrations/mlflow_client.md @@ -4,33 +4,31 @@ group: Incubation info: Machine Learning Experiment Tracking is a GitLab Incubation Engineering program. No technical writer assigned to this group. --- -# MLFlow Client Integration **(FREE)** +# MLflow client integration **(FREE)** -> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/8560) in GitLab 15.6 as an [Alpha](../../../policy/alpha-beta-support.md#alpha-features) release [with a flag](../../../administration/feature_flags.md) named `ml_experiment_tracking`. Disabled by default. +> [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/8560) in GitLab 15.11 as an [Experiment](../../../policy/alpha-beta-support.md#experiment) release [with a flag](../../../administration/feature_flags.md) named `ml_experiment_tracking`. Disabled by default. -DISCLAIMER: -MLFlow Client Integration is an experimental feature being developed by the Incubation Engineering Department, -and will receive significant changes over time. +NOTE: +Model experiment tracking is an [experimental feature](../../../policy/alpha-beta-support.md). +Refer to for feedback and feature requests. -[MLFlow](https://mlflow.org/) is one of the most popular open source tools for Machine Learning Experiment Tracking. -GitLabs works as a backend to the MLFlow Client, [logging experiments](../ml/experiment_tracking/index.md). +[MLflow](https://mlflow.org/) is a popular open source tool for Machine Learning Experiment Tracking. +GitLab works as a backend to the MLflow Client, [logging experiments](../ml/experiment_tracking/index.md). Setting up your integrations requires minimal changes to existing code. -GitLab plays the role of proxy server, both for artifact storage and tracking data. It reflects the -MLFlow [Scenario 5](https://www.mlflow.org/docs/latest/tracking.html#scenario-5-mlflow-tracking-server-enabled-with-proxied-artifact-storage-access). +GitLab plays the role of a MLflow server. Running `mlflow server` is not necessary. -## Enable MLFlow Client Integration - -Complete this task to enable MLFlow Client Integration. +## Enable MLflow client integration Prerequisites: - A [personal access token](../../../user/profile/personal_access_tokens.md) for the project, with minimum access level of `api`. - The project ID. To find the project ID, on the top bar, select **Main menu > Projects** and find your project. On the left sidebar, select **Settings > General**. -1. Set the tracking URI and token environment variables on the host that runs the code (your local environment, CI pipeline, or remote host). +To enable MLflow client integration: - For example: +1. Set the tracking URI and token environment variables on the host that runs the code. + This can be your local environment, CI pipeline, or remote host. For example: ```shell export MLFLOW_TRACKING_URI="http:///api/v4/projects//ml/mlflow" @@ -39,43 +37,66 @@ Prerequisites: 1. If your training code contains the call to `mlflow.set_tracking_uri()`, remove it. -When running the training code, MLFlow will create experiments, runs, log parameters, metrics, +When running the training code, MLflow creates experiments, runs, log parameters, metrics, metadata and artifacts on GitLab. -After experiments are logged, they are listed under `//-/ml/experiments`. Runs are registered as Model Candidates, -that can be explored by selecting an experiment. +After experiments are logged, they are listed under `//-/ml/experiments`. +Runs are registered as: + +- Model Candidates, which can be explored by selecting an experiment. +- Tags, which are registered as metadata. + +## Associating a candidate to a CI/CD job + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/119454) in GitLab 16.1. + +If your training code is being run from a CI/CD job, GitLab can use that information to enhance +candidate metadata. To do so, add the following snippet to your training code within the run +execution context: + +```python +with mlflow.start_run(run_name=f"Candidate {index}"): + # Your training code + + # Start of snippet to be included + if os.getenv('GITLAB_CI'): + mlflow.set_tag('gitlab.CI_JOB_ID', os.getenv('CI_JOB_ID')) + # End of snippet to be included +``` + +## Supported MLflow client methods and caveats + +GitLab supports these methods from the MLflow client. Other methods might be supported but were not +tested. More information can be found in the [MLflow Documentation](https://www.mlflow.org/docs/1.28.0/python_api/mlflow.html). + +| Method | Supported | Version Added | Comments | +|--------------------------|------------------|----------------|----------| +| `get_experiment` | Yes | 15.11 | | +| `get_experiment_by_name` | Yes | 15.11 | | +| `set_experiment` | Yes | 15.11 | | +| `get_run` | Yes | 15.11 | | +| `start_run` | Yes | 15.11 | | +| `log_artifact` | Yes with caveat | 15.11 | (15.11) `artifact_path` must be empty string. Does not support directories. +| `log_artifacts` | Yes with caveat | 15.11 | (15.11) `artifact_path` must be empty string. Does not support directories. +| `log_batch` | Yes | 15.11 | | +| `log_metric` | Yes | 15.11 | | +| `log_metrics` | Yes | 15.11 | | +| `log_param` | Yes | 15.11 | | +| `log_params` | Yes | 15.11 | | +| `log_figure` | Yes | 15.11 | | +| `log_image` | Yes | 15.11 | | +| `log_text` | Yes with caveat | 15.11 | (15.11) Does not support directories. +| `log_dict` | Yes with caveat | 15.11 | (15.11) Does not support directories. +| `set_tag` | Yes | 15.11 | | +| `set_tags` | Yes | 15.11 | | +| `set_terminated` | Yes | 15.11 | | +| `end_run` | Yes | 15.11 | | +| `update_run` | Yes | 15.11 | | +| `log_model` | Partial | 15.11 | (15.11) Saves the artifacts, but not the model data. `artifact_path` must be empty. ## Limitations -- The API GitLab supports is the one defined at MLFlow version 1.28.0. +- The API GitLab supports is the one defined at MLflow version 1.28.0. - API endpoints not listed above are not supported. -- During creation of experiments and runs, tags are ExperimentTags and RunTags are stored, even though they are not displayed. -- MLFlow Model Registry is not supported. - -## Supported methods and caveats - -This is a list of methods we support from the MLFlow client. Other methods might be supported but were not -tested. More information can be found in the [MLFlow Documentation](https://www.mlflow.org/docs/1.28.0/python_api/mlflow.html). - -### `set_experiment()` - -Accepts both `experiment_name` and `experiment_id` - -### `start_run()` - -- Nested runs have not been tested. -- `run_name` is not supported - -### `log_param()`, `log_params()`, `log_metric()`, `log_metrics()` - -Work as defined by the documentation - -### `log_artifact()`, `log_artifacts()` - -`artifact_path` must be empty string. - -### `log_model()` - -This is an experimental method in MLFlow, and partial support is offered. It stores the model artifacts, but does -not log the model information. The `artifact_path` parameter must be set to `''`, because Generic Packages do not support folder -structure. +- During creation of experiments and runs, ExperimentTags are stored, even though they are not displayed. +- MLflow Model Registry is not supported. diff --git a/doc/user/project/integrations/mock_ci.md b/doc/user/project/integrations/mock_ci.md index ae1737f8d3f..da09d621d07 100644 --- a/doc/user/project/integrations/mock_ci.md +++ b/doc/user/project/integrations/mock_ci.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -9,12 +9,12 @@ info: To determine the technical writer assigned to the Stage/Group associated w NOTE: This integration only appears if you're in a [development environment](https://gitlab.com/gitlab-org/gitlab-mock-ci-service#setup-mockci-integration). -To set up the mock CI service server, respond to the following endpoints +To set up the mock CI service server, respond to the following endpoints: - `commit_status`: `#{project.namespace.path}/#{project.path}/status/#{sha}.json` - - Have your service return `200 { status: ['failed'|'canceled'|'running'|'pending'|'success'|'success-with-warnings'|'skipped'|'not_found'] }` - - If the service returns a 404, it is interpreted as `pending` + - Have your service return `200 { status: ['failed'|'canceled'|'running'|'pending'|'success'|'success-with-warnings'|'skipped'|'not_found'] }`. + - If the service returns a 404, the service is interpreted as `pending`. - `build_page`: `#{project.namespace.path}/#{project.path}/status/#{sha}` - - Just where the build is linked to, doesn't matter if implemented + - Where the build is linked to (whether or not it's implemented). -For an example of a mock CI server, see [`gitlab-org/gitlab-mock-ci-service`](https://gitlab.com/gitlab-org/gitlab-mock-ci-service) +For an example of a mock CI server, see [`gitlab-org/gitlab-mock-ci-service`](https://gitlab.com/gitlab-org/gitlab-mock-ci-service). diff --git a/doc/user/project/integrations/pipeline_status_emails.md b/doc/user/project/integrations/pipeline_status_emails.md index 009bb6662ff..9a87b5b3110 100644 --- a/doc/user/project/integrations/pipeline_status_emails.md +++ b/doc/user/project/integrations/pipeline_status_emails.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/project/integrations/pivotal_tracker.md b/doc/user/project/integrations/pivotal_tracker.md index 034be8ab3d8..e1d48037ba8 100644 --- a/doc/user/project/integrations/pivotal_tracker.md +++ b/doc/user/project/integrations/pivotal_tracker.md @@ -1,12 +1,12 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Pivotal Tracker **(FREE)** -This integration adds commit messages as comments to Pivotal Tracker stories. +The Pivotal Tracker integration adds commit messages as comments to Pivotal Tracker stories. Once enabled, commit messages are checked for square brackets containing a hash mark followed by the story ID (for example, `[#555]`). Every story ID found gets the commit comment added to it. diff --git a/doc/user/project/integrations/prometheus.md b/doc/user/project/integrations/prometheus.md index cd92e49cada..d17e8e6bfca 100644 --- a/doc/user/project/integrations/prometheus.md +++ b/doc/user/project/integrations/prometheus.md @@ -2,129 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: 'index.md' --- -# Prometheus **(FREE)** +# Prometheus (removed) **(FREE)** -GitLab offers powerful integration with [Prometheus](https://prometheus.io) for -monitoring key metrics of your apps, directly in GitLab. -Metrics for each environment are retrieved from Prometheus, and then displayed -in the GitLab interface. - -![Environment Dashboard](img/prometheus_dashboard.png) - -There are two ways to set up Prometheus integration, depending on where your apps are running: - -- For deployments on Kubernetes, GitLab can be [integrated with an in-cluster Prometheus](#prometheus-cluster-integration) -- For other deployment targets, [specify the Prometheus server](#manual-configuration-of-prometheus). - -Once enabled, GitLab detects metrics from known services in the -[metric library](prometheus_library/index.md). You can also -[add your own metrics](../../../operations/metrics/index.md#adding-custom-metrics) and create -[custom dashboards](../../../operations/metrics/dashboards/index.md). - -## Enabling Prometheus Integration - -### Prometheus cluster integration - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/55244) in GitLab 13.11. -> - [Replaced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62725) the Prometheus cluster applications in GitLab 14.0. - -GitLab can query an in-cluster Prometheus for your metrics. -See [Prometheus cluster integration](../../clusters/integrations.md#prometheus-cluster-integration) for details. - -### Manual configuration of Prometheus - -#### Requirements - -Integration with Prometheus requires the following: - -- Prometheus must be configured to collect one of the [supported metrics](prometheus_library/index.md) -- Each metric must have a label to indicate the environment -- GitLab must have network connectivity to the Prometheus server - -#### Getting started - -Installing and configuring Prometheus to monitor applications is fairly straightforward. - -1. [Install Prometheus](https://prometheus.io/docs/prometheus/latest/installation/) -1. Set up one of the [supported monitoring targets](prometheus_library/index.md) -1. Configure the Prometheus server to - [collect their metrics](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) - -#### Configuration in GitLab - -The actual configuration of Prometheus integration in GitLab -requires the domain name or IP address of the Prometheus server you'd like -to integrate with. If the Prometheus resource is secured with Google's Identity-Aware Proxy (IAP), -you can pass information like Client ID and Service Account credentials. -GitLab can use these to access the resource. More information about authentication from a -service account can be found at Google's documentation for -[Authenticating from a service account](https://cloud.google.com/iap/docs/authentication-howto#authenticating_from_a_service_account). - -1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Settings > Integrations**. -1. Select **Prometheus**. -1. For **API URL**, provide the domain name or IP address of your server, such as - `http://prometheus.example.com/` or `http://192.0.2.1/`. -1. (Optional) In **Google IAP Audience Client ID**, provide the Client ID of the - Prometheus OAuth Client secured with Google IAP. -1. (Optional) In **Google IAP Service Account JSON**, provide the contents of the - Service Account credentials file that is authorized to access the Prometheus resource. - The JSON key `token_credential_uri` is discarded to prevent - [Server-side Request Forgery (SSRF)](https://www.hackerone.com/application-security/how-server-side-request-forgery-ssrf). -1. Select **Save changes**. - -![Configure Prometheus Service](img/prometheus_manual_configuration_v13_2.png) - -#### Thanos configuration in GitLab - -You can configure [Thanos](https://thanos.io/) as a drop-in replacement for Prometheus -with GitLab. Use the domain name or IP address of the Thanos server you'd like -to integrate with. - -1. On the top bar, select **Main menu > Projects** and find your project. -1. On the left sidebar, select **Settings > Integrations**. -1. Select **Prometheus**. -1. Provide the domain name or IP address of your server, for example - `http://thanos.example.com/` or `http://192.0.2.1/`. -1. Select **Save changes**. - -### Precedence with multiple Prometheus configurations - -Although you can enable both a [manual configuration](#manual-configuration-of-prometheus) -and [cluster integration](#prometheus-cluster-integration) of Prometheus, you -can use only one: - -- If you have enabled a - [Prometheus manual configuration](#manual-configuration-of-prometheus) - and a [Prometheus cluster integration](#prometheus-cluster-integration), - the manual configuration takes precedence and is used to run queries from - [custom dashboards](../../../operations/metrics/dashboards/index.md) and - [custom metrics](../../../operations/metrics/index.md#adding-custom-metrics). -- If you have managed Prometheus applications installed on Kubernetes clusters - at **different** levels (project, group, instance), the order of precedence is described in - [Cluster precedence](../../instance/clusters/index.md#cluster-precedence). -- If you have managed Prometheus applications installed on multiple Kubernetes - clusters at the **same** level, the Prometheus application of a cluster with a - matching [environment scope](../../../ci/environments/index.md#limit-the-environment-scope-of-a-cicd-variable) is used. - -## Determining the performance impact of a merge - -Developers can view the performance impact of their changes in the merge -request workflow. This feature requires [Kubernetes](prometheus_library/kubernetes.md) metrics. - -When a source branch has been deployed to an environment, a sparkline and -numeric comparison of the average memory consumption displays. On the -sparkline, a dot indicates when the current changes were deployed, with up to 30 minutes of -performance data displayed before and after. The comparison shows the difference -between the 30 minute average before and after the deployment. This information -is updated after each commit has been deployed. - -Once merged and the target branch has been redeployed, the metrics switches -to show the new environments this revision has been deployed to. - -Performance data is available for the duration it is persisted on the -Prometheus server. - -![Merge Request with Performance Impact](img/merge_request_performance.png) +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/user/project/integrations/prometheus_library/cloudwatch.md b/doc/user/project/integrations/prometheus_library/cloudwatch.md index 1e9319fa7c7..848ccbb85f6 100644 --- a/doc/user/project/integrations/prometheus_library/cloudwatch.md +++ b/doc/user/project/integrations/prometheus_library/cloudwatch.md @@ -2,48 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../../../../operations/index.md' --- -# Monitoring AWS resources (DEPRECATED) **(FREE)** +# Monitoring AWS resources (removed) **(FREE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) -in GitLab 14.7, and is planned for removal in GitLab 16.0. - -GitLab supports automatically detecting and monitoring AWS resources, starting -with the [Elastic Load Balancer](https://aws.amazon.com/elasticloadbalancing/) (ELB). -This is provided by leveraging the official [Cloudwatch exporter](https://github.com/prometheus/cloudwatch_exporter), which translates [Cloudwatch metrics](https://aws.amazon.com/cloudwatch/) into -a Prometheus readable form. - -## Requirements - -You must enable the [Prometheus service](../prometheus.md). - -## Supported metrics - -| Name | Query | -|----------------------|-------| -| Throughput (req/sec) | `sum(aws_elb_request_count_sum{%{environment_filter}}) / 60` | -| Latency (ms) | `avg(aws_elb_latency_average{%{environment_filter}}) * 1000` | -| HTTP Error Rate (%) | `sum(aws_elb_httpcode_backend_5_xx_sum{%{environment_filter}}) / sum(aws_elb_request_count_sum{%{environment_filter}})` | - -## Configuring Prometheus to monitor for Cloudwatch metrics - -To get started with Cloudwatch monitoring, install and configure the -[Cloudwatch exporter](https://github.com/prometheus/cloudwatch_exporter). The -Cloudwatch exporter retrieves and parses the specified Cloudwatch metrics, and -translates them into a Prometheus monitoring endpoint. - -The only supported AWS resource is the Elastic Load Balancer, whose Cloudwatch -metrics are listed in [this AWS documentation](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-cloudwatch-metrics.html). - -You can [download a sample Cloudwatch Exporter configuration file](../samples/cloudwatch.yml) -that's configured for basic AWS ELB monitoring. - -## Specifying the Environment label - -To isolate and display only the relevant metrics for a given environment, -GitLab needs a method to detect which labels are associated. To do this, GitLab -[looks for an `environment` label](index.md#identifying-environments). +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/user/project/integrations/prometheus_library/haproxy.md b/doc/user/project/integrations/prometheus_library/haproxy.md index b4533d83acd..ca015d7dc8a 100644 --- a/doc/user/project/integrations/prometheus_library/haproxy.md +++ b/doc/user/project/integrations/prometheus_library/haproxy.md @@ -2,34 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../../../../operations/index.md' --- -# Monitoring HAProxy (DEPRECATED) **(FREE)** +# Monitoring HAProxy (removed) **(FREE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) -in GitLab 14.7, and is planned for removal in GitLab 16.0. - -GitLab has support for automatically detecting and monitoring HAProxy. This is provided by leveraging the [HAProxy Exporter](https://github.com/prometheus/haproxy_exporter), which translates HAProxy statistics into a Prometheus readable form. - -## Requirements - -The [Prometheus service](../prometheus.md) must be enabled. - -## Metrics supported - -| Name | Query | -| ---- | ----- | -| Throughput (req/sec) | `sum(rate(haproxy_frontend_http_requests_total{%{environment_filter}}[2m])) by (code)` | -| HTTP Error Rate (%) | `sum(rate(haproxy_frontend_http_requests_total{code="5xx",%{environment_filter}}[2m])) / sum(rate(haproxy_frontend_http_requests_total{%{environment_filter}}[2m]))` | - -## Configuring Prometheus to monitor for HAProxy metrics - -To get started with NGINX monitoring, you should install and configure the [HAProxy exporter](https://github.com/prometheus/haproxy_exporter) which parses these statistics and translates them into a Prometheus monitoring endpoint. - -## Specifying the Environment label - -In order to isolate and only display relevant metrics for a given environment -however, GitLab needs a method to detect which labels are associated. To do this, GitLab [looks for an `environment` label](index.md#identifying-environments). +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/user/project/integrations/prometheus_library/index.md b/doc/user/project/integrations/prometheus_library/index.md index afefe80271e..ee0b0d1fccb 100644 --- a/doc/user/project/integrations/prometheus_library/index.md +++ b/doc/user/project/integrations/prometheus_library/index.md @@ -2,43 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../../../../operations/index.md' --- -# Prometheus Metrics library (DEPRECATED) **(FREE)** +# Prometheus Metrics library (removed) **(FREE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) -in GitLab 14.7, and is planned for removal in GitLab 16.0. - -GitLab offers automatic detection of select [Prometheus exporters](https://prometheus.io/docs/instrumenting/exporters/). - -## Exporters - -Currently supported exporters are: - -- [Kubernetes](kubernetes.md) -- [NGINX](nginx.md) -- [NGINX Ingress Controller 0.9.0-0.15.x](nginx_ingress_vts.md) -- [NGINX Ingress Controller 0.16.0+](nginx_ingress.md) -- [HAProxy](haproxy.md) -- [Amazon Cloud Watch](cloudwatch.md) - -We have tried to surface the most important metrics for each exporter, and -continue to add support for additional exporters in future releases. If you -would like to add support for other official exporters, contributions are welcome. - -## Identifying Environments - -GitLab retrieves performance data from the configured Prometheus server, and -attempts to identifying the presence of known metrics. Once identified, GitLab -then needs to be able to map the data to a particular environment. - -To isolate and only display relevant metrics for a given environment, -GitLab needs a method to detect which labels are associated. To do that, -GitLab uses the defined queries and fills in the environment specific variables. -Typically this involves looking for the -[`$CI_ENVIRONMENT_SLUG`](../../../../ci/variables/index.md#predefined-cicd-variables), -but may also include other information such as the project's Kubernetes namespace. -Each search query is defined in the [exporter specific documentation](#exporters). +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/user/project/integrations/prometheus_library/kubernetes.md b/doc/user/project/integrations/prometheus_library/kubernetes.md index 34a6823f007..4b8b9b4bc21 100644 --- a/doc/user/project/integrations/prometheus_library/kubernetes.md +++ b/doc/user/project/integrations/prometheus_library/kubernetes.md @@ -2,66 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../../../../operations/index.md' --- -# Monitoring Kubernetes (DEPRECATED) **(FREE)** +# Monitoring Kubernetes (removed) **(FREE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) -in GitLab 14.7, and is planned for removal in GitLab 16.0. - -GitLab has support for automatically detecting and monitoring Kubernetes metrics. - -## Requirements - -The [Prometheus](../prometheus.md) and [Kubernetes](../../../infrastructure/clusters/index.md) -integration services must be enabled. - -## Metrics supported - -- Average Memory Usage (MB): - - ```prometheus - avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}) without (job)) /1024/1024 - ``` - -- Average CPU Utilization (%): - - ```prometheus - avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}[15m])) by (pod_name)) - ``` - -## Configuring Prometheus to monitor for Kubernetes metrics - -Prometheus needs to be deployed into the cluster and configured properly to gather Kubernetes metrics. GitLab supports two methods for doing so: - -- GitLab [integrates with Kubernetes](../../clusters/index.md), and can [query a Prometheus in a connected cluster](../../../clusters/integrations.md#prometheus-cluster-integration). The in-cluster Prometheus can be configured to automatically collect application metrics from your cluster. -- To configure your own Prometheus server, you can follow the [Prometheus documentation](https://prometheus.io/docs/introduction/overview/). - -## Specifying the Environment - -To isolate and only display relevant CPU and Memory metrics for a given environment, GitLab needs a method to detect which containers it is running. Because these metrics are tracked at the container level, traditional Kubernetes labels are not available. - -Instead, the [Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) or [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) name should begin with [CI_ENVIRONMENT_SLUG](../../../../ci/variables/index.md#predefined-cicd-variables). It can be followed by a `-` and additional content if desired. For example, a deployment name of `review-homepage-5620p5` would match the `review/homepage` environment. - -## Displaying Canary metrics **(PREMIUM)** - -GitLab also gathers Kubernetes metrics for [canary deployments](../../canary_deployments.md), allowing easy comparison between the current deployed version and the canary. - -These metrics expect the [Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) or [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) name to begin with `$CI_ENVIRONMENT_SLUG-canary`, to isolate the canary metrics. - -### Canary metrics supported - -- Average Memory Usage (MB) - - ```prometheus - avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-canary-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-canary-(.*)",namespace="%{kube_namespace}"}) without (job)) /1024/1024 - ``` - -- Average CPU Utilization (%) - - ```prometheus - avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-canary-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-canary-(.*)",namespace="%{kube_namespace}"}[15m])) by (pod_name)) - ``` +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/user/project/integrations/prometheus_library/nginx.md b/doc/user/project/integrations/prometheus_library/nginx.md index f0a3b25f11a..995de6a28f3 100644 --- a/doc/user/project/integrations/prometheus_library/nginx.md +++ b/doc/user/project/integrations/prometheus_library/nginx.md @@ -2,42 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../../../../operations/index.md' --- -# Monitoring NGINX (DEPRECATED) **(FREE)** +# Monitoring NGINX (removed) **(FREE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) -in GitLab 14.7, and is planned for removal in GitLab 16.0. - -GitLab has support for automatically detecting and monitoring NGINX. This is provided by leveraging the [NGINX VTS exporter](https://github.com/hnlq715/nginx-vts-exporter), which translates [VTS statistics](https://github.com/vozlt/nginx-module-vts) into a Prometheus readable form. - -## Requirements - -The [Prometheus service](../prometheus.md) must be enabled. - -## Metrics supported - -NGINX server metrics are detected, which tracks the pages and content directly served by NGINX. - -[`environment_filter`](../../../../operations/metrics/dashboards/variables.md#environment_filter) is one of the predefined variables that metrics dashboards support. - -| Name | Query | -| ---- | ----- | -| Throughput (req/sec) | `sum(rate(nginx_server_requests{server_zone!="*", server_zone!="_", %{environment_filter}}[2m])) by (code)` | -| Latency (ms) | `avg(nginx_server_requestMsec{%{environment_filter}})` | -| HTTP Error Rate (HTTP Errors / sec) | `sum(rate(nginx_server_requests{code="5xx", %{environment_filter}}[2m]))` | -| HTTP Error (%)| `sum(rate(nginx_server_requests{code=~"5.*", host="*", %{environment_filter}}[2m])) / sum(rate(nginx_server_requests{code="total", host="*", %{environment_filter}}[2m])) * 100` | - -## Configuring Prometheus to monitor for NGINX metrics - -To get started with NGINX monitoring, you should first enable the [VTS statistics](https://github.com/vozlt/nginx-module-vts) module for your NGINX server. This captures and displays statistics in an HTML readable form. Next, you should install and configure the [NGINX VTS exporter](https://github.com/hnlq715/nginx-vts-exporter) which parses these statistics and translates them into a Prometheus monitoring endpoint. - -If you are using NGINX as your Kubernetes Ingress, GitLab [automatically detects](nginx_ingress.md) the metrics once enabled in 0.9.0 and later releases. - -## Specifying the Environment label - -In order to isolate and only display relevant metrics for a given environment -however, GitLab needs a method to detect which labels are associated. To do this, GitLab [looks for an `environment` label](index.md#identifying-environments). +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/user/project/integrations/prometheus_library/nginx_ingress.md b/doc/user/project/integrations/prometheus_library/nginx_ingress.md index 947210541f4..03f88e6f7c5 100644 --- a/doc/user/project/integrations/prometheus_library/nginx_ingress.md +++ b/doc/user/project/integrations/prometheus_library/nginx_ingress.md @@ -2,48 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../../../../operations/index.md' --- -# Monitoring NGINX Ingress Controller (DEPRECATED) **(FREE)** +# Monitoring NGINX Ingress Controller (removed) **(FREE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) -in GitLab 14.7, and is planned for removal in GitLab 16.0. - -GitLab has support for automatically detecting and monitoring the Kubernetes NGINX Ingress controller. This is provided by leveraging the built-in Prometheus metrics included with Kubernetes NGINX Ingress controller [version 0.16.0](https://github.com/kubernetes/ingress-nginx/blob/master/Changelog.md#0160) onward. - -NOTE: -NGINX Ingress versions prior to 0.16.0 offer an included [VTS Prometheus metrics exporter](nginx_ingress_vts.md), which exports metrics different than the built-in metrics. - -## Requirements - -[Prometheus integration](../prometheus.md) must be active. - -## Metrics supported - -| Name | Query | -| ---- | ----- | -| Throughput (req/sec) | `sum(label_replace(rate(nginx_ingress_controller_requests{namespace="%{kube_namespace}",ingress=~".*%{ci_environment_slug}.*"}[2m]), "status_code", "${1}xx", "status", "(.)..")) by (status_code)` | -| Latency (ms) | `sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_sum{namespace="%{kube_namespace}",ingress=~".*%{ci_environment_slug}.*"}[2m])) / sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_count{namespace="%{kube_namespace}",ingress=~".*%{ci_environment_slug}.*"}[2m])) * 1000` | -| HTTP Error Rate (%) | `sum(rate(nginx_ingress_controller_requests{status=~"5.*",namespace="%{kube_namespace}",ingress=~".*%{ci_environment_slug}.*"}[2m])) / sum(rate(nginx_ingress_controller_requests{namespace="%{kube_namespace}",ingress=~".*%{ci_environment_slug}.*"}[2m])) * 100` | - -## Configuring NGINX Ingress monitoring - -Version 0.9.0 and above of [NGINX Ingress](https://github.com/kubernetes/ingress-nginx) have built-in support for exporting Prometheus metrics. To enable, a ConfigMap setting must be passed: `enable-vts-status: "true"`. Once enabled, a Prometheus metrics endpoint starts running on port 10254. - -Next, the Ingress needs to be annotated for Prometheus monitoring. Two new annotations need to be added: - -- `prometheus.io/scrape: "true"` -- `prometheus.io/port: "10254"` - -Managing these settings depends on how NGINX Ingress has been deployed. If you have deployed via the [official Helm chart](https://github.com/kubernetes/ingress-nginx/tree/master/charts/ingress-nginx), metrics can be enabled with `controller.stats.enabled` along with the required annotations. Alternatively it is possible to edit the NGINX Ingress YAML directly in the [Kubernetes dashboard](https://github.com/kubernetes/dashboard). - -## Specifying the Environment label - -To isolate and display only relevant metrics for a given environment, GitLab needs a method to -detect which labels are associated. To do this, GitLab searches for metrics with appropriate labels. -In this case, the `ingress` label must include the value ``. - -If you have used [Auto Deploy](../../../../topics/autodevops/stages.md#auto-deploy) to deploy your app, this format is used automatically and metrics are detected with no action on your part. +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md b/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md index f8057866592..837be9d1e0a 100644 --- a/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md +++ b/doc/user/project/integrations/prometheus_library/nginx_ingress_vts.md @@ -2,46 +2,11 @@ stage: Monitor group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +remove_date: '2023-08-22' +redirect_to: '../../../../operations/index.md' --- -# Monitoring NGINX Ingress Controller with VTS metrics (DEPRECATED) **(FREE)** +# Monitoring NGINX Ingress Controller with VTS metrics (removed) **(FREE)** -> [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7. - -WARNING: -This feature is in its end-of-life process. It is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) -in GitLab 14.7, and is planned for removal in GitLab 16.0. - -NOTE: -[NGINX Ingress version 0.16](nginx_ingress.md) and above have built-in Prometheus metrics, which are different than the VTS based metrics. - -GitLab has support for automatically detecting and monitoring the Kubernetes NGINX Ingress controller. This is provided by leveraging the included VTS Prometheus metrics exporter in [version 0.9.0](https://github.com/kubernetes/ingress-nginx/blob/master/Changelog.md#09-beta1) through [0.15.x](https://github.com/kubernetes/ingress-nginx/blob/master/Changelog.md#0150). - -## Requirements - -[Prometheus integration](../prometheus.md) must be active. - -## Metrics supported - -| Name | Query | -| ---- | ----- | -| Throughput (req/sec) | `sum(rate(nginx_upstream_responses_total{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m])) by (status_code)` | -| Latency (ms) | `avg(nginx_upstream_response_msecs_avg{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"})` | -| HTTP Error Rate (%) | `sum(rate(nginx_upstream_responses_total{status_code="5xx", upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m])) / sum(rate(nginx_upstream_responses_total{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m])) * 100` | - -## Configuring NGINX Ingress monitoring - -Version 0.9.0 and above of [NGINX Ingress](https://github.com/kubernetes/ingress-nginx) has built-in support for exporting Prometheus metrics. To enable, a ConfigMap setting must be passed: `enable-vts-status: "true"`. Once enabled, a Prometheus metrics endpoint begins running on port 10254. - -Next, the Ingress needs to be annotated for Prometheus monitoring. Two new annotations need to be added: - -- `prometheus.io/scrape: "true"` -- `prometheus.io/port: "10254"` - -Managing these settings depends on how NGINX Ingress has been deployed. If you have deployed via the [official Helm chart](https://github.com/kubernetes/ingress-nginx/tree/master/charts/ingress-nginx), metrics can be enabled with `controller.stats.enabled` along with the required annotations. Alternatively it is possible edit the NGINX Ingress YAML directly in the [Kubernetes dashboard](https://github.com/kubernetes/dashboard). - -## Specifying the Environment label - -To isolate and only display relevant metrics for a given environment, GitLab needs a method to detect which labels are associated. To do this, GitLab searches for metrics with appropriate labels. In this case, the `upstream` label must be of the form `--*`. - -If you have used [Auto Deploy](../../../../topics/autodevops/stages.md#auto-deploy) to deploy your app, this format is used automatically and metrics are detected with no action on your part. +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/346541) in GitLab 14.7 +and [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/399231) in 16.0. diff --git a/doc/user/project/integrations/pumble.md b/doc/user/project/integrations/pumble.md index f9c0c79be1b..e05ff9489ca 100644 --- a/doc/user/project/integrations/pumble.md +++ b/doc/user/project/integrations/pumble.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/project/integrations/redmine.md b/doc/user/project/integrations/redmine.md index 0a399d3481f..df8a6681e2b 100644 --- a/doc/user/project/integrations/redmine.md +++ b/doc/user/project/integrations/redmine.md @@ -1,13 +1,12 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- # Redmine **(FREE)** -Use [Redmine](https://www.redmine.org/) as the issue tracker. - +You can use [Redmine](https://www.redmine.org/) as an external issue tracker. To enable the Redmine integration in a project: 1. On the top bar, select **Main menu > Projects** and find your project. diff --git a/doc/user/project/integrations/servicenow.md b/doc/user/project/integrations/servicenow.md index a34655c8e35..a0beb71d83f 100644 --- a/doc/user/project/integrations/servicenow.md +++ b/doc/user/project/integrations/servicenow.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -9,6 +9,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w ServiceNow offers several integrations to help centralize and automate your management of GitLab workflows. +To simplify your stack and streamline your processes, you should use GitLab [deployment approvals](../../../ci/environments/deployment_approvals.md) whenever possible. + ## GitLab spoke With the GitLab spoke in ServiceNow, you can automate actions for GitLab diff --git a/doc/user/project/integrations/shimo.md b/doc/user/project/integrations/shimo.md index ca09de06dc6..cf9745929a1 100644 --- a/doc/user/project/integrations/shimo.md +++ b/doc/user/project/integrations/shimo.md @@ -1,10 +1,10 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- - + # Shimo (deprecated) **(FREE)** diff --git a/doc/user/project/integrations/slack.md b/doc/user/project/integrations/slack.md index 09e7189d4f5..5b769b84663 100644 --- a/doc/user/project/integrations/slack.md +++ b/doc/user/project/integrations/slack.md @@ -1,16 +1,18 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- + -# Slack notifications **(FREE)** +# Slack notifications (deprecated) **(FREE)** WARNING: -This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/372411) on GitLab.com -in GitLab 15.9 and is [planned for removal](https://gitlab.com/groups/gitlab-org/-/epics/8673). -For GitLab.com, use the [GitLab for Slack app](gitlab_slack_application.md) instead. -For self-managed GitLab instances, you can continue to use this feature. +This feature was [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/372411) in GitLab 15.9 +and is planned for removal in 17.0. Use the [GitLab for Slack app](gitlab_slack_application.md) instead. +This change is a breaking change. +For the planned support of the GitLab for Slack app for self-managed instances, +see [epic 1211](https://gitlab.com/groups/gitlab-org/-/epics/1211). The Slack notifications integration enables your GitLab project to send events (such as issue creation) to your existing Slack team as notifications. Setting up @@ -125,7 +127,7 @@ To view which of these problems is the cause of the issue: ``` If GitLab does not trust HTTPS connections to itself, -[add your certificate to the GitLab trusted certificates](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). +[add your certificate to the GitLab trusted certificates](https://docs.gitlab.com/omnibus/settings/ssl/index.html#install-custom-public-certificates). If GitLab does not trust connections to Slack, the GitLab OpenSSL trust store is incorrect. Typical causes are: @@ -150,3 +152,5 @@ p.each do |project| project.slack_integration.update!(:active, false) end ``` + + diff --git a/doc/user/project/integrations/slack_slash_commands.md b/doc/user/project/integrations/slack_slash_commands.md index 2563cec2b05..4fa4b75422e 100644 --- a/doc/user/project/integrations/slack_slash_commands.md +++ b/doc/user/project/integrations/slack_slash_commands.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -15,7 +15,7 @@ working in Slack, you can use Slack slash commands. To use Slack slash commands, you must configure both Slack and GitLab. GitLab can also send events (for example, `issue created`) to Slack as notifications. -The [Slack notifications service](slack.md) is configured separately. +The [Slack notifications integration](slack.md) is configured separately. ## Configure GitLab and Slack diff --git a/doc/user/project/integrations/squash_tm.md b/doc/user/project/integrations/squash_tm.md new file mode 100644 index 00000000000..2a1106edb0c --- /dev/null +++ b/doc/user/project/integrations/squash_tm.md @@ -0,0 +1,37 @@ +--- +stage: Manage +group: Import and Integrate +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Squash TM **(FREE SELF)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/337855) in GitLab 15.10. + +When [Squash TM](https://www.squashtest.com/squash-gitlab-integration?lang=en) (Test Management) +integration is enabled and configured in GitLab, issues (typically user stories) created in GitLab +are synchronized as requirements in Squash TM and test progress is reported in GitLab issues. + +## Configure Squash TM + +1. Optional. Ask your system administrator to [configure a token in the properties file](https://tm-en.doc.squashtest.com/latest/redirect/gitlab-integration-token.html). +1. Follow the [Squash TM documentation](https://tm-en.doc.squashtest.com/latest/redirect/gitlab-integration-configuration.html) to: + 1. Create a GitLab server. + 1. Enable the `Xsquash4GitLab` plugin + 1. Configure a synchronization. + 1. From the **Real-time synchronization** panel, copy the following fields to use later in GitLab: + + - **Webhook URL**. + - **Secret token** if your Squash TM system administrator configured one at step 1. + +## Configure GitLab + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Integrations**. +1. Select **Squash TM**. +1. Ensure that the **Active** toggle is enabled. +1. In the **Trigger** section, indicate which type of issue is concerned by the real-time synchronization. +1. Complete the fields: + + - Enter the **Squash TM webhook URL**, + - Enter the **secret token** if your Squash TM system administrator configured it earlier. diff --git a/doc/user/project/integrations/unify_circuit.md b/doc/user/project/integrations/unify_circuit.md index d465b4e50f0..d20b19a3aaa 100644 --- a/doc/user/project/integrations/unify_circuit.md +++ b/doc/user/project/integrations/unify_circuit.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w The Unify Circuit integration sends notifications from GitLab to a Circuit conversation. -## Set up Unify Circuit service +## Set up Unify Circuit In Unify Circuit, [add a webhook](https://www.circuit.com/unifyportalfaqdetail?articleId=164448) and copy its URL. diff --git a/doc/user/project/integrations/webex_teams.md b/doc/user/project/integrations/webex_teams.md index 233209966aa..c755c7a5c17 100644 --- a/doc/user/project/integrations/webex_teams.md +++ b/doc/user/project/integrations/webex_teams.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/project/integrations/webhook_events.md b/doc/user/project/integrations/webhook_events.md index 53177004888..fda778aa167 100644 --- a/doc/user/project/integrations/webhook_events.md +++ b/doc/user/project/integrations/webhook_events.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -271,6 +271,7 @@ Payload example: "human_time_estimate": null, "human_time_change": null, "weight": null, + "health_status": "at_risk", "iid": 23, "url": "http://example.com/diaspora/issues/23", "state": "opened", @@ -861,6 +862,7 @@ Payload example: "visibility_level":20, "path_with_namespace":"gitlabhq/gitlab-test", "default_branch":"master", + "ci_config_path":"", "homepage":"http://example.com/gitlabhq/gitlab-test", "url":"http://example.com/gitlabhq/gitlab-test.git", "ssh_url":"git@example.com:gitlabhq/gitlab-test.git", @@ -885,7 +887,10 @@ Payload example: "title": "MS-Viewport", "created_at": "2013-12-03T17:23:34Z", "updated_at": "2013-12-03T17:23:34Z", + "last_edited_at": "2013-12-03T17:23:34Z", + "last_edited_by_id": 1, "milestone_id": null, + "state_id": 1, "state": "opened", "blocking_discussions_resolved": true, "work_in_progress": false, @@ -893,6 +898,11 @@ Payload example: "merge_status": "unchecked", "target_project_id": 14, "description": "", + "total_time_spent": 1800, + "time_change": 30, + "human_total_time_spent": "30m", + "human_time_change": "30s", + "human_time_estimate": "30m", "url": "http://example.com/diaspora/merge_requests/1", "source": { "name":"Awesome Project", @@ -929,6 +939,7 @@ Payload example: "last_commit": { "id": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", "message": "fixed readme", + "title": "Update file README.md", "timestamp": "2012-01-03T23:36:29+02:00", "url": "http://example.com/awesome_space/awesome_project/commits/da1560886d4f094c3e6c9ef40349f7d38b5d27d7", "author": { @@ -997,7 +1008,15 @@ Payload example: "type": "ProjectLabel", "group_id": 41 }] - } + }, + "last_edited_at": { + "previous": null, + "current": "2023-03-15 00:00:10 UTC" + }, + "last_edited_by_id": { + "previous": null, + "current": 3278533 + }, }, "assignees": [ { @@ -1074,7 +1093,8 @@ Payload example: "message": "adding an awesome page to the wiki", "slug": "awesome", "url": "http://example.com/root/awesome-project/-/wikis/awesome", - "action": "create" + "action": "create", + "diff_url": "http://example.com/root/awesome-project/-/wikis/home/diff?version_id=78ee4a6705abfbff4f4132c6646dbaae9c8fb6ec" } } ``` @@ -1392,6 +1412,7 @@ Payload example: "build_started_at": null, "build_finished_at": null, "build_duration": null, + "build_queued_duration": 1095.588715, // duration in seconds "build_allow_failure": false, "build_failure_reason": "script_failure", "retries_count": 2, // the second retry of this job @@ -1487,6 +1508,7 @@ Payload example: "deployable_id": 796, "deployable_url": "http://10.126.0.2:3000/root/test-deployment-webhooks/-/jobs/796", "environment": "staging", + "environment_tier": "staging", "environment_slug": "staging", "environment_external_url": "https://staging.example.com", "project": { diff --git a/doc/user/project/integrations/webhooks.md b/doc/user/project/integrations/webhooks.md index 3d971af5c2a..7fcb2680504 100644 --- a/doc/user/project/integrations/webhooks.md +++ b/doc/user/project/integrations/webhooks.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -99,7 +99,7 @@ You can define URL variables directly using the REST API. ## Configure your webhook receiver endpoint Webhook receiver endpoints should be fast and stable. -Slow and unstable receivers can be [disabled automatically](#failing-webhooks) to ensure system reliability. Webhooks that fail can lead to retries, [which cause duplicate events](#webhook-fails-or-multiple-webhook-requests-are-triggered). +Slow and unstable receivers can be [disabled automatically](#failing-webhooks) to ensure system reliability. Webhooks that fail might lead to [duplicate events](#webhook-fails-or-multiple-webhook-requests-are-triggered). Endpoints should follow these best practices: @@ -118,24 +118,28 @@ Endpoints should follow these best practices: - Never return `500` server error status responses if the event has been handled as this can cause the webhook to be [temporarily disabled](#failing-webhooks). - Invalid HTTP responses are treated as failed requests. -### Failing webhooks +## Failing webhooks -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60837) in GitLab 13.12 [with a flag](../../../administration/feature_flags.md) named `web_hooks_disable_failed`. Disabled by default. -> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/329849) in GitLab 15.7. -> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/329849) in GitLab 15.7. Feature flag `web_hooks_disable_failed` removed. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/60837) for project webhooks in GitLab 13.12 [with a flag](../../../administration/feature_flags.md) named `web_hooks_disable_failed`. Disabled by default. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/329849) for project webhooks in GitLab 15.7. Feature flag `web_hooks_disable_failed` removed. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/385902) for group webhooks in GitLab 15.10. +> - [Disabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/390157) in GitLab 15.10 [with a flag](../../../administration/feature_flags.md) named `auto_disabling_web_hooks`. -If a webhook fails repeatedly, it may be disabled automatically. +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `auto_disabling_web_hooks`. +On GitLab.com, this feature is available. -Webhooks that return response codes in the `5xx` range are understood to be failing +Project or group webhooks that fail four consecutive times are automatically disabled. + +Project or group webhooks that return response codes in the `5xx` range are understood to be failing intermittently and are temporarily disabled. These webhooks are initially disabled -for 1 minute, which is extended on each retry up to a maximum of 24 hours. +for one minute, which is extended on each subsequent failure up to a maximum of 24 hours. -Webhooks that return response codes in the `4xx` range are understood to be +Project or group webhooks that return response codes in the `4xx` range are understood to be misconfigured and are permanently disabled until you manually re-enable them yourself. -See [Troubleshooting](#troubleshoot-webhooks) for more information on -disabled webhooks and how to re-enable them. +For more information about disabled webhooks, see [troubleshooting](#troubleshooting). ## Test a webhook @@ -146,7 +150,7 @@ For example, to test `push events`, your project should have at least one commit NOTE: Testing is not supported for some types of events for project and groups webhooks. -Read more in [issue 379201](https://gitlab.com/gitlab-org/gitlab/-/issues/379201). +For more information, see [issue 379201](https://gitlab.com/gitlab-org/gitlab/-/issues/379201). Prerequisites: @@ -254,7 +258,7 @@ Image URLs are not rewritten if: ## Events -For more information about supported events for Webhooks, go to [Webhook events](webhook_events.md). +For more information about supported events for webhooks, see [webhook events](webhook_events.md). ## Delivery headers @@ -270,7 +274,7 @@ Webhook requests to your endpoint include the following headers: | `X-Gitlab-Event` | Name of the webhook type. Corresponds to [event types](webhook_events.md) but in the format `" Hook"`. | `"Push Hook"` | | `X-Gitlab-Event-UUID` | Unique ID per webhook that is not recursive. A hook is recursive if triggered by an earlier webhook that hit the GitLab instance. Recursive webhooks have the same value for this header. | `"13792a34-cac6-4fda-95a8-c58e00a3954e"` | -## Troubleshoot webhooks +## Troubleshooting > **Recent events** for group webhooks [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/325642) in GitLab 15.3. @@ -288,8 +292,9 @@ To view the table: 1. Scroll down to the webhooks. 1. Each [failing webhook](#failing-webhooks) has a badge listing it as: - - **Failed to connect** if it is misconfigured, and needs manual intervention to re-enable it. - - **Fails to connect** if it is temporarily disabled and is retrying later. + - **Failed to connect** if it's misconfigured and must be manually re-enabled. + - **Fails to connect** if it's temporarily disabled and is automatically + re-enabled after the timeout limit has elapsed. ![Badges on failing webhooks](img/failed_badges.png) @@ -325,8 +330,7 @@ Missing intermediate certificates are common causes of verification failure. ### Webhook fails or multiple webhook requests are triggered -If you are receiving multiple webhook requests, the webhook might have timed out and -been retried. +If you're receiving multiple webhook requests, the webhook might have timed out. GitLab expects a response in [10 seconds](../../../user/gitlab_com/index.md#other-limits). On self-managed GitLab instances, you can [change the webhook timeout limit](../../../administration/instance_limits.md#webhook-timeout). diff --git a/doc/user/project/integrations/youtrack.md b/doc/user/project/integrations/youtrack.md index a020cc61533..ee5ca8eca78 100644 --- a/doc/user/project/integrations/youtrack.md +++ b/doc/user/project/integrations/youtrack.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/project/integrations/zentao.md b/doc/user/project/integrations/zentao.md index ca25c1214b7..19f6a3e315c 100644 --- a/doc/user/project/integrations/zentao.md +++ b/doc/user/project/integrations/zentao.md @@ -1,10 +1,10 @@ --- stage: Manage -group: Integrations +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- - + # ZenTao (deprecated) **(PREMIUM)** diff --git a/doc/user/project/issue_board.md b/doc/user/project/issue_board.md index 234faa893eb..f5d0382ccd8 100644 --- a/doc/user/project/issue_board.md +++ b/doc/user/project/issue_board.md @@ -53,7 +53,7 @@ the issue board feature. > - Multiple issue boards per project [moved](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/53811) to GitLab Free in 12.1. > - Multiple issue boards per group are available in GitLab Premium. -Multiple issue boards allow for more than one issue board for a given project in GitLab Free or group in GitLab Premium and higher tiers. +Multiple issue boards allow for more than one issue board for a given project in the Free tier or group in the Premium and Ultimate tier. This is great for large projects with more than one team or when a repository hosts the code of multiple products. Using the search box at the top of the menu, you can filter the listed boards. @@ -74,7 +74,7 @@ Prerequisites: To create a new issue board: -1. Select the dropdown list with the current board name in the upper left corner of the issue boards page. +1. In the upper-left corner of the issue board page, select the dropdown list with the current board name. 1. Select **Create new board**. 1. Enter the new board's name and select its scope: milestone, labels, assignee, or weight. @@ -86,7 +86,7 @@ Prerequisites: To delete the currently active issue board: -1. Select the dropdown list with the current board name in the upper left corner of the issue boards page. +1. In the upper-left corner of the issue board page, select the dropdown list with the current board name. 1. Select **Delete board**. 1. Select **Delete** to confirm. @@ -97,9 +97,9 @@ Here are some common use cases for issue boards. For examples of using issue boards along with [epics](../group/epics/index.md), [issue health status](issues/managing_issues.md#health-status), and -[scoped labels](labels.md#scoped-labels) for various Agile frameworks, check: +[scoped labels](labels.md#scoped-labels) for various Agile frameworks, see: -- The [How to use GitLab for Agile portfolio planning and project management](https://about.gitlab.com/blog/2020/11/11/gitlab-for-agile-portfolio-planning-project-management/) blog post (November 2020) +- [How to use GitLab for Agile portfolio planning and project management](https://about.gitlab.com/blog/2020/11/11/gitlab-for-agile-portfolio-planning-project-management/) blog post (November 2020) - [Cross-project Agile work management with GitLab](https://www.youtube.com/watch?v=5J0bonGoECs) (15 min, July 2020) @@ -132,10 +132,17 @@ If you have the labels **Backend**, **Frontend**, **Staging**, and With [multiple issue boards](#multiple-issue-boards), each team can have their own board to organize their workflow individually. +
    + See the video: Portfolio Planning - Portfolio Management. +
    +
    + +
    + #### Scrum team With multiple issue boards, each team has one board. Now you can move issues through each -part of the process. For instance: **To Do**, **Doing**, and **Done**. +part of the process. For example: **To Do**, **Doing**, and **Done**. #### Organization of topics @@ -143,7 +150,7 @@ Create lists to order issues by topic and quickly change them between topics or such as between **UX**, **Frontend**, and **Backend**. The changes are reflected across boards, as changing lists updates the labels on each issue accordingly. -#### Advanced team handover +#### Issue board workflow between teams For example, suppose we have a UX team with an issue board that contains: @@ -160,6 +167,9 @@ When finished with something, they move the card to **Frontend**. The Frontend t Cards finished by the UX team automatically appear in the **Frontend** column when they are ready for them. +For a tutorial how to set up your boards in a similar way with [scoped labels](labels.md#scoped-labels), see +[Tutorial: Set up issue boards for team hand-off](../../tutorials/boards_for_teams/index.md). + NOTE: For a broader use case, see the blog post [What is GitLab Flow?](https://about.gitlab.com/topics/version-control/what-is-gitlab-flow/). @@ -207,7 +217,7 @@ Prerequisites: - You must have at least the Reporter role for the project. When an issue is created, the system assigns a relative order value that is greater than the maximum value -of that issue's project or root group. This means the issue will be at the bottom of any issue list that +of that issue's project or root group. This means the issue is at the bottom of any issue list that it appears in. When you visit a board, issues appear ordered in any list. You're able to change @@ -226,6 +236,18 @@ This ordering also affects [issue lists](issues/sorting_issue_lists.md). Changing the order in an issue board changes the ordering in an issue list, and vice versa. +## Focus mode + +To enable or disable focus mode, in the upper-right corner, select **Toggle focus mode** (**{maximize}**). +In focus mode, the navigation UI is hidden, allowing you to focus on issues in the board. + +## Group issue boards + +Accessible at the group navigation level, a group issue board offers the same features as a project-level board. +It can display issues from all projects that fall under the group and its descendant subgroups. + +Users on GitLab Free can use a single group issue board. + ## GitLab Enterprise features for issue boards GitLab issue boards are available on the GitLab Free tier, but some @@ -245,7 +267,7 @@ This allows you to create unique boards according to your team's need. You can define the scope of your board when creating it or by selecting the **Edit board** button. After a milestone, iteration, assignee, or weight is assigned to an issue board, you can no longer -filter through these in the search bar. In order to do that, you need to remove the desired scope +filter through these in the search bar. To do that, you need to remove the desired scope (for example, milestone, assignee, or weight) from the issue board. If you don't have editing permission in a board, you're still able to see the configuration by @@ -255,14 +277,6 @@ selecting **View scope**. Watch a [video presentation](https://youtu.be/m5UTNCSqaDk) of the configurable issue board feature. -### Focus mode - -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28597) to GitLab Free SaaS in 12.10. -> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212331) to GitLab Free self-managed in 13.0. - -To enable or disable focus mode, select the **Toggle focus mode** button (**{maximize}**) at the top -right. In focus mode, the navigation UI is hidden, allowing you to focus on issues in the board. - ### Sum of issue weights **(PREMIUM)** > Moved to GitLab Premium in 13.9. @@ -273,13 +287,6 @@ especially in combination with [assignee lists](#assignee-lists). ![issue board summed weights](img/issue_board_summed_weights_v13_6.png) -### Group issue boards - -Accessible at the group navigation level, a group issue board offers the same features as a project-level board. -It can display issues from all projects that fall under the group and its descendant subgroups. - -Users on GitLab Free can use a single group issue board. - ### Assignee lists **(PREMIUM)** As in a regular list showing all issues with a chosen label, you can add @@ -394,11 +401,11 @@ You can also [drag issues](#move-issues-and-lists) to change their position and ![Drag issues between swimlanes](img/epics_swimlanes_drag_and_drop.png) -## Work In Progress limits **(PREMIUM)** +## Work in progress limits **(PREMIUM)** > Moved to GitLab Premium in 13.9. -You can set a Work In Progress (WIP) limit for each issue list on an issue board. When a limit is +You can set a work in progress (WIP) limit for each issue list on an issue board. When a limit is set, the list's header shows the number of issues in the list and the soft limit of issues. You cannot set a WIP limit on the default lists (**Open** and **Closed**). @@ -413,11 +420,11 @@ Prerequisites: - You must have at least the Reporter role for the project. -To set a WIP limit for a list: +To set a WIP limit for a list, in an issue board: -1. Navigate to a Project or Group board of which you're a member. -1. Select the settings icon in a list's header. -1. Next to **Work In Progress Limit**, select **Edit**. +1. On the top of the list you want to edit, select **List actions** (**{ellipsis_v}**) **> Edit list settings**. + The list settings sidebar opens on the right. +1. Next to **Work in progress limit**, select **Edit**. 1. Enter the maximum number of issues. 1. Press Enter to save. @@ -439,7 +446,6 @@ When you hover over the blocked icon (**{issue-block}**), a detailed information - [Remove an existing list](#remove-a-list). - [Remove an issue from a list](#remove-an-issue-from-a-list). - [Filter issues](#filter-issues) that appear across your issue board. -- [Create workflows](#create-workflows). - [Move issues and lists](#move-issues-and-lists). - [Multi-select issue cards](#multi-select-issue-cards). - Drag and reorder the lists. @@ -475,7 +481,7 @@ Additionally, you can also see the time tracking value. ### Create a new list -Create a new list by selecting the **Create** button in the upper right corner of the issue board. +To create a new list, in the upper-right corner of the issue board, select **Create**. ![creating a new list in an issue board](img/issue_board_add_list_v14_1.png) @@ -493,10 +499,10 @@ Prerequisites: To remove a list from an issue board: -1. On the top of the list you want to remove, select the **List settings** icon (**{settings}**). +1. On the top of the list you want to remove, select **List actions** (**{ellipsis_v}**). The list settings sidebar opens on the right. 1. Select **Remove list**. A confirmation dialog appears. -1. Select **OK**. +1. Select **Remove list** again. ### Add issues to a list @@ -567,44 +573,6 @@ When [filtering issues](#filter-issues) in a **group** board, keep this behavior When you edit issues individually using the right sidebar, you can additionally select the milestones and labels from the **project** that the issue is from. -### Create workflows - -By reordering your lists, you can create workflows. As lists in issue boards are -based on labels, it works out of the box with your existing issues. - -So if you've already labeled things with **Backend** and **Frontend**, the issue appears in -the lists as you create them. In addition, this means you can move something between lists by -changing a label. - -A typical workflow of using an issue board would be: - -1. You [create](labels.md#create-a-label) and [prioritize](labels.md#set-label-priority) - labels to categorize your issues. -1. You have a bunch of issues (ideally labeled). -1. You visit the issue board and start [creating lists](#create-a-new-list) to - create a workflow. -1. You move issues around in lists so that your team knows who should be working - on what issue. -1. When the work by one team is done, the issue can be dragged to the next list - so someone else can pick it up. -1. When the issue is finally resolved, the issue is moved to the **Done** list - and gets automatically closed. - -For example, you can create a list based on the label of **Frontend** and one for -**Backend**. A designer can start working on an issue by adding it to the -**Frontend** list. That way, everyone knows that this issue is now being -worked on by the designers. - -Then, when they're done, all they have to do is -drag it to the next list, **Backend**. Then, a backend developer can -eventually pick it up. When they're done, they move it to **Done**, to close the -issue. - -This process can be seen clearly when visiting an issue. With every move -to another list, the label changes and a system note is recorded. - -![issue board system notes](img/issue_board_system_notes_v13_6.png) - ### Move issues and lists You can move issues and lists by dragging them. diff --git a/doc/user/project/issues/associate_zoom_meeting.md b/doc/user/project/issues/associate_zoom_meeting.md index c7187323850..d286e784e0a 100644 --- a/doc/user/project/issues/associate_zoom_meeting.md +++ b/doc/user/project/issues/associate_zoom_meeting.md @@ -36,7 +36,7 @@ You are only allowed to attach a single Zoom meeting to an issue. If you attempt to add a second Zoom meeting using the `/zoom` quick action, it doesn't work. You need to [remove it](#removing-an-existing-zoom-meeting-from-an-issue) first. -Users on GitLab Premium and higher can also +Users on GitLab Premium and Ultimate can also [add multiple Zoom links to incidents](../../../operations/incident_management/linked_resources.md#link-zoom-meetings-from-an-incident). ## Removing an existing Zoom meeting from an issue diff --git a/doc/user/project/issues/confidential_issues.md b/doc/user/project/issues/confidential_issues.md index 2b302a60d63..79278d1b3ad 100644 --- a/doc/user/project/issues/confidential_issues.md +++ b/doc/user/project/issues/confidential_issues.md @@ -85,18 +85,21 @@ Learn how to create [merge requests for confidential issues](../merge_requests/c There are two kinds of level access for confidential issues. The general rule is that confidential issues are visible only to members of a project with at -least the Reporter role. However, a guest user can also create +least the **Reporter role**. + +However, a user with the **Guest role** can create confidential issues, but can only view the ones that they created themselves. -Users with the Guest role or non-members can also read the confidential issue if they are assigned to the issue. + +Users with the Guest role or non-members can read the confidential issue if they are assigned to the issue. When a Guest user or non-member is unassigned from a confidential issue, they can no longer view it. -Confidential issues are also hidden in search results for unprivileged users. +Confidential issues are hidden in search results for unprivileged users. For example, here's what a user with the Maintainer role and the Guest role -sees in the project's search results respectively. +sees in the project's search results: -| Maintainer role | Guest role | -|:---------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------| +| Maintainer role | Guest role | +|:----------------|:-----------| | ![Confidential issues search by maintainer](img/confidential_issues_search_master.png) | ![Confidential issues search by guest](img/confidential_issues_search_guest.png) | ## Related topics diff --git a/doc/user/project/issues/create_issues.md b/doc/user/project/issues/create_issues.md index 5ebb2fc2e1c..4511c89b0ff 100644 --- a/doc/user/project/issues/create_issues.md +++ b/doc/user/project/issues/create_issues.md @@ -78,7 +78,7 @@ Prerequisites: To create an issue from another issue: -1. In an existing issue, select the vertical ellipsis (**{ellipsis_v}**). +1. In an existing issue, select **Issue actions** (**{ellipsis_v}**). 1. Select **New related issue**. 1. Complete the [fields](#fields-in-the-new-issue-form). The new issue form has a **Relate to issue #123** checkbox, where `123` is the ID of the @@ -100,7 +100,7 @@ To create an issue from a project issue board: 1. On the top bar, select **Main menu > Projects** and find your project. 1. Select **Issues > Boards**. -1. At the top of a board list, select **New issue** (**{plus-square}**). +1. At the top of a board list, select **List actions** (**{ellipsis_v}**) **> Create new issue**. 1. Enter the issue's title. 1. Select **Create issue**. @@ -108,7 +108,7 @@ To create an issue from a group issue board: 1. On the top bar, select **Main menu > Groups** and find your group. 1. Select **Issues > Boards**. -1. At the top of a board list, select **New issue** (**{plus-square}**). +1. At the top of a board list, select **List actions** (**{ellipsis_v}**) **> Create new issue**. 1. Enter the issue's title. 1. Under **Projects**, select the project in the group that the issue should belong to. 1. Select **Create issue**. @@ -118,9 +118,6 @@ example, if the list is scoped to a label `Frontend`, the new issue also has thi ## By sending an email -> - Generated email address format changed in GitLab 11.7. -> - The older format is still supported, so existing aliases and contacts still work. - You can send an email to create an issue in a project on the project's **Issues List** page. diff --git a/doc/user/project/issues/crosslinking_issues.md b/doc/user/project/issues/crosslinking_issues.md index 52da1acd32a..f1f41de7cb4 100644 --- a/doc/user/project/issues/crosslinking_issues.md +++ b/doc/user/project/issues/crosslinking_issues.md @@ -39,10 +39,10 @@ git commit -m "this is my commit message. Ref projectname#xxx" ``` If they are not in the same group, you can add the full URL to the issue -(`https://gitlab.com///issues/`). +(`https://gitlab.com///-/issues/`). ```shell -git commit -m "this is my commit message. Related to https://gitlab.com///issues/" +git commit -m "this is my commit message. Related to https://gitlab.com///-/issues/" ``` Of course, you can replace `gitlab.com` with the URL of your own GitLab instance. diff --git a/doc/user/project/issues/csv_import.md b/doc/user/project/issues/csv_import.md index 8a6683a55b5..d23650e973a 100644 --- a/doc/user/project/issues/csv_import.md +++ b/doc/user/project/issues/csv_import.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Import +group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -36,8 +36,8 @@ To import issues: 1. Go to your project's Issues list page. 1. Open the import feature, depending if the project has issues: - - Existing issues are present: Select the import icon in the upper right, next to **Edit issues**. - - Project has no issues: Select **Import CSV** in the middle of the page. + - The project has existing issues: in the upper-right corner, next to **Bulk edit**, select **Actions** (**{ellipsis_v}**) **> Import CSV**. + - The project has no issues: in the middle of the page, select **Import CSV**. 1. Select the file you want to import, and then select **Import issues**. The file is processed in the background, and a notification email is sent diff --git a/doc/user/project/issues/design_management.md b/doc/user/project/issues/design_management.md index f43f87119a6..10b29feb072 100644 --- a/doc/user/project/issues/design_management.md +++ b/doc/user/project/issues/design_management.md @@ -227,6 +227,19 @@ New discussion threads get different pin numbers, which you can use to refer to In GitLab 12.5 and later, new discussions are output to the issue activity, so that everyone involved can participate in the discussion. +## Delete a comment from a design + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/385100) in GitLab 15.9. + +Prerequisites: + +- You must have at least the Reporter role for the project. + +To delete a comment from a design: + +1. On the comment you want to delete, select **More actions** **{ellipsis_v}** **> Delete comment**. +1. On the confirmation dialog box, select **Delete comment**. + ## Resolve a discussion thread on a design > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13049) in GitLab 13.1. diff --git a/doc/user/project/issues/due_dates.md b/doc/user/project/issues/due_dates.md index f41c90a74a5..63b60e4538f 100644 --- a/doc/user/project/issues/due_dates.md +++ b/doc/user/project/issues/due_dates.md @@ -49,7 +49,8 @@ server's time zone. Issues with due dates can also be exported as an iCalendar feed. The URL of the feed can be added to calendar applications. The feed is accessible by selecting -the **Subscribe to calendar** button on the following pages: +the **Subscribe to calendar** option in the **Actions** (**{ellipsis_v}**) dropdown +list on the following pages: - The **Assigned Issues** page linked on the right side of the GitLab header - The **Project Issues** page diff --git a/doc/user/project/issues/issue_weight.md b/doc/user/project/issues/issue_weight.md index d852ad3262b..05c348acf58 100644 --- a/doc/user/project/issues/issue_weight.md +++ b/doc/user/project/issues/issue_weight.md @@ -1,5 +1,4 @@ --- -disqus_identifier: 'https://docs.gitlab.com/ee/workflow/issue_weight.html' stage: Plan group: Project Management info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments diff --git a/doc/user/project/issues/managing_issues.md b/doc/user/project/issues/managing_issues.md index c16074ea1d8..276cc3bc7a5 100644 --- a/doc/user/project/issues/managing_issues.md +++ b/doc/user/project/issues/managing_issues.md @@ -54,7 +54,7 @@ To edit multiple issues at the same time: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Issues**. -1. Select **Edit issues**. A sidebar on the right of your screen appears. +1. Select **Bulk edit**. A sidebar on the right of your screen appears. 1. Select the checkboxes next to each issue you want to edit. 1. From the sidebar, edit the available fields. 1. Select **Update all**. @@ -87,7 +87,7 @@ To edit multiple issues at the same time: 1. On the top bar, select **Main menu > Groups** and find your group. 1. On the left sidebar, select **Issues**. -1. Select **Edit issues**. A sidebar on the right of your screen appears. +1. Select **Bulk edit**. A sidebar on the right of your screen appears. 1. Select the checkboxes next to each issue you want to edit. 1. From the sidebar, edit the available fields. 1. Select **Update all**. @@ -103,7 +103,7 @@ When bulk editing issues in a group, you can edit the following attributes: ## Move an issue When you move an issue, it's closed and copied to the target project. -The original issue is not deleted. A system note, which indicates +The original issue is not deleted. A [system note](../system_notes.md), which indicates where it came from and went to, is added to both issues. Be careful when moving an issue to a project with different access rules. Before moving the issue, make sure it does not contain sensitive data. @@ -136,7 +136,7 @@ To move multiple issues at the same time: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Issues**. -1. Select **Edit issues**. A sidebar on the right of your screen appears. +1. Select **Bulk edit**. A sidebar on the right of your screen appears. 1. Select the checkboxes next to each issue you want to move. 1. From the right sidebar, select **Move selected**. 1. From the dropdown list, select the destination project. @@ -209,6 +209,10 @@ To close an issue, you can do the following: - At the top of the issue, select **Close issue**. - In an [issue board](../issue_board.md), drag an issue card from its list into the **Closed** list. + +If you don't see this action at the top of an issue, your project or instance might have +enabled a feature flag for [moved actions](../merge_requests/index.md#move-sidebar-actions). + ### Reopen a closed issue Prerequisites: @@ -298,7 +302,7 @@ To disable automatic issue closing: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Settings > Repository**. -1. Expand **Default branch**. +1. Expand **Branch defaults**. 1. Clear the **Auto-close referenced issues on default branch** checkbox. 1. Select **Save changes**. @@ -344,7 +348,7 @@ Prerequisites: To delete an issue: -1. In an issue, select the vertical ellipsis (**{ellipsis_v}**). +1. In an issue, select **Issue actions** (**{ellipsis_v}**). 1. Select **Delete issue**. Alternatively: @@ -362,7 +366,7 @@ You can promote an issue to an [epic](../../group/epics/index.md) in the immedia To promote an issue to an epic: -1. In an issue, select the vertical ellipsis (**{ellipsis_v}**). +1. In an issue, select **Issue actions** (**{ellipsis_v}**). 1. Select **Promote to epic**. Alternatively, you can use the `/promote` [quick action](../quick_actions.md#issues-merge-requests-and-epics). @@ -400,7 +404,7 @@ To view all issues assigned to you: Or: - To use a [keyboard shortcut](../../shortcuts.md), press Shift + i. -- On the top bar, in the upper right, select **{issues}** **Issues**. +- On the top bar, in the upper-right corner, select **Issues** (**{issues}**). ## Filter the list of issues @@ -472,6 +476,10 @@ You can now paste the reference into another description or comment. Read more about issue references in [GitLab-Flavored Markdown](../../markdown.md#gitlab-specific-references). + +If you don't see this action on the right sidebar, your project or instance might have +enabled a feature flag for [moved actions](../merge_requests/index.md#move-sidebar-actions). + ## Copy issue email address > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18816) in GitLab 13.8. diff --git a/doc/user/project/issues/sorting_issue_lists.md b/doc/user/project/issues/sorting_issue_lists.md index a2f90d5c444..829e44eae9a 100644 --- a/doc/user/project/issues/sorting_issue_lists.md +++ b/doc/user/project/issues/sorting_issue_lists.md @@ -73,7 +73,7 @@ then issues with a milestone without a due date. ## Sorting by popularity When you sort by **Popularity**, the issue order changes to sort descending by the -number of upvotes ([awarded](../../award_emojis.md) a "thumbs up" emoji) +number of upvotes ([emoji reactions](../../award_emojis.md) with the "thumbs up") on each issue. You can use this to identify issues that are in high demand. ## Sorting by priority diff --git a/doc/user/project/labels.md b/doc/user/project/labels.md index f9e3f1b9225..2af15e06b98 100644 --- a/doc/user/project/labels.md +++ b/doc/user/project/labels.md @@ -136,7 +136,7 @@ To do so: 1. Fill in the name field. You can't specify a description if creating a label this way. You can add a description later by [editing the label](#edit-a-label). 1. Select a color by selecting from the available colors, or enter a hex color value for a specific color. -1. Select **Create**. +1. Select **Create**. Your label is created and selected. ### Create a group label @@ -381,7 +381,7 @@ issue is now under review, they assign the `workflow::review`, and the `workflow is removed. The same happens when you move issues across label lists in an -[issue board](issue_board.md#create-workflows). With scoped labels, team members not working in an +[issue board](issue_board.md). With scoped labels, team members not working in an issue board can also advance workflow states consistently in issues themselves. For a video explanation, see: diff --git a/doc/user/project/members/index.md b/doc/user/project/members/index.md index c7d45b0bd15..6e20492db05 100644 --- a/doc/user/project/members/index.md +++ b/doc/user/project/members/index.md @@ -1,6 +1,6 @@ --- -stage: Manage -group: Organization +stage: Data Stores +group: Tenant Scale info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/project/members/share_project_with_groups.md b/doc/user/project/members/share_project_with_groups.md index 4dda68a6d08..452363c3d9a 100644 --- a/doc/user/project/members/share_project_with_groups.md +++ b/doc/user/project/members/share_project_with_groups.md @@ -1,6 +1,6 @@ --- -stage: Manage -group: Organization +stage: Data Stores +group: Tenant Scale info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -28,17 +28,21 @@ To invite a group to a project, you must be at least one of the following: In addition: -- The group you're inviting must have a more restrictive - [visibility level](../../public_access.md#project-and-group-visibility) - than the project. For example, you can invite: - - A private group to a public project. - - An internal group to a public project. - - A private group to an internal project. +- You must be a member of the group or the subgroup being invited. -- The group or subgroup must be in the project's [namespace](../../namespace/index.md). +- The [visibility level](../../public_access.md#project-and-group-visibility) of the group you're inviting +must be at least as restrictive as that of the project. For example, you can invite: + - A _private_ group to a _private_ project + - A _private_ group to an _internal_ project. + - A _private_ group to a _public_ project. + - An _internal_ group to an _internal_ project. + - An _internal_ group to a _public_ project. + - A _public_ group to a _public_ project. + +- If the project's root ancestor group [does not allow the project to be shared outside the hierarchy](../../group/access_and_permissions.md#prevent-group-sharing-outside-the-group-hierarchy), the invited group or subgroup must be in the project's [namespace](../../namespace/index.md). For example, a project in the namespace `group/subgroup01/project`: - Can be shared with `group/subgroup02` or `group/subgroup01/subgroup03`. - - Cannot be shared with `group`. + - Cannot be shared with `group_abc`. ## Share a project with a group @@ -70,17 +74,21 @@ are given access to the project. In addition: ## Maximum role +When you invite a group to a project, the maximum role is the highest level of access the invited group members are allowed to have in the project. + When multiple groups contain the same members, and the groups have access to the same project, the group members are -given the most restrictive role for the project. - -This most restrictive role is called the *maximum role*, or **Max role**. +given the highest access level of the two for the project. The member's **Max role** is the more restrictive of: - The role the user is assigned for the group. - The role you chose when you invited the group to the project. +NOTE: +The Max role does not elevate the privileges of users. +For example, if a group member has the role of Developer, and the group is invited to a project with a Max role of Maintainer, the member's role is not elevated to Maintainer. + ### View the member's Max role To view the maximum role assigned to a member: diff --git a/doc/user/project/merge_requests/allow_collaboration.md b/doc/user/project/merge_requests/allow_collaboration.md index 0729e024df4..d5851050fd4 100644 --- a/doc/user/project/merge_requests/allow_collaboration.md +++ b/doc/user/project/merge_requests/allow_collaboration.md @@ -61,14 +61,14 @@ In the following example: To change or add a commit to the contributor's merge request: 1. Go to the merge request. -1. In the upper right corner, select **Code**, then select **Check out branch**. +1. In the upper-right corner, select **Code**, then select **Check out branch**. 1. In the modal window, select **Copy** (**{copy-to-clipboard}**). 1. In your terminal, go to your cloned version of the repository, and paste the commands. For example: ```shell git fetch "git@gitlab.com:contributor/forked-project.git" 'fork-branch' - git checkout -b contributor/fork-branch' FETCH_HEAD + git checkout -b 'contributor/fork-branch' FETCH_HEAD ``` Those commands fetch the branch from the forked project, and create a local branch diff --git a/doc/user/project/merge_requests/approvals/index.md b/doc/user/project/merge_requests/approvals/index.md index 21e2055cb61..717358df9f3 100644 --- a/doc/user/project/merge_requests/approvals/index.md +++ b/doc/user/project/merge_requests/approvals/index.md @@ -3,7 +3,6 @@ stage: Create group: Source Code info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments" type: reference, concepts -disqus_identifier: 'https://docs.gitlab.com/ee/user/project/merge_requests/approvals/index.html' --- # Merge request approvals **(FREE)** @@ -17,7 +16,7 @@ approve merge requests, these approvals are [optional](#optional-approvals). flexibility: - Create required [rules](rules.md) about the number and type of approvers before work can merge. -- Specify a list of users who act as [code owners](../../code_owners.md) for specific files, +- Specify a list of users who act as [code owners](../../codeowners/index.md) for specific files, and require their approval before work can merge. You can configure merge request approvals on a per-project basis, and some approvals can be configured @@ -36,7 +35,7 @@ required approvals before work can merge into your project. You can also extend rules to define what types of users can approve work. Some examples of rules you can create include: - Users with specific permissions can always approve work. -- [Code owners](../../code_owners.md) can approve work for files they own. +- [Code owners](../../codeowners/index.md) can approve work for files they own. - Users with specific permissions can approve work, [even if they don't have merge rights](rules.md#merge-request-approval-segregation-of-duties) to the repository. @@ -102,20 +101,31 @@ Without the approvals, the work cannot merge. Required approvals enable multiple database, for all proposed code changes. - Use the [code owners of changed files](rules.md#code-owners-as-eligible-approvers), to determine who should review the work. -- Require an [approval before merging code that causes test coverage to decline](../../../../ci/pipelines/settings.md#coverage-check-approval-rule) +- Require an [approval before merging code that causes test coverage to decline](../../../../ci/testing/code_coverage.md#coverage-check-approval-rule). - Users on GitLab Ultimate can also [require approval from a security team](../../../application_security/index.md#security-approvals-in-merge-requests) before merging code that could introduce a vulnerability. ## Invalid rules -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/334698) in GitLab 15.1. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/334698) in GitLab 15.1. +> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/389905) in GitLab 15.11 [with a flag](../../../../administration/feature_flags.md) named `invalid_scan_result_policy_prevents_merge`. Disabled by default. -Whenever an approval rule cannot be satisfied, the rule will be displayed as `Invalid`. This applies to the following conditions: +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available per project or for your entire instance, +ask an administrator to [enable the feature flag](../../../../administration/feature_flags.md) named `invalid_scan_result_policy_prevents_merge`. +On GitLab.com, this feature is available but can be configured by GitLab.com administrators only. + +Whenever an approval rule cannot be satisfied, the rule is displayed as **(!) Auto approved**. This applies to the following conditions: - The only eligible approver is the author of the merge request. - No eligible approvers (either groups or users) have been assigned to the approval rule. +- The number of required approvals is more than the number of eligible approvers. + +These rules are automatically approved to unblock their respective merge requests, +unless they were created through a security policy. -These rules will be automatically approved to unblock their respective merge requests. +Invalid approval rules created through a security policy are presented with **(!) Action Required** +and are not automatically approved, blocking their respective merge requests. ## Related topics diff --git a/doc/user/project/merge_requests/approvals/rules.md b/doc/user/project/merge_requests/approvals/rules.md index 0a5e7c29860..d55ca5dff04 100644 --- a/doc/user/project/merge_requests/approvals/rules.md +++ b/doc/user/project/merge_requests/approvals/rules.md @@ -53,7 +53,7 @@ To add a merge request approval rule: 1. Select **Add approval rule**. -Users of GitLab Premium and higher tiers can create [additional approval rules](#add-multiple-approval-rules). +Users of GitLab Premium and Ultimate tiers can create [additional approval rules](#add-multiple-approval-rules). Your configuration for approval rule overrides determines if the new rule is applied to existing merge requests: @@ -91,7 +91,7 @@ To edit a merge request approval rule: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1979) in GitLab 11.10. -In GitLab Premium and higher tiers, you can enforce multiple approval rules on a +In GitLab Premium and Ultimate tiers, you can enforce multiple approval rules on a merge request, and multiple default approval rules for a project. If your tier supports multiple default rules: @@ -106,6 +106,9 @@ reduces the number of approvals left (the **Approvals** column) for all rules th ![Approvals premium merge request widget](img/approvals_premium_mr_widget_v13_3.png) + +For an overview, see [Multiple Approvers](https://www.youtube.com/watch?v=8JQJ5821FrA). + ## Eligible approvers > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/10294) in GitLab 13.3, when an eligible approver comments on a merge request, it appears in the **Commented by** column of the Approvals widget. @@ -131,7 +134,7 @@ who commented on the merge request. It helps authors and reviewers identify who contact with questions about the merge request's content. If the number of required approvals is greater than the number of assigned approvers, -approvals from other users with Developer [permissions](../../../permissions.md) or higher +approvals from other users with at least the Developer [role](../../../permissions.md) in the project counts toward meeting the required number of approvals, even if the users were not explicitly listed in the approval rules. @@ -158,7 +161,7 @@ approve in these ways: > Moved to GitLab Premium in 13.9. -If you add [code owners](../../code_owners.md) to your repository, the owners of files +If you add [code owners](../../codeowners/index.md) to your repository, the owners of files become eligible approvers in the project. To enable this merge request approval rule: 1. Go to your project and select **Settings > Merge requests**. @@ -241,6 +244,28 @@ approval rule for certain branches: 1. To enable this configuration, read [Code Owner's approvals for protected branches](../../protected_branches.md#require-code-owner-approval-on-a-protected-branch). +## Code coverage check approval + +You can require specific approvals in the case that a merge request would reduce code test +coverage. + +For more information, see [Coverage check approval rule](../../../../ci/testing/code_coverage.md#coverage-check-approval-rule). + +## Security Approvals **(ULTIMATE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/357021) in GitLab 15.0. + +You can use [scan result policies](../../../application_security/policies/scan-result-policies.md#scan-result-policy-editor) to define security approvals based on the status of vulnerabilities in the merge request and the default branch. +Details for each security policy is shown in the Security Approvals section of your Merge Request configuration. + +The security approval rules are applied to all merge requests until the pipeline is complete. The application of the +security approval rules prevents users from merging in code before the security scans run. After the pipeline is +complete, the security approval rules are checked to determine if the security approvals are still required. + +![Security Approvals](img/security_approvals_v15_0.png) + +These policies are both created and edited in the [security policy editor](../../../application_security/policies/index.md#policy-editor). + ## Troubleshooting ### Approval rule name can't be blank @@ -262,18 +287,3 @@ isn't recognized as a valid Code Owner for the project, nor does it display in t project's **Approvals** list. To fix this problem, add the approval group as a shared group high enough in the shared hierarchy so the project requiring review inherits this group of users. - -## Security Approvals **(ULTIMATE)** - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/357021) in GitLab 15.0. - -You can use [scan result policies](../../../application_security/policies/scan-result-policies.md#scan-result-policy-editor) to define security approvals based on the status of vulnerabilities in the merge request and the default branch. -Details for each security policy is shown in the Security Approvals section of your Merge Request configuration. - -The security approval rules are applied to all merge requests until the pipeline is complete. The application of the -security approval rules prevents users from merging in code before the security scans run. Once the pipeline is -complete, the security approval rules are checked to determine if the security approvals are still required. - -![Security Approvals](img/security_approvals_v15_0.png) - -These policies are both created and edited in the [security policy editor](../../../application_security/policies/index.md#policy-editor). diff --git a/doc/user/project/merge_requests/approvals/settings.md b/doc/user/project/merge_requests/approvals/settings.md index 1ab564c108b..6c079dc9319 100644 --- a/doc/user/project/merge_requests/approvals/settings.md +++ b/doc/user/project/merge_requests/approvals/settings.md @@ -78,13 +78,13 @@ their own. To do this: it can't be changed at the project level. 1. Select **Save changes**. -Depending on your version of GitLab, [code owners](../../code_owners.md) who commit +Depending on your version of GitLab, [code owners](../../codeowners/index.md) who commit to a merge request may or may not be able to approve the work: -- In GitLab 13.10 and earlier, [code owners](../../code_owners.md) who commit +- In GitLab 13.10 and earlier, code owners who commit to a merge request can approve it, even if the merge request affects files they own. - In [GitLab 13.11 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/331548), - [code owners](../../code_owners.md) who commit + code owners who commit to a merge request cannot approve it, when the merge request affects files they own. For more information, see the [official Git documentation](https://git-scm.com/book/en/v2/Git-Basics-Viewing-the-Commit-History). @@ -121,7 +121,7 @@ permission enables an electronic signature for approvals, such as the one define ## Remove all approvals when commits are added to the source branch By default, an approval on a merge request is removed when you add more changes -after the approval. In GitLab Premium and higher tiers, to keep existing approvals +after the approval. In GitLab Premium and Ultimate tiers, to keep existing approvals after more changes are added to the merge request: 1. On the left sidebar, select **Settings > Merge requests**. @@ -140,7 +140,7 @@ However, approvals are reset if the target branch is changed. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/90578) in GitLab 15.3. -If you only want to remove approvals by Code Owners whose files have been changed: +If you only want to remove approvals by Code Owners whose files have been changed when a commit is added: Prerequisite: @@ -153,13 +153,6 @@ To do this: select **Remove approvals by Code Owners if their files changed**. 1. Select **Save changes**. -## Code coverage check approvals - -You can require specific approvals if a merge request would result in a decline in code test -coverage. - -For more information, see [Coverage check approval rule](../../../../ci/pipelines/settings.md#coverage-check-approval-rule). - ## Settings cascading > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/285410) in GitLab 14.4. [Deployed behind the `group_merge_request_approval_settings_feature_flag` flag](../../../../administration/feature_flags.md), disabled by default. diff --git a/doc/user/project/merge_requests/cherry_pick_changes.md b/doc/user/project/merge_requests/cherry_pick_changes.md index 9fac872ac4b..004d24778b7 100644 --- a/doc/user/project/merge_requests/cherry_pick_changes.md +++ b/doc/user/project/merge_requests/cherry_pick_changes.md @@ -55,7 +55,7 @@ by the merge request: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Merge requests**, and find your merge request. 1. Scroll to the merge request reports section, and find the **Merged by** report. -1. In the upper right, select **Cherry-pick**: +1. In the upper-right corner, select **Cherry-pick**: ![Cherry-pick merge request](img/cherry_pick_v15_4.png) 1. In the modal window, select the project and branch to cherry-pick into. @@ -72,7 +72,8 @@ To cherry-pick a commit from the list of all commits for a project: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Repository > Commits**. -1. Select the title of the commit you want to cherry-pick. +1. Select the [title](https://git-scm.com/docs/git-commit#_discussion) of the commit you want to cherry-pick. +1. In the upper-right corner, select **Options > Cherry-pick** to show the cherry-pick modal. 1. In the modal window, select the project and branch to cherry-pick into. 1. Optional. Select **Start a new merge request with these changes**. 1. Select **Cherry-pick**. @@ -86,7 +87,7 @@ list of commits included in a merge request: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Merge requests**, and find your merge request. 1. In the merge request's secondary menu, select **Commits** to display the commit details page. -1. Select the title of the commit you want to cherry-pick. +1. Select the [title](https://git-scm.com/docs/git-commit#_discussion) of the commit you want to cherry-pick. 1. In the upper-right corner, select **Options > Cherry-pick** to show the cherry-pick modal. 1. In the modal window, select the project and branch to cherry-pick into. 1. Optional. Select **Start a new merge request with these changes**. @@ -100,7 +101,8 @@ when you view that file in your project's Git repository: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Repository > Files** and go to the file changed by the commit. -1. Select **History**, then select the title of the commit you want to cherry-pick. +1. Select **History**, then select the [title](https://git-scm.com/docs/git-commit#_discussion) + of the commit you want to cherry-pick. 1. In the upper-right corner, select **Options > Cherry-pick** to show the cherry-pick modal. 1. In the modal window, select the project and branch to cherry-pick into. 1. Optional. Select **Start a new merge request with these changes**. @@ -123,7 +125,7 @@ project, from the GitLab user interface: ## View system notes for cherry-picked commits -When you cherry-pick a merge commit in the GitLab UI or API, GitLab adds a system note +When you cherry-pick a merge commit in the GitLab UI or API, GitLab adds a [system note](../system_notes.md) to the related merge request thread in the format **{cherry-pick-commit}** `[USER]` **picked the changes into the branch** `[BRANCHNAME]` with commit** `[SHA]` `[DATE]`: diff --git a/doc/user/project/merge_requests/commits.md b/doc/user/project/merge_requests/commits.md index a9f67c39ae8..cc6ecd8398f 100644 --- a/doc/user/project/merge_requests/commits.md +++ b/doc/user/project/merge_requests/commits.md @@ -1,11 +1,59 @@ --- -redirect_to: '../merge_requests/index.md' -remove_date: '2023-03-12' +stage: Create +group: Code Review +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +type: index, reference --- -This document was removed. +# Merge request commits **(FREE)** - - - - +Each merge request has a history of the commits made to the source branch +after the merge request was created. + +These commits are displayed on the merge request's **Commits** tab. +From this tab, you can review commit messages and copy a commit's SHA when you need to +[cherry-pick changes](cherry_pick_changes.md). + +## Navigate merge request commits + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18140) in GitLab 13.0. + +To navigate commits in a merge request: + +1. Select the **Commits** tab. +1. Select the commit link. The most recent commit is displayed. +1. Navigate through the commits by either: + + - Selecting **Prev** and **Next** buttons below the tab buttons. + - Using the X and C keyboard shortcuts. + +![Merge requests commit navigation](img/commit_nav_v16_0.png) + +## View merge request commits in context + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/29274) in GitLab 13.12 [with a flag](../../../administration/feature_flags.md) named `context_commits`. Enabled by default. +> - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/320757) in GitLab 14.8. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/320757) in GitLab 14.9. [Feature flag `context_commits`](https://gitlab.com/gitlab-org/gitlab/-/issues/320757) removed. + +When reviewing a merge request, it helps to have more context about the changes +made. That includes unchanged lines in unchanged files, and previous commits +that have already merged that the change is built on. + +To add previously merged commits to a merge request for more context: + +1. Go to your merge request. +1. Select the **Commits** tab. +1. Scroll to the end of the list of commits, and select **Add previously merged commits**: +1. Select the commits that you want to add. +1. Select **Save changes**. + +## View diffs between commits + +To view the changes between previously merged commits: + +1. On your merge request, select the **Changes** tab. +1. By **Compare**, select the commit you want to view: + + ![Previously merged commits](img/previously_merged_commits_v16_0.png) + +If you selected to add previously merged commits, they are displayed in the list. diff --git a/doc/user/project/merge_requests/confidential.md b/doc/user/project/merge_requests/confidential.md index 307998f697d..edcc5711b98 100644 --- a/doc/user/project/merge_requests/confidential.md +++ b/doc/user/project/merge_requests/confidential.md @@ -65,7 +65,7 @@ Your branch, merge requests, and commits remain in your private fork. This preve prematurely revealing confidential information. Open a merge request -[from your fork to the upstream repository](../repository/forking_workflow.md#merging-upstream) when: +[from your fork to the upstream repository](../repository/forking_workflow.md#merge-changes-back-upstream) when: - You are satisfied the problem is resolved in your private fork. - You are ready to make the confidential commits public. diff --git a/doc/user/project/merge_requests/conflicts.md b/doc/user/project/merge_requests/conflicts.md index 932eec5e631..cc8f8cb2fe6 100644 --- a/doc/user/project/merge_requests/conflicts.md +++ b/doc/user/project/merge_requests/conflicts.md @@ -162,9 +162,8 @@ merge commit. Verify it contains no unintended changes and doesn't break your bu ## Related topics -- [Introduction to Git rebase and force-push](../../../topics/git/git_rebase.md). -- [Git GUI apps](https://git-scm.com/downloads/guis) to help you visualize the - differences between branches and resolve them. +- [Introduction to Git rebase and force-push](../../../topics/git/git_rebase.md) +- [Git applications for visualizing the Git workflow](https://git-scm.com/downloads/guis) - - - diff --git a/doc/user/project/merge_requests/img/commit_nav_v16_0.png b/doc/user/project/merge_requests/img/commit_nav_v16_0.png new file mode 100644 index 00000000000..6005e516fff Binary files /dev/null and b/doc/user/project/merge_requests/img/commit_nav_v16_0.png differ diff --git a/doc/user/project/merge_requests/img/previously_merged_commits_v16_0.png b/doc/user/project/merge_requests/img/previously_merged_commits_v16_0.png new file mode 100644 index 00000000000..31204b6e801 Binary files /dev/null and b/doc/user/project/merge_requests/img/previously_merged_commits_v16_0.png differ diff --git a/doc/user/project/merge_requests/img/remove_source_branch_status.png b/doc/user/project/merge_requests/img/remove_source_branch_status.png deleted file mode 100644 index afd93207e02..00000000000 Binary files a/doc/user/project/merge_requests/img/remove_source_branch_status.png and /dev/null differ diff --git a/doc/user/project/merge_requests/index.md b/doc/user/project/merge_requests/index.md index a633366cc62..a65c5518658 100644 --- a/doc/user/project/merge_requests/index.md +++ b/doc/user/project/merge_requests/index.md @@ -26,11 +26,27 @@ view [this GitLab Flow video](https://www.youtube.com/watch?v=InKNIvky2KE). Learn the various ways to [create a merge request](creating_merge_requests.md). +### Use merge request templates + +When you create a merge request, GitLab checks for the existence of a +[description template](../description_templates.md) to add data to your merge request. +GitLab checks these locations in order from 1 to 5, and applies the first template +found to your merge request: + +| Name | Project UI
    setting | Group
    `default.md` | Instance
    `default.md` | Project
    `default.md` | No template | +| :-- | :--: | :--: | :--: | :--: | :--: | +| Standard commit message | 1 | 2 | 3 | 4 | 5 | +| Commit message with an [issue closing pattern](../issues/managing_issues.md#closing-issues-automatically) like `Closes #1234` | 1 | 2 | 3 | 4 | 5 \* | +| Branch name [prefixed with an issue ID](../repository/branches/index.md#prefix-branch-names-with-issue-numbers), like `1234-example` | 1 \* | 2 \* | 3 \* | 4 \* | 5 \* | + +NOTE: +Items marked with an asterisk (\*) also append an [issue closing pattern](../issues/managing_issues.md#closing-issues-automatically). + ## View merge requests You can view merge requests for your project, group, or yourself. -### View merge requests for a project +### For a project To view all merge requests for a project: @@ -39,7 +55,7 @@ To view all merge requests for a project: Or, to use a [keyboard shortcut](../../shortcuts.md), press g + m. -### View merge requests for all projects in a group +### For all projects in a group To view merge requests for all projects in a group: @@ -48,7 +64,7 @@ To view merge requests for all projects in a group: If your group contains subgroups, this view also displays merge requests from the subgroup projects. -### View all merge requests assigned to you +### Assigned to you To view all merge requests assigned to you: @@ -65,7 +81,7 @@ or: or: -1. On the top bar, in the upper right, select **{merge-request-open}** **Merge requests**. +1. On the top bar, in the upper-right corner, select **Merge requests** (**{merge-request-open}**). 1. From the dropdown list, select **Assigned to you**. ## Filter the list of merge requests @@ -79,12 +95,12 @@ To filter the list of merge requests: 1. Above the list of merge requests, select **Search or filter results...**. 1. From the dropdown list, select the attribute you wish to filter by. Some examples: - - [**By environment or deployment date**](#filter-merge-requests-by-environment-or-deployment-date). + - [**By environment or deployment date**](#by-environment-or-deployment-date). - **ID**: Enter filter `#30` to return only merge request 30. - User filters: Type (or select from the dropdown list) any of these filters to display a list of users: - **Approved-By**, for merge requests already approved by a user. **(PREMIUM)**. - **Approver**, for merge requests that this user is eligible to approve. - (For more information, read about [Code owners](../code_owners.md)). **(PREMIUM)** + (For more information, read about [Code owners](../codeowners/index.md)). **(PREMIUM)** - **Reviewer**, for merge requests reviewed by this user. 1. Select or type the operator to use for filtering the attribute. The following operators are available: @@ -100,7 +116,7 @@ To filter the list of merge requests: GitLab displays the results on-screen, but you can also [retrieve them as an RSS feed](../../search/index.md#retrieve-search-results-as-feed). -### Filter merge requests by environment or deployment date +### By environment or deployment date > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/44041) in GitLab 13.6. @@ -251,14 +267,15 @@ after merging does not retarget open merge requests. This improvement is -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85584) in GitLab 14.10 [with a flag](../../../administration/feature_flags.md) named `moved_mr_sidebar`. Disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85584) in GitLab 14.10 [with a flag](../../../administration/feature_flags.md) named `moved_mr_sidebar`. Disabled by default. +> - [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/373757) to also move actions on issues, incidents, and epics in GitLab 16.0. FLAG: On self-managed GitLab, by default this feature is not available. To make it available per project or for your entire instance, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `moved_mr_sidebar`. On GitLab.com, this feature is enabled in the following projects: `gitlab-org/gitlab`, `gitlab-com/www-gitlab-com`, and `gitlab-org/customers-gitlab-com`. -When this feature flag is enabled, you can find the following actions in -**Merge request actions** (**{ellipsis_v}**) in the upper right: +When this feature flag is enabled, in the upper-right corner, +**Merge request actions** (**{ellipsis_v}**) contains the following actions: - The [notifications](../../profile/notifications.md#edit-notification-settings-for-issues-merge-requests-and-epics) toggle - Mark merge request as ready or [draft](../merge_requests/drafts.md) @@ -266,6 +283,8 @@ When this feature flag is enabled, you can find the following actions in - [Lock discussion](../../discussions/index.md#prevent-comments-by-locking-the-discussion) - Copy reference +In GitLab 16.0 and later, similar action menus are available on issues, incidents, and epics. + When this feature flag is disabled, these actions are in the right sidebar. ## Merge request workflows @@ -295,6 +314,43 @@ For a web developer writing a webpage for your company's website: 1. Once approved, your merge request is [squashed and merged](squash_and_merge.md), and [deployed to staging with GitLab Pages](https://about.gitlab.com/blog/2021/02/05/ci-deployment-and-environments/). 1. Your production team [cherry-picks](cherry_pick_changes.md) the merge commit into production. +## Filter activity in a merge request + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/115383) in GitLab 15.11 [with a flag](../../../administration/feature_flags.md) named `mr_activity_filters`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. +To make it available per user, ask an administrator to [enable the feature flag](../../../administration/feature_flags.md) named `mr_activity_filters` for individual or groups of users. +On GitLab.com, this feature unavailable. + +To understand the history of a merge request, filter its activity feed to show you +only the items that are relevant to you. + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Merge requests**. +1. Select a merge request. +1. Scroll to **Activity**. +1. On the right side of the page, select **Activity filter** to show the filter options. + If you've selected filter options previously, this field shows a summary of your + choices, like **Activity + 5 more**. +1. Select the types of activity you want to see. Options include: + + - Assignees & Reviewers + - Approvals + - Comments + - Commits & branches + - Edits + - Labels + - Lock status + - Mentions + - Merge request status + - Tracking + +1. Optional. Select **Sort** (**{sort-lowest}**) to reverse the sort order. + +Your selection persists across all merge requests. You can also change the +sort order by clicking the sort button on the right. + ## Related topics - [Create a merge request](creating_merge_requests.md) @@ -304,7 +360,6 @@ For a web developer writing a webpage for your company's website: - [GitLab keyboard shortcuts](../../shortcuts.md) - [Comments and threads](../../discussions/index.md) - [Suggest code changes](reviews/suggestions.md) -- [Commits](commits.md) - [CI/CD pipelines](../../../ci/index.md) - [Push options](../push_options.md) for merge requests diff --git a/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md b/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md index f0359446b06..7588af70bd4 100644 --- a/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md +++ b/doc/user/project/merge_requests/merge_when_pipeline_succeeds.md @@ -5,7 +5,12 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference, concepts --- -# Merge when pipeline succeeds **(FREE)** +# Auto-merge **(FREE)** + +> **Merge when pipeline succeeds** and **Add to merge train when pipeline succeeds** [renamed](https://gitlab.com/gitlab-org/gitlab/-/issues/409530) to **Auto-merge** in GitLab 16.0 [with a flag](../../../administration/feature_flags.md) named `auto_merge_labels_mr_widget`. Enabled by default. + +NOTE: +[In GitLab 16.0 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/359057), **Merge when pipeline succeeds** and **Add to merge train when pipeline succeeds** become **Set to auto-merge**. If you review a merge request and it's ready to merge, but the pipeline hasn't completed yet, you can set it to merge when the pipeline succeeds (MWPS). You don't @@ -147,3 +152,11 @@ merge-request-pipeline-job: Instead, use branch (`push`) pipelines or merge request pipelines, when possible. For details on avoiding two pipelines for a single merge request, read the [`rules` documentation](../../../ci/jobs/job_control.md#avoid-duplicate-pipelines). + +### Merged results pipeline allows merge, despite a failed branch pipeline + +When [the **Pipelines must succeed** setting](#require-a-successful-pipeline-for-merge) +is combined with +[the **Merged results pipelines** feature](../../../ci/pipelines/merged_results_pipelines.md), +failed branch pipeline may be ignored. +[Issue 385841](https://gitlab.com/gitlab-org/gitlab/-/issues/385841) is open to track this. diff --git a/doc/user/project/merge_requests/methods/index.md b/doc/user/project/merge_requests/methods/index.md index 1f7e15ee982..02bd4ed0502 100644 --- a/doc/user/project/merge_requests/methods/index.md +++ b/doc/user/project/merge_requests/methods/index.md @@ -228,5 +228,4 @@ workflow that requires frequent rebases. ## Related topics -- [Commits history](../commits.md) - [Squash and merge](../squash_and_merge.md) diff --git a/doc/user/project/merge_requests/revert_changes.md b/doc/user/project/merge_requests/revert_changes.md index 5878873f209..77fd78ee0d0 100644 --- a/doc/user/project/merge_requests/revert_changes.md +++ b/doc/user/project/merge_requests/revert_changes.md @@ -58,7 +58,8 @@ To do this: 1. On the top bar, select **Main menu > Projects** and find your project. 1. If you know the merge request that contains the commit: 1. On the left sidebar, select **Merge requests** and identify your merge request. - 1. Select **Commits**, then select the title of the commit you want to revert. GitLab displays the contents of the commit. + 1. Select **Commits**, then select the title of the commit you want to revert. This displays the commit in the **Changes** tab of your merge request. + 1. Select the commit hash you want to revert. GitLab displays the contents of the commit. 1. If you don't know the merge request the commit originated from: 1. On the left sidebar, select **Repository > Commits**. 1. Select the title of the commit to display full information about the commit. diff --git a/doc/user/project/merge_requests/reviews/data_usage.md b/doc/user/project/merge_requests/reviews/data_usage.md index dd07f0b4a6e..f0eb3c015b6 100644 --- a/doc/user/project/merge_requests/reviews/data_usage.md +++ b/doc/user/project/merge_requests/reviews/data_usage.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: index, reference --- -# Suggested Reviewers Data Usage +# Suggested Reviewers Data Usage **(ULTIMATE SAAS)** ## How it works diff --git a/doc/user/project/merge_requests/reviews/img/add_another_suggestion_to_batch_v13_1.jpg b/doc/user/project/merge_requests/reviews/img/add_another_suggestion_to_batch_v13_1.jpg deleted file mode 100644 index e8aa4b7c730..00000000000 Binary files a/doc/user/project/merge_requests/reviews/img/add_another_suggestion_to_batch_v13_1.jpg and /dev/null differ diff --git a/doc/user/project/merge_requests/reviews/img/add_first_suggestion_to_batch_v13_1.jpg b/doc/user/project/merge_requests/reviews/img/add_first_suggestion_to_batch_v13_1.jpg deleted file mode 100644 index d15f5d53e55..00000000000 Binary files a/doc/user/project/merge_requests/reviews/img/add_first_suggestion_to_batch_v13_1.jpg and /dev/null differ diff --git a/doc/user/project/merge_requests/reviews/img/apply_batch_of_suggestions_v13_1.jpg b/doc/user/project/merge_requests/reviews/img/apply_batch_of_suggestions_v13_1.jpg deleted file mode 100644 index 3e1e9c20af9..00000000000 Binary files a/doc/user/project/merge_requests/reviews/img/apply_batch_of_suggestions_v13_1.jpg and /dev/null differ diff --git a/doc/user/project/merge_requests/reviews/img/apply_suggestion_v13_9.png b/doc/user/project/merge_requests/reviews/img/apply_suggestion_v13_9.png deleted file mode 100644 index e27fa629672..00000000000 Binary files a/doc/user/project/merge_requests/reviews/img/apply_suggestion_v13_9.png and /dev/null differ diff --git a/doc/user/project/merge_requests/reviews/img/custom_commit_v13_9.png b/doc/user/project/merge_requests/reviews/img/custom_commit_v13_9.png deleted file mode 100644 index 170c04542dd..00000000000 Binary files a/doc/user/project/merge_requests/reviews/img/custom_commit_v13_9.png and /dev/null differ diff --git a/doc/user/project/merge_requests/reviews/img/make_suggestion_v13_9.png b/doc/user/project/merge_requests/reviews/img/make_suggestion_v13_9.png deleted file mode 100644 index 92d5ba5ddda..00000000000 Binary files a/doc/user/project/merge_requests/reviews/img/make_suggestion_v13_9.png and /dev/null differ diff --git a/doc/user/project/merge_requests/reviews/img/multi-line-suggestion-syntax.png b/doc/user/project/merge_requests/reviews/img/multi-line-suggestion-syntax.png deleted file mode 100644 index 80424d1f056..00000000000 Binary files a/doc/user/project/merge_requests/reviews/img/multi-line-suggestion-syntax.png and /dev/null differ diff --git a/doc/user/project/merge_requests/reviews/img/remove_suggestion_from_batch_v13_1.jpg b/doc/user/project/merge_requests/reviews/img/remove_suggestion_from_batch_v13_1.jpg deleted file mode 100644 index fa8331effdb..00000000000 Binary files a/doc/user/project/merge_requests/reviews/img/remove_suggestion_from_batch_v13_1.jpg and /dev/null differ diff --git a/doc/user/project/merge_requests/reviews/img/suggestion_button_v13_9.png b/doc/user/project/merge_requests/reviews/img/suggestion_button_v13_9.png deleted file mode 100644 index 58e0508d8cf..00000000000 Binary files a/doc/user/project/merge_requests/reviews/img/suggestion_button_v13_9.png and /dev/null differ diff --git a/doc/user/project/merge_requests/reviews/img/suggestions_custom_commit_messages_v14_7.png b/doc/user/project/merge_requests/reviews/img/suggestions_custom_commit_messages_v14_7.png deleted file mode 100644 index 2805ef19f2d..00000000000 Binary files a/doc/user/project/merge_requests/reviews/img/suggestions_custom_commit_messages_v14_7.png and /dev/null differ diff --git a/doc/user/project/merge_requests/reviews/index.md b/doc/user/project/merge_requests/reviews/index.md index 5291a845437..0de38874304 100644 --- a/doc/user/project/merge_requests/reviews/index.md +++ b/doc/user/project/merge_requests/reviews/index.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: index, reference --- -# Review a merge request **(FREE)** +# Merge request reviews **(FREE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/216054) in GitLab 13.5. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/245190) in GitLab 13.9. @@ -21,16 +21,19 @@ You can review merge requests from the GitLab interface. If you install the [GitLab Workflow VS Code extension](../../repository/vscode.md), you can also review merge requests in Visual Studio Code. + +For an overview, see [Merge request review](https://www.youtube.com/watch?v=2MayfXKpU08&list=PLFGfElNsQthYDx0A_FaNNfUm9NHsK6zED&index=183). + ## Suggested reviewers **(ULTIMATE SAAS)** -> [Introduced](https://gitlab.com/groups/gitlab-org/modelops/applied-ml/review-recommender/-/epics/3) in GitLab 15.4. +> - [Introduced](https://gitlab.com/groups/gitlab-org/modelops/applied-ml/review-recommender/-/epics/3) in GitLab 15.4 as a [Beta](../../../../policy/alpha-beta-support.md#beta) feature [with a flag](../../../../administration/feature_flags.md) named `suggested_reviewers_control`. Disabled by default. +> - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/368356) in GitLab 15.6. +> - Beta designation [removed from the UI](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/113058) in GitLab 15.10. GitLab can suggest reviewers. Using the changes in a merge request and a project's contribution graph, machine learning suggestions appear in the reviewer section of the right sidebar. ![Suggested Reviewers](img/suggested_reviewers_v15_9.png) -This feature is currently in [Open Beta](https://about.gitlab.com/handbook/product/gitlab-the-product/#open-beta) behind a [feature flag](https://gitlab.com/gitlab-org/gitlab/-/issues/368356). - For more information, see [Data usage in Suggested Reviewers](data_usage.md). ### Enable suggested reviewers @@ -84,7 +87,7 @@ To download the changes included in a merge request as a diff: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Merge requests**. 1. Select your merge request. -1. In the upper right, select **Code > Plain diff**. +1. In the upper-right corner, select **Code > Plain diff**. If you know the URL of the merge request, you can also download the diff from the command line by appending `.diff` to the URL. This example downloads the diff @@ -107,7 +110,7 @@ To download the changes included in a merge request as a patch file: 1. On the top bar, select **Main menu > Projects** and find your project. 1. On the left sidebar, select **Merge requests**. 1. Select your merge request. -1. In the upper right, select **Code > Email patches**. +1. In the upper-right corner, select **Code > Patches**. If you know the URL of the merge request, you can also download the patch from the command line by appending `.patch` to the URL. This example downloads the patch @@ -172,7 +175,7 @@ If you have a review in progress, you can also add a comment from the **Overview When editing the **Reviewers** field in a new or existing merge request, GitLab displays the name of the matching [approval rule](../approvals/rules.md) -below the name of each suggested reviewer. [Code Owners](../../code_owners.md) are displayed as `Codeowner` without group detail. +below the name of each suggested reviewer. [Code Owners](../../codeowners/index.md) are displayed as `Codeowner` without group detail. This example shows reviewers and approval rules when creating a new merge request: @@ -234,7 +237,7 @@ When bulk-editing merge requests in a project, you can edit the following attrib To update multiple project merge requests at the same time: 1. In a project, go to **Merge requests**. -1. Select **Edit merge requests**. A sidebar on the right-hand side of your screen appears with +1. Select **Bulk edit**. A sidebar on the right-hand side of your screen appears with editable fields. 1. Select the checkboxes next to each merge request you want to edit. 1. Select the appropriate fields and their values from the sidebar. @@ -254,7 +257,7 @@ When bulk editing merge requests in a group, you can edit the following attribut To update multiple group merge requests at the same time: 1. In a group, go to **Merge requests**. -1. Select **Edit merge requests**. A sidebar on the right-hand side of your screen appears with +1. Select **Bulk edit**. A sidebar on the right-hand side of your screen appears with editable fields. 1. Select the checkboxes next to each merge request you want to edit. 1. Select the appropriate fields and their values from the sidebar. @@ -424,3 +427,4 @@ than 1000. The cached value is rounded to thousands (or millions) and updated ev ## Related topics - [Merge methods](../methods/index.md) +- [Draft Notes API](../../../../api/draft_notes.md) diff --git a/doc/user/project/merge_requests/reviews/suggestions.md b/doc/user/project/merge_requests/reviews/suggestions.md index 668dece9fda..9187c5fad44 100644 --- a/doc/user/project/merge_requests/reviews/suggestions.md +++ b/doc/user/project/merge_requests/reviews/suggestions.md @@ -8,156 +8,176 @@ type: index, reference # Suggest changes **(FREE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/25381) custom commit messages for suggestions in GitLab 13.9 [with a flag](../../../../administration/feature_flags.md) named `suggestions_custom_commit`. Disabled by default. -> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/297404) in GitLab 13.10. - -As a reviewer, you're able to suggest code changes with a Markdown syntax in merge request -diff threads. Then, the merge request author (or other users with appropriate -[permission](../../../permissions.md)) can apply these suggestions. -This action generates a commit in the merge request, authored by the user that suggested the changes. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/297404) in GitLab 13.10. Feature flag `suggestions_custom_commit` removed. + +Reviewers can suggest code changes with a Markdown syntax in merge request diff threads. +The merge request author (or other users with the appropriate role) can apply any or +all of the suggestions from the GitLab UI. Applying suggestions adds a commit to the +merge request, authored by the user who suggested the changes. + +## Create suggestions + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Merge requests** and find your merge request. +1. On the secondary menu, select **Changes**. +1. Find the lines of code you want to change. + - To select a single line: + - Hover over the line number, and + select **Add a comment to this line** (**{comment}**). + - To select multiple lines: + 1. Hover over the line number, and select **Add a comment to this line** (**{comment}**). + 1. Select and drag your selection until all desired lines are included. To + learn more, see [Multi-line suggestions](#multi-line-suggestions). +1. In the comment toolbar, select **Insert suggestion** (**{doc-code}**). GitLab + inserts a pre-populated code block into your comment, like this: + + ````markdown + ```suggestion:-0+0 + The content of the line you selected is shown here. + ``` + ```` + +1. Edit the pre-populated code block to add your suggestion. +1. Select either **Start a review** or **Add to review** to add your comment to a + [review](index.md), or **Add comment now** to add the comment to the thread immediately. -1. Choose a line of code to be changed, add a new comment, then select - the **Insert suggestion** icon in the toolbar: +### Multi-line suggestions - ![Add a new comment](img/suggestion_button_v13_9.png) +> [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/232339) in GitLab 13.11: suggestions in multi-line comments also become multi-line. -1. In the comment, add your suggestion to the pre-populated code block: +When you review a merge request diff, you can propose changes to multiple lines (up to 200) +in a single suggestion, by either: - ![Add a suggestion into a code block tagged properly](img/make_suggestion_v13_9.png) +- Selecting and dragging, as described in [Create suggestions](#create-suggestions). + GitLab creates a suggestion block for you. +- Selecting a single line, then manually adjusting the range offsets. -1. Select either **Start a review** or **Add to review** to add your comment to a - [review](index.md), or **Add comment now** to add the comment to the thread immediately. +The range offsets in the first line of the suggestion describe line numbers relative +to the line you selected. The offsets specify the lines your suggestion intends to replace. +For example, this suggestion covers 3 lines above and 4 lines below the +commented line: - The suggestion in the comment can be applied by the merge request author - directly from the merge request: +````markdown +```suggestion:-3+4 + "--talk-name=ca.desrt.dconf", + "--socket=wayland", +``` +```` - ![Apply suggestions](img/apply_suggestion_v13_9.png) +When applied, the suggestion replaces from 3 lines above to 4 lines below the commented line: -1. Optionally specify a custom commit message for individual suggestions (GitLab 13.9 and later) to - describe your change. If you don't specify it, the default commit message is used. +![Multi-line suggestion preview](img/multi-line-suggestion-preview.png) - ![Custom commit](img/custom_commit_v13_9.png) +Suggestions for multiple lines are limited to 100 lines _above_ and 100 +lines _below_ the commented diff line. This allows for up to 200 changed lines per +suggestion. -After the author applies a suggestion: +## Apply suggestions -1. The suggestion is marked as **Applied**. -1. The thread is resolved. -1. GitLab creates a new commit with the changes. -1. If the user has the Developer role, GitLab pushes - the suggested change directly into the codebase in the merge request's branch. +Prerequisites: -## Multi-line suggestions +- You must be the author of the merge request, or have at least the Developer role in the project. -> [Changed](https://gitlab.com/gitlab-org/gitlab/-/issues/232339) in GitLab 13.11: suggestions in multi-line comments also become multi-line. +To apply suggested changes directly from the merge request: -Reviewers can also suggest changes to multiple lines with a single suggestion -within merge request diff threads by selecting and dragging selection to all -relevant line numbers or by adjusting the range offsets. The -offsets are relative to the position of the diff thread, and specify the -range to be replaced by the suggestion when it is applied. +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Merge requests** and find your merge request. +1. Find the comment containing the suggestion you want to apply. + - To apply suggestions individually, select **Apply suggestion**. + - To apply multiple suggestions in a single commit, select **Add suggestion to batch**. +1. Optional. Provide a custom commit message to describe your change. If you don't provide a custom message, the default commit message is used. +1. Select **Apply**. -![Multi-line suggestion syntax](img/multi-line-suggestion-syntax.png) +After a suggestion is applied: -In the previous example, the suggestion covers three lines above and four lines -below the commented line. When applied, it would replace from 3 lines _above_ -to 4 lines _below_ the commented line, with the suggested change. +- The suggestion is marked as **Applied**. +- The comment thread is resolved. +- GitLab creates a new commit with the changes. +- If the user has the Developer role, GitLab pushes + the suggested change directly into the codebase in the merge request's branch. -![Multi-line suggestion preview](img/multi-line-suggestion-preview.png) - -NOTE: -Suggestions for multiple lines are limited to 100 lines _above_ and 100 -lines _below_ the commented diff line. This allows for up to 200 changed lines per -suggestion. - -## Code block nested in suggestions +## Nest code blocks in suggestions To add a suggestion that includes a [fenced code block](../../../markdown.md#code-spans-and-blocks), wrap your suggestion in four backticks instead of three: -~~~markdown +`````markdown ````suggestion:-0+2 ```shell git config --global receive.advertisepushoptions true ``` ```` -~~~ +````` ![Output of a comment with a suggestion with a fenced code block](img/suggestion_code_block_output_v12_8.png) ## Configure the commit message for applied suggestions -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13086) in GitLab 12.7. +GitLab uses a default commit message when applying suggestions. This message +supports placeholders, and can be changed. For example, the default message +`Apply %{suggestions_count} suggestion(s) to %{files_count} file(s)` renders +like this if you apply three suggestions to two different files: -GitLab uses a default commit message -when applying suggestions: `Apply %{suggestions_count} suggestion(s) to %{files_count} file(s)` - - +```plaintext +Apply 3 suggestion(s) to 2 file(s) +``` -For example, consider that a user applied 3 suggestions to 2 different files, the -default commit message is: **Apply 3 suggestion(s) to 2 file(s)** +Merge requests created from forks use the template defined in the target project. - +To meet your project's needs, you can customize these messages and include other +placeholder variables: -These commit messages can be customized to follow any guidelines you might have. -To do so, expand the **Merge requests** tab within your project's **General** -settings and change the **Merge suggestions** text: +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > Merge requests**. +1. Scroll to **Merge suggestions**, and alter the text to meet your needs. + See [Supported variables](#supported-variables) for a list of placeholders + you can use in this message. -![Custom commit message for applied suggestions](img/suggestions_custom_commit_messages_v14_7.png) +### Supported variables -You can also use following variables besides static text: +The template for commit messages for applied suggestions supports these variables: | Variable | Description | Output example | |------------------------|-------------|----------------| | `%{branch_name}` | The name of the branch to which suggestions were applied. | `my-feature-branch` | -| `%{files_count}` | The number of files to which suggestions were applied.| **2** | +| `%{files_count}` | The number of files to which suggestions were applied.| `2` | | `%{file_paths}` | The paths of the file to which suggestions were applied. Paths are separated by commas.| `docs/index.md, docs/about.md` | | `%{project_path}` | The project path. | `my-group/my-project` | -| `%{project_name}` | The human-readable name of the project. | **My Project** | -| `%{suggestions_count}` | The number of suggestions applied.| **3** | +| `%{project_name}` | The human-readable name of the project. | `My Project` | +| `%{suggestions_count}` | The number of suggestions applied.| `3` | | `%{username}` | The username of the user applying suggestions. | `user_1` | -| `%{user_full_name}` | The full name of the user applying suggestions. | **User 1** | +| `%{user_full_name}` | The full name of the user applying suggestions. | `User 1` | For example, to customize the commit message to output -**Addresses user_1's review**, set the custom text to +`Addresses user_1's review`, set the custom text to `Addresses %{username}'s review`. -For merge requests created from forks, GitLab uses the template defined in target project. - -NOTE: -Custom commit messages for each applied suggestion is -introduced by [#25381](https://gitlab.com/gitlab-org/gitlab/-/issues/25381). - ## Batch suggestions -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/25486) in GitLab 13.1 as an [alpha feature](../../../../policy/alpha-beta-support.md#alpha-features) with a flag named `batch_suggestions`, disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/25486) in GitLab 13.1 as an [Experiment](../../../../policy/alpha-beta-support.md#experiment) with a flag named `batch_suggestions`, disabled by default. > - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/227799) in GitLab 13.2. > - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/320755) in GitLab 13.11. [Feature flag `batch_suggestions`](https://gitlab.com/gitlab-org/gitlab/-/issues/320755) removed. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/326168) custom commit messages for batch suggestions in GitLab 14.4. -You can apply multiple suggestions at once to reduce the number of commits added -to your branch to address your reviewers' requests. - -1. To start a batch of suggestions to apply with a single commit, select **Add suggestion to batch**: - - ![A code change suggestion displayed, with the add-suggestion option highlighted.](img/add_first_suggestion_to_batch_v13_1.jpg "Add a suggestion to a batch") - -1. Add as many additional suggestions to the batch as you wish: - - ![A code change suggestion displayed, with the add-more suggestion option highlighted.](img/add_another_suggestion_to_batch_v13_1.jpg "Add another suggestion to a batch") - -1. To remove suggestions, select **Remove from batch**: - - ![A code change suggestion displayed, with the option to remove that suggestion from its batch highlighted.](img/remove_suggestion_from_batch_v13_1.jpg "Remove a suggestion from a batch") +To reduce the number of commits added to your branch, you can apply multiple +suggestions in a single commit. -1. Having added all the suggestions to your liking, when ready, select **Apply suggestions**. You - can optionally specify a custom commit message for [batch suggestions](#batch-suggestions) - (GitLab 14.4 and later) to describe your change. If you don't specify it, the default commit - message is used. +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Merge requests** and find your merge request. +1. For each suggestion you want to apply, and select **Add suggestion to batch**. +1. Optional. To remove a suggestion, select **Remove from batch**. +1. After you add your desired suggestions, select **Apply suggestions**. - ![A code change suggestion displayed, with the option to apply the batch of suggestions highlighted.](img/apply_batch_of_suggestions_v13_1.jpg "Apply a batch of suggestions") + WARNING: + If you apply a batch of suggestions containing changes from multiple authors, + you are credited as the resulting commit's author. If your project is configured + to [prevent approvals from users who add commits](../approvals/settings.md#prevent-approvals-by-users-who-add-commits), you are no longer an eligible + approver for this merge request. -WARNING: -Suggestions applied from multiple authors creates a commit authored by the user applying the suggestions. +1. Optional. Provide a custom commit message for [batch suggestions](#batch-suggestions) + (GitLab 14.4 and later) to describe your change. If you don't specify one, + the default commit message is used. ## Related topics diff --git a/doc/user/project/merge_requests/squash_and_merge.md b/doc/user/project/merge_requests/squash_and_merge.md index 9f87f1e2e0d..075716e90c8 100644 --- a/doc/user/project/merge_requests/squash_and_merge.md +++ b/doc/user/project/merge_requests/squash_and_merge.md @@ -33,7 +33,14 @@ squash commits and merge commits. ## Set default squash options for a merge request Users with permission to create or edit a merge request can set the default squash options -for a merge request. To do this: +for a merge request. + +Prerequisites: + +- Your project must be [configured](#configure-squash-options-for-a-project) to allow or + encourage squashing. + +To do this: 1. Go to the merge request and select **Edit**. 1. Select or clear the **Squash commits when merge request is accepted** checkbox. @@ -58,6 +65,10 @@ squash the commits as part of the merge process: > - [Enabled on GitLab.com and self-managed by default](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39382) in GitLab 13.3. > - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/232536) in GitLab 13.8. Feature flag `squash_options` removed. +Prerequisites: + +- You must have at least the Maintainer role for this project. + To configure the default squashing behavior for all merge requests in your project: 1. On the top bar, select **Main menu > Projects** and find your project. diff --git a/doc/user/project/merge_requests/status_checks.md b/doc/user/project/merge_requests/status_checks.md index 62a2baa049b..0e339c65ed5 100644 --- a/doc/user/project/merge_requests/status_checks.md +++ b/doc/user/project/merge_requests/status_checks.md @@ -3,7 +3,6 @@ stage: Govern group: Compliance info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments" type: reference, concepts -disqus_identifier: 'https://docs.gitlab.com/ee/user/project/merge_requests/status_checks.html' --- # External status checks **(ULTIMATE)** @@ -12,6 +11,8 @@ disqus_identifier: 'https://docs.gitlab.com/ee/user/project/merge_requests/statu > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/320783) in GitLab 14.1. > - `failed` status [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/329636) in GitLab 14.9. +Status checks are API calls to external systems that request the status of an external requirement. + You can create a status check that sends merge request data to third-party tools. When users create, change, or close merge requests, GitLab sends a notification. The users or automated workflows can then update the status of merge requests from outside of GitLab. @@ -70,7 +71,7 @@ using the API. You don't need to wait for a merge request webhook payload to be ## View the status checks on a project -Within each project's settings, you can see a list of status checks added to the project: +Within each project's settings, you can see a list of status check services added to the project: 1. In your project, go to **Settings > Merge requests** section. 1. Scroll down to **Status checks**. @@ -80,9 +81,9 @@ Within each project's settings, you can see a list of status checks added to the This list shows the service name, API URL, and targeted branch. It also provides actions to allow you to create, edit, or remove status checks. -## Add or update a status check +## Add or update a status check service -### Add a status check +### Add a status check service Within the **Status checks** sub-section, select the **Add status check** button. The **Add status check** form is then shown. @@ -91,7 +92,7 @@ The **Add status check** form is then shown. Filling in the form and selecting the **Add status check** button creates a new status check. -### Update a status check +### Update a status check service Within the **Status checks** sub-section, select the **Edit** button next to the status check you want to edit. @@ -134,7 +135,7 @@ for doesn't appear immediately. The search box requires If you want the status check to be applied to **all** merge requests, you can select the **All branches** option. -## Delete a status check +## Delete a status check service Within the **Status checks** sub-section, select the **Remove...** button next to the status check you want to delete. @@ -151,12 +152,15 @@ the status check and it **is not** recoverable. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327634) in GitLab 14.1. > - UI [updated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91504) in GitLab 15.2. > - Ability to retry failed external status checks [added](https://gitlab.com/gitlab-org/gitlab/-/issues/383200) in GitLab 15.8. +> - Widget [updated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/111763) to poll for updates when there are pending status checks in GitLab 15.11. The status checks widget displays in merge requests and displays the following statuses: - **pending** (**{status-neutral}**), while GitLab waits for a response from an external status check. - **success** (**{status-success}**) or **failed** (**{status-failed}**), when GitLab receives a response from an external status check. +When there are pending status checks, the widget polls for updates every few seconds until it receives a **success** or **failed** response. + To retry a failed status check: 1. Expand the merge request widget to show the list of external status checks. diff --git a/doc/user/project/milestones/burndown_and_burnup_charts.md b/doc/user/project/milestones/burndown_and_burnup_charts.md index 81b334c0a02..c7ed6069cb6 100644 --- a/doc/user/project/milestones/burndown_and_burnup_charts.md +++ b/doc/user/project/milestones/burndown_and_burnup_charts.md @@ -137,14 +137,16 @@ To switch between the two settings, select either **Issues** or **Issue weight** When sorting by weight, make sure all your issues have weight assigned, because issues with no weight don't show on the chart. - +A limitation of these charts is that [the days are in the UTC time zone](https://gitlab.com/gitlab-org/gitlab/-/issues/267967). + +This can cause the graphs to be inaccurate in other timezones. For example: + +- All the issues in a milestone are recorded as being closed on or before the last day. +- One issue was closed on the last day at 6 PM PST (Pacific time), which is UTC-7. +- The issue activity log displays the closure time at 6 PM on the last day of the milestone. +- The charts plot the time in UTC, so for this issue, the close time is 1 AM the following day. +- The charts show the milestone as incomplete and missing one closed issue. diff --git a/doc/user/project/milestones/index.md b/doc/user/project/milestones/index.md index 5f9a2961df5..4641af262ca 100644 --- a/doc/user/project/milestones/index.md +++ b/doc/user/project/milestones/index.md @@ -65,7 +65,10 @@ Improving this experience is tracked in issue [339009](https://gitlab.com/gitlab You can view all the milestones you have access to in the entire GitLab namespace. You might not see some milestones because they're in projects or groups you're not a member of. -To do so, on the top bar select **Main menu > Milestones**. +To do so: + +1. On the top bar select **Main menu > Your work**. +1. On the left sidebar, select **Milestones**. ### View milestone details @@ -139,7 +142,7 @@ To edit a milestone: 1. On the top bar, select **Main menu > Projects** and find your project or **Main menu > Groups** and find your group. 1. Select a milestone's title. -1. Select **Edit**. +1. In the top right corner, select **Milestone actions** (**{ellipsis_v}**) and then select **Edit**. 1. Edit the title, start date, due date, or description. 1. Select **Save changes**. @@ -155,7 +158,7 @@ To edit a milestone: 1. On the top bar, select **Main menu > Projects** and find your project or **Main menu > Groups** and find your group. 1. Select a milestone's title. -1. Select **Delete**. +1. In the top right corner, select **Milestone actions** (**{ellipsis_v}**) and then select **Delete**. 1. Select **Delete milestone**. ## Promote a project milestone to a group milestone @@ -182,7 +185,7 @@ To promote a project milestone: 1. On the left sidebar, select **Issues > Milestones**. 1. Either: - Select **Promote to Group Milestone** (**{level-up}**) next to the milestone you want to promote. - - Select the milestone title, and then select **Promote**. + - Select the milestone title, and then select **Milestone actions** (**{ellipsis_v}**) > **Promote**. 1. Select **Promote Milestone**. ## Assign a milestone to an issue or merge request diff --git a/doc/user/project/ml/experiment_tracking/img/candidate_v15_11.png b/doc/user/project/ml/experiment_tracking/img/candidate_v15_11.png new file mode 100644 index 00000000000..50bcd1e8479 Binary files /dev/null and b/doc/user/project/ml/experiment_tracking/img/candidate_v15_11.png differ diff --git a/doc/user/project/ml/experiment_tracking/img/candidate_v15_7.png b/doc/user/project/ml/experiment_tracking/img/candidate_v15_7.png deleted file mode 100644 index fb2e2e706d6..00000000000 Binary files a/doc/user/project/ml/experiment_tracking/img/candidate_v15_7.png and /dev/null differ diff --git a/doc/user/project/ml/experiment_tracking/img/candidates_v15_11.png b/doc/user/project/ml/experiment_tracking/img/candidates_v15_11.png new file mode 100644 index 00000000000..3a2944c92ae Binary files /dev/null and b/doc/user/project/ml/experiment_tracking/img/candidates_v15_11.png differ diff --git a/doc/user/project/ml/experiment_tracking/img/candidates_v15_7.png b/doc/user/project/ml/experiment_tracking/img/candidates_v15_7.png deleted file mode 100644 index 58dfe94a108..00000000000 Binary files a/doc/user/project/ml/experiment_tracking/img/candidates_v15_7.png and /dev/null differ diff --git a/doc/user/project/ml/experiment_tracking/img/experiments_v15_11.png b/doc/user/project/ml/experiment_tracking/img/experiments_v15_11.png new file mode 100644 index 00000000000..f7e25c1a0f1 Binary files /dev/null and b/doc/user/project/ml/experiment_tracking/img/experiments_v15_11.png differ diff --git a/doc/user/project/ml/experiment_tracking/img/experiments_v15_7.png b/doc/user/project/ml/experiment_tracking/img/experiments_v15_7.png deleted file mode 100644 index a7d4a3e559f..00000000000 Binary files a/doc/user/project/ml/experiment_tracking/img/experiments_v15_7.png and /dev/null differ diff --git a/doc/user/project/ml/experiment_tracking/index.md b/doc/user/project/ml/experiment_tracking/index.md index a7096d633a0..a2c2ab0cc40 100644 --- a/doc/user/project/ml/experiment_tracking/index.md +++ b/doc/user/project/ml/experiment_tracking/index.md @@ -4,78 +4,85 @@ group: Incubation info: Machine Learning Experiment Tracking is a GitLab Incubation Engineering program. No technical writer assigned to this group. --- -# Machine Learning Experiment Tracking **(FREE)** +# Machine learning model experiments **(FREE)** -DISCLAIMER: -Machine Learning Experiment Tracking is an experimental feature being developed by the Incubation Engineering Department, -and will receive significant changes over time. This feature is being release with the aim of getting user feedback, but -is not stable and can lead to performance degradation. See below on how to disable this feature. +FLAG: +On self-managed GitLab, model experiment tracking is disabled by default. +To enable the feature, ask an administrator to [enable the feature flag](../../../../administration/feature_flags.md) named `ml_experiment_tracking`. +On GitLab.com, this feature is in private testing only. -When creating machine learning models, data scientists often experiment with different parameters, configurations, feature -engineering, and so on, to improve the performance of the model. Keeping track of all this metadata and the associated +NOTE: +Model experiment tracking is an [experimental feature](../../../../policy/alpha-beta-support.md). Refer to for feedback and feature requests. + +When creating machine learning models, data scientists often experiment with different parameters, configurations, and feature +engineering to improve the performance of the model. Keeping track of all this metadata and the associated artifacts so that the data scientist can later replicate the experiment is not trivial. Machine learning experiment tracking enables them to log parameters, metrics, and artifacts directly into GitLab, giving easy access later on. -![List of Experiments](img/experiments_v15_7.png) +These features have been proposed: -![Experiment Candidates](img/candidates_v15_7.png) +- Searching experiments. +- Visual comparison of candidates. +- Creating, deleting, and updating candidates through the GitLab UI. -![Candidate Detail](img/candidate_v15_7.png) +For feature requests, see [epic 9341](https://gitlab.com/groups/gitlab-org/-/epics/9341). ## What is an experiment? -An experiment is a collection of comparable model candidates. Experiments can be long lived (for example, when they represent -a use case), or short lived (results from hyperparameter tuning triggered by a merge request), but usually hold model candidates -that have a similar set of parameters and metrics. +In a project, an experiment is a collection of comparable model candidates. +Experiments can be long-lived (for example, when they represent a use case), or +short-lived (results from hyperparameter tuning triggered by a merge request), +but usually hold model candidates that have a similar set of parameters measured +by the same metrics. + +![List of Experiments](img/experiments_v15_11.png) ## Model candidate A model candidate is a variation of the training of a machine learning model, that can be eventually promoted to a version -of the model. The goal of a data scientist is to find the model candidate whose parameter values lead to the best model +of the model. + +![Experiment Candidates](img/candidates_v15_11.png) + +The goal of a data scientist is to find the model candidate whose parameter values lead to the best model performance, as indicated by the given metrics. -Example parameters: +![Candidate Detail](img/candidate_v15_11.png) + +Some example parameters: -- Algorithm (linear regression, decision tree, and so on). +- Algorithm (such as linear regression or decision tree). - Hyperparameters for the algorithm (learning rate, tree depth, number of epochs). - Features included. -## Usage +## Track new experiments and candidates -### User access management +Experiment and trials can only be tracked through the +[MLflow](https://www.mlflow.org/docs/latest/tracking.html) client integration. +See [MLflow client integration](../../integrations/mlflow_client.md) for more information +on how to use GitLab as a backend for the MLflow Client. -An experiment is always associated to a project. Only users with access to the project an experiment is associated with -can view that experiment data. +## Explore model candidates -### Tracking new experiments and trials +Prerequisites: -Experiment and trials can only be tracked through the [MLFlow](https://www.mlflow.org/docs/latest/tracking.html) client -integration. More information on how to use GitLab as a backend for MLFlow Client can be found [at the documentation page](../../integrations/mlflow_client.md). +- You must have at least the Developer role to view experiment data. -### Exploring model candidates +To list the current active experiments, either go to `https/-/ml/experiments` or: -To list the current active experiments, navigate to `https/-/ml/experiments`. To display all trials -that have been logged, along with their metrics and parameters, select an experiment. To display details for a candidate, -select **Details**. +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Packages & registries > Model experiments**. +1. To display all candidates that have been logged, along with their metrics, parameters, and metadata, select an experiment. +1. To display details for a candidate, select **Details**. -### Logging artifacts +## View log artifacts Trial artifacts are saved as [generic packages](../../../packages/generic_packages/index.md), and follow all their -conventions. After an artifact is logged for a candidate, all artifacts logged for the candidate are listed in the -package registry. The package name for a candidate is `ml_candidate_`, with version `-`. The link to the -artifacts can also be accessed from the **Experiment Candidates** list or **Candidate detail**. - -### Limitations and future - -- Searching experiments, searching trials, visual comparison of trials, and creating, deleting and updating experiments and trials through GitLab UI is under development. - -## Disabling or enabling the Feature - -On self-managed GitLab, ML Experiment Tracking is disabled by default. To enable the feature, ask an administrator to [disable the feature flag](../../../../administration/feature_flags.md) named `ml_experiment_tracking`. -On GitLab.com, this feature is currently on private testing. - -## Feedback, roadmap and reports +limitations. After an artifact is logged for a candidate, all artifacts logged for the candidate are listed in the +package registry. The package name for a candidate is `ml_experiment_`, where the version is the candidate +IID. The link to the artifacts can also be accessed from the **Experiment Candidates** list or **Candidate detail**. -For updates on the development, refer to the [development epic](https://gitlab.com/groups/gitlab-org/-/epics/8560). +## Related topics -For feedback, bug reports and feature requests, refer to the [feedback issue](https://gitlab.com/gitlab-org/gitlab/-/issues/381660). +- Development details in [epic 8560](https://gitlab.com/groups/gitlab-org/-/epics/8560). +- Add feedback in [issue 381660](https://gitlab.com/gitlab-org/gitlab/-/issues/381660). diff --git a/doc/user/project/organize_work_with_projects.md b/doc/user/project/organize_work_with_projects.md index 2b4ce6d2fd0..85a1dfda679 100644 --- a/doc/user/project/organize_work_with_projects.md +++ b/doc/user/project/organize_work_with_projects.md @@ -1,6 +1,6 @@ --- -stage: Manage -group: Organization +stage: Data Stores +group: Tenant Scale info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md index e55cf337d16..5cdf493fe6f 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/dns_concepts.md @@ -1,14 +1,11 @@ --- type: concepts -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# DNS records overview **(FREE)** - -_Read this document for a brief overview of DNS records in the scope -of GitLab Pages, for beginners in web development._ +# GitLab Pages DNS records **(FREE)** A Domain Name System (DNS) web service routes visitors to websites by translating domain names (such as `www.example.com`) into the @@ -74,7 +71,7 @@ Example: This way, visitors visiting `www.example.com` are redirected to `example.com`. -## MX record +## `MX` record MX records are used to define the mail exchanges that are used for the domain. This helps email messages arrive at your mail server correctly. diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md index 24e9e6e15a4..a97fc1171fc 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md @@ -1,11 +1,10 @@ --- -disqus_identifier: 'https://docs.gitlab.com/ee/user/project/pages/getting_started_part_three.html' -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Custom domains and SSL/TLS certificates **(FREE)** +# GitLab Pages custom domains **(FREE)** > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/238461) in GitLab 15.4, you can use verified domains to [bypass user email confirmation for SAML- or SCIM-provisioned users](../../../group/saml_sso/index.md#bypass-user-email-confirmation-with-verified-domains). @@ -13,7 +12,7 @@ You can use custom domains: - With GitLab Pages. - To [bypass user email confirmation for SAML- or SCIM-provisioned users](../../../group/saml_sso/index.md#bypass-user-email-confirmation-with-verified-domains). - When using custom domains this way, you use the GitLab Pages feature but can skip the [requirements](#requirements). + When using custom domains this way, you use the GitLab Pages feature but can skip the [prerequisites](#prerequisites). To use one or more custom domain names: @@ -24,7 +23,7 @@ To use one or more custom domain names: To set up Pages with a custom domain name, read the requirements and steps below. -### Requirements +### Prerequisites - A GitLab Pages website up and running, served under the default Pages domain (`*.gitlab.io`, for GitLab.com). @@ -34,7 +33,7 @@ To set up Pages with a custom domain name, read the requirements and steps below there are multiple DNS records on that name, you must use an `ALIAS` record. - A DNS `TXT` record to verify your domain's ownership. - Set either `external_http` or `external_https` in `/etc/gitlab/gitlab.rb` to the IP and port of - your [Pages Daemon](../../../../administration/pages/index.md#overview). + your [Pages daemon](../../../../administration/pages/index.md#the-gitlab-pages-daemon). If you don't have IPv6, you can omit the IPv6 address. Example: @@ -197,38 +196,6 @@ from the GitLab project. in place. Your domain is periodically reverified, and may be disabled if the record is removed. -##### Troubleshoot domain verification - -To manually verify that you have properly configured the domain verification -`TXT` DNS entry, you can run the following command in your terminal: - -```shell -dig _gitlab-pages-verification-code. TXT -``` - -Expect the output: - -```plaintext -;; ANSWER SECTION: -_gitlab-pages-verification-code.. 300 IN TXT "gitlab-pages-verification-code=" -``` - -In some cases it can help to add the verification code with the same domain name as you are trying to register. - -For a root domain: - -| From | DNS Record | To | -| ------------------------------------------------- | ---------- | ---------------------- | -| `example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | -| `_gitlab-pages-verification-code.example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | - -For a subdomain: - -| From | DNS Record | To | -| ------------------------------------------------- | ---------- | ---------------------- | -| `www.example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | -| `_gitlab-pages-verification-code.www.example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | - ### Add more domain aliases You can add more than one alias (custom domains and subdomains) to the same project. @@ -352,14 +319,36 @@ To enable this setting: If you use Cloudflare CDN in front of GitLab Pages, make sure to set the SSL connection setting to `full` instead of `flexible`. For more details, see the [Cloudflare CDN directions](https://developers.cloudflare.com/ssl/origin-configuration/ssl-modes#h_4e0d1a7c-eb71-4204-9e22-9d3ef9ef7fef). - +| From | DNS Record | To | +| ------------------------------------------------- | ---------- | ---------------------- | +| `example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | +| `_gitlab-pages-verification-code.example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | + +For a subdomain: + +| From | DNS Record | To | +| ------------------------------------------------- | ---------- | ---------------------- | +| `www.example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | +| `_gitlab-pages-verification-code.www.example.com` | `TXT` | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` | diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md index 95ac2e50f29..91633e01ad2 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md @@ -1,12 +1,12 @@ --- type: reference description: "Automatic Let's Encrypt SSL certificates for GitLab Pages." -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# GitLab Pages integration with Let's Encrypt **(FREE)** +# GitLab Pages Let's Encrypt certificates **(FREE)** > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/28996) in GitLab 12.1. @@ -21,7 +21,7 @@ open source Certificate Authority. WARNING: This feature covers only certificates for **custom domains**, not the wildcard certificate required to run [Pages daemon](../../../../administration/pages/index.md) **(FREE SELF)**. Wildcard certificate generation is tracked in [this issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3342). -## Requirements +## Prerequisites Before you can enable automatic provisioning of an SSL certificate for your domain, make sure you have: diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/ssl_tls_concepts.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/ssl_tls_concepts.md index 398d8dc6e1e..484dc784fdb 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/ssl_tls_concepts.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/ssl_tls_concepts.md @@ -1,14 +1,11 @@ --- type: concepts -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# SSL/TLS certificates **(FREE)** - -_Read this document for a brief overview of SSL/TLS certificates in -the scope of GitLab Pages, for beginners in web development._ +# GitLab Pages SSL/TLS certificates **(FREE)** Every GitLab Pages project on GitLab.com is available under HTTPS for the default Pages domain (`*.gitlab.io`). Once you set diff --git a/doc/user/project/pages/getting_started/pages_ci_cd_template.md b/doc/user/project/pages/getting_started/pages_ci_cd_template.md index f596e418b02..17618146350 100644 --- a/doc/user/project/pages/getting_started/pages_ci_cd_template.md +++ b/doc/user/project/pages/getting_started/pages_ci_cd_template.md @@ -1,11 +1,11 @@ --- type: reference, howto -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Create a Pages website by using a CI/CD template **(FREE)** +# Create a GitLab Pages website from a CI/CD template **(FREE)** GitLab provides `.gitlab-ci.yml` templates for the most popular Static Site Generators (SSGs). You can create your own `.gitlab-ci.yml` file from one of these templates, and run @@ -37,4 +37,4 @@ For every change pushed to your repository, GitLab CI/CD runs a new pipeline that immediately publishes your changes to the Pages site. To view the HTML and other assets that were created for the site, -[download the job artifacts](../../../../ci/pipelines/job_artifacts.md#download-job-artifacts). +[download the job artifacts](../../../../ci/jobs/job_artifacts.md#download-job-artifacts). diff --git a/doc/user/project/pages/getting_started/pages_forked_sample_project.md b/doc/user/project/pages/getting_started/pages_forked_sample_project.md index 509609e9b89..e8de80ac137 100644 --- a/doc/user/project/pages/getting_started/pages_forked_sample_project.md +++ b/doc/user/project/pages/getting_started/pages_forked_sample_project.md @@ -1,11 +1,11 @@ --- type: reference, howto -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Create a Pages website from a forked sample **(FREE)** +# Create a GitLab Pages website from a forked sample project **(FREE)** GitLab provides [sample projects for the most popular Static Site Generators (SSG)](https://gitlab.com/pages). You can fork one of the sample projects and run the CI/CD pipeline to generate a Pages website. @@ -18,8 +18,8 @@ configured to generate a Pages site. To fork a sample project and create a Pages website: 1. View the sample projects by navigating to the [GitLab Pages examples](https://gitlab.com/pages) group. -1. Select the name of the project you want to [fork](../../repository/forking_workflow.md#creating-a-fork). -1. In the upper right, select **Fork** and then choose a namespace to fork to. +1. Select the name of the project you want to [fork](../../repository/forking_workflow.md#create-a-fork). +1. In the upper-right corner, select **Fork**, then choose a namespace to fork to. 1. For your project, on the left sidebar, select **CI/CD > Pipelines** and then **Run pipeline**. GitLab CI/CD builds and deploys your site. @@ -65,4 +65,4 @@ you can rename it to `.gitlab.io`, where `` is your GitLab ## Related topics -- [Download the job artifacts](../../../../ci/pipelines/job_artifacts.md#download-job-artifacts) +- [Download the job artifacts](../../../../ci/jobs/job_artifacts.md#download-job-artifacts) diff --git a/doc/user/project/pages/getting_started/pages_from_scratch.md b/doc/user/project/pages/getting_started/pages_from_scratch.md index a3d6c8f75f9..c62ead69216 100644 --- a/doc/user/project/pages/getting_started/pages_from_scratch.md +++ b/doc/user/project/pages/getting_started/pages_from_scratch.md @@ -1,6 +1,6 @@ --- -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -423,16 +423,11 @@ Now GitLab CI/CD not only builds the website, but also: - **Continuously deploys** every push to the `main` branch. To view the HTML and other assets that were created for the site, -[download the job artifacts](../../../../ci/pipelines/job_artifacts.md#download-job-artifacts). +[download the job artifacts](../../../../ci/jobs/job_artifacts.md#download-job-artifacts). ## Related topics -For more information, see the following blog posts. - -- Use GitLab CI/CD `environments` to - [deploy your web app to staging and production](https://about.gitlab.com/blog/2021/02/05/ci-deployment-and-environments/). -- Learn how to run jobs - [sequentially, in parallel, or build a custom pipeline](https://about.gitlab.com/blog/2020/12/10/basics-of-gitlab-ci-updated/). -- Learn [how to pull specific directories from different projects](https://about.gitlab.com/blog/2016/12/07/building-a-new-gitlab-docs-site-with-nanoc-gitlab-ci-and-gitlab-pages/) - to deploy this website, . -- Learn [how to use GitLab Pages to produce a code coverage report](https://about.gitlab.com/blog/2016/11/03/publish-code-coverage-report-with-gitlab-pages/). +- [Deploy your web app to staging and production](https://about.gitlab.com/blog/2021/02/05/ci-deployment-and-environments/) +- [Run jobs sequentially, in parallel, or build a custom pipeline](https://about.gitlab.com/blog/2020/12/10/basics-of-gitlab-ci-updated/) +- [Pull specific directories from different projects](https://about.gitlab.com/blog/2016/12/07/building-a-new-gitlab-docs-site-with-nanoc-gitlab-ci-and-gitlab-pages/) +- [Use GitLab Pages to produce a code coverage report](https://about.gitlab.com/blog/2016/11/03/publish-code-coverage-report-with-gitlab-pages/) diff --git a/doc/user/project/pages/getting_started/pages_new_project_template.md b/doc/user/project/pages/getting_started/pages_new_project_template.md index a301d2b64be..cb1da3fb21f 100644 --- a/doc/user/project/pages/getting_started/pages_new_project_template.md +++ b/doc/user/project/pages/getting_started/pages_new_project_template.md @@ -1,10 +1,10 @@ --- -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Create a Pages website from a template **(FREE)** +# Create a GitLab Pages website from a project template **(FREE)** GitLab provides templates for the most popular Static Site Generators (SSGs). You can create a new project from a template and run the CI/CD pipeline to generate a Pages website. @@ -32,4 +32,4 @@ For every change pushed to your repository, GitLab CI/CD runs a new pipeline that immediately publishes your changes to the Pages site. To view the HTML and other assets that were created for the site, -[download the job artifacts](../../../../ci/pipelines/job_artifacts.md#download-job-artifacts). +[download the job artifacts](../../../../ci/jobs/job_artifacts.md#download-job-artifacts). diff --git a/doc/user/project/pages/getting_started/pages_ui.md b/doc/user/project/pages/getting_started/pages_ui.md index 91c2c532a9a..00635fe6db2 100644 --- a/doc/user/project/pages/getting_started/pages_ui.md +++ b/doc/user/project/pages/getting_started/pages_ui.md @@ -4,7 +4,7 @@ group: Incubation info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Create a Pages deployment for your static site **(FREE)** +# Create a GitLab Pages deployment for a static site **(FREE)** If you already have a GitLab project that contains your static site or framework, you can generate a GitLab Pages website from it. diff --git a/doc/user/project/pages/getting_started_part_one.md b/doc/user/project/pages/getting_started_part_one.md index a0c8073d6eb..6eb996db210 100644 --- a/doc/user/project/pages/getting_started_part_one.md +++ b/doc/user/project/pages/getting_started_part_one.md @@ -1,10 +1,10 @@ --- -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# GitLab Pages domain names, URLs, and base URLs **(FREE)** +# GitLab Pages default domain names and URLs **(FREE)** On this document, learn how to name your project for GitLab Pages according to your intended website's URL. diff --git a/doc/user/project/pages/index.md b/doc/user/project/pages/index.md index 691757e3eca..a68ad604989 100644 --- a/doc/user/project/pages/index.md +++ b/doc/user/project/pages/index.md @@ -1,7 +1,7 @@ --- description: 'Learn how to use GitLab Pages to deploy a static website at no additional cost.' -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -126,6 +126,13 @@ If you are running a self-managed instance of GitLab, Watch a [video tutorial](https://www.youtube.com/watch?v=dD8c7WNcc6s) about how to get started with GitLab Pages administration. +### Configure GitLab Pages in a Helm Chart (Kubernetes) instance + +To configure GitLab Pages on instances deployed via Helm chart (Kubernetes), use either: + +- [The `gitlab-pages` subchart](https://docs.gitlab.com/charts/charts/gitlab/gitlab-pages/). +- [An external GitLab Pages instance](https://docs.gitlab.com/charts/advanced/external-gitlab-pages/). + ## Security for GitLab Pages If your username is `example`, your GitLab Pages website is located at `example.gitlab.io`. diff --git a/doc/user/project/pages/introduction.md b/doc/user/project/pages/introduction.md index 51c42eeecb8..05d0b461fea 100644 --- a/doc/user/project/pages/introduction.md +++ b/doc/user/project/pages/introduction.md @@ -1,10 +1,10 @@ --- -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Exploring GitLab Pages **(FREE)** +# GitLab Pages settings **(FREE)** This document is a user guide to explore the options and settings GitLab Pages offers. @@ -271,7 +271,7 @@ This problem most likely results from a missing `index.html` file in the public a 404 is encountered, confirm that the public directory contains an `index.html` file. If the file contains a different name such as `test.html`, the Pages site can still be accessed, but the full path would be needed. For example: `https//group-name.pages.example.com/project-name/test.html`. -The contents of the public directory can be confirmed by [browsing the artifacts](../../../ci/pipelines/job_artifacts.md#download-job-artifacts) from the latest pipeline. +The contents of the public directory can be confirmed by [browsing the artifacts](../../../ci/jobs/job_artifacts.md#download-job-artifacts) from the latest pipeline. Files listed under the public directory can be accessed through the Pages URL for the project. diff --git a/doc/user/project/pages/pages_access_control.md b/doc/user/project/pages/pages_access_control.md index 248a74a8abc..1b046d03f59 100644 --- a/doc/user/project/pages/pages_access_control.md +++ b/doc/user/project/pages/pages_access_control.md @@ -1,6 +1,6 @@ --- -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/user/project/pages/public_folder.md b/doc/user/project/pages/public_folder.md index 751db136e34..6ee8ea17aee 100644 --- a/doc/user/project/pages/public_folder.md +++ b/doc/user/project/pages/public_folder.md @@ -1,12 +1,12 @@ --- description: 'Learn how to configure the build output folder for the most common static site generators' -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Configure the public files folder **(FREE)** +# GitLab Pages public folder **(FREE)** All the files that should be accessible by the browser must be in a root-level folder called `public`. @@ -91,14 +91,37 @@ NOTE: GitLab Pages supports only static sites. For Next.js, you can use Next's [Static HTML export functionality](https://nextjs.org/docs/advanced-features/static-html-export). -Use the `-o public` flag after `next export` as the build command, for -example: +With the release of [Next.js 13](https://nextjs.org/blog/next-13) a lot has changed on how Next.js works. +It is recommended to use the following `next.config.js` so all static assets can be exported properly: -```shell -next export -o public +```javascript +/** @type {import('next').NextConfig} */ +const nextConfig = { + reactStrictMode: true, + images: { + unoptimized: true, + }, + assetPrefix: "https://example.gitlab.io/namespace-here/my-gitlab-project/" +} + +module.exports = nextConfig +``` + +An example `.gitlab-ci.yml` can be as minimal as: + +```yaml +pages: + before_script: + - npm install + script: + - npm run build + - mv out/* public + artifacts: + paths: + - public ``` -### Nuxt.js +## Nuxt.js NOTE: GitLab Pages supports only static sites. @@ -146,7 +169,7 @@ module.exports = { ## Should you commit the `public` folder? Not necessarily. However, when the GitLab Pages deploy pipeline runs, it looks -for an [artifact](../../../ci/pipelines/job_artifacts.md) of that name. +for an [artifact](../../../ci/jobs/job_artifacts.md) of that name. If you set up a job that creates the `public` folder before deploy, such as by running `npm run build`, committing the folder isn't required. diff --git a/doc/user/project/pages/redirects.md b/doc/user/project/pages/redirects.md index f5447fd67ca..bd8206b3bda 100644 --- a/doc/user/project/pages/redirects.md +++ b/doc/user/project/pages/redirects.md @@ -1,10 +1,10 @@ --- -stage: Create -group: Editor +stage: Plan +group: Knowledge info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Create redirects for GitLab Pages **(FREE)** +# GitLab Pages redirects **(FREE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab-pages/-/issues/24) in GitLab Pages 1.25.0 and GitLab 13.4 behind a feature flag, disabled by default. > - [Became enabled by default](https://gitlab.com/gitlab-org/gitlab-pages/-/merge_requests/367) in GitLab 13.5. diff --git a/doc/user/project/protected_branches.md b/doc/user/project/protected_branches.md index da53fe2f5e9..1d279436d2c 100644 --- a/doc/user/project/protected_branches.md +++ b/doc/user/project/protected_branches.md @@ -22,12 +22,14 @@ The [default branch](repository/branches/default.md) for your repository is prot ## Who can modify a protected branch +> Branch push permission [changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118532) to require GitLab administrators to also have the **allowed** permission in GitLab 16.0. + When a branch is protected, the default behavior enforces these restrictions on the branch. | Action | Who can do it | |:-------------------------|:------------------------------------------------------------------| | Protect a branch | At least the Maintainer role. | -| Push to the branch | GitLab administrators and anyone with **Allowed** permission. (1) | +| Push to the branch | Anyone with **Allowed** permission. (1) | | Force push to the branch | No one. | | Delete the branch | No one. (2) | @@ -36,6 +38,40 @@ When a branch is protected, the default behavior enforces these restrictions on 1. No one can delete a protected branch using Git commands, however, users with at least Maintainer role can [delete a protected branch from the UI or API](#delete-a-protected-branch). +### When a branch matches multiple rules + +When a branch matches multiple rules, the **most permissive rule** determines the +level of protection for the branch. For example, consider these rules, which include +[wildcards](#configure-multiple-protected-branches-by-using-a-wildcard): + +| Branch name pattern | Allowed to merge | Allowed to push and merge | +|---------------------|------------------------|-----------------| +| `v1.x` | Maintainer | Maintainer | +| `v1.*` | Maintainer + Developer | Maintainer | +| `v*` | No one | No one | + +A branch named `v1.x` matches all three branch name patterns: `v1.x`, `v1.*`, and `v*`. +As the most permissive option determines the behavior, the resulting permissions for branch `v1.x` are: + +- **Allowed to merge:** Of the three settings, `Maintainer + Developer` is most permissive, + and controls branch behavior as a result. Even though the branch also matched `v1.x` and `v*` + (which each have stricter permissions), users with the Developer role can merge into the branch. +- **Allowed to push and merge:** Of the three settings, `Maintainer` is the most permissive, and controls + branch behavior as a result. Even though branches matching `v*` are set to `No one`, branches + that _also_ match `v1.x` or `v1.*` receive the more permissive `Maintainer` permission. + +To be certain that a rule controls the behavior of a branch, +_all_ other patterns that match must apply less or equally permissive rules. + +If you want to ensure that `No one` is allowed to push to branch `v1.x`, every pattern +that matches `v1.x` must set `Allowed to push and merge` to `No one`, like this: + +| Branch name pattern | Allowed to merge | Allowed to push and merge | +|---------------------|------------------------|-----------------| +| `v1.x` | Maintainer | No one | +| `v1.*` | Maintainer + Developer | No one | +| `v*` | No one | No one | + ### Set the default branch protection level Administrators can set a default branch protection level in the @@ -43,10 +79,41 @@ Administrators can set a default branch protection level in the ## Configure a protected branch +Configure protected branches for all projects in a group, or just for a project. + +### For all projects in a group **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106532) in GitLab 15.9 behind a feature flag, disabled by default. + +Group owners can create protected branches for a group. These settings are inherited by all projects in the group and can't be overridden by project settings. + +FLAG: +On self-managed GitLab, by default this feature is not available. +To make it available, ask an administrator to [enable the feature flag](../../administration/feature_flags.md) +named `group_protected_branches`. On GitLab.com, this feature is not available. + +Prerequisite: + +- You must have the Owner role in the group. + +To protect a branch for all the projects in a group: + +1. On the top bar, select **Main menu > Groups** and find your group. +1. On the left sidebar, select **Settings > Repository**. +1. Expand **Protected branches**. +1. In the **Branch** text box, type the branch name or a wildcard. +1. From the **Allowed to merge** list, select a role that can merge into this branch. +1. From the **Allowed to push and merge** list, select a role that can push to this branch. +1. Select **Protect**. + +The protected branch is added to the list of protected branches. + +### For a project + Prerequisite: - You must have at least the Maintainer role. -- When granting a group **Allowed to merge** or **Allowed to push** permissions +- When granting a group **Allowed to merge** or **Allowed to push and merge** permissions on a protected branch, the group must be added to the project. To protect a branch: @@ -56,7 +123,7 @@ To protect a branch: 1. Expand **Protected branches**. 1. From the **Branch** dropdown list, select the branch you want to protect. 1. From the **Allowed to merge** list, select a role, or group that can merge into this branch. In GitLab Premium, you can also add users. -1. From the **Allowed to push** list, select a role, group, or user that can push to this branch. In GitLab Premium, you can also add users. +1. From the **Allowed to push and merge** list, select a role, group, or user that can push to this branch. In GitLab Premium, you can also add users. 1. Select **Protect**. The protected branch displays in the list of protected branches. @@ -65,7 +132,7 @@ The protected branch displays in the list of protected branches. If both a specific rule and a wildcard rule apply to the same branch, the most permissive rule controls how the branch behaves. For merge controls to work properly, -set **Allowed to push** to a broader set of users than **Allowed to merge**. +set **Allowed to push and merge** to a broader set of users than **Allowed to merge**. Prerequisite: @@ -86,7 +153,7 @@ To protect multiple branches at the same time: | `*gitlab*` | `gitlab`, `gitlab/staging`, `master/gitlab/production` | 1. From the **Allowed to merge** list, select a role, or group that can merge into this branch. In GitLab Premium, you can also add users. -1. From the **Allowed to push** list, select a role, group, or user that can push to this branch. In GitLab Premium, you can also add users. +1. From the **Allowed to push and merge** list, select a role, group, or user that can push to this branch. In GitLab Premium, you can also add users. 1. Select **Protect**. The protected branch displays in the list of protected branches. @@ -97,7 +164,7 @@ Users with at least the Developer role can create a protected branch. Prerequisites: -- **Allowed to push** is set to **No one** +- **Allowed to push and merge** is set to **No one** - **Allowed to merge** is set to **Developers**. You can create a protected branch by using the UI or API only. @@ -124,11 +191,7 @@ like the [GitLab workflow](../../topics/gitlab_flow.md). 1. Expand **Protected branches**. 1. From the **Branch** dropdown list, select the branch you want to protect. 1. From the **Allowed to merge** list, select **Developers + Maintainers**. -1. From the **Allowed to push** list, select **No one**. - - NOTE: - Setting a role, group or user as **Allowed to push** also allows those users to merge. - +1. From the **Allowed to push and merge** list, select **No one**. 1. Select **Protect**. ## Allow everyone to push directly to a protected branch @@ -139,7 +202,7 @@ You can allow everyone with write access to push to the protected branch. 1. On the left sidebar, select **Settings > Repository**. 1. Expand **Protected branches**. 1. From the **Branch** dropdown list, select the branch you want to protect. -1. From the **Allowed to push** list, select **Developers + Maintainers**. +1. From the **Allowed to push and merge** list, select **Developers + Maintainers**. 1. Select **Protect**. ## Allow deploy keys to push to a protected branch @@ -167,7 +230,7 @@ To allow a deploy key to push to a protected branch: 1. On the left sidebar, select **Settings > Repository**. 1. Expand **Protected branches**. 1. From the **Branch** dropdown list, select the branch you want to protect. -1. From the **Allowed to push** list, select the deploy key. +1. From the **Allowed to push and merge** list, select the deploy key. 1. Select **Protect**. Deploy keys are not available in the **Allowed to merge** dropdown list. @@ -186,7 +249,7 @@ To protect a new branch and enable force push: 1. On the left sidebar, select **Settings > Repository**. 1. Expand **Protected branches**. 1. From the **Branch** dropdown list, select the branch you want to protect. -1. From the **Allowed to push** and **Allowed to merge** lists, select the settings you want. +1. From the **Allowed to push and merge** and **Allowed to merge** lists, select the settings you want. 1. To allow all users with push access to force push, turn on the **Allowed to force push** toggle. 1. To reject code pushes that change files listed in the `CODEOWNERS` file, turn on the **Require approval from code owners** toggle. @@ -203,10 +266,9 @@ Members who can push to this branch can now also force push. ## Require Code Owner approval on a protected branch **(PREMIUM)** -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/13251) in GitLab 12.4. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35097) in GitLab 13.5, users and groups who can push to protected branches do not have to use a merge request to merge their feature branches. This means they can skip merge request approval rules. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35097) in GitLab 13.5, users and groups who can push to protected branches do not have to use a merge request to merge their feature branches. This means they can skip merge request approval rules. -For a protected branch, you can require at least one approval by a [Code Owner](code_owners.md). +For a protected branch, you can require at least one approval by a [Code Owner](codeowners/index.md). To protect a new branch and enable Code Owner's approval: @@ -214,7 +276,7 @@ To protect a new branch and enable Code Owner's approval: 1. On the left sidebar, select **Settings > Repository**. 1. Expand **Protected branches**. 1. From the **Branch** dropdown list, select the branch you want to protect. -1. From the **Allowed to push** and **Allowed to merge** lists, select the settings you want. +1. From the **Allowed to push and merge** and **Allowed to merge** lists, select the settings you want. 1. Turn on the **Require approval from code owners** toggle. 1. Select **Protect**. @@ -262,6 +324,12 @@ Protected branches can only be deleted by using GitLab either from the UI or API This prevents accidentally deleting a branch through local Git commands or third-party Git clients. +## Related topics + +- [Protected branches API](../../api/protected_branches.md) +- [Branches](repository/branches/index.md) +- [Branches API](../../api/branches.md) + - -| Command | Issue | Merge request | Epic | Action | -|:-------------------------------------------------------------------------------------------------|:-----------------------|:-----------------------|:-----------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `/add_contacts [contact:email1@example.com] [contact:email2@example.com]` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Add one or more active [CRM contacts](../crm/index.md) ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73413) in GitLab 14.6). | -| `/approve` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Approve the merge request. | -| `/assign @user1 @user2` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Assign one or more users. | -| `/assign me` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Assign yourself. | -| `/assign_reviewer @user1 @user2` or `/reviewer @user1 @user2` or `/request_review @user1 @user2` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Assign one or more users as reviewers. | -| `/assign_reviewer me` or `/reviewer me` or `/request_review me` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Assign yourself as a reviewer. | -| `/award :emoji:` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Toggle emoji award. | + + +| Command | Issue | Merge request | Epic | Action | +|:-------------------------------------------------------------------------------------------------|:-----------------------|:-----------------------|:-----------------------|:-------| +| `/add_contacts [contact:email1@example.com] [contact:email2@example.com]` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Add one or more active [CRM contacts](../crm/index.md) ([introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73413) in GitLab 14.6). +| `/approve` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Approve the merge request. +| `/assign @user1 @user2` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Assign one or more users. +| `/assign me` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Assign yourself. +| `/assign_reviewer @user1 @user2` or `/reviewer @user1 @user2` or `/request_review @user1 @user2` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Assign one or more users as reviewers. +| `/assign_reviewer me` or `/reviewer me` or `/request_review me` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Assign yourself as a reviewer. +| `/award :emoji:` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Toggle an emoji reaction. | `/cc @user` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Mention a user. In GitLab 15.0 and later, this command performs no action. You can instead type `CC @user` or only `@user`. [In GitLab 14.9 and earlier](https://gitlab.com/gitlab-org/gitlab/-/issues/31200), mentioning a user at the start of a line created a specific type of to-do item notification. | -| `/child_epic ` | **{dotted-circle}** No | **{dotted-circle}** No | **{check-circle}** Yes | Add child epic to ``. The `` value should be in the format of `&epic`, `group&epic`, or a URL to an epic ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7330) in GitLab 12.0). | -| `/clear_health_status` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Clear [health status](issues/managing_issues.md#health-status) ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213814) in GitLab 14.7). | -| `/clear_weight` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Clear weight. | -| `/clone [--with_notes]` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Clone the issue to given project, or the current one if no arguments are given ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9421) in GitLab 13.7). Copies as much data as possible as long as the target project contains equivalent labels, milestones, and so on. Does not copy comments or system notes unless `--with_notes` is provided as an argument. | -| `/close` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Close. | +| `/child_epic ` | **{dotted-circle}** No | **{dotted-circle}** No | **{check-circle}** Yes | Add child epic to ``. The `` value should be in the format of `&epic`, `group&epic`, or a URL to an epic. +| `/clear_health_status` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Clear [health status](issues/managing_issues.md#health-status) ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213814) in GitLab 14.7). +| `/clear_weight` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Clear weight. +| `/clone [--with_notes]` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Clone the issue to given project, or the current one if no arguments are given ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/9421) in GitLab 13.7). Copies as much data as possible as long as the target project contains equivalent labels, milestones, and so on. Does not copy comments or system notes unless `--with_notes` is provided as an argument. +| `/close` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Close. | `/confidential` | **{check-circle}** Yes | **{dotted-circle}** No | **{check-circle}** Yes | Mark issue or epic as confidential. Support for epics [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213741) in GitLab 15.6. | -| `/copy_metadata ` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Copy labels and milestone from another merge request in the project. | -| `/copy_metadata <#issue>` | **{check-circle}** Yes | **{check-circle}** Yes | **{dotted-circle}** No | Copy labels and milestone from another issue in the project. | -| `/create_merge_request ` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Create a new merge request starting from the current issue. | -| `/done` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | Mark to do as done. | -| `/draft` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | Set the [draft status](merge_requests/drafts.md). Use for toggling the draft status ([deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/92654) in GitLab 15.4.) | -| `/due ` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Set due date. Examples of valid `` include `in 2 days`, `this Friday` and `December 31st`. See [Chronic](https://gitlab.com/gitlab-org/ruby/gems/gitlab-chronic#examples) for more examples. | -| `/duplicate <#issue>` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Close this issue. Mark as a duplicate of, and related to, issue `<#issue>`. | -| `/epic ` | **{check-circle}** Yes | **{dotted-circle}** No | **{dotted-circle}** No | Add to epic ``. The `` value should be in the format of `&epic`, `group&epic`, or a URL to an epic. | -| `/estimate
    LanguageLanguage VersionsPackage ManagerLanguage versionsPackage manager Supported files Processes multiple files?
    Y
    JavaJava and Kotlin (not Android)1 8 LTS, 11 LTS, - 131, - 141, - 151, - 161, or 17 LTS Gradle2N
    JavaScript and TypeScriptAll versionsJavaScript and TypeScriptAll versions npm
      @@ -224,11 +223,15 @@ table.supported-languages ul {
    		Y
    All versions yarn yarn.lock Y
    pnpm3pnpm-lock.yamlY
    PHP All versions
    Python3.93.9, 3.104 setuptools setup.py NPipenv N Maven
    JavaScript and TypeScriptJavaScript and TypeScript npm
    pnpm
    yarn