diff options
| -rw-r--r-- | .gitlab/ci/review.gitlab-ci.yml | 4 | ||||
| -rw-r--r-- | app/helpers/application_settings_helper.rb | 1 | ||||
| -rw-r--r-- | app/views/layouts/header/_registration_enabled_callout.html.haml | 6 | ||||
| -rw-r--r-- | doc/api/settings.md | 3 | ||||
| -rw-r--r-- | doc/architecture/blueprints/ci_pipeline_components/index.md | 18 | ||||
| -rw-r--r-- | doc/architecture/blueprints/pods/pods-feature-ci-runners.md | 142 | ||||
| -rw-r--r-- | doc/user/read_only_namespaces.md | 50 | ||||
| -rw-r--r-- | locale/gitlab.pot | 15 | ||||
| -rw-r--r-- | spec/features/callouts/registration_enabled_spec.rb | 4 | ||||
| -rw-r--r-- | spec/requests/api/settings_spec.rb | 1 | ||||
| -rw-r--r-- | spec/support/database_cleaner.rb | 10 | ||||
| -rw-r--r-- | spec/support/migration.rb | 2 |
12 files changed, 221 insertions, 35 deletions
diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index 08673eff14d..507bb73c723 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -42,8 +42,6 @@ review-k8s-resources-count-checks: environment: name: review/k8s-resources-count-checks action: verify - before_script: - - !reference [".use-kube-context", before_script] script: - scripts/review_apps/k8s-resources-count-checks.sh || (scripts/slack review-apps-monitoring "☠️ \`${CI_JOB_NAME}\` failed! ☠️ See ${CI_JOB_URL} - <https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/blob/main/runbooks/review-apps.md#review-k8s-resources-count-checks-job-failed|📗 RUNBOOK 📕>" warning "GitLab Bot" && exit 1); @@ -55,8 +53,6 @@ review-gcp-quotas-checks: environment: name: review/gcp-quotas-checks action: verify - before_script: - - !reference [".use-kube-context", before_script] script: - ruby scripts/review_apps/gcp-quotas-checks.rb || (scripts/slack review-apps-monitoring "☠️ \`${CI_JOB_NAME}\` failed! ☠️ See ${CI_JOB_URL} - <https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/blob/main/runbooks/review-apps.md#review-gcp-quotas-checks-job-failed|📗 RUNBOOK 📕>" warning "GitLab Bot" && exit 1); diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index 7f13f609353..f573aa34730 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -216,6 +216,7 @@ module ApplicationSettingsHelper :default_branch_protection, :default_ci_config_path, :default_group_visibility, + :default_preferred_language, :default_project_creation, :default_project_visibility, :default_projects_limit, diff --git a/app/views/layouts/header/_registration_enabled_callout.html.haml b/app/views/layouts/header/_registration_enabled_callout.html.haml index dd3d14a5678..52c39fce961 100644 --- a/app/views/layouts/header/_registration_enabled_callout.html.haml +++ b/app/views/layouts/header/_registration_enabled_callout.html.haml @@ -1,17 +1,17 @@ - return unless show_registration_enabled_user_callout? -= render Pajamas::AlertComponent.new(title: _('Anyone can register for an account.'), += render Pajamas::AlertComponent.new(title: _('Check your sign-up restrictions'), variant: :warning, alert_options: { class: 'js-registration-enabled-callout', data: { feature_id: Users::CalloutsHelper::REGISTRATION_ENABLED_CALLOUT, dismiss_endpoint: callouts_path }}, close_button_options: { data: { testid: 'close-registration-enabled-callout' }}) do |c| = c.body do - = _('Only allow anyone to register for accounts on GitLab instances that you intend to be used by anyone. Allowing anyone to register makes GitLab instances more vulnerable.') + = _("Your GitLab instance allows anyone to register for an account, which is a security risk on public-facing GitLab instances. You should deactivate new sign ups if public users aren't expected to register for an account.") = c.actions do = link_to general_admin_application_settings_path(anchor: 'js-signup-settings'), class: 'btn gl-alert-action btn-confirm btn-md gl-button' do %span.gl-button-text - = _('Turn off') + = _('Deactivate') %button.btn.gl-alert-action.btn-default.btn-md.gl-button.js-close %span.gl-button-text = _('Acknowledge') diff --git a/doc/api/settings.md b/doc/api/settings.md index 1ff47c64f0c..dfcfba4edf4 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -36,6 +36,7 @@ Example response: "signup_enabled" : true, "id" : 1, "default_branch_protection" : 2, + "default_preferred_language" : "en", "restricted_visibility_levels" : [], "password_authentication_enabled_for_web" : true, "after_sign_out_path" : null, @@ -146,6 +147,7 @@ Example response: { "id": 1, "default_projects_limit": 100000, + "default_preferred_language": "en", "signup_enabled": false, "password_authentication_enabled_for_web": true, "gravatar_enabled": true, @@ -290,6 +292,7 @@ listed in the descriptions of the relevant settings. | `default_branch_protection` | integer | no | Determine if developers can push to the default branch. Can take: `0` _(not protected, both users with the Developer role or Maintainer role can push new commits and force push)_, `1` _(partially protected, users with the Developer role or Maintainer role can push new commits, but cannot force push)_ or `2` _(fully protected, users with the Developer or Maintainer role cannot push new commits, but users with the Developer or Maintainer role can; no one can force push)_ as a parameter. Default is `2`. | | `default_ci_config_path` | string | no | Default CI/CD configuration file and path for new projects (`.gitlab-ci.yml` if not set). | | `default_group_visibility` | string | no | What visibility level new groups receive. Can take `private`, `internal` and `public` as a parameter. Default is `private`. | +| `default_preferred_language` | string | no | Default preferred language for users who are not logged in. | | `default_project_creation` | integer | no | Default project creation protection. Can take: `0` _(No one)_, `1` _(Maintainers)_ or `2` _(Developers + Maintainers)_| | `default_project_visibility` | string | no | What visibility level new projects receive. Can take `private`, `internal` and `public` as a parameter. Default is `private`. | | `default_projects_limit` | integer | no | Project limit per user. Default is `100000`. | diff --git a/doc/architecture/blueprints/ci_pipeline_components/index.md b/doc/architecture/blueprints/ci_pipeline_components/index.md index f98a82dfe5b..08a285acadc 100644 --- a/doc/architecture/blueprints/ci_pipeline_components/index.md +++ b/doc/architecture/blueprints/ci_pipeline_components/index.md @@ -3,7 +3,7 @@ status: proposed creation-date: "2022-09-14" authors: [ "@fabio", "@grzesiek" ] coach: "@kamil" -approvers: [ "@dhershkovitch" ] +approvers: [ "@dhershkovitch", "@marknuzzo" ] owning-stage: "~devops::verify" participating-stages: [] --- @@ -314,21 +314,21 @@ Proposal: <!-- vale gitlab.Spelling = NO --> -| Role | Who -|------------------------------|-------------------------| -| Author | Fabio Pitino | -| Engineering Leader | ? | -| Product Manager | Dov Hershkovitch | -| Architecture Evolution Coach | Kamil Trzciński, Grzegorz Bizon | +| Role | Who +|--------------------------------|-------------------------| +| Author | Fabio Pitino | +| Engineering Leaders | Cheryl Li, Mark Nuzzo | +| Product Manager | Dov Hershkovitch | +| Architecture Evolution Coaches | Kamil Trzciński, Grzegorz Bizon | DRIs: | Role | Who |------------------------------|------------------------| -| Leadership | ? | +| Leadership | Mark Nuzzo | | Product | Dov Hershkovitch | | Engineering | Fabio Pitino | -| UX | Kevin Comoli | +| UX | Kevin Comoli | Domain experts: diff --git a/doc/architecture/blueprints/pods/pods-feature-ci-runners.md b/doc/architecture/blueprints/pods/pods-feature-ci-runners.md index d7bb853b995..2c6746ab2e1 100644 --- a/doc/architecture/blueprints/pods/pods-feature-ci-runners.md +++ b/doc/architecture/blueprints/pods/pods-feature-ci-runners.md @@ -14,16 +14,156 @@ we can document the reasons for not choosing this approach. # Pods: CI Runners -> TL;DR +GitLab in order to execute CI jobs [GitLab Runner](https://gitlab.com/gitlab-org/gitlab-runner/), +very often managed by customer in their infrastructure. + +All CI jobs created as part of CI pipeline are run in a context of project +it poses a challenge how to manage GitLab Runners. ## 1. Definition +There are 3 different types of runners: + +- instance-wide: runners that are registered globally with specific tags (selection criteria) +- group runners: runners that execute jobs from a given top-level group or subprojects of that group +- project runners: runners that execute jobs from projects or many projects: some runners might + have projects assigned from projects in different top-level groups. + +This alongside with existing data structure where `ci_runners` is a table describing +all types of runners poses a challenge how the `ci_runners` should be managed in a Pods environment. + ## 2. Data flow +GitLab Runners use a set of globally scoped endpoints to: + +- registration of a new runner via registration token `https://gitlab.com/api/v4/runners` + ([subject for removal](../runner_tokens/index.md)) (`registration token`) +- requests jobs via an authenticated `https://gitlab.com/api/v4/jobs/request` endpoint (`runner token`) +- upload job status via `https://gitlab.com/api/v4/jobs/:job_id` (`build token`) +- upload trace via `https://gitlab.com/api/v4/jobs/:job_id/trace` (`build token`) +- download and upload artifacts via `https://gitlab.com/api/v4/jobs/:job_id/artifacts` (`build token`) + +Currently three types of authentication tokens are used: + +- runner registration token ([subject for removal](../runner_tokens/index.md)) +- runner token representing an registered runner in a system with specific configuration (`tags`, `locked`, etc.) +- build token representing an ephemeral token giving a limited access to updating a specific + job, uploading artifacts, downloading dependent artifacts, downloading and uploading + container registry images + +Each of those endpoints do receive an authentication token via header (`JOB-TOKEN` for `/trace`) +or body parameter (`token` all other endpoints). + +Since the CI pipeline would be created in a context of a specific Pod it would be required +that pick of a build would have to be processed by that particular Pod. This requires +that build picking depending on a solution would have to be either: + +- routed to correct Pod for a first time +- be made to be two phase: request build from global pool, claim build on a specific Pod using a Pod specific URL + ## 3. Proposal +This section describes various proposals. Reader should consider that those +proposals do describe solutions for different problems. Many or some aspects +of those proposals might be the solution to the stated problem. + +### 3.1. Authentication tokens + +Even though the paths for CI Runners are not routable they can be made routable with +those two possible solutions: + +- The `https://gitlab.com/api/v4/jobs/request` uses a long polling mechanism with + a ticketing mechanism (based on `X-GitLab-Last-Update` header). Runner when first + starts sends a request to GitLab to which GitLab responds with either a build to pick + by runner. This value is completely controlled by GitLab. This allows GitLab + to use JWT or any other means to encode `pod` identifier that could be easily + decodable by Router. +- The majority of communication (in terms of volume) is using `build token` making it + the easiest target to change since GitLab is sole owner of the token that Runner later + uses for specific job. There were prior discussions about not storing `build token` + but rather using `JWT` token with defined scopes. Such token could encode the `pod` + to which router could easily route all requests. + +### 3.2. Request body + +- The most of used endpoints pass authentication token in request body. It might be desired + to use HTTP Headers as an easier way to access this information by Router without + a need to proxy requests. + +### 3.3. Instance-wide are Pod local + +We can pick a design where all runners are always registered and local to a given Pod: + +- Each Pod has it's own set of instance-wide runners that are updated at it's own pace +- The project runners can only be linked to projects from the same organization + creating strong isolation. +- In this model the `ci_runners` table is local to the Pod. +- In this model we would require the above endpoints to be scoped to a Pod in some way + or made routable. It might be via prefixing them, adding additional Pod parameter, + or providing much more robust way to decode runner token and match it to Pod. +- If routable token is used, we could move away from cryptographical random stored in + database to rather prefer to use JWT tokens that would encode +- The Admin Area showing registered Runners would have to be scoped to a Pod + +This model might be desired since it provides strong isolation guarnatees. +This model does significantly increase maintanance overhead since each Pod is managed +separately. + +This model may require adjustments to runner tags feature so that projects have consistent runner experience across pods. + +### 3.4. Instance-wide are cluster-wide + +Contrary to proposal where all runners are Pod local, we can consider that runners +are global, or just instance-wide runners are global. + +However, this requires significant overhaul of system and to change the following aspects: + +- `ci_runners` table would likely have to be split decomposed into `ci_instance_runners`, ... +- all interfaces would have to be adopted to use correct table +- build queuing would have to be reworked to be two phase where each Pod would know of all pending + and running builds, but the actual claim of a build would happen against a Pod containing data +- likely `ci_pending_builds` and `ci_running_builds` would have to be made `cluster-wide` tables + increasing likelity of creating hotspots in a system related to CI queueing + +This model makes it complex to implement from engineering side. Does make some data being shared +between Pods. Creates hotspots / scalability issues in a system (ex. during abuse) that +might impact experience of organizations on other Pods. + +### 3.5. GitLab CI Daemon + +Another potential solution to explore is to have a dedicated service responsible for builds queueing +owning it's database and working in a model of either shareded or podded service. There were prior +discussions about [CI/CD Daemon](https://gitlab.com/gitlab-org/gitlab/-/issues/19435). + +If the service would be sharded: + +- depending on a model if runners are cluster-wide or pod-local this service would have to fetch + data from all Pods +- if the sharded service would be used we could adapt a model of either sharing database containig + `ci_pending_builds/ci_running_builds` with the service +- if the sharded service would be used we could consider a push model where each Pod pushes to CI/CD Daemon + builds that should be picked by Runner +- the sharded service would be aware which Pod is responsible for processing the given build and could + route processing requests to designated Pod + +If the service would be podded: + +- all expectations of routable endpoints are still valid + +In general usage of CI Daemon does not help significantly with the stated problem. However, this offers +a few upsides related to more efficient processing and decoupling model: push model and it opens a way +to offer stateful communication with GitLab Runners (ex. gRPC or Websockets). + ## 4. Evaluation +Considering all solutions it appears that solution giving the most promise is: + +- use "instance-wide are Pod local" +- refine endpoints to have routable identities (either via specific paths, or better tokens) + +Other potential upsides is to get rid of `ci_builds.token` and rather use a `JWT token` +that can much better and easier encode wider set of scopes allowed by CI runner. + ## 4.1. Pros ## 4.2. Cons diff --git a/doc/user/read_only_namespaces.md b/doc/user/read_only_namespaces.md new file mode 100644 index 00000000000..c68a81e42b1 --- /dev/null +++ b/doc/user/read_only_namespaces.md @@ -0,0 +1,50 @@ +--- +stage: Growth +group: Acquisition +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments +noindex: true +--- + +# Read-only namespaces **(FREE SAAS)** + +In GitLab SaaS, a top-level namespace is placed in a read-only state when it either: + +- Exceeds the [free user limit](free_user_limit.md) when the namespace visibility is private. +- Exceeds the [storage usage quota](usage_quotas.md), regardless of namespace visibility. +placed in a read-only state when it exceeds the [free user limit](free_user_limit.md) +or the [storage usage quota](usage_quotas.md). + +While a namespace is in a read-only state, a banner appears at the +top of the page. + +Your ability to write new data to read-only namespaces is restricted. For more +information, see [Restricted actions](#restricted-actions). + +## Remove the read-only state + +To restore a namespace to its normal state, you can: + +- For exceeded free user limits: + - [Reduce the number of members](free_user_limit.md#manage-members-in-your-namespace) in your namespace. + - [Start a free trial](https://gitlab.com/-/trial_registrations/new), which includes an unlimited number of members. + - [Purchase a paid tier](https://about.gitlab.com/pricing/). +- For exceeded storage quota: + - [Purchase more storage for the namespace](../subscriptions/gitlab_com/index.md#purchase-more-storage-and-transfer). + - [Manage your storage usage](usage_quotas.md#manage-your-storage-usage). + +## Restricted actions + +| Feature | Action restricted | +|---------|-------------------| +| Container Registry | Create, edit, and delete cleanup policies <br> Push an image to the container registry | +| Merge Requests | Create and update an MR | +| Package Registry | Publish a package | +| Repositories | Add tags <br> Create new branches <br> Create and update commit status <br> Push and force push to non-protected branches <br> Push and force push to protected branches <br> Upload files <br> Create merge requests | +| CI/CD | Create, edit, admin, and run pipelines <br> Create, edit, admin, and run builds <br> Create and edit admin environments <br> Create and edit admin deployments <br> Create and edit admin clusters <br> Create and edit admin releases | +| Namespaces | **For exceeded free user limits:** Invite new users | + +## Related topics + +- [Frequently Asked Questions - GitLab SaaS Free Tier](https://about.gitlab.com/pricing/faq-efficient-free-tier/) +- [Free user limit](free_user_limit.md) +- [Storage usage quotas](usage_quotas.md) diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 745a2075934..053b4b150e9 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -4680,9 +4680,6 @@ msgstr "" msgid "Any namespace" msgstr "" -msgid "Anyone can register for an account." -msgstr "" - msgid "App ID" msgstr "" @@ -8102,6 +8099,9 @@ msgstr "" msgid "Check your Kubernetes cluster images for known vulnerabilities." msgstr "" +msgid "Check your sign-up restrictions" +msgstr "" + msgid "Checking %{text} availability…" msgstr "" @@ -12781,6 +12781,9 @@ msgstr "" msgid "Days to merge" msgstr "" +msgid "Deactivate" +msgstr "" + msgid "Deactivate dormant users after a period of inactivity" msgstr "" @@ -28637,9 +28640,6 @@ msgstr "" msgid "Only active projects show up in the search and on the dashboard." msgstr "" -msgid "Only allow anyone to register for accounts on GitLab instances that you intend to be used by anyone. Allowing anyone to register makes GitLab instances more vulnerable." -msgstr "" - msgid "Only effective when remote storage is enabled. Set to 0 for no size limit." msgstr "" @@ -47566,6 +47566,9 @@ msgstr "" msgid "Your GitLab group" msgstr "" +msgid "Your GitLab instance allows anyone to register for an account, which is a security risk on public-facing GitLab instances. You should deactivate new sign ups if public users aren't expected to register for an account." +msgstr "" + msgid "Your Groups" msgstr "" diff --git a/spec/features/callouts/registration_enabled_spec.rb b/spec/features/callouts/registration_enabled_spec.rb index 4fc73863de6..1ea52dbf12a 100644 --- a/spec/features/callouts/registration_enabled_spec.rb +++ b/spec/features/callouts/registration_enabled_spec.rb @@ -6,7 +6,7 @@ RSpec.describe 'Registration enabled callout', feature_category: :authentication let_it_be(:admin) { create(:admin) } let_it_be(:non_admin) { create(:user) } let_it_be(:project) { create(:project) } - let_it_be(:callout_title) { _('Anyone can register for an account.') } + let_it_be(:callout_title) { _('Check your sign-up restrictions') } context 'when "Sign-up enabled" setting is `true`' do before do @@ -22,7 +22,7 @@ RSpec.describe 'Registration enabled callout', feature_category: :authentication visit root_path expect(page).to have_content callout_title - expect(page).to have_link _('Turn off'), href: general_admin_application_settings_path(anchor: 'js-signup-settings') + expect(page).to have_link _('Deactivate'), href: general_admin_application_settings_path(anchor: 'js-signup-settings') visit root_dashboard_path diff --git a/spec/requests/api/settings_spec.rb b/spec/requests/api/settings_spec.rb index 4eb03204de3..a169db87b1a 100644 --- a/spec/requests/api/settings_spec.rb +++ b/spec/requests/api/settings_spec.rb @@ -25,6 +25,7 @@ RSpec.describe API::Settings, 'Settings', :do_not_mock_admin_mode_setting do expect(json_response['secret_detection_token_revocation_url']).to be_nil expect(json_response['secret_detection_revocation_token_types_url']).to be_nil expect(json_response['sourcegraph_public_only']).to be_truthy + expect(json_response['default_preferred_language']).to be_a String expect(json_response['default_project_visibility']).to be_a String expect(json_response['default_snippet_visibility']).to be_a String expect(json_response['default_group_visibility']).to be_a String diff --git a/spec/support/database_cleaner.rb b/spec/support/database_cleaner.rb index 222cbe9feeb..7bd1f0c5dfa 100644 --- a/spec/support/database_cleaner.rb +++ b/spec/support/database_cleaner.rb @@ -22,14 +22,4 @@ RSpec.configure do |config| self.class.use_transactional_tests = true end - - config.around(:each, :migration) do |example| - self.class.use_transactional_tests = false - - example.run - - delete_from_all_tables!(except: deletion_except_tables) - - self.class.use_transactional_tests = true - end end diff --git a/spec/support/migration.rb b/spec/support/migration.rb index 2a69630a29a..4d4a293e9ff 100644 --- a/spec/support/migration.rb +++ b/spec/support/migration.rb @@ -45,6 +45,8 @@ RSpec.configure do |config| example.run end + delete_from_all_tables!(except: deletion_except_tables) + self.class.use_transactional_tests = true end |