diff options
36 files changed, 349 insertions, 1128 deletions
diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 199a92d1a25..1e60e6684cb 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -107,7 +107,7 @@ update-storybook-yarn-cache: - run_timed_command "gem install knapsack --no-document" - run_timed_command "scripts/gitaly-test-spawn" - source ./scripts/rspec_helpers.sh - - rspec_paralellized_job "--tag frontend_fixture" + - rspec_paralellized_job artifacts: name: frontend-fixtures expire_in: 31d diff --git a/app/assets/javascripts/pipelines/components/graph/stage_column_component.vue b/app/assets/javascripts/pipelines/components/graph/stage_column_component.vue index 1a7464e8ed1..d34ae8036ed 100644 --- a/app/assets/javascripts/pipelines/components/graph/stage_column_component.vue +++ b/app/assets/javascripts/pipelines/components/graph/stage_column_component.vue @@ -79,21 +79,6 @@ export default { columnSpacingClass() { return this.isStageView ? 'gl-px-6' : 'gl-px-9'; }, - /* - currentGroups and filteredGroups are part of - a test to hunt down a bug - (see: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/57142). - - They should be removed when the bug is rectified. - */ - currentGroups() { - return this.glFeatures.pipelineFilterJobs ? this.filteredGroups : this.groups; - }, - filteredGroups() { - return this.groups.map((group) => { - return { ...group, jobs: group.jobs.filter(Boolean) }; - }); - }, formattedTitle() { return capitalize(escape(this.name)); }, @@ -156,7 +141,7 @@ export default { </template> <template #jobs> <div - v-for="group in currentGroups" + v-for="group in groups" :id="groupId(group)" :key="getGroupId(group)" data-testid="stage-column-group" diff --git a/app/assets/javascripts/projects/settings/components/shared_runners_toggle.vue b/app/assets/javascripts/projects/settings/components/shared_runners_toggle.vue index 0786a74f6b1..6880f8ef097 100644 --- a/app/assets/javascripts/projects/settings/components/shared_runners_toggle.vue +++ b/app/assets/javascripts/projects/settings/components/shared_runners_toggle.vue @@ -1,15 +1,23 @@ <script> import { GlAlert, GlToggle, GlTooltip } from '@gitlab/ui'; import axios from '~/lib/utils/axios_utils'; -import { __ } from '~/locale'; +import { __, s__ } from '~/locale'; const DEFAULT_ERROR_MESSAGE = __('An error occurred while updating the configuration.'); +const REQUIRES_VALIDATION_TEXT = s__( + `Billings|Shared runners cannot be enabled until a valid credit card is on file.`, +); export default { + i18n: { + REQUIRES_VALIDATION_TEXT, + }, components: { GlAlert, GlToggle, GlTooltip, + CcValidationRequiredAlert: () => + import('ee_component/billings/components/cc_validation_required_alert.vue'), }, props: { isDisabledAndUnoverridable: { @@ -20,6 +28,10 @@ export default { type: Boolean, required: true, }, + isCreditCardValidationRequired: { + type: Boolean, + required: false, + }, updatePath: { type: String, required: true, @@ -30,12 +42,17 @@ export default { isLoading: false, isSharedRunnerEnabled: false, errorMessage: null, + isCcValidationRequired: false, }; }, created() { this.isSharedRunnerEnabled = this.isEnabled; + this.isCcValidationRequired = this.isCreditCardValidationRequired; }, methods: { + creditCardValidated() { + this.isCcValidationRequired = false; + }, toggleSharedRunners() { this.isLoading = true; this.errorMessage = null; @@ -45,6 +62,7 @@ export default { .then(() => { this.isLoading = false; this.isSharedRunnerEnabled = !this.isSharedRunnerEnabled; + this.isCcValidationRequired = this.isCreditCardValidationRequired; }) .catch((error) => { this.isLoading = false; @@ -61,16 +79,25 @@ export default { <gl-alert v-if="errorMessage" class="gl-mb-3" variant="danger" :dismissible="false"> {{ errorMessage }} </gl-alert> - <div ref="sharedRunnersToggle"> - <gl-toggle - :disabled="isDisabledAndUnoverridable" - :is-loading="isLoading" - :label="__('Enable shared runners for this project')" - :value="isSharedRunnerEnabled" - data-testid="toggle-shared-runners" - @change="toggleSharedRunners" - /> - </div> + + <cc-validation-required-alert + v-if="isCcValidationRequired && !isSharedRunnerEnabled" + class="gl-pb-5" + :custom-message="$options.i18n.REQUIRES_VALIDATION_TEXT" + @verifiedCreditCard="creditCardValidated" + /> + + <gl-toggle + v-else + ref="sharedRunnersToggle" + :disabled="isDisabledAndUnoverridable" + :is-loading="isLoading" + :label="__('Enable shared runners for this project')" + :value="isSharedRunnerEnabled" + data-testid="toggle-shared-runners" + @change="toggleSharedRunners" + /> + <gl-tooltip v-if="isDisabledAndUnoverridable" :target="() => $refs.sharedRunnersToggle"> {{ __('Shared runners are disabled on group level') }} </gl-tooltip> diff --git a/app/assets/javascripts/projects/settings/mount_shared_runners_toggle.js b/app/assets/javascripts/projects/settings/mount_shared_runners_toggle.js index eaeb5848b68..5ca864a412b 100644 --- a/app/assets/javascripts/projects/settings/mount_shared_runners_toggle.js +++ b/app/assets/javascripts/projects/settings/mount_shared_runners_toggle.js @@ -4,7 +4,12 @@ import SharedRunnersToggle from '~/projects/settings/components/shared_runners_t export default (containerId = 'toggle-shared-runners-form') => { const containerEl = document.getElementById(containerId); - const { isDisabledAndUnoverridable, isEnabled, updatePath } = containerEl.dataset; + const { + isDisabledAndUnoverridable, + isEnabled, + updatePath, + isCreditCardValidationRequired, + } = containerEl.dataset; return new Vue({ el: containerEl, @@ -13,6 +18,7 @@ export default (containerId = 'toggle-shared-runners-form') => { props: { isDisabledAndUnoverridable: parseBoolean(isDisabledAndUnoverridable), isEnabled: parseBoolean(isEnabled), + isCreditCardValidationRequired: parseBoolean(isCreditCardValidationRequired), updatePath, }, }); diff --git a/app/controllers/projects/pipelines_controller.rb b/app/controllers/projects/pipelines_controller.rb index 3caea14479c..98feadd8b23 100644 --- a/app/controllers/projects/pipelines_controller.rb +++ b/app/controllers/projects/pipelines_controller.rb @@ -14,7 +14,6 @@ class Projects::PipelinesController < Projects::ApplicationController before_action :authorize_update_pipeline!, only: [:retry, :cancel] before_action do push_frontend_feature_flag(:pipeline_graph_layers_view, project, type: :development, default_enabled: :yaml) - push_frontend_feature_flag(:pipeline_filter_jobs, project, default_enabled: :yaml) push_frontend_feature_flag(:graphql_pipeline_details, project, type: :development, default_enabled: :yaml) push_frontend_feature_flag(:graphql_pipeline_details_users, current_user, type: :development, default_enabled: :yaml) end diff --git a/app/models/user.rb b/app/models/user.rb index 6b8df50e181..f252acb5f0d 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1234,7 +1234,7 @@ class User < ApplicationRecord end def matches_identity?(provider, extern_uid) - identities.where(provider: provider, extern_uid: extern_uid).exists? + identities.with_extern_uid(provider, extern_uid).exists? end def project_deploy_keys diff --git a/app/views/admin/application_settings/_third_party_offers.html.haml b/app/views/admin/application_settings/_third_party_offers.html.haml index 5df2454ed2e..9a34400092e 100644 --- a/app/views/admin/application_settings/_third_party_offers.html.haml +++ b/app/views/admin/application_settings/_third_party_offers.html.haml @@ -2,11 +2,11 @@ %section.settings.as-third-party-offers.no-animate#js-third-party-offers-settings{ class: ('expanded' if expanded) } .settings-header %h4 - = _('Third party offers') + = _('Third-party offers') %button.btn.gl-button.btn-default.js-settings-toggle{ type: 'button' } = expanded ? _('Collapse') : _('Expand') %p - = _('Control the display of third party offers.') + = _('Control whether to display third-party offers in GitLab.') .settings-content = form_for @application_setting, url: general_admin_application_settings_path(anchor: 'js-third-party-offers-settings'), html: { class: 'fieldset-form', id: 'third-party-offers-settings' } do |f| = form_errors(@application_setting) if expanded @@ -15,6 +15,6 @@ .form-group .form-check = f.check_box :hide_third_party_offers, class: 'form-check-input' - = f.label :hide_third_party_offers, _('Do not display offers from third parties within GitLab'), class: 'form-check-label' + = f.label :hide_third_party_offers, _('Do not display offers from third parties'), class: 'form-check-label' = f.submit _('Save changes'), class: "gl-button btn btn-confirm" diff --git a/app/views/shared/issuable/_form.html.haml b/app/views/shared/issuable/_form.html.haml index e79719d41b0..6aa80e6808d 100644 --- a/app/views/shared/issuable/_form.html.haml +++ b/app/views/shared/issuable/_form.html.haml @@ -6,11 +6,16 @@ = form_errors(issuable) - if @conflict - .gl-alert.gl-alert-danger.gl-mb-5 - Someone edited the #{issuable.class.model_name.human.downcase} the same time you did. - Please check out - = link_to "the #{issuable.class.model_name.human.downcase}", polymorphic_path([@project, issuable]), target: "_blank", rel: 'noopener noreferrer' - and make sure your changes will not unintentionally remove theirs + = render 'shared/global_alert', + variant: :danger, + dismissible: false, + is_contained: true, + alert_class: 'gl-mb-5' do + .gl-alert-body + Someone edited the #{issuable.class.model_name.human.downcase} the same time you did. + Please check out + = link_to "the #{issuable.class.model_name.human.downcase}", polymorphic_path([@project, issuable]), target: "_blank", rel: 'noopener noreferrer' + and make sure your changes will not unintentionally remove theirs = render 'shared/issuable/form/branch_chooser', issuable: issuable, form: form diff --git a/config/feature_flags/development/pipeline_filter_jobs.yml b/config/feature_flags/development/pipeline_filter_jobs.yml deleted file mode 100644 index 6fb989a6815..00000000000 --- a/config/feature_flags/development/pipeline_filter_jobs.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: pipeline_filter_jobs -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/57142 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/325693 -milestone: '13.11' -type: development -group: group::pipeline authoring -default_enabled: false diff --git a/config/feature_flags/development/usage_data_code_review_aggregation.yml b/config/feature_flags/development/usage_data_code_review_aggregation.yml deleted file mode 100644 index 23c8f86360a..00000000000 --- a/config/feature_flags/development/usage_data_code_review_aggregation.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: usage_data_code_review_aggregation -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53553 -rollout_issue_url: -milestone: '13.9' -type: development -group: group::code review -default_enabled: true diff --git a/config/metrics/aggregates/code_review.yml b/config/metrics/aggregates/code_review.yml index c002bd88b7d..1461931e268 100644 --- a/config/metrics/aggregates/code_review.yml +++ b/config/metrics/aggregates/code_review.yml @@ -9,7 +9,6 @@ --- - name: code_review_group_monthly_active_users operator: OR - feature_flag: usage_data_code_review_aggregation source: redis time_frame: [7d, 28d] events: @@ -68,7 +67,6 @@ - 'i_code_review_user_resolve_conflict' - name: code_review_category_monthly_active_users operator: OR - feature_flag: usage_data_code_review_aggregation source: redis time_frame: [7d, 28d] events: @@ -126,7 +124,6 @@ - 'i_code_review_user_resolve_conflict' - name: code_review_extension_category_monthly_active_users operator: OR - feature_flag: usage_data_code_review_aggregation source: redis time_frame: [7d, 28d] events: diff --git a/db/migrate/20210623163342_add_index_to_compliance_management_frameworks_pipeline_configuration.rb b/db/migrate/20210623163342_add_index_to_compliance_management_frameworks_pipeline_configuration.rb new file mode 100644 index 00000000000..81903230f43 --- /dev/null +++ b/db/migrate/20210623163342_add_index_to_compliance_management_frameworks_pipeline_configuration.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddIndexToComplianceManagementFrameworksPipelineConfiguration < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + INDEX_NAME = 'index_compliance_frameworks_id_where_frameworks_not_null' + + def up + add_concurrent_index :compliance_management_frameworks, :id, name: INDEX_NAME, where: 'pipeline_configuration_full_path IS NOT NULL' + end + + def down + remove_concurrent_index_by_name :compliance_management_frameworks, INDEX_NAME + end +end diff --git a/db/schema_migrations/20210623163342 b/db/schema_migrations/20210623163342 new file mode 100644 index 00000000000..f60112edd81 --- /dev/null +++ b/db/schema_migrations/20210623163342 @@ -0,0 +1 @@ +6732401a959a7a691b8729e2bfb38e010dd4da4cd04418aada22946c42fdd6dc
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index a3cab52973e..62c559c1c02 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -23110,6 +23110,8 @@ CREATE INDEX index_clusters_on_user_id ON clusters USING btree (user_id); CREATE UNIQUE INDEX index_commit_user_mentions_on_note_id ON commit_user_mentions USING btree (note_id); +CREATE INDEX index_compliance_frameworks_id_where_frameworks_not_null ON compliance_management_frameworks USING btree (id) WHERE (pipeline_configuration_full_path IS NOT NULL); + CREATE INDEX index_composer_cache_files_where_namespace_id_is_null ON packages_composer_cache_files USING btree (id) WHERE (namespace_id IS NULL); CREATE INDEX index_container_expiration_policies_on_next_run_at_and_enabled ON container_expiration_policies USING btree (next_run_at, enabled); diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index 5e1a2efbb4d..c2c32f059c9 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -537,7 +537,7 @@ the original form is omitted. To move to the consolidated form, remove the original configuration (for example, `artifacts_object_store_enabled`, or `uploads_object_store_connection`) -## Storage-specific configuration +### Storage-specific configuration For configuring object storage in GitLab 13.1 and earlier, or for storage types not supported by consolidated configuration form, refer to the following guides: @@ -580,7 +580,7 @@ There are plans to [enable the use of a single bucket](https://gitlab.com/gitlab in the future. Helm-based installs require separate buckets to -[handle backup restorations](https://docs.gitlab.com/charts/advanced/external-object-storage/#lfs-artifacts-uploads-packages-external-diffs-pseudonymizer) +[handle backup restorations](https://docs.gitlab.com/charts/advanced/external-object-storage/#lfs-artifacts-uploads-packages-external-diffs-pseudonymizer). ### S3 API compatibility issues diff --git a/doc/development/agent/gitops.md b/doc/development/agent/gitops.md index f183ba86aa1..3d59f5bd845 100644 --- a/doc/development/agent/gitops.md +++ b/doc/development/agent/gitops.md @@ -1,148 +1,9 @@ --- -stage: Configure -group: Configure -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: 'https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/gitops.md' +remove_date: '2022-06-24' --- -# GitOps with the Kubernetes Agent **(PREMIUM SELF)** +This file was moved to [another location](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/gitops.md). -The [GitLab Kubernetes Agent](../../user/clusters/agent/index.md) supports the -[pull-based version](https://www.gitops.tech/#pull-based-deployments) of -[GitOps](https://www.gitops.tech/). To be useful, the feature must be able to perform these tasks: - -- Connect one or more Kubernetes clusters to a GitLab project or group. -- Synchronize cluster-wide state from a Git repository. -- Synchronize namespace-scoped state from a Git repository. -- Control the following settings: - - - The kinds of objects an agent can manage. - - Enabling the namespaced mode of operation for managing objects only in a specific namespace. - - Enabling the non-namespaced mode of operation for managing objects in any namespace, and - managing non-namespaced objects. - -- Synchronize state from one or more Git repositories into a cluster. -- Configure multiple agents running in different clusters to synchronize state - from the same repository. - -## GitOps architecture - -In this architecture, the Kubernetes cluster (`agentk`) periodically fetches -configuration from (`kas`), spawning a goroutine for each configured GitOps -repository. Each goroutine makes a streaming `GetObjectsToSynchronize()` gRPC call. -`kas` accepts these requests, then checks if this agent is authorized to access -this GitLab repository. If authorized, `kas` polls Gitaly for repository updates -and sends the latest manifests to the agent. - -Before each poll, `kas` verifies with GitLab that the agent's token is still valid. -When `agentk` receives an updated manifest, it performs a synchronization using -[`gitops-engine`](https://github.com/argoproj/gitops-engine). - -If a repository is removed from the list, `agentk` stops the `GetObjectsToSynchronize()` -calls to that repository. - -```mermaid -graph TB - agentk -- fetch configuration --> kas - agentk -- fetch GitOps manifests --> kas - - subgraph "GitLab" - kas[kas] - GitLabRoR[GitLab RoR] - Gitaly[Gitaly] - kas -- poll GitOps repositories --> Gitaly - kas -- authZ for agentk --> GitLabRoR - kas -- fetch configuration --> Gitaly - end - - subgraph "Kubernetes cluster" - agentk[agentk] - end -``` - -## Architecture considered but not implemented - -As part of the implementation process, this architecture was considered, but ultimately -not implemented. - -In this architecture, `agentk` periodically fetches configuration from `kas`. For each -configured GitOps repository, it spawns a goroutine. Each goroutine then spawns a -copy of [`git-sync`](https://github.com/kubernetes/git-sync). It polls a particular -repository and invokes a corresponding webhook on `agentk` when it changes. When that -happens, `agentk` performs a synchronization using -[`gitops-engine`](https://github.com/argoproj/gitops-engine). - -For repositories no longer in the list, `agentk` stops corresponding goroutines -and `git-sync` copies, also deleting their cloned repositories from disk: - -```mermaid -graph TB - agentk -- fetch configuration --> kas - git-sync -- poll GitOps repositories --> GitLabRoR - - subgraph "GitLab" - kas[kas] - GitLabRoR[GitLab RoR] - kas -- authZ for agentk --> GitLabRoR - kas -- fetch configuration --> Gitaly[Gitaly] - end - - subgraph "Kubernetes cluster" - agentk[agentk] - git-sync[git-sync] - agentk -- control --> git-sync - git-sync -- notify about changes --> agentk - end -``` - -## Comparing implemented and non-implemented architectures - -Both architectures attempt to answer the same question: how to grant an agent -access to a non-public repository? - -In the **implemented** architecture: - -- Favorable: Fewer moving parts, as `git-sync` and `git` are not used, making this - design more reliable. -- Favorable: Uses existing connectivity and authentication mechanisms are used (gRPC + `agentk` token). -- Favorable: No polling through external infrastructure. Saves traffic and avoids - noise in access logs. - -In the **unimplemented** architecture: - -- Favorable: `agentk` uses `git-sync` to access repositories with standard protocols - (either HTTPS, or SSH and Git) with accepted authentication and authorization methods. - - - Unfavorable: The user must put credentials into a `secret`. GitLab doesn't have - a mechanism for per-repository tokens for robots. - - Unfavorable: Rotating all credentials is more work than rotating a single `agentk` token. - -- Unfavorable: A dependency on an external component (`git-sync`) that can be avoided. -- Unfavorable: More network traffic and connections than the implemented design - -### Ideas considered for the unimplemented design - -As part of the design process, these ideas were considered, and discarded: - -- Running `git-sync` and `gitops-engine` as part of `kas`. - - - Favorable: More code and infrastructure under our control for GitLab.com - - Unfavorable: Running an arbitrary number of `git-sync` processes would require - an unbounded amount of RAM and disk space. - - Unfavorable: Unclear which `kas` replica is responsible for which agent and - repository synchronization. If done as part of `agentk`, leader election can be - done using [client-go](https://pkg.go.dev/k8s.io/client-go/tools/leaderelection?tab=doc). - -- Running `git-sync` and a "`gitops-engine` driver" helper program as a separate - Kubernetes `Deployment`. - - - Favorable: Better isolation and higher resiliency. For example, if the node - with `agentk` dies, not all synchronization stops. - - Favorable: Each deployment has its own memory and disk limits. - - Favorable: Per-repository synchronization identity (distinct `ServiceAccount`) - can be implemented. - - Unfavorable: Time consuming to implement properly: - - - Each `Deployment` needs CRUD (create, update, and delete) permissions. - - Users may want to customize a `Deployment`, or add and remove satellite objects - like `PodDisruptionBudget`, `HorizontalPodAutoscaler`, and `PodSecurityPolicy`. - - Metrics, monitoring, logs for the `Deployment`. +<!-- This redirect file can be deleted after <2022-06-24>. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/development/agent/identity.md b/doc/development/agent/identity.md index 83af4318de9..67084a6d995 100644 --- a/doc/development/agent/identity.md +++ b/doc/development/agent/identity.md @@ -1,106 +1,9 @@ --- -stage: Configure -group: Configure -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: 'https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/identity_and_auth.md' +remove_date: '2022-06-24' --- -# Kubernetes Agent identity and authentication **(PREMIUM SELF)** +This file was moved to [another location](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/identity_and_auth.md). -This page uses the word `agent` to describe the concept of the -GitLab Kubernetes Agent. The program that implements the concept is called `agentk`. -Read the -[architecture page](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/architecture.md) -for more information. - -## Agent identity and name - -In a GitLab installation, each agent must have a unique, immutable name. This -name must be unique in the project the agent is attached to, and this name must -follow the [DNS label standard from RFC 1123](https://tools.ietf.org/html/rfc1123). -The name must: - -- Contain at most 63 characters. -- Contain only lowercase alphanumeric characters or `-`. -- Start with an alphanumeric character. -- End with an alphanumeric character. - -Kubernetes uses the -[same naming restriction](https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names) -for some names. - -The regex for names is: `/\A[a-z0-9]([-a-z0-9]*[a-z0-9])?\z/`. - -## Multiple agents in a cluster - -A Kubernetes cluster may have 0 or more agents running in it. Each agent likely -has a different configuration. Some may enable features A and B, and some may -enable features B and C. This flexibility enables different groups of people to -use different features of the agent in the same cluster. - -For example, [Priyanka (Platform Engineer)](https://about.gitlab.com/handbook/marketing/strategic-marketing/roles-personas/#priyanka-platform-engineer) -may want to use cluster-wide features of the agent, while -[Sasha (Software Developer)](https://about.gitlab.com/handbook/marketing/strategic-marketing/roles-personas/#sasha-software-developer) -uses the agent that only has access to a particular namespace. - -Each agent is likely running using a -[`ServiceAccount`](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/), -a distinct Kubernetes identity, with a distinct set of permissions attached to it. -These permissions enable the agent administrator to follow the -[principle of least privilege](https://en.wikipedia.org/wiki/Principle_of_least_privilege) -and minimize the permissions each particular agent needs. - -## Kubernetes Agent authentication - -When adding a new agent, GitLab provides the user with a bearer access token. The -agent uses this token to authenticate with GitLab. This token is a random string -and does not encode any information in it, but it is secret and must -be treated with care. Store it as a `Secret` in Kubernetes. - -Each agent can have 0 or more tokens in a GitLab database. Having several valid -tokens helps you rotate tokens without needing to re-register an agent. Each token -record in the database has the following fields: - -- Agent identity it belongs to. -- Token value. Encrypted at rest. -- Creation time. -- Who created it. -- Revocation flag to mark token as revoked. -- Revocation time. -- Who revoked it. -- A text field to store any comments the administrator may want to make about the token for future self. - -Tokens can be managed by users with `maintainer` and higher level of -[permissions](../../user/permissions.md). - -Tokens are immutable, and only the following fields can be updated: - -- Revocation flag. Can only be updated to `true` once, but immutable after that. -- Revocation time. Set to the current time when revocation flag is set, but immutable after that. -- Comments field. Can be updated any number of times, including after the token has been revoked. - -The agent sends its token, along with each request, to GitLab to authenticate itself. -For each request, GitLab checks the token's validity: - -- Does the token exist in the database? -- Has the token been revoked? - -This information may be cached for some time to reduce load on the database. - -## Kubernetes Agent authorization - -GitLab provides the following information in its response for a given Agent access token: - -- Agent configuration Git repository. (The agent doesn't support per-folder authorization.) -- Agent name. - -## Create an agent - -You can create an agent by following the [user documentation](../../user/clusters/agent/index.md#create-an-agent-record-in-gitlab), or via Rails console: - -```ruby -project = ::Project.find_by_full_path("path-to/your-configuration-project") -# agent-name should be the same as specified above in the config.yaml -agent = ::Clusters::Agent.create(name: "<agent-name>", project: project) -token = ::Clusters::AgentToken.create(agent: agent) -token.token # this will print out the token you need to use on the next step -``` +<!-- This redirect file can be deleted after <2022-06-24>. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/development/agent/index.md b/doc/development/agent/index.md index 112162f8f90..2cb05e2dd8f 100644 --- a/doc/development/agent/index.md +++ b/doc/development/agent/index.md @@ -1,87 +1,9 @@ --- -stage: Configure -group: Configure -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: 'https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/architecture.md' +remove_date: '2022-06-24' --- -# Kubernetes Agent development **(PREMIUM SELF)** +This file was moved to [another location](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/architecture.md). -This page contains developer-specific information about the GitLab Kubernetes Agent. -[End-user documentation about the GitLab Kubernetes Agent](../../user/clusters/agent/index.md) -is also available. - -The agent can help you perform tasks like these: - -- Integrate a cluster, located behind a firewall or NAT, with GitLab. To - learn more, read [issue #212810, Invert the model GitLab.com uses for Kubernetes integration by leveraging long lived reverse tunnels](https://gitlab.com/gitlab-org/gitlab/-/issues/212810). -- Access API endpoints in a cluster in real time. For an example use case, read - [issue #218220, Allow Prometheus in K8s cluster to be installed manually](https://gitlab.com/gitlab-org/gitlab/-/issues/218220#note_348729266). -- Enable real-time features by pushing information about events happening in a cluster. - For example, you could build a cluster view dashboard to visualize changes in progress - in a cluster. For more information about these efforts, read about the - [Real-Time Working Group](https://about.gitlab.com/company/team/structure/working-groups/real-time/). -- Enable a [cache of Kubernetes objects through informers](https://github.com/kubernetes/client-go/blob/ccd5becdffb7fd8006e31341baaaacd14db2dcb7/tools/cache/shared_informer.go#L34-L183), - kept up-to-date with very low latency. This cache helps you: - - - Reduce or eliminate information propagation latency by avoiding Kubernetes API calls - and polling, and only fetching data from an up-to-date cache. - - Lower the load placed on the Kubernetes API by removing polling. - - Eliminate any rate-limiting errors by removing polling. - - Simplify backend code by replacing polling code with cache access. While it's another - API call, no polling is needed. This example describes [fetching cached data synchronously from the front end](https://gitlab.com/gitlab-org/gitlab/-/issues/217792#note_348582537) instead of fetching data from the Kubernetes API. - -## Architecture of the Kubernetes Agent - -The GitLab Kubernetes Agent and the GitLab Kubernetes Agent Server use -[bidirectional streaming](https://grpc.io/docs/what-is-grpc/core-concepts/#bidirectional-streaming-rpc) -to allow the connection acceptor (the gRPC server, GitLab Kubernetes Agent Server) to -act as a client. The connection acceptor sends requests as gRPC replies. The client-server -relationship is inverted because the connection must be initiated from inside the -Kubernetes cluster to bypass any firewall or NAT the cluster may be located behind. -To learn more about this inversion, read -[issue #212810](https://gitlab.com/gitlab-org/gitlab/-/issues/212810). - -This diagram describes how GitLab (`GitLab RoR`), the GitLab Kubernetes Agent (`agentk`), and the GitLab Kubernetes Agent Server (`kas`) work together. - -```mermaid -graph TB - agentk -- gRPC bidirectional streaming --> kas - - subgraph "GitLab" - kas[kas] - GitLabRoR[GitLab RoR] -- gRPC --> kas - kas -- gRPC --> Gitaly[Gitaly] - kas -- REST API --> GitLabRoR - end - - subgraph "Kubernetes cluster" - agentk[agentk] - end -``` - -- `GitLab RoR` is the main GitLab application. It uses gRPC to talk to `kas`. -- `agentk` is the GitLab Kubernetes Agent. It keeps a connection established to a - `kas` instance, waiting for requests to process. It may also actively send information - about things happening in the cluster. -- `kas` is the GitLab Kubernetes Agent Server, and is responsible for: - - Accepting requests from `agentk`. - - [Authentication of requests](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/identity_and_auth.md) from `agentk` by querying `GitLab RoR`. - - Fetching agent's configuration from a corresponding Git repository by querying Gitaly. - - Matching incoming requests from `GitLab RoR` with existing connections from - the right `agentk`, forwarding requests to it and forwarding responses back. - - (Optional) Sending notifications through ActionCable for events received from `agentk`. - - Polling manifest repositories for [GitOps support](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/gitops.md) by communicating with Gitaly. - -<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> -To learn more about how the repository is structured, see -[GitLab Kubernetes Agent repository overview](https://www.youtube.com/watch?v=j8CyaCWroUY). - -## Guiding principles - -GitLab prefers to add logic into `kas` rather than `agentk`. `agentk` should be kept -streamlined and small to minimize the need for upgrades. On GitLab.com, `kas` is -managed by GitLab, so upgrades and features can be added without requiring you -to upgrade `agentk` in your clusters. - -`agentk` can't be viewed as a dumb reverse proxy because features are planned to be built -[on top of the cache with informers](https://github.com/kubernetes/client-go/blob/ccd5becdffb7fd8006e31341baaaacd14db2dcb7/tools/cache/shared_informer.go#L34-L183). +<!-- This redirect file can be deleted after <2022-06-24>. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/development/agent/local.md b/doc/development/agent/local.md index 50959b5c450..1fff5607de4 100644 --- a/doc/development/agent/local.md +++ b/doc/development/agent/local.md @@ -1,155 +1,9 @@ --- -stage: Configure -group: Configure -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: 'https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/local.md' +remove_date: '2022-06-24' --- -# Run the Kubernetes Agent locally **(PREMIUM SELF)** +This file was moved to [another location](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/local.md). -You can run `kas` and `agentk` locally to test the [Kubernetes Agent](index.md) yourself. - -1. Create a `cfg.yaml` file from the contents of - [`config_example.yaml`](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/pkg/kascfg/config_example.yaml), or this example: - - ```yaml - agent: - listen: - network: tcp - address: 127.0.0.1:8150 - websocket: false - gitops: - poll_period: "10s" - gitlab: - address: http://localhost:3000 - authentication_secret_file: /Users/tkuah/code/ee-gdk/gitlab/.gitlab_kas_secret - ``` - -1. Create a `token.txt`. This is the token for - [the agent you created](../../user/clusters/agent/index.md#create-an-agent-record-in-gitlab). This file must not contain a newline character. You can create the file with this command: - - ```shell - echo -n "<TOKEN>" > token.txt - ``` - -1. Start the binaries with the following commands: - - ```shell - # Need GitLab to start - gdk start - # Stop GDK's version of kas - gdk stop gitlab-k8s-agent - - # Start kas - bazel run //cmd/kas -- --configuration-file="$(pwd)/cfg.yaml" - ``` - -1. In a new terminal window, run this command to start `agentk`: - - ```shell - bazel run //cmd/agentk -- --kas-address=grpc://127.0.0.1:8150 --token-file="$(pwd)/token.txt" - ``` - -You can also inspect the -[Makefile](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/Makefile) -for more targets. - -<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> -To learn more about how the repository is structured, see -[GitLab Kubernetes Agent repository overview](https://www.youtube.com/watch?v=j8CyaCWroUY). - -## Run tests locally - -You can run all tests, or a subset of tests, locally. - -- **To run all tests**: Run the command `make test`. -- **To run all test targets in the directory**: Run the command - `bazel test //internal/module/gitops/server:all`. - - You can use `*` in the command, instead of `all`, but it must be quoted to - avoid shell expansion: `bazel test '//internal/module/gitops/server:*'`. -- **To run all tests in a directory and its subdirectories**: Run the command - `bazel test //internal/module/gitops/server/...`. - -### Run specific test scenarios - -To run only a specific test scenario, you need the directory name and the target -name of the test. For example, to run the tests at -`internal/module/gitops/server/module_test.go`, the `BUILD.bazel` file that -defines the test's target name lives at `internal/module/gitops/server/BUILD.bazel`. -In the latter, the target name is defined like: - -```bazel -go_test( - name = "server_test", - size = "small", - srcs = [ - "module_test.go", -``` - -The target name is `server_test` and the directory is `internal/module/gitops/server/`. -Run the test scenario with this command: - -```shell -bazel test //internal/module/gitops/server:server_test -``` - -### Additional resources - -- Bazel documentation about [specifying targets to build](https://docs.bazel.build/versions/master/guide.html#specifying-targets-to-build). -- [The Bazel query](https://docs.bazel.build/versions/master/query.html) -- [Bazel query how to](https://docs.bazel.build/versions/master/query-how-to.html) - -## KAS QA tests - -This section describes how to run KAS tests against different GitLab environments based on the -[GitLab QA orchestrator](https://gitlab.com/gitlab-org/gitlab-qa). - -### Status - -The `kas` QA tests currently have some limitations. You can run them manually on GDK, but they don't -run automatically with the nightly jobs against the live environment. See the section below -to learn how to run them against different environments. - -### Prepare - -Before performing any of these tests, if you have a `k3s` instance running, make sure to -stop it manually before running them. Otherwise, the tests might fail with the message -`failed to remove k3s cluster`. - -You might need to specify the correct Agent image version that matches the `kas` image version. You can use the `GITLAB_AGENTK_VERSION` local environment for this. - -### Against `staging` - -1. Go to your local `qa/qa/service/cluster_provider/k3s.rb` and comment out - [this line](https://gitlab.com/gitlab-org/gitlab/-/blob/5b15540ea78298a106150c3a1d6ed26416109b9d/qa/qa/service/cluster_provider/k3s.rb#L8) and - [this line](https://gitlab.com/gitlab-org/gitlab/-/blob/5b15540ea78298a106150c3a1d6ed26416109b9d/qa/qa/service/cluster_provider/k3s.rb#L36). - We don't allow local connections on `staging` as they require an admin user. -1. Ensure you don't have an `EE_LICENSE` environment variable set as this would force an admin login. -1. Go to your GDK root folder and `cd gitlab/qa`. -1. Login with your user in staging and create a group to be used as sandbox. - Something like: `username-qa-sandbox`. -1. Create an access token for your user with the `api` permission. -1. Replace the values given below with your own and run: - - ```shell - GITLAB_SANDBOX_NAME="<THE GROUP ID YOU CREATED ON STEP 2>" \ - GITLAB_QA_ACCESS_TOKEN="<THE ACCESS TOKEN YOU CREATED ON STEP 3>" \ - GITLAB_USERNAME="<YOUR STAGING USERNAME>" \ - GITLAB_PASSWORD="<YOUR STAGING PASSWORD>" \ - bundle exec bin/qa Test::Instance::All https://staging.gitlab.com -- --tag quarantine qa/specs/features/ee/api/7_configure/kubernetes/kubernetes_agent_spec.rb - ``` - -### Against GDK - -1. Go to your `qa/qa/fixtures/kubernetes_agent/agentk-manifest.yaml.erb` and comment out [this line](https://gitlab.com/gitlab-org/gitlab/-/blob/a55b78532cfd29426cf4e5b4edda81407da9d449/qa/qa/fixtures/kubernetes_agent/agentk-manifest.yaml.erb#L27) and uncomment [this line](https://gitlab.com/gitlab-org/gitlab/-/blob/a55b78532cfd29426cf4e5b4edda81407da9d449/qa/qa/fixtures/kubernetes_agent/agentk-manifest.yaml.erb#L28). - GDK's `kas` listens on `grpc`, not on `wss`. -1. Go to the GDK's root folder and `cd gitlab/qa`. -1. On the contrary to staging, run the QA test in GDK as admin, which is the default choice. To do so, use the default sandbox group and run the command below. Make sure to adjust your credentials if necessary, otherwise, the test might fail: - - ```shell - GITLAB_USERNAME=root \ - GITLAB_PASSWORD="5iveL\!fe" \ - GITLAB_ADMIN_USERNAME=root \ - GITLAB_ADMIN_PASSWORD="5iveL\!fe" \ - bundle exec bin/qa Test::Instance::All http://gdk.test:3000 -- --tag quarantine qa/specs/features/ee/api/7_configure/kubernetes/kubernetes_agent_spec.rb - ``` +<!-- This redirect file can be deleted after <2022-06-24>. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/development/agent/repository_overview.md b/doc/development/agent/repository_overview.md index b9eea286a3e..43ea99889c5 100644 --- a/doc/development/agent/repository_overview.md +++ b/doc/development/agent/repository_overview.md @@ -1,98 +1,9 @@ --- -stage: Configure -group: Configure -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: 'https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/repository_overview.md' +remove_date: '2022-06-24' --- -# Kubernetes Agent repository overview **(PREMIUM SELF)** +This file was moved to [another location](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/repository_overview.md). -This page describes the subfolders of the Kubernetes Agent repository. -[Development information](index.md) and -[end-user documentation](../../user/clusters/agent/index.md) are both available. - -<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> -For a video overview, see -[GitLab Kubernetes Agent repository overview](https://www.youtube.com/watch?v=j8CyaCWroUY). - -## `build` - -Various files for the build process. - -### `build/deployment` - -A [`kpt`](https://googlecontainertools.github.io/kpt/) package that bundles some -[Kustomize](https://kustomize.io/) layers and components. Can be used as-is, or -to create a custom package to install `agentk`. - -## `cmd` - -Commands are binaries that this repository produces. They are: - -- `kas` is the GitLab Kubernetes Agent Server binary. -- `agentk` is the GitLab Kubernetes Agent binary. - -Each of these directories contain application bootstrap code for: - -- Reading configuration. -- Applying defaults to it. -- Constructing the dependency graph of objects that constitute the program. -- Running it. - -### `cmd/agentk` - -- `agentk` initialization logic. -- Implementation of the agent modules API. - -### `cmd/kas` - -- `kas` initialization logic. -- Implementation of the server modules API. - -## `examples` - -Git submodules for the example projects. - -## `internal` - -The main code of both `gitlab-kas` and `agentk`, and various supporting building blocks. - -### `internal/api` - -Structs that represent some important pieces of data. - -### `internal/gitaly` - -Items to work with [Gitaly](../../administration/gitaly/index.md). - -### `internal/gitlab` - -GitLab REST client. - -### `internal/module` - -Modules that implement server and agent-side functionality. - -### `internal/tool` - -Various building blocks. `internal/tool/testing` contains mocks and helpers -for testing. Mocks are generated with [`gomock`](https://pkg.go.dev/github.com/golang/mock). - -## `it` - -Contains scaffolding for integration tests. Unused at the moment. - -## `pkg` - -Contains exported packages. - -### `pkg/agentcfg` - -Contains protobuf definitions of the `agentk` configuration file. Used to configure -the agent through a configuration repository. - -### `pkg/kascfg` - -Contains protobuf definitions of the `gitlab-kas` configuration file. Contains an -example of that configuration file along with the test for it. The test ensures -the configuration file example is in sync with the protobuf definitions of the -file and defaults, which are applied when the file is loaded. +<!-- This redirect file can be deleted after <2022-06-24>. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/development/agent/routing.md b/doc/development/agent/routing.md index 9a7d6422d47..7792d6d56a4 100644 --- a/doc/development/agent/routing.md +++ b/doc/development/agent/routing.md @@ -1,230 +1,9 @@ --- -stage: Configure -group: Configure -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: 'https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/kas_request_routing.md' +remove_date: '2022-06-24' --- -# Routing `kas` requests in the Kubernetes Agent **(PREMIUM SELF)** +This file was moved to [another location](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/kas_request_routing.md). -This document describes how `kas` routes requests to concrete `agentk` instances. -GitLab must talk to GitLab Kubernetes Agent Server (`kas`) to: - -- Get information about connected agents. [Read more](https://gitlab.com/gitlab-org/gitlab/-/issues/249560). -- Interact with agents. [Read more](https://gitlab.com/gitlab-org/gitlab/-/issues/230571). -- Interact with Kubernetes clusters. [Read more](https://gitlab.com/gitlab-org/gitlab/-/issues/240918). - -Each agent connects to an instance of `kas` and keeps an open connection. When -GitLab must talk to a particular agent, a `kas` instance connected to this agent must -be found, and the request routed to it. - -## System design - -For an architecture overview please see -[architecture.md](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/architecture.md). - -```mermaid -flowchart LR - subgraph "Kubernetes 1" - agentk1p1["agentk 1, Pod1"] - agentk1p2["agentk 1, Pod2"] - end - - subgraph "Kubernetes 2" - agentk2p1["agentk 2, Pod1"] - end - - subgraph "Kubernetes 3" - agentk3p1["agentk 3, Pod1"] - end - - subgraph kas - kas1["kas 1"] - kas2["kas 2"] - kas3["kas 3"] - end - - GitLab["GitLab Rails"] - Redis - - GitLab -- "gRPC to any kas" --> kas - kas1 -- register connected agents --> Redis - kas2 -- register connected agents --> Redis - kas1 -- lookup agent --> Redis - - agentk1p1 -- "gRPC" --> kas1 - agentk1p2 -- "gRPC" --> kas2 - agentk2p1 -- "gRPC" --> kas1 - agentk3p1 -- "gRPC" --> kas2 -``` - -For this architecture, this diagram shows a request to `agentk 3, Pod1` for the list of pods: - -```mermaid -sequenceDiagram - GitLab->>+kas1: Get list of running<br />Pods from agentk<br />with agent_id=3 - Note right of kas1: kas1 checks for<br />agent connected with agent_id=3.<br />It does not.<br />Queries Redis - kas1->>+Redis: Get list of connected agents<br />with agent_id=3 - Redis-->-kas1: List of connected agents<br />with agent_id=3 - Note right of kas1: kas1 picks a specific agentk instance<br />to address and talks to<br />the corresponding kas instance,<br />specifying which agentk instance<br />to route the request to. - kas1->>+kas2: Get the list of running Pods<br />from agentk 3, Pod1 - kas2->>+agentk 3 Pod1: Get list of Pods - agentk 3 Pod1->>-kas2: Get list of Pods - kas2-->>-kas1: List of running Pods<br />from agentk 3, Pod1 - kas1-->>-GitLab: List of running Pods<br />from agentk with agent_id=3 -``` - -Each `kas` instance tracks the agents connected to it in Redis. For each agent, it -stores a serialized protobuf object with information about the agent. When an agent -disconnects, `kas` removes all corresponding information from Redis. For both events, -`kas` publishes a notification to a Redis [pub-sub channel](https://redis.io/topics/pubsub). - -Each agent, while logically a single entity, can have multiple replicas (multiple pods) -in a cluster. `kas` accommodates that and records per-replica (generally per-connection) -information. Each open `GetConfiguration()` streaming request is given -a unique identifier which, combined with agent ID, identifies an `agentk` instance. - -gRPC can keep multiple TCP connections open for a single target host. `agentk` only -runs one `GetConfiguration()` streaming request. `kas` uses that connection, and -doesn't see idle TCP connections because they are handled by the gRPC framework. - -Each `kas` instance provides information to Redis, so other `kas` instances can discover and access it. - -Information is stored in Redis with an [expiration time](https://redis.io/commands/expire), -to expire information for `kas` instances that become unavailable. To prevent -information from expiring too quickly, `kas` periodically updates the expiration time -for valid entries. Before terminating, `kas` cleans up the information it adds into Redis. - -When `kas` must atomically update multiple data structures in Redis, it uses -[transactions](https://redis.io/topics/transactions) to ensure data consistency. -Grouped data items must have the same expiration time. - -In addition to the existing `agentk -> kas` gRPC endpoint, `kas` exposes two new, -separate gRPC endpoints for GitLab and for `kas -> kas` requests. Each endpoint -is a separate network listener, making it easier to control network access to endpoints -and allowing separate configuration for each endpoint. - -Databases, like PostgreSQL, aren't used because the data is transient, with no need -to reliably persist it. - -### `GitLab : kas` external endpoint - -GitLab authenticates with `kas` using JWT and the same shared secret used by the -`kas -> GitLab` communication. The JWT issuer should be `gitlab` and the audience -should be `gitlab-kas`. - -When accessed through this endpoint, `kas` plays the role of request router. - -If a request from GitLab comes but no connected agent can handle it, `kas` blocks -and waits for a suitable agent to connect to it or to another `kas` instance. It -stops waiting when the client disconnects, or when some long timeout happens, such -as client timeout. `kas` is notified of new agent connections through a -[pub-sub channel](https://redis.io/topics/pubsub) to avoid frequent polling. -When a suitable agent connects, `kas` routes the request to it. - -### `kas : kas` internal endpoint - -This endpoint is an implementation detail, an internal API, and should not be used -by any other system. It's protected by JWT using a secret, shared among all `kas` -instances. No other system must have access to this secret. - -When accessed through this endpoint, `kas` uses the request itself to determine -which `agentk` to send the request to. It prevents request cycles by only following -the instructions in the request, rather than doing discovery. It's the responsibility -of the `kas` receiving the request from the _external_ endpoint to retry and re-route -requests. This method ensures a single central component for each request can determine -how a request is routed, rather than distributing the decision across several `kas` instances. - -### Reverse gRPC tunnel - -This section explains how the `agentk` -> `kas` reverse gRPC tunnel is implemented. - -<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> -For a video overview of how some of the blocks map to code, see -[GitLab Kubernetes Agent reverse gRPC tunnel architecture and code overview -](https://www.youtube.com/watch?v=9pnQF76hyZc). - -#### High level schema - -In this example, `Server side of module A` exposes its API to get the `Pod` list -on the `Public API gRPC server`. When it receives a request, it must determine -the agent ID from it, then call the proxying code which forwards the request to -a suitable `agentk` that can handle it. - -The `Agent side of module A` exposes the same API on the `Internal gRPC server`. -When it receives the request, it needs to handle it (such as retrieving and returning -the `Pod` list). - -This schema describes how reverse tunneling is handled fully transparently -for modules, so you can add new features: - -```mermaid -graph TB - subgraph kas - server-internal-grpc-server[Internal gRPC server] - server-api-grpc-server[Public API gRPC server] - server-module-a[Server side of module A] - server-module-b[Server side of module B] - end - subgraph agentk - agent-internal-grpc-server[Internal gRPC server] - agent-module-a[Agent side of module A] - agent-module-b[Agent side of module B] - end - - agent-internal-grpc-server -- request --> agent-module-a - agent-internal-grpc-server -- request --> agent-module-b - - server-module-a-. expose API on .-> server-internal-grpc-server - server-module-b-. expose API on .-> server-api-grpc-server - - server-internal-grpc-server -- proxy request --> agent-internal-grpc-server - server-api-grpc-server -- proxy request --> agent-internal-grpc-server -``` - -#### Implementation schema - -`HandleTunnelConnection()` is called with the server-side interface of the reverse -tunnel. It registers the connection and blocks, waiting for a request to proxy -through the connection. - -`HandleIncomingConnection()` is called with the server-side interface of the incoming -connection. It registers the connection and blocks, waiting for a matching tunnel -to proxy the connection through. - -After it has two connections that match, `Connection registry` starts bi-directional -data streaming: - -```mermaid -graph TB - subgraph kas - server-tunnel-module[Server tunnel module] - connection-registry[Connection registry] - server-internal-grpc-server[Internal gRPC server] - server-api-grpc-server[Public API gRPC server] - server-module-a[Server side of module A] - server-module-b[Server side of module B] - end - subgraph agentk - agent-internal-grpc-server[Internal gRPC server] - agent-tunnel-module[Agent tunnel module] - agent-module-a[Agent side of module A] - agent-module-b[Agent side of module B] - end - - server-tunnel-module -- "HandleTunnelConnection()" --> connection-registry - server-internal-grpc-server -- "HandleIncomingConnection()" --> connection-registry - server-api-grpc-server -- "HandleIncomingConnection()" --> connection-registry - server-module-a-. expose API on .-> server-internal-grpc-server - server-module-b-. expose API on .-> server-api-grpc-server - - agent-tunnel-module -- "establish tunnel, receive request" --> server-tunnel-module - agent-tunnel-module -- make request --> agent-internal-grpc-server - agent-internal-grpc-server -- request --> agent-module-a - agent-internal-grpc-server -- request --> agent-module-b -``` - -### API definitions - -- [`agent_tracker/agent_tracker.proto`](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/internal/module/agent_tracker/agent_tracker.proto) -- [`agent_tracker/rpc/rpc.proto`](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/internal/module/agent_tracker/rpc/rpc.proto) -- [`reverse_tunnel/rpc/rpc.proto`](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/internal/module/reverse_tunnel/rpc/rpc.proto) +<!-- This redirect file can be deleted after <2022-06-24>. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/development/agent/user_stories.md b/doc/development/agent/user_stories.md index ab135cad9d3..ce38064b31b 100644 --- a/doc/development/agent/user_stories.md +++ b/doc/development/agent/user_stories.md @@ -1,77 +1,9 @@ --- -stage: Configure -group: Configure -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +redirect_to: 'https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/user_stories.md' +remove_date: '2022-06-24' --- -# Kubernetes Agent user stories **(PREMIUM SELF)** +This file was moved to [another location](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/user_stories.md). -The [personas in action](https://about.gitlab.com/handbook/marketing/strategic-marketing/roles-personas/#user-personas) -for the Kubernetes Agent are: - -- [Sasha, the Software Developer](https://about.gitlab.com/handbook/marketing/strategic-marketing/roles-personas/#sasha-software-developer). -- [Allison, the Application Operator](https://about.gitlab.com/handbook/marketing/strategic-marketing/roles-personas/#allison-application-ops). -- [Priyanka, the Platform Engineer](https://about.gitlab.com/handbook/marketing/strategic-marketing/roles-personas/#priyanka-platform-engineer). - -[Devon, the DevOps engineer](https://about.gitlab.com/handbook/marketing/strategic-marketing/roles-personas/#devon-devops-engineer) -is intentionally excluded here, as DevOps is more of a role than a persona. - -There are various workflows to support, so some user stories might seem to contradict each other. They don't. - -## Software Developer user stories - -<!-- vale gitlab.FirstPerson = NO --> - -- As a Software Developer, I want to push my code, and move to the next development task, - to work on business applications. -- As a Software Developer, I want to set necessary dependencies and resource requirements - together with my application code, so my code runs fine after deployment. - -<!-- vale gitlab.FirstPerson = YES --> - -## Application Operator user stories - -<!-- vale gitlab.FirstPerson = NO --> - -- As an Application Operator, I want to standardize the deployments used by my teams, - so I can support all teams with minimal effort. -- As an Application Operator, I want to have a single place to define all the deployments, - so I can assure security fixes are applied everywhere. -- As an Application Operator, I want to offer a set of predefined templates to - Software Developers, so they can get started quickly and can deploy to production - without my intervention, and I am not a bottleneck. -- As an Application Operator, I want to know exactly what changes are being deployed, - so I can fulfill my SLAs. -- As an Application Operator, I want deep insights into what versions of my applications - are running and want to be able to debug them, so I can fix operational issues. -- As an Application Operator, I want application code to be automatically deployed - to staging environments when new versions are available. -- As an Application Operator, I want to follow my preferred deployment strategy, - so I can move code into production in a reliable way. -- As an Application Operator, I want review all code before it's deployed into production, - so I can fulfill my SLAs. -- As an Application Operator, I want to be notified before deployment when new code needs my attention, - so I can review it swiftly. - -<!-- vale gitlab.FirstPerson = YES --> - -## Platform Engineer user stories - -<!-- vale gitlab.FirstPerson = NO --> - -- As a Platform Engineer, I want to restrict customizations to preselected values - for Operators, so I can fulfill my SLAs. -- As a Platform Engineer, I want to allow some level of customization to Operators, - so I don't become a bottleneck. -- As a Platform Engineer, I want to define all deployments in a single place, so - I can assure security fixes are applied everywhere. -- As a Platform Engineer, I want to define the infrastructure by code, so my - infrastructure management is testable, reproducible, traceable, and scalable. -- As a Platform Engineer, I want to define various policies that applications must - follow, so that I can fulfill my SLAs. -- As a Platform Engineer, I want approved tooling for log management and persistent storage, - so I can scale, secure, and manage them as needed. -- As a Platform Engineer, I want to be alerted when my infrastructure differs from - its definition, so I can make sure that everything is configured as expected. - -<!-- vale gitlab.FirstPerson = YES --> +<!-- This redirect file can be deleted after <2022-06-24>. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/development/usage_ping/dictionary.md b/doc/development/usage_ping/dictionary.md index e3f02388b40..bfe0fd56df3 100644 --- a/doc/development/usage_ping/dictionary.md +++ b/doc/development/usage_ping/dictionary.md @@ -16332,6 +16332,18 @@ Status: `data_available` Tiers: `free` +### `usage_activity_by_stage.manage.compliance_frameworks_with_pipeline` + +Count of compliance frameworks that have a pipeline configuration + +[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/config/metrics/counts_all/20210622123800_compliance_frameworks_with_pipeline.yml) + +Group: `compliance` + +Status: `implemented` + +Tiers: `ultimate` + ### `usage_activity_by_stage.manage.custom_compliance_frameworks` Total count of all custom compliance framework labels @@ -18398,6 +18410,18 @@ Status: `data_available` Tiers: `free` +### `usage_activity_by_stage_monthly.manage.compliance_frameworks_with_pipeline` + +Count of compliance frameworks that have a pipeline configuration + +[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/config/metrics/counts_28d/20210622091519_compliance_frameworks_with_pipeline.yml) + +Group: `compliance` + +Status: `implemented` + +Tiers: `ultimate` + ### `usage_activity_by_stage_monthly.manage.custom_compliance_frameworks` Monthly count of all custom compliance framework labels diff --git a/doc/integration/shibboleth.md b/doc/integration/shibboleth.md index c52535c297d..4a3aa6b3dc5 100644 --- a/doc/integration/shibboleth.md +++ b/doc/integration/shibboleth.md @@ -1,151 +1,9 @@ --- -stage: Create -group: Ecosystem -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: 'saml.md' +remove_date: '2021-09-29' --- -# Shibboleth OmniAuth Provider **(FREE)** +This file was moved to [another location](saml.md). -NOTE: -The preferred approach for integrating a Shibboleth authentication system -with GitLab 10 or newer is to use the [GitLab SAML integration](saml.md). This documentation is for Omnibus GitLab 9.x installs or older. - -To enable Shibboleth support in GitLab we need to use Apache instead of NGINX. (It may be possible to use NGINX, however this is difficult to configure using the bundled NGINX provided in the Omnibus GitLab package.) Apache uses `mod_shib2` module for Shibboleth authentication and can pass attributes as headers to OmniAuth Shibboleth provider. - -To enable the Shibboleth OmniAuth provider you must configure Apache Shibboleth module. -The installation and configuration of the module itself is out of the scope of this document. -Check [the Shibboleth documentation](https://wiki.shibboleth.net/confluence/display/SP3/Apache) for more information. - -You can find Apache configuration in [GitLab Recipes](https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/web-server/apache). - -The following changes are needed to enable Shibboleth: - -1. Protect the OmniAuth Shibboleth callback URL: - - ```apache - <Location /users/auth/shibboleth/callback> - AuthType shibboleth - ShibRequestSetting requireSession 1 - ShibUseHeaders On - require valid-user - </Location> - - Alias /shibboleth-sp /usr/share/shibboleth - <Location /shibboleth-sp> - Satisfy any - </Location> - - <Location /Shibboleth.sso> - SetHandler shib - </Location> - ``` - -1. Exclude Shibboleth URLs from rewriting. Add `RewriteCond %{REQUEST_URI} !/Shibboleth.sso` and `RewriteCond %{REQUEST_URI} !/shibboleth-sp`. Configuration should look like this: - - ```apache - # Apache equivalent of Nginx try files - RewriteEngine on - RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f - RewriteCond %{REQUEST_URI} !/Shibboleth.sso - RewriteCond %{REQUEST_URI} !/shibboleth-sp - RewriteRule .* http://127.0.0.1:8080%{REQUEST_URI} [P,QSA] - RequestHeader set X_FORWARDED_PROTO 'https' - ``` - - NOTE: - In GitLab versions 11.4 and later, OmniAuth is enabled by default. If you're using an - earlier version, you must explicitly enable it in `/etc/gitlab/gitlab.rb`. - -1. In addition, add Shibboleth to `/etc/gitlab/gitlab.rb` as an OmniAuth provider. - User attributes are sent from the - Apache reverse proxy to GitLab as headers with the names from the Shibboleth - attribute mapping. Therefore the values of the `args` hash - should be in the form of `"HTTP_ATTRIBUTE"`. The keys in the hash are arguments - to the [OmniAuth::Strategies::Shibboleth class](https://github.com/toyokazu/omniauth-shibboleth/blob/master/lib/omniauth/strategies/shibboleth.rb) - and are documented by the [`omniauth-shibboleth` gem](https://github.com/toyokazu/omniauth-shibboleth) - (take care to note the version of the gem packaged with GitLab). If some of - your users appear to be authenticated by Shibboleth and Apache, but GitLab - rejects their account with a URI that contains "e-mail is invalid" then your - Shibboleth Identity Provider or Attribute Authority may be asserting multiple - email addresses. In this instance, you might consider setting the - `multi_values` argument to `first`. - - The file should look like this: - - ```ruby - external_url 'https://gitlab.example.com' - gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' - - # disable Nginx - nginx['enable'] = false - - gitlab_rails['omniauth_allow_single_sign_on'] = true - gitlab_rails['omniauth_block_auto_created_users'] = false - gitlab_rails['omniauth_providers'] = [ - { - "name" => "'shibboleth"', - "label" => "Text for Login Button", - "args" => { - "shib_session_id_field" => "HTTP_SHIB_SESSION_ID", - "shib_application_id_field" => "HTTP_SHIB_APPLICATION_ID", - "uid_field" => 'HTTP_EPPN', - "name_field" => 'HTTP_CN', - "info_fields" => { "email" => 'HTTP_MAIL'} - } - } - ] - - ``` - -1. [Reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart](../administration/restart_gitlab.md#installations-from-source) GitLab for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. - -On the sign in page, there should now be a **Sign in with: Shibboleth** icon below the regular sign in form. Click the icon to begin the authentication process. You are redirected to IdP server (depends on your Shibboleth module configuration). If everything goes well the user is returned to GitLab and is signed in. - -## Apache 2.4 / GitLab 8.6 update - -The order of the first 2 Location directives is important. If they are reversed, -requesting a Shibboleth session fails! - -```plaintext -<Location /> - Require all granted - ProxyPassReverse http://127.0.0.1:8181 - ProxyPassReverse http://YOUR_SERVER_FQDN/ -</Location> - -<Location /users/auth/shibboleth/callback> - AuthType shibboleth - ShibRequestSetting requireSession 1 - ShibUseHeaders On - Require shib-session -</Location> - -Alias /shibboleth-sp /usr/share/shibboleth - -<Location /shibboleth-sp> - Require all granted -</Location> - -<Location /Shibboleth.sso> - SetHandler shib -</Location> - -RewriteEngine on - -#Don't escape encoded characters in api requests -RewriteCond %{REQUEST_URI} ^/api/v4/.* -RewriteCond %{REQUEST_URI} !/Shibboleth.sso -RewriteCond %{REQUEST_URI} !/shibboleth-sp -RewriteRule .* http://127.0.0.1:8181%{REQUEST_URI} [P,QSA,NE] - -#Forward all requests to gitlab-workhorse except existing files -RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f [OR] -RewriteCond %{REQUEST_URI} ^/uploads/.* -RewriteCond %{REQUEST_URI} !/Shibboleth.sso -RewriteCond %{REQUEST_URI} !/shibboleth-sp -RewriteRule .* http://127.0.0.1:8181%{REQUEST_URI} [P,QSA] - -RequestHeader set X_FORWARDED_PROTO 'https' -RequestHeader set X-Forwarded-Ssl on -``` +<!-- This redirect file can be deleted after <2021-09-29>. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/update/index.md b/doc/update/index.md index 789b2c22087..c81d8981753 100644 --- a/doc/update/index.md +++ b/doc/update/index.md @@ -371,13 +371,14 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap ### 14.0.0 -In GitLab 13.3 some [pipeline processing methods were deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/218536) -and this code was completely removed in GitLab 14.0. If you plan to upgrade from -**GitLab 13.2 or older** directly to 14.0, you should not have any pipelines running -when you upgrade. The pipelines might report the wrong status when the upgrade completes. -You should shut down GitLab and wait for all pipelines on runners to complete, then upgrade -GitLab to 14.0. Alternatively, you can first upgrade GitLab to a version between 13.3 and -13.12, then upgrade to 14.0. +- In GitLab 13.3 some [pipeline processing methods were deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/218536) + and this code was completely removed in GitLab 14.0. If you plan to upgrade from + **GitLab 13.2 or older** directly to 14.0, you should not have any pipelines running + when you upgrade. The pipelines might report the wrong status when the upgrade completes. + You should shut down GitLab and wait for all pipelines on runners to complete, then upgrade + GitLab to 14.0. Alternatively, you can first upgrade GitLab to a version between 13.3 and + 13.12, then upgrade to 14.0. +- The support of PostgreSQL 11 [has been dropped](../install/requirements.md#database). Make sure to [update your database](https://docs.gitlab.com/omnibus/settings/database.html#upgrade-packaged-postgresql-server) to version 12 before updating to GitLab 14.0. ### 13.11.0 diff --git a/doc/user/admin_area/settings/third_party_offers.md b/doc/user/admin_area/settings/third_party_offers.md index e7fa8b1dc40..6f7cb081315 100644 --- a/doc/user/admin_area/settings/third_party_offers.md +++ b/doc/user/admin_area/settings/third_party_offers.md @@ -5,7 +5,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w type: reference --- -# Third party offers **(FREE SELF)** +# Third-party offers **(FREE SELF)** > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/20379) in GitLab Free 11.1. @@ -16,8 +16,9 @@ for using [Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine/ To toggle the display of third-party offers: 1. On the top bar, select **Menu >** **{admin}** **Admin**. -1. In the left sidebar, select **Settings**, and expand **Third party offers**. -1. Select **Do not display offers from third parties within GitLab**. +1. On the left sidebar, select **Settings**, and expand **Third-party offers**. +1. Select **Do not display offers from third parties**. +1. Select **Save changes**. <!-- ## Troubleshooting diff --git a/doc/user/clusters/agent/index.md b/doc/user/clusters/agent/index.md index a7bd1a8322f..4c95ebcd5ba 100644 --- a/doc/user/clusters/agent/index.md +++ b/doc/user/clusters/agent/index.md @@ -23,7 +23,7 @@ tasks in a secure and cloud-native way. It enables: - [CI/CD Tunnel](ci_cd_tunnel.md) that enables users to access Kubernetes clusters from GitLab CI/CD jobs even if there is no network connectivity between GitLab Runner and a cluster. Many more features are planned. Please review [our roadmap](https://gitlab.com/groups/gitlab-org/-/epics/3329) -and [our development documentation](../../../development/agent/index.md). +and [our development documentation](https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/tree/master/doc). ## GitLab Agent GitOps workflow diff --git a/doc/user/project/releases/index.md b/doc/user/project/releases/index.md index f5fbe515f0f..728608e557f 100644 --- a/doc/user/project/releases/index.md +++ b/doc/user/project/releases/index.md @@ -14,8 +14,11 @@ To introduce a checkpoint in your source code history, you can assign a However, in most cases, your users need more than just the raw source code. They need compiled objects or other assets output by your CI/CD system. -A GitLab *Release* is a snapshot of the source, build output, artifacts, and other metadata -associated with a released version of your code. +A GitLab Release can be: + +- A snapshot of the source code of your repository. +- [Generic packages](../../packages/generic_packages/index.md) created from job artifacts. +- Other metadata associated with a released version of your code. You can create a GitLab release on any branch. When you create a release: @@ -273,14 +276,28 @@ Release note descriptions are unrelated. Description supports [Markdown](../../m A release contains the following types of assets: - [Source code](#source-code) -- [Permanent links to release assets](#permanent-links-to-release-assets) +- [Link](#links) #### Source code GitLab automatically generates `zip`, `tar.gz`, `tar.bz2`, and `tar` archived source code from the given Git tag. These are read-only assets. -#### Permanent links to release assets +#### Links + +A link is any URL which can point to whatever you like: documentation, built +binaries, or other related materials. These can be both internal or external +links from your GitLab instance. +Each link as an asset has the following attributes: + +| Attribute | Description | Required | +| ---- | ----------- | --- | +| `name` | The name of the link. | Yes | +| `url` | The URL to download a file. | Yes | +| `filepath` | The redirect link to the `url`. See [this section](#permanent-links-to-release-assets) for more information. | No | +| `link_type` | The content kind of what users can download via `url`. See [this section](#link-types) for more information. | No | + +##### Permanent links to release assets > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/27300) in GitLab 12.9. @@ -289,20 +306,6 @@ GitLab always redirects this URL to the actual asset location, so even if the assets move to a different location, you can continue to use the same URL. This is defined during [link creation](../../../api/releases/links.md#create-a-link) or [updating](../../../api/releases/links.md#update-a-link). -Each asset has a `name`, a `url` of the *actual* asset location, and optionally, -`filepath` and `link_type` parameters. - -A `filepath` creates a URL pointing to the asset for the Release. - -The `link_type` parameter accepts one of the following four values: - -- `runbook` -- `package` -- `image` -- `other` (default) - -This field has no effect on the URL and it's only used for visual purposes in the Releases page of your project. - The format of the URL is: ```plaintext @@ -329,13 +332,104 @@ https://gitlab.com/gitlab-org/gitlab-runner/releases/v11.9.0-rc2/downloads/binar The physical location of the asset can change at any time and the direct link remains unchanged. -### Links +##### Link Types -A link is any URL which can point to whatever you like: documentation, built -binaries, or other related materials. These can be both internal or external -links from your GitLab instance. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/207257) in GitLab 13.1. The four types of links are "Runbook," "Package," "Image," and "Other." +The `link_type` parameter accepts one of the following four values: + +- `runbook` +- `package` +- `image` +- `other` (default) + +This field has no effect on the URL and it's only used for visual purposes in the Releases page of your project. + +##### Use a generic package for attaching binaries + +You can use [generic packages](../../packages/generic_packages/index.md) +to store any artifacts from a release or tag pipeline, +that can also be used for attaching binary files to an individual release entry. +You basically need to: + +1. [Push the artifacts to the Generic Package Registry](../../packages/generic_packages/index.md#publish-a-package-file). +1. [Attach the package link to the release](#links). + +The following example generates release assets, publishes them +as a generic package, and then creates a release: + +```yaml +stages: + - build + - upload + - release + +variables: + # Package version can only contain numbers (0-9), and dots (.). + # Must be in the format of X.Y.Z, i.e. should match /\A\d+\.\d+\.\d+\z/ regular expresion. + # See https://docs.gitlab.com/ee/user/packages/generic_packages/#publish-a-package-file + PACKAGE_VERSION: "1.2.3" + DARWIN_AMD64_BINARY: "myawesomerelease-darwin-amd64-${PACKAGE_VERSION}" + LINUX_AMD64_BINARY: "myawesomerelease-linux-amd64-${PACKAGE_VERSION}" + PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/myawesomerelease/${PACKAGE_VERSION}" + +build: + stage: build + image: alpine:latest + rules: + - if: $CI_COMMIT_TAG + script: + - mkdir bin + - echo "Mock binary for ${DARWIN_AMD64_BINARY}" > bin/${DARWIN_AMD64_BINARY} + - echo "Mock binary for ${LINUX_AMD64_BINARY}" > bin/${LINUX_AMD64_BINARY} + artifacts: + paths: + - bin/ + +upload: + stage: upload + image: curlimages/curl:latest + rules: + - if: $CI_COMMIT_TAG + script: + - | + curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${DARWIN_AMD64_BINARY} ${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY} + - | + curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file bin/${LINUX_AMD64_BINARY} ${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY} + +release: + # Caution, as of 2021-02-02 these assets links require a login, see: + # https://gitlab.com/gitlab-org/gitlab/-/issues/299384 + stage: release + image: registry.gitlab.com/gitlab-org/release-cli:latest + rules: + - if: $CI_COMMIT_TAG + script: + - | + release-cli create --name "Release $CI_COMMIT_TAG" --tag-name $CI_COMMIT_TAG \ + --assets-link "{\"name\":\"${DARWIN_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${DARWIN_AMD64_BINARY}\"}" \ + --assets-link "{\"name\":\"${LINUX_AMD64_BINARY}\",\"url\":\"${PACKAGE_REGISTRY_URL}/${LINUX_AMD64_BINARY}\"}" +``` + +PowerShell users may need to escape the double quote `"` inside a JSON +string with a `` ` `` (back tick) for `--assets-link` and `ConvertTo-Json` +before passing on to the `release-cli`. +For example: + +```yaml +release: + script: + - $env:asset = "{`"name`":`"MyFooAsset`",`"url`":`"https://gitlab.com/upack/artifacts/download/$env:UPACK_GROUP/$env:UPACK_NAME/$($env:GitVersion_SemVer)?contentOnly=zip`"}" + - $env:assetjson = $env:asset | ConvertTo-Json + - release-cli create --name $CI_COMMIT_TAG --description "Release $CI_COMMIT_TAG" --ref $CI_COMMIT_TAG --tag-name $CI_COMMIT_TAG --assets-link=$env:assetjson +``` + +NOTE: +Directly attaching [job artifacts](../../../ci/pipelines/job_artifacts.md) +links to a release is not recommended, because artifacts are ephemeral and +are used to pass data in the same pipeline. This means there's a risk that +they could either expire or someone might manually delete them. ## Release evidence diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 4b1dcd35c60..ff0edf30588 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -5117,6 +5117,9 @@ msgstr "" msgid "BillingPlan|Upgrade for free" msgstr "" +msgid "Billings|Shared runners cannot be enabled until a valid credit card is on file." +msgstr "" + msgid "Billings|To use free pipeline minutes on shared runners, you’ll need to validate your account with a credit or debit card. If you prefer not to provide one, you can run pipelines by bringing your own runners and disabling shared runners for your project. This is required to discourage and reduce abuse on GitLab infrastructure. %{strongStart}GitLab will not charge or store your card, it will only be used for validation.%{strongEnd} %{linkStart}Learn more%{linkEnd}." msgstr "" @@ -8736,7 +8739,7 @@ msgstr "" msgid "Control emails linked to your account" msgstr "" -msgid "Control the display of third party offers." +msgid "Control whether to display third-party offers in GitLab." msgstr "" msgid "Cookie domain" @@ -11431,7 +11434,7 @@ msgstr "" msgid "Display source" msgstr "" -msgid "Do not display offers from third parties within GitLab" +msgid "Do not display offers from third parties" msgstr "" msgid "Do you want to remove this deploy key?" @@ -32934,7 +32937,7 @@ msgstr "" msgid "Third Party Advisory Link" msgstr "" -msgid "Third party offers" +msgid "Third-party offers" msgstr "" msgid "This %{issuableDisplayName} is locked. Only project members can comment." diff --git a/spec/fast_spec_helper.rb b/spec/fast_spec_helper.rb index 469c29cd2e0..0672a47c73c 100644 --- a/spec/fast_spec_helper.rb +++ b/spec/fast_spec_helper.rb @@ -24,6 +24,8 @@ ActiveSupport::Dependencies.autoload_paths << 'ee/lib' ActiveSupport::XmlMini.backend = 'Nokogiri' RSpec.configure do |config| - config.filter_run focus: true - config.run_all_when_everything_filtered = true + unless ENV['CI'] + config.filter_run focus: true + config.run_all_when_everything_filtered = true + end end diff --git a/spec/features/admin/admin_settings_spec.rb b/spec/features/admin/admin_settings_spec.rb index ef4877b390e..5a4c47b9dc5 100644 --- a/spec/features/admin/admin_settings_spec.rb +++ b/spec/features/admin/admin_settings_spec.rb @@ -275,7 +275,7 @@ RSpec.describe 'Admin updates settings' do it 'enable hiding third party offers' do page.within('.as-third-party-offers') do - check 'Do not display offers from third parties within GitLab' + check 'Do not display offers from third parties' click_button 'Save changes' end diff --git a/spec/features/projects/clusters/gcp_spec.rb b/spec/features/projects/clusters/gcp_spec.rb index 8c497cded8e..21e587288f5 100644 --- a/spec/features/projects/clusters/gcp_spec.rb +++ b/spec/features/projects/clusters/gcp_spec.rb @@ -214,7 +214,7 @@ RSpec.describe 'Gcp Cluster', :js do it 'user does not see the offer' do page.within('.as-third-party-offers') do click_button 'Expand' - check 'Do not display offers from third parties within GitLab' + check 'Do not display offers from third parties' click_button 'Save changes' end diff --git a/spec/frontend/projects/settings/components/shared_runners_toggle_spec.js b/spec/frontend/projects/settings/components/shared_runners_toggle_spec.js index 2d6efe7ae83..b592db8a64e 100644 --- a/spec/frontend/projects/settings/components/shared_runners_toggle_spec.js +++ b/spec/frontend/projects/settings/components/shared_runners_toggle_spec.js @@ -1,6 +1,8 @@ import { GlAlert, GlToggle, GlTooltip } from '@gitlab/ui'; import { shallowMount } from '@vue/test-utils'; import MockAxiosAdapter from 'axios-mock-adapter'; +import CcValidationRequiredAlert from 'ee_component/billings/components/cc_validation_required_alert.vue'; +import { TEST_HOST } from 'helpers/test_constants'; import waitForPromises from 'helpers/wait_for_promises'; import axios from '~/lib/utils/axios_utils'; import SharedRunnersToggleComponent from '~/projects/settings/components/shared_runners_toggle.vue'; @@ -20,6 +22,7 @@ describe('projects/settings/components/shared_runners', () => { isDisabledAndUnoverridable: false, isLoading: false, updatePath: TEST_UPDATE_PATH, + isCreditCardValidationRequired: false, ...props, }, }); @@ -28,6 +31,7 @@ describe('projects/settings/components/shared_runners', () => { const findErrorAlert = () => wrapper.find(GlAlert); const findSharedRunnersToggle = () => wrapper.find(GlToggle); const findToggleTooltip = () => wrapper.find(GlTooltip); + const findCcValidationRequiredAlert = () => wrapper.findComponent(CcValidationRequiredAlert); const getToggleValue = () => findSharedRunnersToggle().props('value'); const isToggleLoading = () => findSharedRunnersToggle().props('isLoading'); const isToggleDisabled = () => findSharedRunnersToggle().props('disabled'); @@ -154,4 +158,40 @@ describe('projects/settings/components/shared_runners', () => { }); }); }); + + describe('with credit card validation required and shared runners DISABLED', () => { + beforeEach(() => { + window.gon = { + subscriptions_url: TEST_HOST, + payment_form_url: TEST_HOST, + }; + + createComponent({ + isCreditCardValidationRequired: true, + isEnabled: false, + }); + }); + + it('toggle should not be visible', () => { + expect(findSharedRunnersToggle().exists()).toBe(false); + }); + + it('credit card validation component should exist', () => { + expect(findCcValidationRequiredAlert().exists()).toBe(true); + expect(findCcValidationRequiredAlert().text()).toBe( + SharedRunnersToggleComponent.i18n.REQUIRES_VALIDATION_TEXT, + ); + }); + + describe('when credit card is validated', () => { + it('should show the toggle button', async () => { + findCcValidationRequiredAlert().vm.$emit('verifiedCreditCard'); + await waitForPromises(); + + expect(findSharedRunnersToggle().exists()).toBe(true); + expect(getToggleValue()).toBe(false); + expect(isToggleDisabled()).toBe(false); + }); + }); + }); }); diff --git a/spec/helpers/ci/runners_helper_spec.rb b/spec/helpers/ci/runners_helper_spec.rb index 94d4d620de9..40927d44e24 100644 --- a/spec/helpers/ci/runners_helper_spec.rb +++ b/spec/helpers/ci/runners_helper_spec.rb @@ -3,6 +3,12 @@ require 'spec_helper' RSpec.describe Ci::RunnersHelper do + let_it_be(:user, refind: true) { create(:user) } + + before do + allow(helper).to receive(:current_user).and_return(user) + end + describe '#runner_status_icon', :clean_gitlab_redis_cache do it "returns - not contacted yet" do runner = create(:ci_runner) @@ -90,28 +96,28 @@ RSpec.describe Ci::RunnersHelper do context 'when project has runners' do it 'returns the correct value for is_enabled' do - data = toggle_shared_runners_settings_data(project_with_runners) + data = helper.toggle_shared_runners_settings_data(project_with_runners) expect(data[:is_enabled]).to eq("true") end end context 'when project does not have runners' do it 'returns the correct value for is_enabled' do - data = toggle_shared_runners_settings_data(project_without_runners) + data = helper.toggle_shared_runners_settings_data(project_without_runners) expect(data[:is_enabled]).to eq("false") end end context 'for all projects' do it 'returns the update path for toggling the shared runners setting' do - data = toggle_shared_runners_settings_data(project_with_runners) + data = helper.toggle_shared_runners_settings_data(project_with_runners) expect(data[:update_path]).to eq(toggle_shared_runners_project_runners_path(project_with_runners)) end it 'returns false for is_disabled_and_unoverridable when project has no group' do project = create(:project) - data = toggle_shared_runners_settings_data(project) + data = helper.toggle_shared_runners_settings_data(project) expect(data[:is_disabled_and_unoverridable]).to eq("false") end @@ -129,7 +135,7 @@ RSpec.describe Ci::RunnersHelper do project = create(:project, group: group) allow(group).to receive(:shared_runners_setting).and_return(shared_runners_setting) - data = toggle_shared_runners_settings_data(project) + data = helper.toggle_shared_runners_settings_data(project) expect(data[:is_disabled_and_unoverridable]).to eq(is_disabled_and_unoverridable) end end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 63df2409e8e..58632a4ebe8 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -2813,6 +2813,14 @@ RSpec.describe User do end end + describe '#matches_identity?' do + it 'finds the identity when the DN is formatted differently' do + user = create(:omniauth_user, provider: 'ldapmain', extern_uid: 'uid=john smith,ou=people,dc=example,dc=com') + + expect(user.matches_identity?('ldapmain', 'uid=John Smith, ou=People, dc=example, dc=com')).to eq(true) + end + end + describe '#ldap_block' do let(:user) { create(:omniauth_user, provider: 'ldapmain', name: 'John Smith') } diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index e94ff5bcb45..6a2975e566a 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -76,9 +76,6 @@ require_relative '../tooling/quality/test_level' quality_level = Quality::TestLevel.new RSpec.configure do |config| - config.filter_run focus: true - config.run_all_when_everything_filtered = true - config.use_transactional_fixtures = true config.use_instantiated_fixtures = false config.fixture_path = Rails.root @@ -113,6 +110,8 @@ RSpec.configure do |config| end unless ENV['CI'] + config.filter_run focus: true + config.run_all_when_everything_filtered = true # Re-run failures locally with `--only-failures` config.example_status_persistence_file_path = './spec/examples.txt' end |