From ab128cc125f9db0c3a1bd48845f90c3d61ef42c9 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Fri, 6 Mar 2020 06:08:30 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- .gitlab/issue_templates/Feature proposal.md | 2 + app/models/clusters/cluster.rb | 15 +++-- .../unreleased/dblessing-add-scim-identities.yml | 6 ++ .../fix-deployment-namespace-resolution.yml | 5 ++ .../20200227164113_create_scim_identities.rb | 18 +++++ db/schema.rb | 14 ++++ doc/api/project_clusters.md | 30 ++++----- doc/ci/variables/predefined_variables.md | 2 +- doc/integration/vault.md | 4 +- doc/user/gitlab_com/index.md | 8 +++ doc/user/project/clusters/serverless/index.md | 3 +- lib/gitlab/ci/pipeline/seed/deployment.rb | 8 +-- spec/factories/deployments.rb | 2 +- .../lib/gitlab/ci/pipeline/seed/deployment_spec.rb | 32 +++++++-- spec/models/clusters/cluster_spec.rb | 76 +++++++++++----------- 15 files changed, 152 insertions(+), 73 deletions(-) create mode 100644 changelogs/unreleased/dblessing-add-scim-identities.yml create mode 100644 changelogs/unreleased/fix-deployment-namespace-resolution.yml create mode 100644 db/migrate/20200227164113_create_scim_identities.rb diff --git a/.gitlab/issue_templates/Feature proposal.md b/.gitlab/issue_templates/Feature proposal.md index fa989e45281..4e52a69f415 100644 --- a/.gitlab/issue_templates/Feature proposal.md +++ b/.gitlab/issue_templates/Feature proposal.md @@ -15,6 +15,8 @@ * [Sidney (Systems Administrator)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#sidney-systems-administrator) * [Sam (Security Analyst)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#sam-security-analyst) * [Dana (Data Analyst)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#dana-data-analyst) +* [Simone (Software Engineer in Test)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#simone-software-engineer-in-test) +* [Allison (Application Ops)](https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/#allison-application-ops) Personas are described at https://about.gitlab.com/handbook/marketing/product-marketing/roles-personas/ --> diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 7f155a8d435..78efe2b4337 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -249,9 +249,13 @@ module Clusters platform_kubernetes.kubeclient if kubernetes? end - def kubernetes_namespace_for(environment) + def kubernetes_namespace_for(environment, deployable: environment.last_deployable) + if deployable && environment.project_id != deployable.project_id + raise ArgumentError, 'environment.project_id must match deployable.project_id' + end + managed_namespace(environment) || - ci_configured_namespace(environment) || + ci_configured_namespace(deployable) || default_namespace(environment) end @@ -318,8 +322,11 @@ module Clusters ).execute&.namespace end - def ci_configured_namespace(environment) - environment.last_deployable&.expanded_kubernetes_namespace + def ci_configured_namespace(deployable) + # YAML configuration of namespaces not supported for managed clusters + return if managed? + + deployable&.expanded_kubernetes_namespace end def default_namespace(environment) diff --git a/changelogs/unreleased/dblessing-add-scim-identities.yml b/changelogs/unreleased/dblessing-add-scim-identities.yml new file mode 100644 index 00000000000..4e7663b12a7 --- /dev/null +++ b/changelogs/unreleased/dblessing-add-scim-identities.yml @@ -0,0 +1,6 @@ +--- +title: Create scim_identities table in preparation for newer SCIM features in the + future +merge_request: 26124 +author: +type: added diff --git a/changelogs/unreleased/fix-deployment-namespace-resolution.yml b/changelogs/unreleased/fix-deployment-namespace-resolution.yml new file mode 100644 index 00000000000..389e6c6cb56 --- /dev/null +++ b/changelogs/unreleased/fix-deployment-namespace-resolution.yml @@ -0,0 +1,5 @@ +--- +title: Fix Kubernetes namespace resolution for new DeploymentCluster records +merge_request: 25853 +author: +type: fixed diff --git a/db/migrate/20200227164113_create_scim_identities.rb b/db/migrate/20200227164113_create_scim_identities.rb new file mode 100644 index 00000000000..1942270761b --- /dev/null +++ b/db/migrate/20200227164113_create_scim_identities.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class CreateScimIdentities < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def change + create_table :scim_identities do |t| + t.references :group, foreign_key: { to_table: :namespaces, on_delete: :cascade }, null: false + t.references :user, index: false, foreign_key: { on_delete: :cascade }, null: false + t.timestamps_with_timezone + t.boolean :active, default: false + t.string :extern_uid, null: false, limit: 255 + + t.index 'LOWER(extern_uid),group_id', name: 'index_scim_identities_on_lower_extern_uid_and_group_id', unique: true + t.index [:user_id, :group_id], unique: true + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 2e7e0d8ce7c..e77e8e44b62 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -3819,6 +3819,18 @@ ActiveRecord::Schema.define(version: 2020_03_04_160823) do t.index ["group_id"], name: "index_saml_providers_on_group_id" end + create_table "scim_identities", force: :cascade do |t| + t.bigint "group_id", null: false + t.bigint "user_id", null: false + t.datetime_with_timezone "created_at", null: false + t.datetime_with_timezone "updated_at", null: false + t.boolean "active", default: false + t.string "extern_uid", limit: 255, null: false + t.index "lower((extern_uid)::text), group_id", name: "index_scim_identities_on_lower_extern_uid_and_group_id", unique: true + t.index ["group_id"], name: "index_scim_identities_on_group_id" + t.index ["user_id", "group_id"], name: "index_scim_identities_on_user_id_and_group_id", unique: true + end + create_table "scim_oauth_access_tokens", id: :serial, force: :cascade do |t| t.datetime_with_timezone "created_at", null: false t.datetime_with_timezone "updated_at", null: false @@ -5037,6 +5049,8 @@ ActiveRecord::Schema.define(version: 2020_03_04_160823) do add_foreign_key "reviews", "projects", on_delete: :cascade add_foreign_key "reviews", "users", column: "author_id", on_delete: :nullify add_foreign_key "saml_providers", "namespaces", column: "group_id", on_delete: :cascade + add_foreign_key "scim_identities", "namespaces", column: "group_id", on_delete: :cascade + add_foreign_key "scim_identities", "users", on_delete: :cascade add_foreign_key "scim_oauth_access_tokens", "namespaces", column: "group_id", on_delete: :cascade add_foreign_key "security_scans", "ci_builds", column: "build_id", on_delete: :cascade add_foreign_key "self_managed_prometheus_alert_events", "environments", on_delete: :cascade diff --git a/doc/api/project_clusters.md b/doc/api/project_clusters.md index 6586cfc9f7f..2ed57eceb85 100644 --- a/doc/api/project_clusters.md +++ b/doc/api/project_clusters.md @@ -177,16 +177,16 @@ Parameters: | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `id` | integer | yes | The ID of the project owned by the authenticated user | -| `name` | String | yes | The name of the cluster | -| `domain` | String | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster | +| `name` | string | yes | The name of the cluster | +| `domain` | string | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster | | `enabled` | boolean | no | Determines if cluster is active or not, defaults to true | | `managed` | boolean | no | Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true | -| `platform_kubernetes_attributes[api_url]` | String | yes | The URL to access the Kubernetes API | -| `platform_kubernetes_attributes[token]` | String | yes | The token to authenticate against Kubernetes | -| `platform_kubernetes_attributes[ca_cert]` | String | no | TLS certificate (needed if API is using a self-signed TLS certificate | -| `platform_kubernetes_attributes[namespace]` | String | no | The unique namespace related to the project | -| `platform_kubernetes_attributes[authorization_type]` | String | no | The cluster authorization type: `rbac`, `abac` or `unknown_authorization`. Defaults to `rbac`. | -| `environment_scope` | String | no | The associated environment to the cluster. Defaults to `*` **(PREMIUM)** | +| `platform_kubernetes_attributes[api_url]` | string | yes | The URL to access the Kubernetes API | +| `platform_kubernetes_attributes[token]` | string | yes | The token to authenticate against Kubernetes | +| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. | +| `platform_kubernetes_attributes[namespace]` | string | no | The unique namespace related to the project | +| `platform_kubernetes_attributes[authorization_type]` | string | no | The cluster authorization type: `rbac`, `abac` or `unknown_authorization`. Defaults to `rbac`. | +| `environment_scope` | string | no | The associated environment to the cluster. Defaults to `*` **(PREMIUM)** | Example request: @@ -271,14 +271,14 @@ Parameters: | --------- | ---- | -------- | ----------- | | `id` | integer | yes | The ID of the project owned by the authenticated user | | `cluster_id` | integer | yes | The ID of the cluster | -| `name` | String | no | The name of the cluster | -| `domain` | String | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster | +| `name` | string | no | The name of the cluster | +| `domain` | string | no | The [base domain](../user/project/clusters/index.md#base-domain) of the cluster | | `management_project_id` | integer | no | The ID of the [management project](../user/clusters/management_project.md) for the cluster | -| `platform_kubernetes_attributes[api_url]` | String | no | The URL to access the Kubernetes API | -| `platform_kubernetes_attributes[token]` | String | no | The token to authenticate against Kubernetes | -| `platform_kubernetes_attributes[ca_cert]` | String | no | TLS certificate (needed if API is using a self-signed TLS certificate | -| `platform_kubernetes_attributes[namespace]` | String | no | The unique namespace related to the project | -| `environment_scope` | String | no | The associated environment to the cluster **(PREMIUM)** | +| `platform_kubernetes_attributes[api_url]` | string | no | The URL to access the Kubernetes API | +| `platform_kubernetes_attributes[token]` | string | no | The token to authenticate against Kubernetes | +| `platform_kubernetes_attributes[ca_cert]` | string | no | TLS certificate. Required if API is using a self-signed TLS certificate. | +| `platform_kubernetes_attributes[namespace]` | string | no | The unique namespace related to the project | +| `environment_scope` | string | no | The associated environment to the cluster **(PREMIUM)** | NOTE: **Note:** `name`, `api_url`, `ca_cert` and `token` can only be updated if the cluster was added diff --git a/doc/ci/variables/predefined_variables.md b/doc/ci/variables/predefined_variables.md index 2bc9132aa93..65381d512e5 100644 --- a/doc/ci/variables/predefined_variables.md +++ b/doc/ci/variables/predefined_variables.md @@ -33,7 +33,7 @@ future GitLab releases.** | `CI_COMMIT_DESCRIPTION` | 10.8 | all | The description of the commit: the message without first line, if the title is shorter than 100 characters; full message in other case. | | `CI_COMMIT_MESSAGE` | 10.8 | all | The full commit message. | | `CI_COMMIT_REF_NAME` | 9.0 | all | The branch or tag name for which project is built | -| `CI_COMMIT_REF_PROTECTED` | 11.11 | all | If the job is running on a protected branch | +| `CI_COMMIT_REF_PROTECTED` | 11.11 | all | `true` if the job is running on a protected branch, `false` if not | | `CI_COMMIT_REF_SLUG` | 9.0 | all | `$CI_COMMIT_REF_NAME` lowercased, shortened to 63 bytes, and with everything except `0-9` and `a-z` replaced with `-`. No leading / trailing `-`. Use in URLs, host names and domain names. | | `CI_COMMIT_SHA` | 9.0 | all | The commit revision for which project is built | | `CI_COMMIT_SHORT_SHA` | 11.7 | all | The first eight characters of `CI_COMMIT_SHA` | diff --git a/doc/integration/vault.md b/doc/integration/vault.md index 3ba401d8b35..c216fdfca4a 100644 --- a/doc/integration/vault.md +++ b/doc/integration/vault.md @@ -87,11 +87,11 @@ The following assumes you already have Vault installed and running. 1. If the `OIDC` method is not currently selected, open the dropdown and select it. 1. Click the **Sign in With GitLab** button, which will open a modal window: - ![Sign into Vault with GitLab](img/sign_into_vault_with_gitlab_v12_6.png) + ![Sign into Vault with GitLab](img/sign_into_vault_with_gitlab_v12_6.png) 1. Click **Authorize** on the modal to allow Vault to sign in through GitLab. This will redirect you back to your Vault UI as a signed-in user. - ![Authorize Vault to connect with GitLab](img/authorize_vault_with_gitlab_v12_6.png) + ![Authorize Vault to connect with GitLab](img/authorize_vault_with_gitlab_v12_6.png) 1. **Sign in using the Vault CLI** (optional): diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index 16b5f94162b..bb43cb9ed4a 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -530,6 +530,14 @@ On GitLab.com, projects, groups, and snippets created As of GitLab 12.2 (July 2019), projects, groups, and snippets have the [**Internal** visibility](../../public_access/public_access.md#internal-projects) setting [disabled on GitLab.com](https://gitlab.com/gitlab-org/gitlab/issues/12388). +### SSH maximum number of connections + +GitLab.com defines the maximum number of concurrent, unauthenticated SSH connections by +using the [MaxStartups setting](http://man.openbsd.org/sshd_config.5#MaxStartups). +If more than the maximum number of allowed connections occur concurrently, they are +dropped and users get +[an `ssh_exchange_identification` error](../../topics/git/troubleshooting_git.md#ssh_exchange_identification-error). + ## GitLab.com Logging We use [Fluentd](https://gitlab.com/gitlab-com/runbooks/tree/master/logging/doc#fluentd) to parse our logs. Fluentd sends our logs to diff --git a/doc/user/project/clusters/serverless/index.md b/doc/user/project/clusters/serverless/index.md index cb149446a19..0b5ebf3c74c 100644 --- a/doc/user/project/clusters/serverless/index.md +++ b/doc/user/project/clusters/serverless/index.md @@ -384,7 +384,8 @@ The sample function can now be triggered from any HTTP client using a simple `PO ### Secrets -To access your Kubernetes secrets from within your function, the secrets should be created under the namespace of your serverless deployment. +To access your Kubernetes secrets from within your function, the secrets should be created under the namespace of your serverless deployment and specified in your `serverless.yml` file as above. +You can create secrets in several ways. The following sections show some examples. #### CLI example diff --git a/lib/gitlab/ci/pipeline/seed/deployment.rb b/lib/gitlab/ci/pipeline/seed/deployment.rb index 624189acc8a..69dfd6be8d5 100644 --- a/lib/gitlab/ci/pipeline/seed/deployment.rb +++ b/lib/gitlab/ci/pipeline/seed/deployment.rb @@ -23,12 +23,12 @@ module Gitlab # non-environment job. return unless deployment.valid? && deployment.environment.persisted? - if cluster_id = deployment.environment.deployment_platform&.cluster_id + if cluster = deployment.environment.deployment_platform&.cluster # double write cluster_id until 12.9: https://gitlab.com/gitlab-org/gitlab/issues/202628 - deployment.cluster_id = cluster_id + deployment.cluster_id = cluster.id deployment.deployment_cluster = ::DeploymentCluster.new( - cluster_id: cluster_id, - kubernetes_namespace: deployment.environment.deployment_namespace + cluster_id: cluster.id, + kubernetes_namespace: cluster.kubernetes_namespace_for(deployment.environment, deployable: job) ) end diff --git a/spec/factories/deployments.rb b/spec/factories/deployments.rb index f92e213a385..42046464213 100644 --- a/spec/factories/deployments.rb +++ b/spec/factories/deployments.rb @@ -7,7 +7,7 @@ FactoryBot.define do tag { false } user { nil } project { nil } - deployable factory: :ci_build + deployable { association :ci_build, environment: environment.name, project: environment.project } environment factory: :environment after(:build) do |deployment, evaluator| diff --git a/spec/lib/gitlab/ci/pipeline/seed/deployment_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/deployment_spec.rb index 8d097bdd740..ceb3cb28bc9 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/deployment_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/deployment_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' describe Gitlab::Ci::Pipeline::Seed::Deployment do - let_it_be(:project) { create(:project, :repository) } + let_it_be(:project, refind: true) { create(:project, :repository) } let(:pipeline) do create(:ci_pipeline, project: project, sha: 'b83d6e391c22777fca1ed3012fce84f633d7fed0') @@ -25,10 +25,12 @@ describe Gitlab::Ci::Pipeline::Seed::Deployment do let(:attributes) do { environment: 'production', - options: { environment: { name: 'production' } } + options: { environment: { name: 'production', **kubernetes_options } } } end + let(:kubernetes_options) { {} } + it 'returns a deployment object with environment' do expect(subject).to be_a(Deployment) expect(subject.iid).to be_present @@ -38,14 +40,30 @@ describe Gitlab::Ci::Pipeline::Seed::Deployment do end context 'when environment has deployment platform' do - let!(:cluster) { create(:cluster, :provided_by_gcp, projects: [project]) } + let!(:cluster) { create(:cluster, :provided_by_gcp, projects: [project], managed: managed_cluster) } + let(:managed_cluster) { true } it 'sets the cluster and deployment_cluster' do expect(subject.cluster).to eq(cluster) # until we stop double writing in 12.9: https://gitlab.com/gitlab-org/gitlab/issues/202628 - expect(subject.deployment_cluster).to have_attributes( - cluster_id: cluster.id, - kubernetes_namespace: subject.environment.deployment_namespace - ) + expect(subject.deployment_cluster.cluster).to eq(cluster) + end + + context 'when a custom namespace is given' do + let(:kubernetes_options) { { kubernetes: { namespace: 'the-custom-namespace' } } } + + context 'when cluster is managed' do + it 'does not set the custom namespace' do + expect(subject.deployment_cluster.kubernetes_namespace).not_to eq('the-custom-namespace') + end + end + + context 'when cluster is not managed' do + let(:managed_cluster) { false } + + it 'sets the custom namespace' do + expect(subject.deployment_cluster.kubernetes_namespace).to eq('the-custom-namespace') + end + end end end diff --git a/spec/models/clusters/cluster_spec.rb b/spec/models/clusters/cluster_spec.rb index 23592cb0c70..8d0ede3d9bd 100644 --- a/spec/models/clusters/cluster_spec.rb +++ b/spec/models/clusters/cluster_spec.rb @@ -674,59 +674,59 @@ describe Clusters::Cluster, :use_clean_rails_memory_store_caching do end describe '#kubernetes_namespace_for' do - let(:cluster) { create(:cluster, :group) } - let(:environment) { create(:environment, last_deployable: build) } - let(:build) { create(:ci_build) } + subject { cluster.kubernetes_namespace_for(environment, deployable: build) } - subject { cluster.kubernetes_namespace_for(environment) } + let(:environment_name) { 'the-environment-name' } + let(:environment) { create(:environment, name: environment_name, project: cluster.project, last_deployable: build) } + let(:build) { create(:ci_build, environment: environment_name, project: cluster.project) } + let(:cluster) { create(:cluster, :project, managed: managed_cluster) } + let(:managed_cluster) { true } + let(:default_namespace) { Gitlab::Kubernetes::DefaultNamespace.new(cluster, project: cluster.project).from_environment_slug(environment.slug) } + let(:build_options) { {} } - before do - expect(Clusters::KubernetesNamespaceFinder).to receive(:new) - .with(cluster, project: environment.project, environment_name: environment.name) - .and_return(double(execute: persisted_namespace)) - - allow(build).to receive(:expanded_kubernetes_namespace) - .and_return(ci_configured_namespace) + it 'validates the project id' do + environment.project_id = build.project_id + 1 + expect { subject }.to raise_error ArgumentError, 'environment.project_id must match deployable.project_id' end - context 'no persisted namespace exists and namespace is not specified in CI template' do - let(:persisted_namespace) { nil } - let(:ci_configured_namespace) { nil } + context 'when environment has no last_deployable' do + let(:build) { nil } - let(:namespace_generator) { double } - let(:default_namespace) { 'a-default-namespace' } + it { is_expected.to eq default_namespace } + end + context 'when cluster is managed' do before do - expect(Gitlab::Kubernetes::DefaultNamespace).to receive(:new) - .with(cluster, project: environment.project) - .and_return(namespace_generator) - expect(namespace_generator).to receive(:from_environment_slug) - .with(environment.slug) - .and_return(default_namespace) + build.options = { environment: { kubernetes: { namespace: 'ci yaml namespace' } } } end - it { is_expected.to eq default_namespace } - end - - context 'persisted namespace exists' do - let(:persisted_namespace) { create(:cluster_kubernetes_namespace) } - let(:ci_configured_namespace) { nil } + it 'returns the cached namespace if present, ignoring CI config' do + cached_namespace = create(:cluster_kubernetes_namespace, cluster: cluster, environment: environment, namespace: 'the name', service_account_token: 'some token') + expect(subject).to eq cached_namespace.namespace + end - it { is_expected.to eq persisted_namespace.namespace } + it 'returns the default namespace when no cached namespace, ignoring CI config' do + expect(subject).to eq default_namespace + end end - context 'namespace is specified in CI template' do - let(:persisted_namespace) { nil } - let(:ci_configured_namespace) { 'ci-configured-namespace' } + context 'when cluster is not managed' do + let(:managed_cluster) { false } - it { is_expected.to eq ci_configured_namespace } - end + it 'returns the cached namespace if present, regardless of CI config' do + cached_namespace = create(:cluster_kubernetes_namespace, cluster: cluster, environment: environment, namespace: 'the name', service_account_token: 'some token') + build.options = { environment: { kubernetes: { namespace: 'ci yaml namespace' } } } + expect(subject).to eq cached_namespace.namespace + end - context 'persisted namespace exists and namespace is also specifed in CI template' do - let(:persisted_namespace) { create(:cluster_kubernetes_namespace) } - let(:ci_configured_namespace) { 'ci-configured-namespace' } + it 'returns the CI YAML namespace when configured' do + build.options = { environment: { kubernetes: { namespace: 'ci yaml namespace' } } } + expect(subject).to eq 'ci yaml namespace' + end - it { is_expected.to eq persisted_namespace.namespace } + it 'returns the default namespace when no namespace is configured' do + expect(subject).to eq default_namespace + end end end -- cgit v1.2.3