diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-03-31 14:47:09 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-03-31 14:47:09 +0300 |
| commit | 738634c9fc278002eef7991bc48877e80119b97d (patch) | |
| tree | e2492fbf13954000994aa07c67daebf15e15e6dc | |
| parent | 389512173f18b46cbc80242597c7110882d1dfb1 (diff) | |
Add latest changes from gitlab-org/gitlab@12-9-stable-ee
41 files changed, 529 insertions, 31 deletions
diff --git a/app/controllers/concerns/import_url_params.rb b/app/controllers/concerns/import_url_params.rb index e51e4157f50..28a3876810a 100644 --- a/app/controllers/concerns/import_url_params.rb +++ b/app/controllers/concerns/import_url_params.rb @@ -4,7 +4,13 @@ module ImportUrlParams def import_url_params return {} unless params.dig(:project, :import_url).present? - { import_url: import_params_to_full_url(params[:project]) } + { + import_url: import_params_to_full_url(params[:project]), + # We need to set import_type because attempting to retry an import by URL + # could leave a stale value around. This would erroneously cause an importer + # (e.g. import/export) to run. + import_type: 'git' + } end def import_params_to_full_url(params) diff --git a/app/uploaders/content_type_whitelist.rb b/app/uploaders/content_type_whitelist.rb index b3975d7e2e0..3210d57b00c 100644 --- a/app/uploaders/content_type_whitelist.rb +++ b/app/uploaders/content_type_whitelist.rb @@ -26,14 +26,14 @@ module ContentTypeWhitelist # Here we override and extend CarrierWave's method that does not parse the # magic headers. def check_content_type_whitelist!(new_file) - new_file.content_type = mime_magic_content_type(new_file.path) + if content_type_whitelist + content_type = mime_magic_content_type(new_file.path) - if content_type_whitelist && !whitelisted_content_type?(new_file.content_type) - message = I18n.translate(:"errors.messages.content_type_whitelist_error", allowed_types: Array(content_type_whitelist).join(", ")) - raise CarrierWave::IntegrityError, message + unless whitelisted_content_type?(content_type) + message = I18n.translate(:"errors.messages.content_type_whitelist_error", allowed_types: Array(content_type_whitelist).join(", ")) + raise CarrierWave::IntegrityError, message + end end - - super(new_file) end def whitelisted_content_type?(content_type) diff --git a/app/views/admin/application_settings/metrics_and_profiling.html.haml b/app/views/admin/application_settings/metrics_and_profiling.html.haml index 0b747082de0..6a703d0b70c 100644 --- a/app/views/admin/application_settings/metrics_and_profiling.html.haml +++ b/app/views/admin/application_settings/metrics_and_profiling.html.haml @@ -60,4 +60,6 @@ .settings-content = render 'usage' += render_if_exists 'admin/application_settings/seat_link_setting', expanded: expanded_by_default? + = render_if_exists 'admin/application_settings/pseudonymizer_settings', expanded: expanded_by_default? diff --git a/app/views/projects/settings/ci_cd/_form.html.haml b/app/views/projects/settings/ci_cd/_form.html.haml index 54e46501287..4040b1094aa 100644 --- a/app/views/projects/settings/ci_cd/_form.html.haml +++ b/app/views/projects/settings/ci_cd/_form.html.haml @@ -92,10 +92,10 @@ .form-check = f.check_box :forward_deployment_enabled, { class: 'form-check-input' } = f.label :forward_deployment_enabled, class: 'form-check-label' do - %strong= _("Skip older, pending deployment jobs") + %strong= _("Skip outdated deployment jobs") .form-text.text-muted = _("When a deployment job is successful, skip older deployment jobs that are still pending") - = link_to icon('question-circle'), help_page_path('ci/pipelines/settings', anchor: 'skip-older-pending-deployment-jobs'), target: '_blank' + = link_to icon('question-circle'), help_page_path('ci/pipelines/settings', anchor: 'skip-outdated-deployment-jobs'), target: '_blank' %hr .form-group diff --git a/changelogs/unreleased/212264-gcs-job-log-is-stored-with-content-type-invalid-invalid.yml b/changelogs/unreleased/212264-gcs-job-log-is-stored-with-content-type-invalid-invalid.yml new file mode 100644 index 00000000000..e6b9528780d --- /dev/null +++ b/changelogs/unreleased/212264-gcs-job-log-is-stored-with-content-type-invalid-invalid.yml @@ -0,0 +1,5 @@ +--- +title: Leave upload Content-Type unchaged +merge_request: 27864 +author: +type: fixed diff --git a/changelogs/unreleased/docs-rename-skip-outdated-jobs.yml b/changelogs/unreleased/docs-rename-skip-outdated-jobs.yml new file mode 100644 index 00000000000..2f11cbcc484 --- /dev/null +++ b/changelogs/unreleased/docs-rename-skip-outdated-jobs.yml @@ -0,0 +1,5 @@ +--- +title: Rename feature on the FE and locale +merge_request: +author: +type: changed diff --git a/changelogs/unreleased/georgekoltsov-fix-issues-mrs-state.yml b/changelogs/unreleased/georgekoltsov-fix-issues-mrs-state.yml new file mode 100644 index 00000000000..018a14121e4 --- /dev/null +++ b/changelogs/unreleased/georgekoltsov-fix-issues-mrs-state.yml @@ -0,0 +1,6 @@ +--- +title: Fix issue/MR state not being preserved when importing a project using Project + Import/Export +merge_request: 27816 +author: +type: fixed diff --git a/changelogs/unreleased/issue_11391_2.yml b/changelogs/unreleased/issue_11391_2.yml new file mode 100644 index 00000000000..c59ac2d9ae4 --- /dev/null +++ b/changelogs/unreleased/issue_11391_2.yml @@ -0,0 +1,5 @@ +--- +title: Index issues on sent_notifications table +merge_request: 27034 +author: +type: performance diff --git a/changelogs/unreleased/sh-check-features-table-gitaly.yml b/changelogs/unreleased/sh-check-features-table-gitaly.yml new file mode 100644 index 00000000000..e9ae3e89a08 --- /dev/null +++ b/changelogs/unreleased/sh-check-features-table-gitaly.yml @@ -0,0 +1,5 @@ +--- +title: Fix rake gitlab:setup failing on new installs +merge_request: 28270 +author: +type: fixed diff --git a/changelogs/unreleased/sh-disable-archive-rate-throttle-default.yml b/changelogs/unreleased/sh-disable-archive-rate-throttle-default.yml new file mode 100644 index 00000000000..f9786f4e388 --- /dev/null +++ b/changelogs/unreleased/sh-disable-archive-rate-throttle-default.yml @@ -0,0 +1,5 @@ +--- +title: Disable archive rate limit by default +merge_request: 28264 +author: +type: fixed diff --git a/changelogs/unreleased/sh-fix-import-by-url-retries.yml b/changelogs/unreleased/sh-fix-import-by-url-retries.yml new file mode 100644 index 00000000000..b66026f4891 --- /dev/null +++ b/changelogs/unreleased/sh-fix-import-by-url-retries.yml @@ -0,0 +1,5 @@ +--- +title: Ensure import by URL works after a failed import +merge_request: 27546 +author: +type: fixed diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 8d88d1bcf7c..6e669a59b53 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -219,6 +219,7 @@ Gitlab.ee do Settings.gitlab['mirror_max_delay'] ||= 300 Settings.gitlab['mirror_max_capacity'] ||= 30 Settings.gitlab['mirror_capacity_threshold'] ||= 15 + Settings.gitlab['seat_link_enabled'] = true if Settings.gitlab['seat_link_enabled'].nil? end # diff --git a/db/migrate/20200311141943_insert_ci_pipeline_schedules_plan_limits.rb b/db/migrate/20200311141943_insert_ci_pipeline_schedules_plan_limits.rb index 849d95667a7..2a04781c65e 100644 --- a/db/migrate/20200311141943_insert_ci_pipeline_schedules_plan_limits.rb +++ b/db/migrate/20200311141943_insert_ci_pipeline_schedules_plan_limits.rb @@ -5,7 +5,7 @@ class InsertCiPipelineSchedulesPlanLimits < ActiveRecord::Migration[6.0] DOWNTIME = false - def change + def up return unless Gitlab.com? create_or_update_plan_limit('ci_pipeline_schedules', 'free', 10) diff --git a/db/migrate/20200325152327_add_seat_link_enabled_to_application_settings.rb b/db/migrate/20200325152327_add_seat_link_enabled_to_application_settings.rb new file mode 100644 index 00000000000..52f579c175c --- /dev/null +++ b/db/migrate/20200325152327_add_seat_link_enabled_to_application_settings.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddSeatLinkEnabledToApplicationSettings < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column_with_default(:application_settings, :seat_link_enabled, + :boolean, + default: true, + allow_null: false) + end + + def down + remove_column(:application_settings, :seat_link_enabled) + end +end diff --git a/db/post_migrate/20200311192351_add_index_on_noteable_type_and_noteable_id_to_sent_notifications.rb b/db/post_migrate/20200311192351_add_index_on_noteable_type_and_noteable_id_to_sent_notifications.rb new file mode 100644 index 00000000000..fa0246218c3 --- /dev/null +++ b/db/post_migrate/20200311192351_add_index_on_noteable_type_and_noteable_id_to_sent_notifications.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class AddIndexOnNoteableTypeAndNoteableIdToSentNotifications < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'index_sent_notifications_on_noteable_type_noteable_id' + + disable_ddl_transaction! + + def up + add_concurrent_index :sent_notifications, + [:noteable_id], + name: INDEX_NAME, + where: "noteable_type = 'Issue'" + end + + def down + remove_concurrent_index_by_name :sent_notifications, INDEX_NAME + end +end diff --git a/db/schema.rb b/db/schema.rb index 55c99cb1027..791d16f9111 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_03_13_123934) do +ActiveRecord::Schema.define(version: 2020_03_25_152327) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -352,6 +352,7 @@ ActiveRecord::Schema.define(version: 2020_03_13_123934) do t.boolean "email_restrictions_enabled", default: false, null: false t.text "email_restrictions" t.boolean "npm_package_requests_forwarding", default: true, null: false + t.boolean "seat_link_enabled", default: true, null: false t.index ["custom_project_templates_group_id"], name: "index_application_settings_on_custom_project_templates_group_id" t.index ["file_template_project_id"], name: "index_application_settings_on_file_template_project_id" t.index ["instance_administration_project_id"], name: "index_applicationsettings_on_instance_administration_project_id" @@ -3907,6 +3908,7 @@ ActiveRecord::Schema.define(version: 2020_03_13_123934) do t.string "note_type" t.text "position" t.string "in_reply_to_discussion_id" + t.index ["noteable_id"], name: "index_sent_notifications_on_noteable_type_noteable_id", where: "((noteable_type)::text = 'Issue'::text)" t.index ["reply_key"], name: "index_sent_notifications_on_reply_key", unique: true end diff --git a/doc/ci/pipelines/settings.md b/doc/ci/pipelines/settings.md index 13b8f4ee307..48862986114 100644 --- a/doc/ci/pipelines/settings.md +++ b/doc/ci/pipelines/settings.md @@ -192,7 +192,7 @@ you can enable this in the project settings: 1. Check the **Auto-cancel redundant, pending pipelines** checkbox. 1. Click **Save changes**. -## Skip older, pending deployment jobs +## Skip outdated deployment jobs > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/25276) in GitLab 12.9. @@ -206,7 +206,7 @@ To avoid this scenario: 1. Go to **{settings}** **Settings > CI / CD**. 1. Expand **General pipelines**. -1. Check the **Skip older, pending deployment jobs** checkbox. +1. Check the **Skip outdated deployment jobs** checkbox. 1. Click **Save changes**. The pending deployment jobs will be skipped. diff --git a/doc/subscriptions/index.md b/doc/subscriptions/index.md index 925f162d0fe..3d8a8071470 100644 --- a/doc/subscriptions/index.md +++ b/doc/subscriptions/index.md @@ -245,11 +245,12 @@ Seat Link allows us to provide our self-managed customers with prorated charges Seat Link sends to GitLab daily a count of all users in connected self-managed instances. That information is used to automate prorated reconciliations. The data is sent securely through an encrypted HTTPS connection. -Seat Link is mandatory because we need the user count data to enable prorated billing. Seat Link provides **only** the following information to GitLab: +Seat Link provides **only** the following information to GitLab: - Date -- Historical maximum user count - License key +- Historical maximum user count +- Active users count Here is an example of the POST request: @@ -290,12 +291,39 @@ TjJ4eVlVUkdkWEJtDQpkSHByYWpreVJrcG9UVlo0Y0hKSU9URndiV2RzVFdO VlhHNXRhVmszTkV0SVEzcEpNMWRyZEVoRU4ydHINCmRIRnFRVTlCVUVVM1pV SlRORE4xUjFaYVJGb3JlWGM5UFZ4dUlpd2lhWFlpt2lKV00yRnNVbk5RTjJk Sg0KU1hNMGExaE9SVGR2V2pKQlBUMWNiaUo5DQo=', - max_historical_user_count: 10 + max_historical_user_count: 10, + active_users: 6 } ``` For air-gapped or closed network customers, the existing [true-up model](#users-over-license) will be used. Prorated charges are not possible without user count data. +#### Disable Seat Link + +Seat Link is enabled by default. To disable this feature, go to +**{admin}** **Admin Area > Settings > Metrics and profiling** and +clear the Seat Link checkbox. + +To disable Seat Link in an Omnibus GitLab installation, and prevent it from +being configured in the future through the administration panel, set the following in +[`gitlab.rb`](https://docs.gitlab.com/omnibus/settings/configuration.html#configuration-options): + +```ruby +gitlab_rails['seat_link_enabled'] = false +``` + +To disable Seat Link in a GitLab source installation, and prevent it from +being configured in the future through the administration panel, +set the following in `gitlab.yml`: + +```yaml +production: &base + # ... + gitlab: + # ... + seat_link_enabled: false +``` + ### Renew or change a GitLab.com subscription To renew for more users than are currently active in your GitLab.com system, contact our sales team via `renewals@gitlab.com` for assistance as this can't be done in the Customers Portal. diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index 57cceba8c0d..4c2707fb46d 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -666,9 +666,10 @@ To use Auto Deploy on a Kubernetes 1.16+ cluster, you must: This will opt-in to using a version of the PostgreSQL chart that supports Kubernetes 1.16 and higher. -DANGER: **Danger:** Opting into `AUTO_DEVOPS_POSTGRES_CHANNEL` version `2` will delete -the version `1` PostgreSQL database. Please backup the contents of the PostgreSQL database -first before opting into version `2`, so that you can restore into the version `2` database. +DANGER: **Danger:** Opting into `AUTO_DEVOPS_POSTGRES_CHANNEL` version +`2` will delete the version `1` PostgreSQL database. Please follow the +guide on [upgrading PostgreSQL](upgrading_postgresql.md) to backup and +restore your database before opting into version `2`. #### Migrations @@ -1100,6 +1101,27 @@ the database are preconfigured, but can be customized by setting the associated postgres://user:password@postgres-host:postgres-port/postgres-database ``` +#### Upgrading PostgresSQL + +CAUTION: **Deprecation** +The variable `AUTO_DEVOPS_POSTGRES_CHANNEL` that controls default provisioned +PostgreSQL currently defaults to `1`. This is scheduled to change to `2` in +[GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/210499). + +The version of the chart used to provision PostgreSQL: + +- Is 0.7.1 in GitLab 12.8 and earlier. +- Can be set to from 0.7.1 to 8.2.1 in GitLab 12.9 and later. + +GitLab encourages users to [migrate their database](upgrading_postgresql.md) +to the newer PostgreSQL. + +To use the new PostgreSQL: + +- New projects can set the `AUTO_DEVOPS_POSTGRES_CHANNEL` variable to `2`. +- Old projects can be upgraded by following the guide to + [upgrading PostgresSQL](upgrading_postgresql.md). + #### Using external PostgreSQL database providers While Auto DevOps provides out-of-the-box support for a PostgreSQL container for diff --git a/doc/topics/autodevops/upgrading_postgresql.md b/doc/topics/autodevops/upgrading_postgresql.md new file mode 100644 index 00000000000..796aa69469b --- /dev/null +++ b/doc/topics/autodevops/upgrading_postgresql.md @@ -0,0 +1,239 @@ +# Upgrading PostgreSQL for Auto DevOps + +Auto DevOps provides an [in-cluster PostgreSQL database](index.md#postgresql-database-support) +for your application. + +The version of the chart used to provision PostgreSQL: + +- Is 0.7.1 in GitLab 12.8 and earlier. +- Can be set to from 0.7.1 to 8.2.1 in GitLab 12.9 and later. + +GitLab encourages users to GitLab encourages users to migrate their database to the +newer PostgreSQL chart. + +This guide provides instructions on how to migrate your PostgreSQL database, which +involves: + +1. Taking a database dump of your data. +1. Installing a new PostgreSQL database using the newer version 8.2.1 of the chart + and removing the old PostgreSQL installation. +1. Restoring the database dump into the new PostgreSQL. + +## Prerequisites + +1. Install + [`kubectl`](https://kubernetes.io/docs/tasks/tools/install-kubectl/). +1. Ensure that you can access your Kubernetes cluster using `kubectl`. + This varies based on Kubernetes providers. +1. Prepare for downtime. The steps below include taking the application offline + so that the in-cluster database does not get modified after the database dump is created. + +TIP: **Tip:** If you have configured Auto DevOps to have staging, +consider trying out the backup and restore steps on staging first, or +trying this out on a review app. + +## Take your application offline + +If required, take your application offline to prevent the database from +being modified after the database dump is created. + +1. Get the Kubernetes namespace for the environment. It typically looks like `<project-name>-<project-id>-<environment>`. + In our example, the namespace is called `minimal-ruby-app-4349298-production`. + + ```sh + $ kubectl get ns + + NAME STATUS AGE + minimal-ruby-app-4349298-production Active 7d14h + ``` + +1. For ease of use, export the namespace name: + + ```sh + export APP_NAMESPACE=minimal-ruby-app-4349298-production + ``` + +1. Get the deployment name for your application with the following command. In our example, the deployment name is `production`. + + ```sh + $ kubectl get deployment --namespace "$APP_NAMESPACE" + NAME READY UP-TO-DATE AVAILABLE AGE + production 2/2 2 2 7d21h + production-postgres 1/1 1 1 7d21h + ``` + +1. To prevent the database from being modified, set replicas to 0 for the deployment with the following command. + We use the deployment name from the previous step (`deployments/<DEPLOYMENT_NAME>`). + + ```sh + $ kubectl scale --replicas=0 deployments/production --namespace "$APP_NAMESPACE" + deployment.extensions/production scaled + ``` + +1. You also will need to set replicas to zero for workers if you have any. + +## Backup + +1. Get the service name for PostgreSQL. The name of the service should end with `-postgres`. In our example the service name is `production-postgres`. + + ```sh + $ kubectl get svc --namespace "$APP_NAMESPACE" + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + production-auto-deploy ClusterIP 10.30.13.90 <none> 5000/TCP 7d14h + production-postgres ClusterIP 10.30.4.57 <none> 5432/TCP 7d14h + ``` + +1. Get the pod name for PostgreSQL with the following command. In our example, the pod name is `production-postgres-5db86568d7-qxlxv`. + + ```sh + $ kubectl get pod --namespace "$APP_NAMESPACE" -l app=production-postgres + NAME READY STATUS RESTARTS AGE + production-postgres-5db86568d7-qxlxv 1/1 Running 0 7d14h + ``` + +1. Connect to the pod with: + + ```sh + kubectl exec -it production-postgres-5db86568d7-qxlxv --namespace "$APP_NAMESPACE" bash + ``` + +1. Once, connected, create a dump file with the following command. + + - `SERVICE_NAME` is the service name obtained in a previous step. + - `USERNAME` is the username you have configured for PostgreSQL. The default is `user`. + - `DATABASE_NAME` is usually the environment name. + + - You will be asked for the database password, the default is `testing-password`. + + ```sh + ## Format is: + # pg_dump -h SERVICE_NAME -U USERNAME DATABASE_NAME > /tmp/backup.sql + + pg_dump -h production-postgres -U user production > /tmp/backup.sql + ``` + +1. Once the backup dump is complete, exit the Kubernetes exec process with `Control-D` or `exit`. + +1. Download the dump file with the following command: + + ```sh + kubectl cp --namespace "$APP_NAMESPACE" production-postgres-5db86568d7-qxlxv:/tmp/backup.sql backup.sql + ``` + +## Retain persistent volumes + +By default the [persistent +volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) +used to store the underlying data for PostgreSQL is marked as `Delete` +when the pods and pod claims that use the volume is deleted. + +This is signficant as, when you opt into the newer 8.2.1 PostgreSQL, the older 0.7.1 PostgreSQL is +deleted causing the persistent volumes to be deleted as well. + +You can verify this by using the following command: + +```sh +$ kubectl get pv +NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE +pvc-0da80c08-5239-11ea-9c8d-42010a8e0096 8Gi RWO Delete Bound minimal-ruby-app-4349298-staging/staging-postgres standard 7d22h +pvc-9085e3d3-5239-11ea-9c8d-42010a8e0096 8Gi RWO Delete Bound minimal-ruby-app-4349298-production/production-postgres standard 7d22h +``` + +To retain the persistent volume, even when the older 0.7.1 PostgreSQL is +deleted, we can change the retention policy to `Retain`. In this example, we find +the persistent volume names by looking at the claims names. As we are +interested in keeping the volumes for the staging and production of the +`minimal-ruby-app-4349298` application, the volume names here are +`pvc-0da80c08-5239-11ea-9c8d-42010a8e0096` and `pvc-9085e3d3-5239-11ea-9c8d-42010a8e0096`: + +```sh +$ kubectl patch pv pvc-0da80c08-5239-11ea-9c8d-42010a8e0096 -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' +persistentvolume/pvc-0da80c08-5239-11ea-9c8d-42010a8e0096 patched +$ kubectl patch pv pvc-9085e3d3-5239-11ea-9c8d-42010a8e0096 -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' +persistentvolume/pvc-9085e3d3-5239-11ea-9c8d-42010a8e0096 patched +$ kubectl get pv +NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE +pvc-0da80c08-5239-11ea-9c8d-42010a8e0096 8Gi RWO Retain Bound minimal-ruby-app-4349298-staging/staging-postgres standard 7d22h +pvc-9085e3d3-5239-11ea-9c8d-42010a8e0096 8Gi RWO Retain Bound minimal-ruby-app-4349298-production/production-postgres standard 7d22h +``` + +## Install new PostgreSQL + +CAUTION: **Caution:** Using the newer version of PostgreSQL will delete +the older 0.7.1 PostgreSQL. To prevent the underlying data from being +deleted, you can choose to retain the [persistent +volume](#retain-persistent-volumes). + +TIP: **Tip:** You can also +[scope](../../ci/environments.md#scoping-environments-with-specs) the +`AUTO_DEVOPS_POSTGRES_CHANNEL` and `POSTGRES_VERSION` variables to +specific environments, e.g. `staging`. + +1. Set `AUTO_DEVOPS_POSTGRES_CHANNEL` to `2`. This opts into using the + newer 8.2.1-based PostgreSQL, and removes the older 0.7.1-based + PostgreSQL. +1. Set `POSTGRES_VERSION` to `9.6.16`. This is the minimum PostgreSQL + version supported. +1. Set `PRODUCTION_REPLICAS` to `0`. For other environments, use + `REPLICAS` with an [environment scope](../../ci/environments.md#scoping-environments-with-specs). +1. If you have set the `DB_INITIALIZE` or `DB_MIGRATE` variables, either + remove the variables, or rename the variables temporarily to + `XDB_INITIALIZE` or the `XDB_MIGRATE` to effectively disable them. +1. Run a new CI pipeline for the branch. In this case, we run a new CI + pipeline for `master`. +1. Once the pipeline is successful, your application will now be upgraded + with the new PostgreSQL installed. There will also be zero replicas + which means no traffic will be served for your application (to prevent + new data from coming in). + +## Restore + +1. Get the pod name for the new PostgreSQL, in our example, the pod name is + `production-postgresql-0`: + + ```sh + $ kubectl get pod --namespace "$APP_NAMESPACE" -l app=postgresql + NAME READY STATUS RESTARTS AGE + production-postgresql-0 1/1 Running 0 19m + ```` + +1. Copy the dump file from the backup steps to the pod: + + ```sh + kubectl cp --namespace "$APP_NAMESPACE" backup.sql production-postgresql-0:/tmp/backup.sql + ``` + +1. Connect to the pod: + + ```sh + kubectl exec -it production-postgresql-0 --namespace "$APP_NAMESPACE" bash + ``` + +1. Once connected to the pod, run the following command to restore the database. + + - You will be asked for the database password, the default is `testing-password`. + - `USERNAME` is the username you have configured for postgres. The default is `user`. + - `DATABASE_NAME` is usually the environment name. + + ```sh + ## Format is: + # psql -U USERNAME -d DATABASE_NAME < /tmp/backup.sql + + psql -U user -d production < /tmp/backup.sql + ``` + +1. You can now check that your data restored correctly after the restore + is complete. You can perform spot checks of your data by using the + `psql`. + +## Reinstate your application + +Once you are satisfied the database has been restored, run the following +steps to reinstate your application: + +1. Restore the `DB_INITIALIZE` and `DB_MIGRATE` variables, if previously + removed or disabled. +1. Restore the `PRODUCTION_REPLICAS` or `REPLICAS` variable to its original value. +1. Run a new CI pipeline for the branch. In this case, we run a new CI + pipeline for `master`. After the pipeline is successful, your + application should be serving traffic as before. diff --git a/doc/user/analytics/img/label_based_stage_vsm_v12_9.png b/doc/user/analytics/img/label_based_stage_vsm_v12_9.png Binary files differnew file mode 100644 index 00000000000..f1cbd9e1a8e --- /dev/null +++ b/doc/user/analytics/img/label_based_stage_vsm_v12_9.png diff --git a/doc/user/analytics/img/new_vsm_stage_v12_9.png b/doc/user/analytics/img/new_vsm_stage_v12_9.png Binary files differnew file mode 100644 index 00000000000..dbef25d33ed --- /dev/null +++ b/doc/user/analytics/img/new_vsm_stage_v12_9.png diff --git a/doc/user/analytics/img/vsm_stage_list_v12_9.png b/doc/user/analytics/img/vsm_stage_list_v12_9.png Binary files differnew file mode 100644 index 00000000000..3775cdeddb4 --- /dev/null +++ b/doc/user/analytics/img/vsm_stage_list_v12_9.png diff --git a/doc/user/analytics/value_stream_analytics.md b/doc/user/analytics/value_stream_analytics.md index 9d925b00d8b..385cc39c0c0 100644 --- a/doc/user/analytics/value_stream_analytics.md +++ b/doc/user/analytics/value_stream_analytics.md @@ -151,6 +151,56 @@ A few notes: cycles, calculate their median time and the result is what the dashboard of Value Stream Analytics is showing. +## Customizable Value Stream Analytics + +The default stages are designed to work straight out of the box, but they might not be suitable for all teams. Different teams use different approaches to building software, so some teams might want to customize their Value Stream Analytics. From GitLab 12.9, users can hide default stages and create custom stages that align better to their development workflow. + +### Adding a stage + +In the following example we're creating a new stage that measures and tracks issues from creation time until they are closed. + +1. Navigate to your group page. +1. Open Value Stream Analytics from the sidebar: **Analytics > Value Stream** +1. Click the "Add a stage" button. +1. Fill in the new stage form: + - Name: Issue start to finish + - Start event: Issue created + - End event: Issue closed +1. Click the "Add stage" button. + + + +The new stage is persisted and it will always show up on the value stream analytics page for your group. In case you want to alter or delete the stage you can easily do that for customized stages by hovering over the stage and clicking the three-dot icon that appears. + + + +Creating a custom stage requires specifying two events, a start and an end. Be careful to choose a start event that occurs *before* your end event. For example, consider if a stage started when an issue is added to a board, and ended when the issue is created. This stage would not work because the end event has already happened when the start event occurs. To prevent such invalid stages, the form prohibits incompatible start and end events. After you select the start event, the stop event dropdown will only list the compatible events. + +Note: The ability to re-order the stages is a [planned enhancement](https://gitlab.com/gitlab-org/gitlab/issues/196698). + +### Label based stages + +The pre-defined start and end events can cover many use cases involving both issues and merge requests. For supporting more complex workflows, we can use stages based on group labels. These events are based on labels being added/removed. In particular, [scoped labels](../project/labels.md#scoped-labels-premium) are useful for complex workflows. + +In this example we'd like to measure more accurate code review times. The workflow is the following: + +- When the code review starts, the reviewer adds `workflow::code_review_start` label to the merge request. +- When the code review is finished, the reviewer adds `workflow::code_review_complete` label to the merge request. + +Creating a new stage called "Code Review": + + + +### Hiding unused stages + +Sometimes certain default stages are not relevant to a team. In this case you can easily hide stages so they no longer appear in the list. First, add a custom stage to activate customizability. Then hover over the default stage you want to hide, click the three-dot icon that appears and select "Hide stage". + +To recover a default stage that was previously hidden: + +1. Click "Add a stage" button. +1. In the top right corner open the "Recover hidden stage" dropdown. +1. Select a stage. + ## Days to completion chart > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/21631) in GitLab 12.6. diff --git a/doc/user/application_security/container_scanning/index.md b/doc/user/application_security/container_scanning/index.md index 075536ce9ad..1cb5b45cd0f 100644 --- a/doc/user/application_security/container_scanning/index.md +++ b/doc/user/application_security/container_scanning/index.md @@ -179,6 +179,7 @@ using environment variables. | `CLAIR_DB_IMAGE` | The Docker image name and tag for the [Postgres server hosting the vulnerabilities definitions](https://hub.docker.com/r/arminc/clair-db). It can be useful to override this value with a specific version, for example, to provide a consistent set of vulnerabilities for integration testing purposes, or to refer to a locally hosted vulnerabilities database for an on-premise air-gapped installation. | `arminc/clair-db:latest` | | `CLAIR_DB_IMAGE_TAG` | (**DEPRECATED - use `CLAIR_DB_IMAGE` instead**) The Docker image tag for the [Postgres server hosting the vulnerabilities definitions](https://hub.docker.com/r/arminc/clair-db). It can be useful to override this value with a specific version, for example, to provide a consistent set of vulnerabilities for integration testing purposes. | `latest` | | `DOCKERFILE_PATH` | The path to the `Dockerfile` to be used for generating remediations. By default, the scanner will look for a file named `Dockerfile` in the root directory of the project, so this variable should only be configured if your `Dockerfile` is in a non-standard location, such as a subdirectory. See [Solutions for vulnerabilities](#solutions-for-vulnerabilities-auto-remediation) for more details. | `Dockerfile` | +| `ADDITIONAL_CA_CERT_BUNDLE` | Bundle of CA certs that you want to trust. | "" | ### Overriding the Container Scanning template diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 1a0a7a7711f..83e997bc7a2 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -145,6 +145,7 @@ The following variables allow configuration of global dependency scanning settin | `DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT` | 2m | Time limit for Docker client negotiation. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, or `h`. For example, `300ms`, `1.5h`, or `2h45m`. | | `DS_PULL_ANALYZER_IMAGE_TIMEOUT` | 5m | Time limit when pulling an analyzer's image. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, or `h`. For example, `300ms`, `1.5h`, or `2h45m`. | | `DS_RUN_ANALYZER_TIMEOUT` | 20m | Time limit when running an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, or `h`. For example, `300ms`, `1.5h`, or `2h45m`. | +| `ADDITIONAL_CA_CERT_BUNDLE` | | Bundle of CA certs that you want to trust. | #### Configuring specific analyzers used by Dependency Scanning diff --git a/doc/user/application_security/sast/index.md b/doc/user/application_security/sast/index.md index 70d31f8e1d6..aaade9b03d6 100644 --- a/doc/user/application_security/sast/index.md +++ b/doc/user/application_security/sast/index.md @@ -261,6 +261,11 @@ See [Analyzer settings](#analyzer-settings) for the complete list of available o SAST can be [configured](#customizing-the-sast-settings) using environment variables. +#### Custom Certificate Authority + +To trust a custom Certificate Authority, set the `ADDITIONAL_CA_CERT_BUNDLE` variable to the bundle +of CA certs that you want to trust within the SAST environment. + #### Docker images The following are Docker image-related variables. diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb index d327162b34e..2d0fdf98e8c 100644 --- a/lib/feature/gitaly.rb +++ b/lib/feature/gitaly.rb @@ -14,6 +14,9 @@ class Feature end def server_feature_flags + # We need to check that both the DB connection and table exists + return {} unless ::Gitlab::Database.cached_table_exists?(FlipperFeature.table_name) + Feature.persisted_names .select { |f| f.start_with?(PREFIX) } .map do |f| diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index aa6085de4f9..56f7ffbe4f8 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -340,6 +340,7 @@ methods: - :diff_head_sha - :source_branch_sha - :target_branch_sha + - :state events: - :action push_event_payload: @@ -350,6 +351,8 @@ methods: - :list_type ci_pipelines: - :notes + issues: + - :state preloads: statuses: diff --git a/lib/gitlab/rate_limit_helpers.rb b/lib/gitlab/rate_limit_helpers.rb index 2dcc888892b..653410a40a5 100644 --- a/lib/gitlab/rate_limit_helpers.rb +++ b/lib/gitlab/rate_limit_helpers.rb @@ -7,7 +7,7 @@ module Gitlab ARCHIVE_RATE_THROTTLE_KEY = :project_repositories_archive def archive_rate_limit_reached?(user, project) - return false unless Feature.enabled?(:archive_rate_limit, default_enabled: true) + return false unless Feature.enabled?(:archive_rate_limit) key = ARCHIVE_RATE_THROTTLE_KEY diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 66503621851..db531f06f11 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -17,11 +17,12 @@ module Gitlab end def group_name_regex - project_name_regex + @group_name_regex ||= /\A[\p{Alnum}\u{00A9}-\u{1f9ff}_][\p{Alnum}\p{Pd}\u{00A9}-\u{1f9ff}_()\. ]*\z/.freeze end def group_name_regex_message - project_name_regex_message + "can contain only letters, digits, emojis, '_', '.', dash, space, parenthesis. " \ + "It must start with letter, digit, emoji or '_'." end ## diff --git a/locale/gitlab.pot b/locale/gitlab.pot index a078cd23917..53d9de6dfc7 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -339,6 +339,9 @@ msgstr "" msgid "%{lineOneStart}Drag and drop to upload your designs%{lineOneEnd} or %{linkStart}click to upload%{linkEnd}." msgstr "" +msgid "%{link_start}Learn more%{link_end} about what information is shared with GitLab Inc." +msgstr "" + msgid "%{link_start}Read more%{link_end} about role permissions" msgstr "" @@ -7402,6 +7405,9 @@ msgstr "" msgid "Enable SAML authentication for this group" msgstr "" +msgid "Enable Seat Link" +msgstr "" + msgid "Enable access to Grafana" msgstr "" @@ -7441,6 +7447,9 @@ msgstr "" msgid "Enable mirror configuration" msgstr "" +msgid "Enable or disable Seat Link." +msgstr "" + msgid "Enable or disable keyboard shortcuts" msgstr "" @@ -8923,6 +8932,9 @@ msgstr "" msgid "For more information, see the documentation on %{deactivating_usage_ping_link_start}deactivating the usage ping%{deactivating_usage_ping_link_end}." msgstr "" +msgid "For more information, see the documentation on %{link_start}disabling Seat Link%{link_end}." +msgstr "" + msgid "For private projects, any member (guest or higher) can view pipelines and access job details (output logs and artifacts)" msgstr "" @@ -17406,6 +17418,12 @@ msgid_plural "SearchResults|wiki results" msgstr[0] "" msgstr[1] "" +msgid "Seat Link" +msgstr "" + +msgid "Seat Link is disabled, and cannot be configured through this form." +msgstr "" + msgid "Seats currently in use" msgstr "" @@ -18287,7 +18305,7 @@ msgstr "" msgid "Size settings for static websites" msgstr "" -msgid "Skip older, pending deployment jobs" +msgid "Skip outdated deployment jobs" msgstr "" msgid "Skip this for now" @@ -20940,6 +20958,9 @@ msgstr "" msgid "To set up this service:" msgstr "" +msgid "To simplify the billing process, GitLab will collect user counts in order to prorate charges for user growth throughout the year using a quarterly reconciliation process." +msgstr "" + msgid "To specify the notification level per project of a group you belong to, you need to visit project page and change notification level there." msgstr "" diff --git a/spec/controllers/concerns/import_url_params_spec.rb b/spec/controllers/concerns/import_url_params_spec.rb index adbe6e5d3bf..41e29d71386 100644 --- a/spec/controllers/concerns/import_url_params_spec.rb +++ b/spec/controllers/concerns/import_url_params_spec.rb @@ -31,7 +31,8 @@ describe ImportUrlParams do describe '#import_url_params' do it 'returns hash with import_url' do expect(import_url_params).to eq( - import_url: "https://user:password@url.com" + import_url: "https://user:password@url.com", + import_type: 'git' ) end end @@ -48,7 +49,8 @@ describe ImportUrlParams do describe '#import_url_params' do it 'does not change the url' do expect(import_url_params).to eq( - import_url: "https://user:password@url.com" + import_url: "https://user:password@url.com", + import_type: 'git' ) end end diff --git a/spec/lib/feature/gitaly_spec.rb b/spec/lib/feature/gitaly_spec.rb index afb522d05e1..08651c42276 100644 --- a/spec/lib/feature/gitaly_spec.rb +++ b/spec/lib/feature/gitaly_spec.rb @@ -32,5 +32,15 @@ describe Feature::Gitaly do it { is_expected.to be_a(Hash) } it { is_expected.to eq("gitaly-feature-mep-mep" => "true") } + + context 'when table does not exist' do + before do + allow(::Gitlab::Database).to receive(:cached_table_exists?).and_return(false) + end + + it 'returns an empty Hash' do + expect(subject).to eq({}) + end + end end end diff --git a/spec/lib/gitlab/regex_spec.rb b/spec/lib/gitlab/regex_spec.rb index f1b5393a2d8..5a2cf2eda8b 100644 --- a/spec/lib/gitlab/regex_spec.rb +++ b/spec/lib/gitlab/regex_spec.rb @@ -13,10 +13,6 @@ describe Gitlab::Regex do it { is_expected.not_to match('?gitlab') } end - shared_examples_for 'project/group name error message' do - it { is_expected.to eq("can contain only letters, digits, emojis, '_', '.', dash, space. It must start with letter, digit, emoji or '_'.") } - end - describe '.project_name_regex' do subject { described_class.project_name_regex } @@ -27,18 +23,26 @@ describe Gitlab::Regex do subject { described_class.group_name_regex } it_behaves_like 'project/group name regex' + + it 'allows parenthesis' do + is_expected.to match('Group One (Test)') + end + + it 'does not start with parenthesis' do + is_expected.not_to match('(Invalid Group name)') + end end describe '.project_name_regex_message' do subject { described_class.project_name_regex_message } - it_behaves_like 'project/group name error message' + it { is_expected.to eq("can contain only letters, digits, emojis, '_', '.', dash, space. It must start with letter, digit, emoji or '_'.") } end describe '.group_name_regex_message' do subject { described_class.group_name_regex_message } - it_behaves_like 'project/group name error message' + it { is_expected.to eq("can contain only letters, digits, emojis, '_', '.', dash, space, parenthesis. It must start with letter, digit, emoji or '_'.") } end describe '.environment_name_regex' do diff --git a/spec/support/shared_examples/uploaders/upload_type_shared_examples.rb b/spec/support/shared_examples/uploaders/upload_type_shared_examples.rb index 81ac6bd94db..e58723324d3 100644 --- a/spec/support/shared_examples/uploaders/upload_type_shared_examples.rb +++ b/spec/support/shared_examples/uploaders/upload_type_shared_examples.rb @@ -26,3 +26,15 @@ shared_examples 'accepted carrierwave upload' do expect { uploader.cache!(fixture_file) }.to change { uploader.file }.from(nil).to(kind_of(CarrierWave::SanitizedFile)) end end + +# @param path [String] the path to file to upload. E.g. File.join('spec', 'fixtures', 'sanitized.svg') +# @param uploader [CarrierWave::Uploader::Base] uploader to handle the upload. +# @param content_type [String] the upload file content type after cache +shared_examples 'upload with content type' do |content_type| + let(:fixture_file) { fixture_file_upload(path, content_type) } + + it 'will not change upload file content type' do + uploader.cache!(fixture_file) + expect(uploader.file.content_type).to eq(content_type) + end +end diff --git a/spec/uploaders/content_type_whitelist_spec.rb b/spec/uploaders/content_type_whitelist_spec.rb index be519ead1c8..4689f83759d 100644 --- a/spec/uploaders/content_type_whitelist_spec.rb +++ b/spec/uploaders/content_type_whitelist_spec.rb @@ -18,6 +18,7 @@ describe ContentTypeWhitelist do let(:path) { File.join('spec', 'fixtures', 'rails_sample.jpg') } it_behaves_like 'accepted carrierwave upload' + it_behaves_like 'upload with content type', 'image/jpeg' end context 'upload non-whitelisted file content type' do diff --git a/spec/uploaders/job_artifact_uploader_spec.rb b/spec/uploaders/job_artifact_uploader_spec.rb index 60b5a6697b1..a03cf3b9dea 100644 --- a/spec/uploaders/job_artifact_uploader_spec.rb +++ b/spec/uploaders/job_artifact_uploader_spec.rb @@ -97,5 +97,12 @@ describe JobArtifactUploader do it_behaves_like "migrates", to_store: described_class::Store::REMOTE it_behaves_like "migrates", from_store: described_class::Store::REMOTE, to_store: described_class::Store::LOCAL + + # CI job artifacts usually are shown as text/plain, but they contain + # escape characters so MIME detectors usually fail to determine what + # the Content-Type is. + it 'does not set Content-Type' do + expect(uploader.file.content_type).to be_blank + end end end diff --git a/vendor/gitignore/C++.gitignore b/vendor/gitignore/C++.gitignore index 259148fa18f..259148fa18f 100644..100755 --- a/vendor/gitignore/C++.gitignore +++ b/vendor/gitignore/C++.gitignore diff --git a/vendor/gitignore/Java.gitignore b/vendor/gitignore/Java.gitignore index a1c2a238a96..a1c2a238a96 100644..100755 --- a/vendor/gitignore/Java.gitignore +++ b/vendor/gitignore/Java.gitignore diff --git a/vendor/project_templates/gatsby.tar.gz b/vendor/project_templates/gatsby.tar.gz Binary files differindex d9025978126..a4ae5f99047 100644 --- a/vendor/project_templates/gatsby.tar.gz +++ b/vendor/project_templates/gatsby.tar.gz |
