From 4588bbc93a7857eb2d031a4f3e0212c0aa46b846 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Mon, 14 Sep 2020 10:26:13 +0000 Subject: Add latest changes from gitlab-org/gitlab@13-3-stable-ee --- changelogs/unreleased/246741-conan-unicorn-bug.yml | 5 ++ .../unreleased/fix-bump-ado-image-to-1-0-3.yml | 6 ++ .../20200515152649_enable_btree_gist_extension.rb | 6 +- doc/install/postgresql_extensions.md | 76 ++++++++++++++++++++++ doc/install/requirements.md | 7 +- doc/update/README.md | 1 + lib/api/helpers/packages/conan/api_helpers.rb | 2 +- .../Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml | 2 +- lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml | 2 +- lib/gitlab/database/migration_helpers.rb | 57 ++++++++++++++++ .../table_management_helpers.rb | 6 +- locale/gitlab.pot | 2 +- spec/lib/gitlab/database/migration_helpers_spec.rb | 52 +++++++++++++++ spec/requests/api/conan_packages_spec.rb | 12 ++++ 14 files changed, 223 insertions(+), 13 deletions(-) create mode 100644 changelogs/unreleased/246741-conan-unicorn-bug.yml create mode 100644 changelogs/unreleased/fix-bump-ado-image-to-1-0-3.yml create mode 100644 doc/install/postgresql_extensions.md diff --git a/changelogs/unreleased/246741-conan-unicorn-bug.yml b/changelogs/unreleased/246741-conan-unicorn-bug.yml new file mode 100644 index 00000000000..669550c1c81 --- /dev/null +++ b/changelogs/unreleased/246741-conan-unicorn-bug.yml @@ -0,0 +1,5 @@ +--- +title: Use 'read' method to get request body in Conan to fix uploads when using Unicorn +merge_request: 41801 +author: +type: fixed diff --git a/changelogs/unreleased/fix-bump-ado-image-to-1-0-3.yml b/changelogs/unreleased/fix-bump-ado-image-to-1-0-3.yml new file mode 100644 index 00000000000..6000a9f493d --- /dev/null +++ b/changelogs/unreleased/fix-bump-ado-image-to-1-0-3.yml @@ -0,0 +1,6 @@ +--- +title: Fixes Auto DevOps deploy script for multiple additional hosts separated by + comma and space +merge_request: 41404 +author: +type: fixed diff --git a/db/migrate/20200515152649_enable_btree_gist_extension.rb b/db/migrate/20200515152649_enable_btree_gist_extension.rb index 686b685fb5d..bd81c921a87 100644 --- a/db/migrate/20200515152649_enable_btree_gist_extension.rb +++ b/db/migrate/20200515152649_enable_btree_gist_extension.rb @@ -1,13 +1,15 @@ # frozen_string_literal: true class EnableBtreeGistExtension < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + DOWNTIME = false def up - execute 'CREATE EXTENSION IF NOT EXISTS btree_gist' + create_extension :btree_gist end def down - execute 'DROP EXTENSION IF EXISTS btree_gist' + drop_extension :btree_gist end end diff --git a/doc/install/postgresql_extensions.md b/doc/install/postgresql_extensions.md new file mode 100644 index 00000000000..4156d72097d --- /dev/null +++ b/doc/install/postgresql_extensions.md @@ -0,0 +1,76 @@ +--- +last_updated: 2020-09-01 +--- + +# Managing PostgreSQL extensions + +This guide documents how to manage PostgreSQL extensions for installations with an external +PostgreSQL database. + +GitLab requires certain extensions to be installed into the GitLab database. For example, +GitLab relies on `pg_trgm` and the `btree_gist` extensions. + +In order to install extensions, PostgreSQL requires the user to have superuser privileges. +Typically, the GitLab database user is not a superuser. Therefore, regular database migrations +cannot be used in installing extensions and instead, extensions have to be installed manually +prior to upgrading GitLab to a newer version. + +## Installing PostgreSQL extensions manually + +In order to install a PostgreSQL extension, this procedure should be followed: + +1. Connect to the GitLab PostgreSQL database using a superuser, for example: + + ```shell + sudo gitlab-psql -d gitlabhq_production + ``` + +1. Install the extension (`btree_gist` in this example) using [`CREATE EXTENSION`](https://www.postgresql.org/docs/11/sql-createextension.html): + + ```sql + CREATE EXTENSION IF NOT EXISTS btree_gist + ``` + +1. Verify installed extensions: + + ```shell + gitlabhq_production=# \dx + List of installed extensions + Name | Version | Schema | Description + ------------+---------+------------+------------------------------------------------------------------- + btree_gist | 1.5 | public | support for indexing common datatypes in GiST + pg_trgm | 1.4 | public | text similarity measurement and index searching based on trigrams + plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language + (3 rows) + ``` + +On some systems you may need to install an additional package (for example, +`postgresql-contrib`) for certain extensions to become available. + +## A typical migration failure scenario + +The following is an example of a situation when the extension hasn't been installed before running migrations. +In this scenario, the database migration fails to create the extension `btree_gist` because of insufficient +privileges. + +```shell +== 20200515152649 EnableBtreeGistExtension: migrating ========================= +-- execute("CREATE EXTENSION IF NOT EXISTS btree_gist") + +GitLab requires the PostgreSQL extension 'btree_gist' installed in database 'gitlabhq_production', but +the database user is not allowed to install the extension. + +You can either install the extension manually using a database superuser: + + CREATE EXTENSION IF NOT EXISTS btree_gist + +Or, you can solve this by logging in to the GitLab database (gitlabhq_production) using a superuser and running: + + ALTER regular WITH SUPERUSER + +This query will grant the user superuser permissions, ensuring any database extensions +can be installed through migrations. +``` + +In order to recover from this situation, the extension needs to be installed manually using a superuser, and +the database migration (or GitLab upgrade) can be retried afterwards. diff --git a/doc/install/requirements.md b/doc/install/requirements.md index 4d9f09bb8b0..10d5853c82e 100644 --- a/doc/install/requirements.md +++ b/doc/install/requirements.md @@ -140,11 +140,8 @@ GitLab version | Minimum PostgreSQL version 12.10 | 11 13.0 | 11 -You must also ensure the `pg_trgm` and `btree_gist` extensions are loaded into every -GitLab database. These extensions [can be enabled](https://www.postgresql.org/docs/11/sql-createextension.html) using a PostgreSQL super user. - -On some systems you may need to install an additional package (for example, -`postgresql-contrib`) for this extension to become available. +You must also ensure the `pg_trgm` and `btree_gist` extensions are [loaded into every +GitLab database](postgresql_extensions.html). NOTE: **Note:** Support for [PostgreSQL 9.6 and 10 has been removed in GitLab 13.0](https://about.gitlab.com/releases/2020/05/22/gitlab-13-0-released/#postgresql-11-is-now-the-minimum-required-version-to-install-gitlab) so that GitLab can benefit from PostgreSQL 11 improvements, such as partitioning. For the schedule of transitioning to PostgreSQL 12, see [the related epic](https://gitlab.com/groups/gitlab-org/-/epics/2184). diff --git a/doc/update/README.md b/doc/update/README.md index 85fc4363673..a7f7aaf5887 100644 --- a/doc/update/README.md +++ b/doc/update/README.md @@ -310,3 +310,4 @@ for more information. - [Restoring from backup after a failed upgrade](restore_after_failure.md) - [Upgrading PostgreSQL Using Slony](upgrading_postgresql_using_slony.md), for upgrading a PostgreSQL database with minimal downtime. +- [Managing PostgreSQL extensions](../install/postgresql_extensions.md) diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb index c9c2f66ef62..1161d1386bb 100644 --- a/lib/api/helpers/packages/conan/api_helpers.rb +++ b/lib/api/helpers/packages/conan/api_helpers.rb @@ -139,7 +139,7 @@ module API end def file_names - json_payload = Gitlab::Json.parse(request.body.string) + json_payload = Gitlab::Json.parse(request.body.read) bad_request!(nil) unless json_payload.is_a?(Hash) diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index e9d77766db3..86f946aafda 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.2" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3" dast_environment_deploy: extends: .dast-auto-deploy diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 41120750ff4..2922e1c6e88 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.2" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3" dependencies: [] include: diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index a618a3017b2..b62b6e20dd5 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -1212,6 +1212,63 @@ into similar problems in the future (e.g. when new tables are created). ) end + def create_extension(extension) + execute('CREATE EXTENSION IF NOT EXISTS %s' % extension) + rescue ActiveRecord::StatementInvalid => e + dbname = Database.database_name + user = Database.username + + warn(<<~MSG) if e.to_s =~ /permission denied/ + GitLab requires the PostgreSQL extension '#{extension}' installed in database '#{dbname}', but + the database user is not allowed to install the extension. + + You can either install the extension manually using a database superuser: + + CREATE EXTENSION IF NOT EXISTS #{extension} + + Or, you can solve this by logging in to the GitLab + database (#{dbname}) using a superuser and running: + + ALTER #{user} WITH SUPERUSER + + This query will grant the user superuser permissions, ensuring any database extensions + can be installed through migrations. + + For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html. + MSG + + raise + end + + def drop_extension(extension) + execute('DROP EXTENSION IF EXISTS %s' % extension) + rescue ActiveRecord::StatementInvalid => e + dbname = Database.database_name + user = Database.username + + warn(<<~MSG) if e.to_s =~ /permission denied/ + This migration attempts to drop the PostgreSQL extension '#{extension}' + installed in database '#{dbname}', but the database user is not allowed + to drop the extension. + + You can either drop the extension manually using a database superuser: + + DROP EXTENSION IF EXISTS #{extension} + + Or, you can solve this by logging in to the GitLab + database (#{dbname}) using a superuser and running: + + ALTER #{user} WITH SUPERUSER + + This query will grant the user superuser permissions, ensuring any database extensions + can be dropped through migrations. + + For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html. + MSG + + raise + end + private def validate_check_constraint_name!(constraint_name) diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index e6d8ec55319..84b6fb9f76e 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -55,8 +55,10 @@ module Gitlab partitioned_table_name = make_partitioned_table_name(table_name) - create_range_partitioned_copy(table_name, partitioned_table_name, partition_column, primary_key) - create_daterange_partitions(partitioned_table_name, partition_column.name, min_date, max_date) + transaction do + create_range_partitioned_copy(table_name, partitioned_table_name, partition_column, primary_key) + create_daterange_partitions(partitioned_table_name, partition_column.name, min_date, max_date) + end create_trigger_to_sync_tables(table_name, partitioned_table_name, primary_key) end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index ed9d4b43f5a..e81e3f3b2e0 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -28682,7 +28682,7 @@ msgstr "" msgid "ciReport|%{linkStartTag}Learn more about SAST %{linkEndTag}" msgstr "" -msgid "ciReport|%{linkStartTag}Learn more about Secret Scanning %{linkEndTag}" +msgid "ciReport|%{linkStartTag}Learn more about Secret Detection %{linkEndTag}" msgstr "" msgid "ciReport|%{linkStartTag}Learn more about codequality reports %{linkEndTag}" diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb index 4b7f371b25a..7d26fbb1132 100644 --- a/spec/lib/gitlab/database/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers_spec.rb @@ -2329,4 +2329,56 @@ RSpec.describe Gitlab::Database::MigrationHelpers do end end end + + describe '#create_extension' do + subject { model.create_extension(extension) } + + let(:extension) { :btree_gist } + + it 'executes CREATE EXTENSION statement' do + expect(model).to receive(:execute).with(/CREATE EXTENSION IF NOT EXISTS #{extension}/) + + subject + end + + context 'without proper permissions' do + before do + allow(model).to receive(:execute).with(/CREATE EXTENSION IF NOT EXISTS #{extension}/).and_raise(ActiveRecord::StatementInvalid, 'InsufficientPrivilege: permission denied') + end + + it 'raises the exception' do + expect { subject }.to raise_error(ActiveRecord::StatementInvalid, /InsufficientPrivilege/) + end + + it 'prints an error message' do + expect { subject }.to output(/user is not allowed/).to_stderr.and raise_error + end + end + end + + describe '#drop_extension' do + subject { model.drop_extension(extension) } + + let(:extension) { 'btree_gist' } + + it 'executes CREATE EXTENSION statement' do + expect(model).to receive(:execute).with(/DROP EXTENSION IF EXISTS #{extension}/) + + subject + end + + context 'without proper permissions' do + before do + allow(model).to receive(:execute).with(/DROP EXTENSION IF EXISTS #{extension}/).and_raise(ActiveRecord::StatementInvalid, 'InsufficientPrivilege: permission denied') + end + + it 'raises the exception' do + expect { subject }.to raise_error(ActiveRecord::StatementInvalid, /InsufficientPrivilege/) + end + + it 'prints an error message' do + expect { subject }.to output(/user is not allowed/).to_stderr.and raise_error + end + end + end end diff --git a/spec/requests/api/conan_packages_spec.rb b/spec/requests/api/conan_packages_spec.rb index 7a97743ede1..d24f6b68048 100644 --- a/spec/requests/api/conan_packages_spec.rb +++ b/spec/requests/api/conan_packages_spec.rb @@ -363,6 +363,16 @@ RSpec.describe API::ConanPackages do end end + shared_examples 'successful response when using Unicorn' do + context 'on Unicorn', :unicorn do + it 'returns successfully' do + subject + + expect(response).to have_gitlab_http_status(:ok) + end + end + end + describe 'GET /api/v4/packages/conan/v1/conans/:package_name/package_version/:package_username/:package_channel' do let(:recipe_path) { package.conan_recipe_path } @@ -458,6 +468,7 @@ RSpec.describe API::ConanPackages do it_behaves_like 'rejects invalid recipe' it_behaves_like 'rejects invalid upload_url params' + it_behaves_like 'successful response when using Unicorn' it 'returns a set of upload urls for the files requested' do subject @@ -521,6 +532,7 @@ RSpec.describe API::ConanPackages do it_behaves_like 'rejects invalid recipe' it_behaves_like 'rejects invalid upload_url params' + it_behaves_like 'successful response when using Unicorn' it 'returns a set of upload urls for the files requested' do expected_response = { -- cgit v1.2.3