From 13d327df4f3c52505bf8ec1144f2dedb6a351ad6 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Fri, 5 Mar 2021 23:11:43 +0000 Subject: Add latest changes from gitlab-org/gitlab@13-9-stable-ee --- GITLAB_SHELL_VERSION | 2 +- Gemfile | 2 +- Gemfile.lock | 4 +-- app/models/application_setting.rb | 3 +- app/models/application_setting_implementation.rb | 9 +++-- .../application_settings/update_service.rb | 2 +- ...roki-gem-with-spelling-mistake-for-wavedrom.yml | 5 +++ ...rrors-in-13-9-for-geo-with-downtime-upgrade.yml | 5 +++ .../unreleased/sh-bump-gitlab-shell-version.yml | 5 +++ .../unreleased/sh-fix-kroki-format-filtering.yml | 5 +++ ...sset_proxy_allowlist_on_application_settings.rb | 21 ++++++++++++ ...oxy_allowlist_rename_on_application_settings.rb | 21 ++++++++++++ db/schema_migrations/20210301200601 | 1 + db/schema_migrations/20210301200959 | 1 + db/structure.sql | 2 +- doc/administration/object_storage.md | 18 +++++++++- .../backfill_artifact_expiry_date.rb | 13 ++++---- lib/gitlab/kroki.rb | 3 -- spec/deprecation_toolkit_env.rb | 1 + spec/lib/gitlab/kroki_spec.rb | 38 ++++++++++++++++++++++ spec/models/application_setting_spec.rb | 26 +++++++++++++++ .../application_settings/update_service_spec.rb | 1 + vendor/gitignore/C++.gitignore | 0 vendor/gitignore/Java.gitignore | 0 24 files changed, 169 insertions(+), 19 deletions(-) create mode 100644 changelogs/unreleased/322861-outdated-asciidoctor-kroki-gem-with-spelling-mistake-for-wavedrom.yml create mode 100644 changelogs/unreleased/322991-asset_proxy_allowlist-errors-in-13-9-for-geo-with-downtime-upgrade.yml create mode 100644 changelogs/unreleased/sh-bump-gitlab-shell-version.yml create mode 100644 changelogs/unreleased/sh-fix-kroki-format-filtering.yml create mode 100644 db/migrate/20210301200601_rename_asset_proxy_allowlist_on_application_settings.rb create mode 100644 db/post_migrate/20210301200959_clean_up_asset_proxy_allowlist_rename_on_application_settings.rb create mode 100644 db/schema_migrations/20210301200601 create mode 100644 db/schema_migrations/20210301200959 create mode 100644 spec/lib/gitlab/kroki_spec.rb mode change 100644 => 100755 vendor/gitignore/C++.gitignore mode change 100644 => 100755 vendor/gitignore/Java.gitignore diff --git a/GITLAB_SHELL_VERSION b/GITLAB_SHELL_VERSION index 592a1a89678..15ede847472 100644 --- a/GITLAB_SHELL_VERSION +++ b/GITLAB_SHELL_VERSION @@ -1 +1 @@ -13.16.1 +13.17.0 diff --git a/Gemfile b/Gemfile index 0445cc391aa..ec253c37b3a 100644 --- a/Gemfile +++ b/Gemfile @@ -156,7 +156,7 @@ gem 'wikicloth', '0.8.1' gem 'asciidoctor', '~> 2.0.10' gem 'asciidoctor-include-ext', '~> 0.3.1', require: false gem 'asciidoctor-plantuml', '~> 0.0.12' -gem 'asciidoctor-kroki', '~> 0.3.0', require: false +gem 'asciidoctor-kroki', '~> 0.4.0', require: false gem 'rouge', '~> 3.26.0' gem 'truncato', '~> 0.7.11' gem 'bootstrap_form', '~> 4.2.0' diff --git a/Gemfile.lock b/Gemfile.lock index 90ee2eaec09..1b6701e8e3d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -84,7 +84,7 @@ GEM asciidoctor (2.0.12) asciidoctor-include-ext (0.3.1) asciidoctor (>= 1.5.6, < 3.0.0) - asciidoctor-kroki (0.3.0) + asciidoctor-kroki (0.4.0) asciidoctor (~> 2.0) asciidoctor-plantuml (0.0.12) asciidoctor (>= 1.5.6, < 3.0.0) @@ -1291,7 +1291,7 @@ DEPENDENCIES asana (~> 0.10.3) asciidoctor (~> 2.0.10) asciidoctor-include-ext (~> 0.3.1) - asciidoctor-kroki (~> 0.3.0) + asciidoctor-kroki (~> 0.4.0) asciidoctor-plantuml (~> 0.0.12) atlassian-jwt (~> 0.2.0) attr_encrypted (~> 3.1.0) diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 33c058dab96..4959401eb27 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -58,8 +58,9 @@ class ApplicationSetting < ApplicationRecord serialize :domain_allowlist, Array # rubocop:disable Cop/ActiveRecordSerialize serialize :domain_denylist, Array # rubocop:disable Cop/ActiveRecordSerialize serialize :repository_storages # rubocop:disable Cop/ActiveRecordSerialize - serialize :asset_proxy_allowlist, Array # rubocop:disable Cop/ActiveRecordSerialize + # See https://gitlab.com/gitlab-org/gitlab/-/issues/300916 + serialize :asset_proxy_allowlist, Array # rubocop:disable Cop/ActiveRecordSerialize serialize :asset_proxy_whitelist, Array # rubocop:disable Cop/ActiveRecordSerialize cache_markdown_field :sign_in_text diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 2911ae6b1c8..08c16930b13 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -280,13 +280,18 @@ module ApplicationSettingImplementation self.notes_create_limit_allowlist = strings_to_array(values).map(&:downcase) end - def asset_proxy_allowlist=(values) + def asset_proxy_whitelist=(values) values = strings_to_array(values) if values.is_a?(String) # make sure we always allow the running host values << Gitlab.config.gitlab.host unless values.include?(Gitlab.config.gitlab.host) - self[:asset_proxy_allowlist] = values + self[:asset_proxy_whitelist] = values + end + alias_method :asset_proxy_allowlist=, :asset_proxy_whitelist= + + def asset_proxy_allowlist + read_attribute(:asset_proxy_whitelist) end def repository_storages diff --git a/app/services/application_settings/update_service.rb b/app/services/application_settings/update_service.rb index 5e5c8ae2177..7792b811b4e 100644 --- a/app/services/application_settings/update_service.rb +++ b/app/services/application_settings/update_service.rb @@ -6,7 +6,7 @@ module ApplicationSettings attr_reader :params, :application_setting - MARKDOWN_CACHE_INVALIDATING_PARAMS = %w(asset_proxy_enabled asset_proxy_url asset_proxy_secret_key asset_proxy_allowlist).freeze + MARKDOWN_CACHE_INVALIDATING_PARAMS = %w(asset_proxy_enabled asset_proxy_url asset_proxy_secret_key asset_proxy_whitelist).freeze def execute result = update_settings diff --git a/changelogs/unreleased/322861-outdated-asciidoctor-kroki-gem-with-spelling-mistake-for-wavedrom.yml b/changelogs/unreleased/322861-outdated-asciidoctor-kroki-gem-with-spelling-mistake-for-wavedrom.yml new file mode 100644 index 00000000000..e5c9eeae035 --- /dev/null +++ b/changelogs/unreleased/322861-outdated-asciidoctor-kroki-gem-with-spelling-mistake-for-wavedrom.yml @@ -0,0 +1,5 @@ +--- +title: Update Kroki to fix Wavedrom graphs +merge_request: 55659 +author: +type: fixed diff --git a/changelogs/unreleased/322991-asset_proxy_allowlist-errors-in-13-9-for-geo-with-downtime-upgrade.yml b/changelogs/unreleased/322991-asset_proxy_allowlist-errors-in-13-9-for-geo-with-downtime-upgrade.yml new file mode 100644 index 00000000000..36d3730448a --- /dev/null +++ b/changelogs/unreleased/322991-asset_proxy_allowlist-errors-in-13-9-for-geo-with-downtime-upgrade.yml @@ -0,0 +1,5 @@ +--- +title: Rename asset_proxy_allowlist column +merge_request: 55884 +author: +type: fixed diff --git a/changelogs/unreleased/sh-bump-gitlab-shell-version.yml b/changelogs/unreleased/sh-bump-gitlab-shell-version.yml new file mode 100644 index 00000000000..fef8885c0a8 --- /dev/null +++ b/changelogs/unreleased/sh-bump-gitlab-shell-version.yml @@ -0,0 +1,5 @@ +--- +title: Upgrade gitlab-shell to v13.17.0 +merge_request: 55295 +author: +type: fixed diff --git a/changelogs/unreleased/sh-fix-kroki-format-filtering.yml b/changelogs/unreleased/sh-fix-kroki-format-filtering.yml new file mode 100644 index 00000000000..7c5f3d210ae --- /dev/null +++ b/changelogs/unreleased/sh-fix-kroki-format-filtering.yml @@ -0,0 +1,5 @@ +--- +title: Fix disabling of Kroki optional formats +merge_request: 55665 +author: +type: fixed diff --git a/db/migrate/20210301200601_rename_asset_proxy_allowlist_on_application_settings.rb b/db/migrate/20210301200601_rename_asset_proxy_allowlist_on_application_settings.rb new file mode 100644 index 00000000000..8a9acd8b86e --- /dev/null +++ b/db/migrate/20210301200601_rename_asset_proxy_allowlist_on_application_settings.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RenameAssetProxyAllowlistOnApplicationSettings < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers::V2 + + DOWNTIME = false + + disable_ddl_transaction! + + def up + rename_column_concurrently :application_settings, + :asset_proxy_allowlist, + :asset_proxy_whitelist + end + + def down + undo_rename_column_concurrently :application_settings, + :asset_proxy_allowlist, + :asset_proxy_whitelist + end +end diff --git a/db/post_migrate/20210301200959_clean_up_asset_proxy_allowlist_rename_on_application_settings.rb b/db/post_migrate/20210301200959_clean_up_asset_proxy_allowlist_rename_on_application_settings.rb new file mode 100644 index 00000000000..d0b372b84f0 --- /dev/null +++ b/db/post_migrate/20210301200959_clean_up_asset_proxy_allowlist_rename_on_application_settings.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class CleanUpAssetProxyAllowlistRenameOnApplicationSettings < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers::V2 + + DOWNTIME = false + + disable_ddl_transaction! + + def up + cleanup_concurrent_column_rename :application_settings, + :asset_proxy_allowlist, + :asset_proxy_whitelist + end + + def down + undo_cleanup_concurrent_column_rename :application_settings, + :asset_proxy_allowlist, + :asset_proxy_whitelist + end +end diff --git a/db/schema_migrations/20210301200601 b/db/schema_migrations/20210301200601 new file mode 100644 index 00000000000..13907f9de8b --- /dev/null +++ b/db/schema_migrations/20210301200601 @@ -0,0 +1 @@ +21ae7ea7cbf1d34c7b9dc300a641eaf975ed1e33f5bc519494cd37c4a661bec8 \ No newline at end of file diff --git a/db/schema_migrations/20210301200959 b/db/schema_migrations/20210301200959 new file mode 100644 index 00000000000..8c5efabd76d --- /dev/null +++ b/db/schema_migrations/20210301200959 @@ -0,0 +1 @@ +28b90c9b7c2e4f2e2b12088f5aee85c16dfb567f89ed6a8e771f2c5d91c818d9 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 97a2850fd5c..c6b3b7ea997 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -9394,11 +9394,11 @@ CREATE TABLE application_settings ( container_registry_cleanup_tags_service_max_list_size integer DEFAULT 200 NOT NULL, enforce_ssh_key_expiration boolean DEFAULT false NOT NULL, git_two_factor_session_expiry integer DEFAULT 15 NOT NULL, - asset_proxy_allowlist text, keep_latest_artifact boolean DEFAULT true NOT NULL, notes_create_limit integer DEFAULT 300 NOT NULL, notes_create_limit_allowlist text[] DEFAULT '{}'::text[] NOT NULL, kroki_formats jsonb DEFAULT '{}'::jsonb NOT NULL, + asset_proxy_whitelist text, CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)), CONSTRAINT app_settings_registry_exp_policies_worker_capacity_positive CHECK ((container_registry_expiration_policies_worker_capacity >= 0)), CONSTRAINT check_17d9558205 CHECK ((char_length((kroki_url)::text) <= 1024)), diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index 3cad18dc497..3b43f53dc82 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -31,6 +31,8 @@ GitLab has been tested on a number of object storage providers: HTTP Range Requests from working with CI job artifacts](https://gitlab.com/gitlab-org/gitlab/-/issues/223806). Be sure to upgrade to GitLab v13.3.0 or above if you use S3 storage with this hardware. +- Ceph S3 prior to [Kraken 11.0.2](https://ceph.com/releases/kraken-11-0-2-released/) does not support the [Upload Copy Part API](https://gitlab.com/gitlab-org/gitlab/-/issues/300604). You may need to [disable multi-threaded copying](#multi-threaded-copying). + ## Configuration guides There are two ways of specifying object storage configuration in GitLab: @@ -752,7 +754,6 @@ To set up an instance profile: "Action": [ "s3:PutObject", "s3:GetObject", - "s3:AbortMultipartUpload", "s3:DeleteObject" ], "Resource": "arn:aws:s3:::test-bucket/*" @@ -764,3 +765,18 @@ To set up an instance profile: 1. [Attach this role](https://aws.amazon.com/premiumsupport/knowledge-center/attach-replace-ec2-instance-profile/) to the EC2 instance hosting your GitLab instance. 1. Configure GitLab to use it via the `use_iam_profile` configuration option. + +### Multi-threaded copying + +GitLab uses the [S3 Upload Part Copy API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html) +to accelerate the copying of files within a bucket. Ceph S3 [prior to Kraken 11.0.2](https://ceph.com/releases/kraken-11-0-2-released/) +does not support this and [returns a 404 error when files are copied during the upload process](https://gitlab.com/gitlab-org/gitlab/-/issues/300604). + +The feature can be disabled using the `:s3_multithreaded_uploads` +feature flag. To disable the feature, ask a GitLab administrator with +[Rails console access](feature_flags.md#how-to-enable-and-disable-features-behind-flags) +to run the following command: + +```ruby +Feature.disable(:s3_multithreaded_uploads) +``` diff --git a/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb b/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb index 0a8c203421b..f6b36571c90 100644 --- a/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb +++ b/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb @@ -6,9 +6,9 @@ module Gitlab class BackfillArtifactExpiryDate include Gitlab::Utils::StrongMemoize - BATCH_SIZE = 1_000 - DEFAULT_EXPIRATION_SWITCH_DATE = Date.new(2020, 6, 22).freeze + SWITCH_DATE = Date.new(2020, 06, 22).freeze OLD_ARTIFACT_AGE = 15.months + BATCH_SIZE = 1_000 OLD_ARTIFACT_EXPIRY_OFFSET = 3.months RECENT_ARTIFACT_EXPIRY_OFFSET = 1.year @@ -18,16 +18,17 @@ module Gitlab self.table_name = 'ci_job_artifacts' - scope :between, -> (start_id, end_id) { where(id: start_id..end_id) } - scope :before_default_expiration_switch, -> { where('created_at < ?', DEFAULT_EXPIRATION_SWITCH_DATE) } scope :without_expiry_date, -> { where(expire_at: nil) } + scope :before_switch, -> { where("date(created_at AT TIME ZONE 'UTC') < ?::date", SWITCH_DATE) } + scope :between, -> (start_id, end_id) { where(id: start_id..end_id) } scope :old, -> { where(self.arel_table[:created_at].lt(OLD_ARTIFACT_AGE.ago)) } scope :recent, -> { where(self.arel_table[:created_at].gt(OLD_ARTIFACT_AGE.ago)) } end def perform(start_id, end_id) - Ci::JobArtifact.between(start_id, end_id) - .without_expiry_date.before_default_expiration_switch + Ci::JobArtifact + .without_expiry_date.before_switch + .between(start_id, end_id) .each_batch(of: BATCH_SIZE) do |batch| batch.old.update_all(expire_at: old_artifact_expiry_date) batch.recent.update_all(expire_at: recent_artifact_expiry_date) diff --git a/lib/gitlab/kroki.rb b/lib/gitlab/kroki.rb index 2948b6ebd5b..fa10e922c80 100644 --- a/lib/gitlab/kroki.rb +++ b/lib/gitlab/kroki.rb @@ -31,9 +31,6 @@ module Gitlab DIAGRAMS_FORMATS end - # No additional diagram formats - return diagram_formats unless current_settings.kroki_formats.present? - # Diagrams that require a companion container must be explicitly enabled from the settings diagram_formats.select do |diagram_type| current_settings.kroki_format_supported?(diagram_type) diff --git a/spec/deprecation_toolkit_env.rb b/spec/deprecation_toolkit_env.rb index d2ff2d2cb37..10eaaf13aaa 100644 --- a/spec/deprecation_toolkit_env.rb +++ b/spec/deprecation_toolkit_env.rb @@ -61,6 +61,7 @@ module DeprecationToolkitEnv batch-loader-1.4.0/lib/batch_loader/graphql.rb carrierwave-1.3.1/lib/carrierwave/sanitized_file.rb activerecord-6.0.3.4/lib/active_record/relation.rb + asciidoctor-2.0.12/lib/asciidoctor/extensions.rb ] end diff --git a/spec/lib/gitlab/kroki_spec.rb b/spec/lib/gitlab/kroki_spec.rb new file mode 100644 index 00000000000..31d3edd158b --- /dev/null +++ b/spec/lib/gitlab/kroki_spec.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true +require 'spec_helper' + +RSpec.describe Gitlab::Kroki do + using RSpec::Parameterized::TableSyntax + + describe '.formats' do + def default_formats + %w[bytefield c4plantuml ditaa erd graphviz nomnoml plantuml svgbob umlet vega vegalite wavedrom].freeze + end + + subject { described_class.formats(Gitlab::CurrentSettings) } + + where(:enabled_formats, :expected_formats) do + '' | default_formats + 'blockdiag' | default_formats + %w[actdiag blockdiag nwdiag packetdiag rackdiag seqdiag] + 'bpmn' | default_formats + %w[bpmn] + 'excalidraw' | default_formats + %w[excalidraw] + end + + with_them do + before do + kroki_formats = + if enabled_formats.present? + { enabled_formats => true } + else + {} + end + + stub_application_setting(kroki_enabled: true, kroki_url: "http://localhost:8000", kroki_formats: kroki_formats) + end + + it 'returns the expected formats' do + expect(subject).to match_array(expected_formats) + end + end + end +end diff --git a/spec/models/application_setting_spec.rb b/spec/models/application_setting_spec.rb index 9a4dd2c799b..5658057f588 100644 --- a/spec/models/application_setting_spec.rb +++ b/spec/models/application_setting_spec.rb @@ -650,6 +650,32 @@ RSpec.describe ApplicationSetting do end end + describe '#asset_proxy_whitelist' do + context 'when given an Array' do + it 'sets the domains and adds current running host' do + setting.asset_proxy_whitelist = ['example.com', 'assets.example.com'] + expect(setting.asset_proxy_whitelist).to eq(['example.com', 'assets.example.com', 'localhost']) + end + end + + context 'when given a String' do + it 'sets multiple domains with spaces' do + setting.asset_proxy_whitelist = 'example.com *.example.com' + expect(setting.asset_proxy_whitelist).to eq(['example.com', '*.example.com', 'localhost']) + end + + it 'sets multiple domains with newlines and a space' do + setting.asset_proxy_whitelist = "example.com\n *.example.com" + expect(setting.asset_proxy_whitelist).to eq(['example.com', '*.example.com', 'localhost']) + end + + it 'sets multiple domains with commas' do + setting.asset_proxy_whitelist = "example.com, *.example.com" + expect(setting.asset_proxy_whitelist).to eq(['example.com', '*.example.com', 'localhost']) + end + end + end + describe '#asset_proxy_allowlist' do context 'when given an Array' do it 'sets the domains and adds current running host' do diff --git a/spec/services/application_settings/update_service_spec.rb b/spec/services/application_settings/update_service_spec.rb index 1352a595ba4..258b3d25aee 100644 --- a/spec/services/application_settings/update_service_spec.rb +++ b/spec/services/application_settings/update_service_spec.rb @@ -123,6 +123,7 @@ RSpec.describe ApplicationSettings::UpdateService do it_behaves_like 'invalidates markdown cache', { asset_proxy_url: 'http://test.com' } it_behaves_like 'invalidates markdown cache', { asset_proxy_secret_key: 'another secret' } it_behaves_like 'invalidates markdown cache', { asset_proxy_allowlist: ['domain.com'] } + it_behaves_like 'invalidates markdown cache', { asset_proxy_whitelist: ['domain.com'] } context 'when also setting the local_markdown_version' do let(:params) { { asset_proxy_enabled: true, local_markdown_version: 12 } } diff --git a/vendor/gitignore/C++.gitignore b/vendor/gitignore/C++.gitignore old mode 100644 new mode 100755 diff --git a/vendor/gitignore/Java.gitignore b/vendor/gitignore/Java.gitignore old mode 100644 new mode 100755 -- cgit v1.2.3