diff options
20 files changed, 211 insertions, 68 deletions
diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index b9588bf08ad..1d63c044cc8 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -56aa9a2196f0003bc011e4152bb093b8d32be83d +af77b534aba831bd0fb21ef71746f363786d9a92 diff --git a/app/helpers/diff_helper.rb b/app/helpers/diff_helper.rb index e430b0f402b..3aa54e3afe9 100644 --- a/app/helpers/diff_helper.rb +++ b/app/helpers/diff_helper.rb @@ -190,10 +190,8 @@ module DiffHelper end def render_overflow_warning?(diffs_collection) - diff_files = diffs_collection.raw_diff_files - - diff_files.overflow?.tap do |overflown| - log_overflow_limits(diff_files) + diffs_collection.overflow?.tap do |overflown| + log_overflow_limits(diff_files: diffs_collection.raw_diff_files, collection_overflow: overflown) end end @@ -285,12 +283,12 @@ module DiffHelper conflicts_service.conflicts.files.index_by(&:our_path) end - def log_overflow_limits(diff_files) + def log_overflow_limits(diff_files:, collection_overflow:) if diff_files.any?(&:too_large?) Gitlab::Metrics.add_event(:diffs_overflow_single_file_limits) end - Gitlab::Metrics.add_event(:diffs_overflow_collection_limits) if diff_files.overflow? + Gitlab::Metrics.add_event(:diffs_overflow_collection_limits) if collection_overflow Gitlab::Metrics.add_event(:diffs_overflow_max_bytes_limits) if diff_files.overflow_max_bytes? Gitlab::Metrics.add_event(:diffs_overflow_max_files_limits) if diff_files.overflow_max_files? Gitlab::Metrics.add_event(:diffs_overflow_max_lines_limits) if diff_files.overflow_max_lines? diff --git a/app/serializers/merge_request_diff_entity.rb b/app/serializers/merge_request_diff_entity.rb index 02b81b707b9..e197c3d0fbb 100644 --- a/app/serializers/merge_request_diff_entity.rb +++ b/app/serializers/merge_request_diff_entity.rb @@ -16,7 +16,6 @@ class MergeRequestDiffEntity < Grape::Entity end expose :created_at - expose :state expose :commits_count expose :latest?, as: :latest diff --git a/danger/feature_flag/Dangerfile b/danger/feature_flag/Dangerfile index ac9ea812ebb..9b67590f117 100644 --- a/danger/feature_flag/Dangerfile +++ b/danger/feature_flag/Dangerfile @@ -41,6 +41,19 @@ def message_for_feature_flag_missing_group!(feature_flag:, mr_group_label:) end end +def message_for_global_rollout(feature_flag) + return unless feature_flag.default_enabled == true + + message = <<~SUGGEST_COMMENT + You're about to [release the feature with the feature flag](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/issue_templates/Feature%20Flag%20Roll%20Out.md#optional-release-the-feature-with-the-feature-flag). + This process can only be done **after** the [global rollout on production](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/issue_templates/Feature%20Flag%20Roll%20Out.md#global-rollout-on-production). + Please make sure in [the rollout issue](#{feature_flag.rollout_issue_url}) that the preliminary steps have already been done. Otherwise, changing the YAML definition might not have the desired effect. + SUGGEST_COMMENT + + mr_line = feature_flag.raw.lines.find_index { |l| l.include?('default_enabled:') } + markdown(message, file: feature_flag.path, line: mr_line.succ) +end + def message_for_feature_flag_with_group!(feature_flag:, mr_group_label:) return if feature_flag.group_match_mr_label?(mr_group_label) @@ -65,6 +78,10 @@ feature_flag.feature_flag_files(change_type: :added).each do |feature_flag| check_feature_flag_yaml(feature_flag) end +feature_flag.feature_flag_files(change_type: :modified).each do |feature_flag| + message_for_global_rollout(feature_flag) +end + if helper.security_mr? && feature_flag_file_added? fail "Feature flags are discouraged from security merge requests. Read the [security documentation](https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/security/utilities/feature_flags.md) for details." end diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index ec95f9f9382..c0111209133 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -693,23 +693,14 @@ database encryption. Proceed with caution. gitlab_pages['access_control'] = true ``` +1. Configure [the object storage and migrate pages data to it](#using-object-storage). + 1. [Reconfigure the **GitLab server**](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. The `gitlab-secrets.json` file is now updated with the new configuration. 1. Set up a new server. This becomes the **Pages server**. -1. Create an [NFS share](../nfs.md) - on the **Pages server** and configure this share to - allow access from your main **GitLab server**. - Note that the example there is more general and - shares several sub-directories from `/home` to several `/nfs/home` mount points. - For our Pages-specific example here, we instead share only the - default GitLab Pages folder `/var/opt/gitlab/gitlab-rails/shared/pages` - from the **Pages server** and we mount it to `/mnt/pages` - on the **GitLab server**. - Therefore, omit "Step 4" there. - 1. On the **Pages server**, install Omnibus GitLab and modify `/etc/gitlab/gitlab.rb` to include: @@ -728,7 +719,7 @@ database encryption. Proceed with caution. ``` 1. Copy the `/etc/gitlab/gitlab-secrets.json` file from the **GitLab server** - to the **Pages server**, for example via the NFS share. + to the **Pages server**. ```shell # On the GitLab server @@ -746,7 +737,6 @@ database encryption. Proceed with caution. pages_external_url "http://<pages_server_URL>" gitlab_pages['enable'] = false pages_nginx['enable'] = false - gitlab_rails['pages_path'] = "/mnt/pages" ``` 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -1324,6 +1314,24 @@ To enable disk access: 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +### `httprange: new resource 403` + +If you see an error similar to: + +```plaintext +{"error":"httprange: new resource 403: \"403 Forbidden\"","host":"root.pages.example.com","level":"error","msg":"vfs.Root","path":"/pages1/","time":"2021-06-10T08:45:19Z"} +``` + +And you run pages on the separate server syncing files via NFS, it may mean that +the shared pages directory is mounted on a different path on the main GitLab server and the +GitLab Pages server. + +In that case, it's highly recommended you to configure +[object storage and migrate any existing pages data to it](#using-object-storage). + +Alternatively, you can mount the GitLab Pages shared directory to the same path on +both servers. + ### GitLab Pages doesn't work after upgrading to GitLab 14.0 or above GitLab 14.0 introduces a number of changes to GitLab Pages which may require manual intervention. diff --git a/doc/api/deployments.md b/doc/api/deployments.md index 586f3edf51e..a2e56fc8557 100644 --- a/doc/api/deployments.md +++ b/doc/api/deployments.md @@ -9,9 +9,6 @@ type: concepts, howto ## List project deployments -> The `updated_after` and `updated_before` attributes were removed and replaced - by `finished_after` and `finished_before` respectively in GitLab 14.0. - Get a list of deployments in a project. ```plaintext @@ -23,8 +20,8 @@ GET /projects/:id/deployments | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user. | | `order_by` | string | no | Return deployments ordered by either one of `id`, `iid`, `created_at`, `updated_at` or `ref` fields. Default is `id`. | | `sort` | string | no | Return deployments sorted in `asc` or `desc` order. Default is `asc`. | -| `finished_after` | datetime | no | Return deployments updated after the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | -| `finished_before` | datetime | no | Return deployments updated before the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `updated_after` | datetime | no | Return deployments updated after the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | +| `updated_before` | datetime | no | Return deployments updated before the specified date. Expected in ISO 8601 format (`2019-03-15T08:00:00Z`). | | `environment` | string | no | The [name of the environment](../ci/environments/index.md) to filter deployments by. | | `status` | string | no | The status to filter deployments by. One of `created`, `running`, `success`, `failed`, `canceled`. diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 78c5ace7b2b..6d04c4874c7 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -85,6 +85,10 @@ module Gitlab # No-op end + def overflow? + raw_diff_files.overflow? + end + private def empty_pagination_data diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb index d2ca86fdfe7..0c68a57daab 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb @@ -48,6 +48,10 @@ module Gitlab @merge_request_diff.real_size end + def overflow? + @merge_request_diff.overflow? + end + private def highlight_cache diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 9c1a2aba45f..34aed91e4de 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -31554,6 +31554,9 @@ msgstr "" msgid "Symbolic link" msgstr "" +msgid "Sync LDAP" +msgstr "" + msgid "Sync now" msgstr "" @@ -31563,6 +31566,9 @@ msgstr "" msgid "Synchronization settings" msgstr "" +msgid "Synchronize LDAP" +msgstr "" + msgid "Syncing…" msgstr "" @@ -36217,6 +36223,9 @@ msgstr "" msgid "Warning: Displaying this diagram might cause performance issues on this page." msgstr "" +msgid "Warning: Synchronizing LDAP removes direct members' access." +msgstr "" + msgid "We are currently unable to fetch data for the pipeline header." msgstr "" @@ -37937,7 +37946,7 @@ msgstr "" msgid "cannot be enabled unless all domains have TLS certificates" msgstr "" -msgid "cannot be enabled until a valid credit credit is on file" +msgid "cannot be enabled until a valid credit card is on file" msgstr "" msgid "cannot be modified" diff --git a/scripts/frontend/startup_css/startup_css_changed.sh b/scripts/frontend/startup_css/startup_css_changed.sh index f214e61cdfb..2713d752974 100755 --- a/scripts/frontend/startup_css/startup_css_changed.sh +++ b/scripts/frontend/startup_css/startup_css_changed.sh @@ -10,10 +10,10 @@ echo "-----------------------------------------------------------" startup_glob="*stylesheets/startup*" echo "Staging changes to '${startup_glob}' so we can check for untracked files..." -git add ${startup_glob} +git add "${startup_glob}" -if [ -n "$(git diff HEAD --name-only -- ${startup_glob})" ]; then - diff=$(git diff HEAD -- ${startup_glob}) +if [ -n "$(git diff HEAD --name-only -- "${startup_glob}")" ]; then + diff=$(git diff HEAD -- "${startup_glob}") cat <<EOF Startup CSS changes detected! @@ -21,14 +21,19 @@ Startup CSS changes detected! It looks like there have been recent changes which require regenerating the Startup CSS files. -**What should I do now?** +IMPORTANT: -IMPORTANT: Please make sure to update your MR title with "[RUN AS-IF-FOSS]" and start a new MR pipeline + - If you are making changes to any Startup CSS file, it is very likely that + **both** the CE and EE Startup CSS files will need to be updated. + - Changing any Startup CSS file will trigger the "as-if-foss" job to also run. + +HOW TO FIX: To fix this job, consider one of the following options: - 1. Regenerating locally with "yarn run generate:startup_css". - 2. Copy and apply the following diff: + 1. (Strongly recommended) Copy and apply the diff below: + 2. Regenerate locally with "yarn run generate:startup_css". + You may need to set "FOSS_ONLY=1" if you are trying to generate for CE. ----- start diff ----- $diff diff --git a/spec/frontend_integration/test_helpers/setup/setup_mock_server.js b/spec/frontend_integration/test_helpers/setup/setup_mock_server.js index 43a21deed25..6f4832992a5 100644 --- a/spec/frontend_integration/test_helpers/setup/setup_mock_server.js +++ b/spec/frontend_integration/test_helpers/setup/setup_mock_server.js @@ -7,6 +7,16 @@ beforeEach(() => { const server = createMockServer(); server.logging = false; + server.pretender.handledRequest = (verb, path, { status, responseText }) => { + if (status >= 500) { + // eslint-disable-next-line no-console + console.log(` +The mock server returned status ${status} with "${verb} ${path}": + +${JSON.stringify({ responseText }, null, 2)} +`); + } + }; global.mockServer = server; }); diff --git a/spec/frontend_integration/test_helpers/setup/setup_testing_library.js b/spec/frontend_integration/test_helpers/setup/setup_testing_library.js index 5081b1c3b62..adc59665306 100644 --- a/spec/frontend_integration/test_helpers/setup/setup_testing_library.js +++ b/spec/frontend_integration/test_helpers/setup/setup_testing_library.js @@ -1,3 +1,15 @@ import { configure } from '@testing-library/dom'; -configure({ asyncUtilTimeout: 10000 }); +const CUSTOM_ERROR_TYPE = 'TestingLibraryError'; + +configure({ + asyncUtilTimeout: 10000, + // Overwrite default error message to reduce noise. + getElementError: (messageArg) => { + // Add to message because the `name` doesn't look like it's used (although it should). + const message = `${CUSTOM_ERROR_TYPE}:\n\n${messageArg}`; + const error = new Error(message); + error.name = CUSTOM_ERROR_TYPE; + return error; + }, +}); diff --git a/spec/helpers/diff_helper_spec.rb b/spec/helpers/diff_helper_spec.rb index dfea1020c52..2678395b1fa 100644 --- a/spec/helpers/diff_helper_spec.rb +++ b/spec/helpers/diff_helper_spec.rb @@ -293,23 +293,22 @@ RSpec.describe DiffHelper do describe '#render_overflow_warning?' do using RSpec::Parameterized::TableSyntax - let(:diffs_collection) { instance_double(Gitlab::Diff::FileCollection::MergeRequestDiff, raw_diff_files: diff_files) } + let(:diffs_collection) { instance_double(Gitlab::Diff::FileCollection::MergeRequestDiff, raw_diff_files: diff_files, overflow?: false) } let(:diff_files) { Gitlab::Git::DiffCollection.new(files) } let(:safe_file) { { too_large: false, diff: '' } } let(:large_file) { { too_large: true, diff: '' } } let(:files) { [safe_file, safe_file] } - before do - allow(diff_files).to receive(:overflow?).and_return(false) - allow(diff_files).to receive(:overflow_max_bytes?).and_return(false) - allow(diff_files).to receive(:overflow_max_files?).and_return(false) - allow(diff_files).to receive(:overflow_max_lines?).and_return(false) - allow(diff_files).to receive(:collapsed_safe_bytes?).and_return(false) - allow(diff_files).to receive(:collapsed_safe_files?).and_return(false) - allow(diff_files).to receive(:collapsed_safe_lines?).and_return(false) - end - context 'when no limits are hit' do + before do + allow(diff_files).to receive(:overflow_max_bytes?).and_return(false) + allow(diff_files).to receive(:overflow_max_files?).and_return(false) + allow(diff_files).to receive(:overflow_max_lines?).and_return(false) + allow(diff_files).to receive(:collapsed_safe_bytes?).and_return(false) + allow(diff_files).to receive(:collapsed_safe_files?).and_return(false) + allow(diff_files).to receive(:collapsed_safe_lines?).and_return(false) + end + it 'returns false and does not log any overflow events' do expect(Gitlab::Metrics).not_to receive(:add_event).with(:diffs_overflow_collection_limits) expect(Gitlab::Metrics).not_to receive(:add_event).with(:diffs_overflow_single_file_limits) @@ -343,7 +342,7 @@ RSpec.describe DiffHelper do context 'when the file collection has an overflow' do before do - allow(diff_files).to receive(:overflow?).and_return(true) + allow(diffs_collection).to receive(:overflow?).and_return(true) end it 'returns true and only logs all the correct collection overflow event' do diff --git a/spec/lib/gitlab/diff/file_collection/base_spec.rb b/spec/lib/gitlab/diff/file_collection/base_spec.rb new file mode 100644 index 00000000000..00d3aa47301 --- /dev/null +++ b/spec/lib/gitlab/diff/file_collection/base_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Diff::FileCollection::Base do + let(:merge_request) { create(:merge_request) } + let(:diffable) { merge_request.merge_request_diff } + let(:diff_options) { {} } + + describe '#overflow?' do + subject(:overflown) { described_class.new(diffable, project: merge_request.project, diff_options: diff_options).overflow? } + + context 'when it is not overflown' do + it 'returns false' do + expect(overflown).to eq(false) + end + end + + context 'when it is overflown' do + let(:diff_options) { { max_files: 1 } } + + it 'returns true' do + expect(overflown).to eq(true) + end + end + end +end diff --git a/spec/lib/gitlab/diff/file_collection/merge_request_diff_base_spec.rb b/spec/lib/gitlab/diff/file_collection/merge_request_diff_base_spec.rb new file mode 100644 index 00000000000..022ee5504d6 --- /dev/null +++ b/spec/lib/gitlab/diff/file_collection/merge_request_diff_base_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Diff::FileCollection::MergeRequestDiffBase do + let(:merge_request) { create(:merge_request) } + let(:diffable) { merge_request.merge_request_diff } + + describe '#overflow?' do + subject(:overflown) { described_class.new(diffable, diff_options: nil).overflow? } + + context 'when it is not overflown' do + it 'returns false' do + expect(overflown).to eq(false) + end + end + + context 'when it is overflown' do + before do + diffable.update!(state: :overflow) + end + + it 'returns true' do + expect(overflown).to eq(true) + end + end + end +end diff --git a/spec/serializers/merge_request_diff_entity_spec.rb b/spec/serializers/merge_request_diff_entity_spec.rb index fae0cd7a0f9..9bf95e68874 100644 --- a/spec/serializers/merge_request_diff_entity_spec.rb +++ b/spec/serializers/merge_request_diff_entity_spec.rb @@ -29,7 +29,7 @@ RSpec.describe MergeRequestDiffEntity do expect(subject).to include( :version_index, :created_at, :commits_count, :latest, :short_commit_sha, :version_path, - :compare_path, :state + :compare_path ) end end diff --git a/spec/tooling/danger/feature_flag_spec.rb b/spec/tooling/danger/feature_flag_spec.rb index 5e495cd43c6..74e19d8f535 100644 --- a/spec/tooling/danger/feature_flag_spec.rb +++ b/spec/tooling/danger/feature_flag_spec.rb @@ -87,7 +87,11 @@ RSpec.describe Tooling::Danger::FeatureFlag do let(:feature_flag_path) { 'config/feature_flags/development/entry.yml' } let(:group) { 'group::source code' } let(:raw_yaml) do - YAML.dump('group' => group) + YAML.dump( + 'group' => group, + 'default_enabled' => true, + 'rollout_issue_url' => 'https://gitlab.com/gitlab-org/gitlab/-/issues/1' + ) end subject(:found) { described_class.new(feature_flag_path) } @@ -109,6 +113,18 @@ RSpec.describe Tooling::Danger::FeatureFlag do end end + describe '#default_enabled' do + it 'returns the default_enabled found in the YAML' do + expect(found.default_enabled).to eq(true) + end + end + + describe '#rollout_issue_url' do + it 'returns the rollout_issue_url found in the YAML' do + expect(found.rollout_issue_url).to eq('https://gitlab.com/gitlab-org/gitlab/-/issues/1') + end + end + describe '#group_match_mr_label?' do subject(:result) { found.group_match_mr_label?(mr_group_label) } diff --git a/spec/tooling/danger/project_helper_spec.rb b/spec/tooling/danger/project_helper_spec.rb index 5edec1c5229..d2d706fad3f 100644 --- a/spec/tooling/danger/project_helper_spec.rb +++ b/spec/tooling/danger/project_helper_spec.rb @@ -56,24 +56,26 @@ RSpec.describe Tooling::Danger::ProjectHelper do 'ee/doc/foo' | [:unknown] 'ee/README' | [:unknown] - 'app/assets/foo' | [:frontend] - 'app/views/foo' | [:frontend] - 'public/foo' | [:frontend] - 'scripts/frontend/foo' | [:frontend] - 'spec/javascripts/foo' | [:frontend] - 'spec/frontend/bar' | [:frontend] - 'vendor/assets/foo' | [:frontend] - 'babel.config.js' | [:frontend] - 'jest.config.js' | [:frontend] - 'package.json' | [:frontend] - 'yarn.lock' | [:frontend] - 'config/foo.js' | [:frontend] - 'config/deep/foo.js' | [:frontend] - - 'ee/app/assets/foo' | [:frontend] - 'ee/app/views/foo' | [:frontend] - 'ee/spec/javascripts/foo' | [:frontend] - 'ee/spec/frontend/bar' | [:frontend] + 'app/assets/foo' | [:frontend] + 'app/views/foo' | [:frontend] + 'public/foo' | [:frontend] + 'scripts/frontend/foo' | [:frontend] + 'spec/javascripts/foo' | [:frontend] + 'spec/frontend/bar' | [:frontend] + 'spec/frontend_integration/bar' | [:frontend] + 'vendor/assets/foo' | [:frontend] + 'babel.config.js' | [:frontend] + 'jest.config.js' | [:frontend] + 'package.json' | [:frontend] + 'yarn.lock' | [:frontend] + 'config/foo.js' | [:frontend] + 'config/deep/foo.js' | [:frontend] + + 'ee/app/assets/foo' | [:frontend] + 'ee/app/views/foo' | [:frontend] + 'ee/spec/javascripts/foo' | [:frontend] + 'ee/spec/frontend/bar' | [:frontend] + 'ee/spec/frontend_integration/bar' | [:frontend] '.gitlab/ci/frontend.gitlab-ci.yml' | %i[frontend engineering_productivity] diff --git a/tooling/danger/feature_flag.rb b/tooling/danger/feature_flag.rb index 2e65831ef9f..06c89f83e23 100644 --- a/tooling/danger/feature_flag.rb +++ b/tooling/danger/feature_flag.rb @@ -29,6 +29,14 @@ module Tooling @group ||= yaml['group'] end + def default_enabled + @default_enabled ||= yaml['default_enabled'] + end + + def rollout_issue_url + @rollout_issue_url ||= yaml['rollout_issue_url'] + end + def group_match_mr_label?(mr_group_label) mr_group_label == group end diff --git a/tooling/danger/project_helper.rb b/tooling/danger/project_helper.rb index 2ef0caad5fe..146decfa6db 100644 --- a/tooling/danger/project_helper.rb +++ b/tooling/danger/project_helper.rb @@ -49,7 +49,7 @@ module Tooling )\z}x => [:frontend, :product_intelligence], %r{\A(ee/)?app/(assets|views)/} => :frontend, %r{\A(ee/)?public/} => :frontend, - %r{\A(ee/)?spec/(javascripts|frontend)/} => :frontend, + %r{\A(ee/)?spec/(javascripts|frontend|frontend_integration)/} => :frontend, %r{\A(ee/)?vendor/assets/} => :frontend, %r{\A(ee/)?scripts/frontend/} => :frontend, %r{(\A|/)( |