diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-10-07 15:06:18 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-10-07 15:06:18 +0300 |
commit | 185f428fa5e6123ffa0f29e307523da138e7b028 (patch) | |
tree | 1d5bb1d4700c0953aed2ad0e5d3515cc7935e550 | |
parent | ab2382923e7a864a3fa27fdf8eeb9b21893b9147 (diff) |
Add latest changes from gitlab-org/gitlab@master
-rw-r--r-- | app/models/concerns/analytics/cycle_analytics/stage.rb | 2 | ||||
-rw-r--r-- | changelogs/unreleased/33337-fix-insights-only-projects-for-project.yml | 5 | ||||
-rw-r--r-- | changelogs/unreleased/ignore-autreply.yml | 5 | ||||
-rw-r--r-- | changelogs/unreleased/jc-remove-cleanup-rake-tasks.yml | 5 | ||||
-rw-r--r-- | doc/raketasks/cleanup.md | 40 | ||||
-rw-r--r-- | doc/user/project/merge_requests/merge_request_dependencies.md | 6 | ||||
-rw-r--r-- | lib/gitlab/analytics/cycle_analytics/default_stages.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/email/receiver.rb | 21 | ||||
-rw-r--r-- | lib/gitlab/import_export/fast_hash_serializer.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/metrics/exporter/base_exporter.rb | 3 | ||||
-rw-r--r-- | lib/tasks/gitlab/cleanup.rake | 63 | ||||
-rwxr-xr-x | scripts/review_apps/review-apps.sh | 24 | ||||
-rw-r--r-- | spec/fixtures/emails/auto_reply.eml | 2 | ||||
-rw-r--r-- | spec/fixtures/emails/auto_submitted.eml | 21 | ||||
-rw-r--r-- | spec/lib/gitlab/email/receiver_spec.rb | 10 | ||||
-rw-r--r-- | spec/support/shared_examples/cycle_analytics_stage_examples.rb | 7 | ||||
-rw-r--r-- | spec/tasks/gitlab/cleanup_rake_spec.rb | 68 |
17 files changed, 89 insertions, 203 deletions
diff --git a/app/models/concerns/analytics/cycle_analytics/stage.rb b/app/models/concerns/analytics/cycle_analytics/stage.rb index 0c603c2d5e6..0011ba10a9a 100644 --- a/app/models/concerns/analytics/cycle_analytics/stage.rb +++ b/app/models/concerns/analytics/cycle_analytics/stage.rb @@ -7,6 +7,7 @@ module Analytics included do validates :name, presence: true + validates :name, exclusion: { in: Gitlab::Analytics::CycleAnalytics::DefaultStages.names }, if: :custom? validates :start_event_identifier, presence: true validates :end_event_identifier, presence: true validate :validate_stage_event_pairs @@ -15,6 +16,7 @@ module Analytics enum end_event_identifier: Gitlab::Analytics::CycleAnalytics::StageEvents.to_enum, _prefix: :end_event_identifier alias_attribute :custom_stage?, :custom + scope :default_stages, -> { where(custom: false) } end def parent=(_) diff --git a/changelogs/unreleased/33337-fix-insights-only-projects-for-project.yml b/changelogs/unreleased/33337-fix-insights-only-projects-for-project.yml new file mode 100644 index 00000000000..7dea4519d10 --- /dev/null +++ b/changelogs/unreleased/33337-fix-insights-only-projects-for-project.yml @@ -0,0 +1,5 @@ +--- +title: Make sure project insights stick on its own +merge_request: 18082 +author: +type: fixed diff --git a/changelogs/unreleased/ignore-autreply.yml b/changelogs/unreleased/ignore-autreply.yml new file mode 100644 index 00000000000..69f9885a16f --- /dev/null +++ b/changelogs/unreleased/ignore-autreply.yml @@ -0,0 +1,5 @@ +--- +title: Ignore incoming emails with X-Autoreply header. +merge_request: 18118 +author: +type: fixed diff --git a/changelogs/unreleased/jc-remove-cleanup-rake-tasks.yml b/changelogs/unreleased/jc-remove-cleanup-rake-tasks.yml new file mode 100644 index 00000000000..fd2d0e323e8 --- /dev/null +++ b/changelogs/unreleased/jc-remove-cleanup-rake-tasks.yml @@ -0,0 +1,5 @@ +--- +title: Removing cleanup:repo, cleanup:dirs +merge_request: 18087 +author: +type: deprecated diff --git a/doc/raketasks/cleanup.md b/doc/raketasks/cleanup.md index 957f583cd7d..67bf7cbd828 100644 --- a/doc/raketasks/cleanup.md +++ b/doc/raketasks/cleanup.md @@ -2,46 +2,6 @@ ## Remove garbage from filesystem -DANGER: **Danger:** -The commands below will remove data permanently from your GitLab instance. Only use -these commands if you are 100% certain that it is safe to delete this data. - -Remove namespaces(dirs) from all repository storage paths if they don't exist in GitLab database. - -``` -# omnibus-gitlab -sudo gitlab-rake gitlab:cleanup:dirs - -# installation from source -bundle exec rake gitlab:cleanup:dirs RAILS_ENV=production -``` - -DANGER: **Danger:** -The following task does not currently work as expected. -The use will probably mark more existing repositories as orphaned. -For more information, see the [issue](https://gitlab.com/gitlab-org/gitlab/issues/24633). - -Rename repositories from all repository storage paths if they don't exist in GitLab database. -The repositories get a `+orphaned+TIMESTAMP` suffix so that they cannot block new repositories from being created. - -``` -# omnibus-gitlab -sudo gitlab-rake gitlab:cleanup:repos - -# installation from source -bundle exec rake gitlab:cleanup:repos RAILS_ENV=production -``` - -Remove old repository copies from repositories moved to another storage. - -``` -# omnibus-gitlab -sudo gitlab-rake gitlab:cleanup:moved - -# installation from source -bundle exec rake gitlab:cleanup:moved RAILS_ENV=production -``` - Clean up local project upload files if they don't exist in GitLab database. The task attempts to fix the file if it can find its project, otherwise it moves the file to a lost and found directory. diff --git a/doc/user/project/merge_requests/merge_request_dependencies.md b/doc/user/project/merge_requests/merge_request_dependencies.md index f1a02e837a5..c99e6663093 100644 --- a/doc/user/project/merge_requests/merge_request_dependencies.md +++ b/doc/user/project/merge_requests/merge_request_dependencies.md @@ -4,7 +4,11 @@ type: reference, concepts # Merge Request dependencies **(PREMIUM)** -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9688) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.2. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/9688) in +[GitLab Premium](https://about.gitlab.com/pricing/) 12.2. +> - [Renamed](https://gitlab.com/gitlab-org/gitlab/merge_requests/17291) from +"Cross-project dependencies" to "Merge Requests dependencies" in +[GitLab Premium](https://about.gitlab.com/pricing/) 12.4. Merge request dependencies allows a required order of merging between merge requests to be expressed. If a merge request "depends on" another, diff --git a/lib/gitlab/analytics/cycle_analytics/default_stages.rb b/lib/gitlab/analytics/cycle_analytics/default_stages.rb index 286c393005f..711645800fb 100644 --- a/lib/gitlab/analytics/cycle_analytics/default_stages.rb +++ b/lib/gitlab/analytics/cycle_analytics/default_stages.rb @@ -23,6 +23,10 @@ module Gitlab ] end + def self.names + all.map { |stage| stage[:name] } + end + def self.params_for_issue_stage { name: 'issue', diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 7da8b385266..847260b2e0f 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -32,7 +32,7 @@ module Gitlab mail = build_mail - ignore_auto_submitted!(mail) + ignore_auto_reply!(mail) mail_key = extract_mail_key(mail) handler = Handler.for(mail, mail_key) @@ -96,14 +96,25 @@ module Gitlab end end - def ignore_auto_submitted!(mail) + def ignore_auto_reply!(mail) + if auto_submitted?(mail) || auto_replied?(mail) + raise AutoGeneratedEmailError + end + end + + def auto_submitted?(mail) # Mail::Header#[] is case-insensitive auto_submitted = mail.header['Auto-Submitted']&.value # Mail::Field#value would strip leading and trailing whitespace - raise AutoGeneratedEmailError if - # See also https://tools.ietf.org/html/rfc3834 - auto_submitted && auto_submitted != 'no' + # See also https://tools.ietf.org/html/rfc3834 + auto_submitted && auto_submitted != 'no' + end + + def auto_replied?(mail) + autoreply = mail.header['X-Autoreply']&.value + + autoreply && autoreply == 'yes' end end end diff --git a/lib/gitlab/import_export/fast_hash_serializer.rb b/lib/gitlab/import_export/fast_hash_serializer.rb index e2d5c5ae055..5a067b5c9f3 100644 --- a/lib/gitlab/import_export/fast_hash_serializer.rb +++ b/lib/gitlab/import_export/fast_hash_serializer.rb @@ -135,10 +135,7 @@ module Gitlab data = [] - # rubocop:disable Cop/InBatches - # If we put `rubocop:disable` inline after `do |batch|`, - # `Cop/LineBreakAroundConditionalBlock` will fail - record.in_batches(of: @batch_size) do |batch| + record.in_batches(of: @batch_size) do |batch| # rubocop:disable Cop/InBatches if Feature.enabled?(:export_fast_serialize_with_raw_json, default_enabled: true) data.append(JSONBatchRelation.new(batch, options, preloads[key]).tap(&:raw_json)) else @@ -146,7 +143,6 @@ module Gitlab data += batch.as_json(options) end end - # rubocop:enable Cop/InBatches data end diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb index 108d1330b47..de7870dfb8c 100644 --- a/lib/gitlab/metrics/exporter/base_exporter.rb +++ b/lib/gitlab/metrics/exporter/base_exporter.rb @@ -36,10 +36,11 @@ module Gitlab end def stop_working - if server # rubocop:disable Cop/LineBreakAroundConditionalBlock + if server server.shutdown server.listeners.each(&:close) end + @server = nil end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 4d854cd178d..0a0ee7b4bfa 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -3,69 +3,6 @@ require 'set' namespace :gitlab do namespace :cleanup do - desc "GitLab | Cleanup | Clean namespaces" - task dirs: :gitlab_environment do - namespaces = Set.new(Namespace.pluck(:path)) - namespaces << Storage::HashedProject::REPOSITORY_PATH_PREFIX - - Gitaly::Server.all.each do |server| - all_dirs = Gitlab::GitalyClient::StorageService - .new(server.storage) - .list_directories(depth: 0) - .reject { |dir| dir.ends_with?('.git') || namespaces.include?(File.basename(dir)) } - - puts "Looking for directories to remove... " - all_dirs.each do |dir_path| - if remove? - begin - Gitlab::GitalyClient::NamespaceService.new(server.storage) - .remove(dir_path) - - puts "Removed...#{dir_path}" - rescue StandardError => e - puts "Cannot remove #{dir_path}: #{e.message}".color(:red) - end - else - puts "Can be removed: #{dir_path}".color(:red) - end - end - end - - unless remove? - puts "To cleanup this directories run this command with REMOVE=true".color(:yellow) - end - end - - desc "GitLab | Cleanup | Clean repositories" - task repos: :gitlab_environment do - move_suffix = "+orphaned+#{Time.now.to_i}" - - Gitaly::Server.all.each do |server| - Gitlab::GitalyClient::StorageService - .new(server.storage) - .list_directories - .each do |path| - repo_with_namespace = path.chomp('.git').chomp('.wiki') - - # TODO ignoring hashed repositories for now. But revisit to fully support - # possible orphaned hashed repos - next if repo_with_namespace.start_with?(Storage::HashedProject::REPOSITORY_PATH_PREFIX) - next if Project.find_by_full_path(repo_with_namespace) - - new_path = path + move_suffix - puts path.inspect + ' -> ' + new_path.inspect - - begin - Gitlab::GitalyClient::NamespaceService - .new(server.storage) - .rename(path, new_path) - rescue StandardError => e - puts "Error occurred while moving the repository: #{e.message}".color(:red) - end - end - end - end - desc "GitLab | Cleanup | Block users that have been removed in LDAP" task block_removed_ldap_users: :gitlab_environment do warn_user_is_not_gitlab diff --git a/scripts/review_apps/review-apps.sh b/scripts/review_apps/review-apps.sh index a4fb4749720..4c44a32c7dd 100755 --- a/scripts/review_apps/review-apps.sh +++ b/scripts/review_apps/review-apps.sh @@ -255,25 +255,13 @@ EOF } function display_deployment_debug() { - kubectl get pods -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG} + # Get all pods that are not ready (this will return completed pods for minio and migrations jobs) + echoinfo "Unready Pods for release ${CI_ENVIRONMENT_SLUG}" + kubectl get pods -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG} --field-selector=status.phase!=Running - migrations_pod=$(get_pod "migrations"); - if [ -z "${migrations_pod}" ]; then - echoerr "Migrations pod not found." - else - echoinfo "Logs tail of the ${migrations_pod} pod..." - - kubectl logs -n "$KUBE_NAMESPACE" "${migrations_pod}" | sed "s/${REVIEW_APPS_ROOT_PASSWORD}/[REDACTED]/g" - fi - - unicorn_pod=$(get_pod "unicorn"); - if [ -z "${unicorn_pod}" ]; then - echoerr "Unicorn pod not found." - else - echoinfo "Logs tail of the ${unicorn_pod} pod..." - - kubectl logs -n "$KUBE_NAMESPACE" -c unicorn "${unicorn_pod}" | sed "s/${REVIEW_APPS_ROOT_PASSWORD}/[REDACTED]/g" - fi + # Get all non-completed jobs + echoinfo "Unsuccessful Jobs for release ${CI_ENVIRONMENT_SLUG}" + kubectl get jobs -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG} --field-selector=status.successful!=1 } function add_license() { diff --git a/spec/fixtures/emails/auto_reply.eml b/spec/fixtures/emails/auto_reply.eml index 7999c8d78b7..4d5a45df467 100644 --- a/spec/fixtures/emails/auto_reply.eml +++ b/spec/fixtures/emails/auto_reply.eml @@ -12,7 +12,7 @@ Mime-Version: 1.0 Content-Type: text/plain; charset=ISO-8859-1 Content-Transfer-Encoding: 7bit -Auto-Submitted: auto-generated +X-Autoreply: yes X-Sieve: CMU Sieve 2.2 X-Received: by 10.0.0.1 with SMTP id n7mr11234144ipb.85.1371157428600; Thu, 13 Jun 2013 14:03:48 -0700 (PDT) diff --git a/spec/fixtures/emails/auto_submitted.eml b/spec/fixtures/emails/auto_submitted.eml new file mode 100644 index 00000000000..7999c8d78b7 --- /dev/null +++ b/spec/fixtures/emails/auto_submitted.eml @@ -0,0 +1,21 @@ +Return-Path: <jake@adventuretime.ooo> +Received: from iceking.adventuretime.ooo ([unix socket]) by iceking (Cyrus v2.2.13-Debian-2.2.13-19+squeeze3) with LMTPA; Thu, 13 Jun 2013 17:03:50 -0400 +Received: from mail-ie0-x234.google.com (mail-ie0-x234.google.com [IPv6:2607:f8b0:4001:c03::234]) by iceking.adventuretime.ooo (8.14.3/8.14.3/Debian-9.4) with ESMTP id r5DL3nFJ016967 (version=TLSv1/SSLv3 cipher=RC4-SHA bits=128 verify=NOT) for <reply+59d8df8370b7e95c5a49fbf86aeb2c93@discourse.example.com>; Thu, 13 Jun 2013 17:03:50 -0400 +Received: by mail-ie0-f180.google.com with SMTP id f4so21977375iea.25 for <reply+59d8df8370b7e95c5a49fbf86aeb2c93@discourse.example.com>; Thu, 13 Jun 2013 14:03:48 -0700 +Received: by 10.0.0.1 with HTTP; Thu, 13 Jun 2013 14:03:48 -0700 +Date: Thu, 13 Jun 2013 17:03:48 -0400 +From: Jake the Dog <jake@adventuretime.ooo> +To: reply+636ca428858779856c226bb145ef4fad@appmail.adventuretime.ooo +Message-ID: <CADkmRc+rNGAGGbV2iE5p918UVy4UyJqVcXRO2=otppgzduJSg@mail.gmail.com> +Subject: re: [Discourse Meta] eviltrout posted in 'Adventure Time Sux' +Mime-Version: 1.0 +Content-Type: text/plain; + charset=ISO-8859-1 +Content-Transfer-Encoding: 7bit +Auto-Submitted: auto-generated +X-Sieve: CMU Sieve 2.2 +X-Received: by 10.0.0.1 with SMTP id n7mr11234144ipb.85.1371157428600; Thu, + 13 Jun 2013 14:03:48 -0700 (PDT) +X-Scanned-By: MIMEDefang 2.69 on IPv6:2001:470:1d:165::1 + +Test reply to Discourse email digest diff --git a/spec/lib/gitlab/email/receiver_spec.rb b/spec/lib/gitlab/email/receiver_spec.rb index 6b5a355e598..43c73242f5f 100644 --- a/spec/lib/gitlab/email/receiver_spec.rb +++ b/spec/lib/gitlab/email/receiver_spec.rb @@ -40,7 +40,15 @@ describe Gitlab::Email::Receiver do end end - context "when the email was auto generated" do + context "when the email was auto generated with Auto-Submitted header" do + let(:email_raw) { fixture_file("emails/auto_submitted.eml") } + + it "raises an AutoGeneratedEmailError" do + expect { receiver.execute }.to raise_error(Gitlab::Email::AutoGeneratedEmailError) + end + end + + context "when the email was auto generated with X-Autoreply header" do let(:email_raw) { fixture_file("emails/auto_reply.eml") } it "raises an AutoGeneratedEmailError" do diff --git a/spec/support/shared_examples/cycle_analytics_stage_examples.rb b/spec/support/shared_examples/cycle_analytics_stage_examples.rb index 151f5325e84..dc2ea229171 100644 --- a/spec/support/shared_examples/cycle_analytics_stage_examples.rb +++ b/spec/support/shared_examples/cycle_analytics_stage_examples.rb @@ -46,6 +46,13 @@ shared_examples_for 'cycle analytics stage' do expect(stage).not_to be_valid expect(stage.errors.details[:end_event]).to eq([{ error: :not_allowed_for_the_given_start_event }]) end + + context 'disallows default stage names when creating custom stage' do + let(:invalid_params) { valid_params.merge(name: Gitlab::Analytics::CycleAnalytics::DefaultStages.names.first, custom: true) } + let(:stage) { described_class.new(invalid_params) } + + it { expect(stage).not_to be_valid } + end end describe '#subject_model' do diff --git a/spec/tasks/gitlab/cleanup_rake_spec.rb b/spec/tasks/gitlab/cleanup_rake_spec.rb index 4aee6d005a8..6c09bb5d9f9 100644 --- a/spec/tasks/gitlab/cleanup_rake_spec.rb +++ b/spec/tasks/gitlab/cleanup_rake_spec.rb @@ -5,74 +5,6 @@ describe 'gitlab:cleanup rake tasks' do Rake.application.rake_require 'tasks/gitlab/cleanup' end - describe 'cleanup namespaces and repos' do - let(:gitlab_shell) { Gitlab::Shell.new } - let(:storage) { storages.keys.first } - let(:storages) do - { - 'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/default_storage')) - } - end - - before(:all) do - @default_storage_hash = Gitlab.config.repositories.storages.default.to_h - end - - before do - allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) - end - - after do - Gitlab::GitalyClient::StorageService.new(storage).delete_all_repositories - end - - describe 'cleanup:repos' do - before do - gitlab_shell.add_namespace(storage, 'broken/project.git') - gitlab_shell.add_namespace(storage, '@hashed/12/34/5678.git') - end - - it 'moves it to an orphaned path' do - now = Time.now - - Timecop.freeze(now) do - run_rake_task('gitlab:cleanup:repos') - repo_list = Gitlab::GitalyClient::StorageService.new(storage).list_directories(depth: 0) - - expect(repo_list.last).to include("broken+orphaned+#{now.to_i}") - end - end - - it 'ignores @hashed repos' do - run_rake_task('gitlab:cleanup:repos') - - expect(gitlab_shell.exists?(storage, '@hashed/12/34/5678.git')).to be(true) - end - end - - describe 'cleanup:dirs' do - it 'removes missing namespaces' do - gitlab_shell.add_namespace(storage, "namespace_1/project.git") - gitlab_shell.add_namespace(storage, "namespace_2/project.git") - allow(Namespace).to receive(:pluck).and_return(['namespace_1']) - - stub_env('REMOVE', 'true') - run_rake_task('gitlab:cleanup:dirs') - - expect(gitlab_shell.exists?(storage, 'namespace_1')).to be(true) - expect(gitlab_shell.exists?(storage, 'namespace_2')).to be(false) - end - - it 'ignores @hashed directory' do - gitlab_shell.add_namespace(storage, '@hashed/12/34/5678.git') - - run_rake_task('gitlab:cleanup:dirs') - - expect(gitlab_shell.exists?(storage, '@hashed/12/34/5678.git')).to be(true) - end - end - end - # A single integration test that is redundant with one part of the # Gitlab::Cleanup::ProjectUploads spec. # |