Add latest changes from gitlab-org/gitlab@13-11-stable-eev13.11.0-rc43

18 files changed, 211 insertions, 82 deletions

diff --git a/config/initializers/0_inject_enterprise_edition_module.rb b/config/initializers/0_inject_enterprise_edition_module.rb
index 7478727f869..f9c82f45040 100644
--- a/config/initializers/0_inject_enterprise_edition_module.rb
+++ b/config/initializers/0_inject_enterprise_edition_module.rb
@@ -31,6 +31,12 @@ module InjectEnterpriseEditionModule
     include(ee_module) if Gitlab.ee?
   end
 
+  def prepend_if_jh(constant, with_descendants: false)
+    return unless Gitlab.jh?
+
+    prepend_module(constant.constantize, with_descendants)
+  end
+
   private
 
   def prepend_module(mod, with_descendants)
diff --git a/config/initializers/0_license.rb b/config/initializers/0_license.rb
index ce3103be2e4..3db5ec0a91a 100644
--- a/config/initializers/0_license.rb
+++ b/config/initializers/0_license.rb
@@ -1,10 +1,18 @@
 # frozen_string_literal: true
 
-Gitlab.ee do
+load_license = lambda do |dir:, license_name:|
   prefix = ENV['GITLAB_LICENSE_MODE'] == 'test' ? 'test_' : ''
-  public_key_file = File.read(Rails.root.join(".#{prefix}license_encryption_key.pub"))
+  public_key_file = File.read(Rails.root.join(dir, ".#{prefix}license_encryption_key.pub"))
   public_key = OpenSSL::PKey::RSA.new(public_key_file)
   Gitlab::License.encryption_key = public_key
 rescue
-  warn "WARNING: No valid license encryption key provided."
+  warn "WARNING: No valid #{license_name} encryption key provided."
+end
+
+Gitlab.ee do
+  load_license.call(dir: '.', license_name: 'license')
+end
+
+Gitlab.jh do
+  load_license.call(dir: 'jh', license_name: 'JH license')
 end
diff --git a/config/initializers/0_marginalia.rb b/config/initializers/0_marginalia.rb
index ab21f936cd8..7e48c9d4fcd 100644
--- a/config/initializers/0_marginalia.rb
+++ b/config/initializers/0_marginalia.rb
@@ -13,7 +13,7 @@ require 'marginalia'
 # matching against the raw SQL, and prepending the comment prevents color
 # coding from working in the development log.
 Marginalia::Comment.prepend_comment = true if Rails.env.production?
-Marginalia::Comment.components = [:application, :controller, :action, :correlation_id, :jid, :job_class, :endpoint_id]
+Marginalia::Comment.components = [:application, :correlation_id, :jid, :endpoint_id]
 
 # As mentioned in https://github.com/basecamp/marginalia/pull/93/files,
 # adding :line has some overhead because a regexp on the backtrace has
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index b8dc464deed..99335321f28 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -68,7 +68,7 @@ if Settings.ldap['enabled'] || Rails.env.test?
     server['tls_options'] ||= {}
 
     if server['ssl_version'] || server['ca_file']
-      Rails.logger.warn 'DEPRECATED: LDAP options `ssl_version` and `ca_file` should be nested within `tls_options`' # rubocop:disable Gitlab/RailsLogger
+      Gitlab::AppLogger.warn 'DEPRECATED: LDAP options `ssl_version` and `ca_file` should be nested within `tls_options`'
     end
 
     if server['ssl_version']
@@ -184,8 +184,8 @@ Settings.gitlab['user'] ||= 'git'
 Settings.gitlab['ssh_user'] ||= Settings.gitlab.user
 Settings.gitlab['user_home'] ||= begin
   Etc.getpwnam(Settings.gitlab['user']).dir
-                                 rescue ArgumentError # no user configured
-                                   '/home/' + Settings.gitlab['user']
+rescue ArgumentError # no user configured
+  '/home/' + Settings.gitlab['user']
 end
 Settings.gitlab['time_zone'] ||= nil
 Settings.gitlab['signup_enabled'] ||= true if Settings.gitlab['signup_enabled'].nil?
@@ -310,6 +310,9 @@ Settings.pages['secret_file'] ||= Rails.root.join('.gitlab_pages_secret')
 # this will allow us to easier migrate existing instances with NFS
 Settings.pages['storage_path']      = Settings.pages['path']
 Settings.pages['object_store']      = ObjectStoreSettings.legacy_parse(Settings.pages['object_store'])
+Settings.pages['local_store'] ||= Settingslogic.new({})
+Settings.pages['local_store']['path'] = Settings.absolute(Settings.pages['local_store']['path'] || File.join(Settings.shared['path'], "pages"))
+Settings.pages['local_store']['enabled'] = true if Settings.pages['local_store']['enabled'].nil?
 
 #
 # GitLab documentation
@@ -512,9 +515,6 @@ Settings.cron_jobs['pages_domain_ssl_renewal_cron_worker']['job_class'] = 'Pages
 Settings.cron_jobs['issue_due_scheduler_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['issue_due_scheduler_worker']['cron'] ||= '50 00 * * *'
 Settings.cron_jobs['issue_due_scheduler_worker']['job_class'] = 'IssueDueSchedulerWorker'
-Settings.cron_jobs['prune_web_hook_logs_worker'] ||= Settingslogic.new({})
-Settings.cron_jobs['prune_web_hook_logs_worker']['cron'] ||= '0 */1 * * *'
-Settings.cron_jobs['prune_web_hook_logs_worker']['job_class'] = 'PruneWebHookLogsWorker'
 Settings.cron_jobs['metrics_dashboard_schedule_annotations_prune_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['metrics_dashboard_schedule_annotations_prune_worker']['cron'] ||= '0 1 * * *'
 Settings.cron_jobs['metrics_dashboard_schedule_annotations_prune_worker']['job_class'] = 'Metrics::Dashboard::ScheduleAnnotationsPruneWorker'
@@ -560,16 +560,25 @@ Settings.cron_jobs['manage_evidence_worker']['job_class'] = 'Releases::ManageEvi
 Settings.cron_jobs['user_status_cleanup_batch_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['user_status_cleanup_batch_worker']['cron'] ||= '* * * * *'
 Settings.cron_jobs['user_status_cleanup_batch_worker']['job_class'] = 'UserStatusCleanup::BatchWorker'
+Settings.cron_jobs['ssh_keys_expired_notification_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['ssh_keys_expired_notification_worker']['cron'] ||= '0 2 * * *'
+Settings.cron_jobs['ssh_keys_expired_notification_worker']['job_class'] = 'SshKeys::ExpiredNotificationWorker'
+Settings.cron_jobs['namespaces_in_product_marketing_emails_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['namespaces_in_product_marketing_emails_worker']['cron'] ||= '0 9 * * *'
+Settings.cron_jobs['namespaces_in_product_marketing_emails_worker']['job_class'] = 'Namespaces::InProductMarketingEmailsWorker'
+Settings.cron_jobs['ssh_keys_expiring_soon_notification_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['ssh_keys_expiring_soon_notification_worker']['cron'] ||= '0 1 * * *'
+Settings.cron_jobs['ssh_keys_expiring_soon_notification_worker']['job_class'] = 'SshKeys::ExpiringSoonNotificationWorker'
 
 Gitlab.com do
-  Settings.cron_jobs['namespaces_in_product_marketing_emails_worker'] ||= Settingslogic.new({})
-  Settings.cron_jobs['namespaces_in_product_marketing_emails_worker']['cron'] ||= '0 9 * * *'
-  Settings.cron_jobs['namespaces_in_product_marketing_emails_worker']['job_class'] = 'Namespaces::InProductMarketingEmailsWorker'
+  Settings.cron_jobs['batched_background_migrations_worker'] ||= Settingslogic.new({})
+  Settings.cron_jobs['batched_background_migrations_worker']['cron'] ||= '* * * * *'
+  Settings.cron_jobs['batched_background_migrations_worker']['job_class'] = 'Database::BatchedBackgroundMigrationWorker'
 end
 
 Gitlab.ee do
   Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker'] ||= Settingslogic.new({})
-  Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['cron'] ||= '0 4 * * *'
+  Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['cron'] ||= '0 4 * * 0'
   Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['job_class'] = 'Analytics::DevopsAdoption::CreateAllSnapshotsWorker'
   Settings.cron_jobs['active_user_count_threshold_worker'] ||= Settingslogic.new({})
   Settings.cron_jobs['active_user_count_threshold_worker']['cron'] ||= '0 12 * * *'
@@ -586,6 +595,9 @@ Gitlab.ee do
   Settings.cron_jobs['geo_verification_cron_worker'] ||= Settingslogic.new({})
   Settings.cron_jobs['geo_verification_cron_worker']['cron'] ||= '* * * * *'
   Settings.cron_jobs['geo_verification_cron_worker']['job_class'] ||= 'Geo::VerificationCronWorker'
+  Settings.cron_jobs['geo_sync_timeout_cron_worker'] ||= Settingslogic.new({})
+  Settings.cron_jobs['geo_sync_timeout_cron_worker']['cron'] ||= '*/10 * * * *'
+  Settings.cron_jobs['geo_sync_timeout_cron_worker']['job_class'] ||= 'Geo::SyncTimeoutCronWorker'
   Settings.cron_jobs['geo_secondary_usage_data_cron_worker'] ||= Settingslogic.new({})
   Settings.cron_jobs['geo_secondary_usage_data_cron_worker']['cron'] ||= '0 0 * * 0'
   Settings.cron_jobs['geo_secondary_usage_data_cron_worker']['job_class'] ||= 'Geo::SecondaryUsageDataCronWorker'
diff --git a/config/initializers/active_record_ping.rb b/config/initializers/active_record_ping.rb
index 196f587f565..7088c690a51 100644
--- a/config/initializers/active_record_ping.rb
+++ b/config/initializers/active_record_ping.rb
@@ -2,6 +2,6 @@
 
 # # frozen_string_literal: true
 
-if Gitlab::Utils.to_boolean(ENV['ENABLE_ACTIVERECORD_EMPTY_PING'], default: false)
+if Gitlab::Utils.to_boolean(ENV['ENABLE_ACTIVERECORD_EMPTY_PING'], default: true)
   ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(Gitlab::Database::PostgresqlAdapter::EmptyQueryPing)
 end
diff --git a/config/initializers/active_record_preloader.rb b/config/initializers/active_record_preloader.rb
index 349ca6c4831..257a8a9e955 100644
--- a/config/initializers/active_record_preloader.rb
+++ b/config/initializers/active_record_preloader.rb
@@ -9,6 +9,7 @@ module ActiveRecord
         end
 
         def self.run
+          self
         end
 
         def self.preloaded_records
diff --git a/config/initializers/bullet.rb b/config/initializers/bullet.rb
index 2d21514b121..3ef426aaadc 100644
--- a/config/initializers/bullet.rb
+++ b/config/initializers/bullet.rb
@@ -1,18 +1,18 @@
 # frozen_string_literal: true
 
-def bullet_enabled?
-  Gitlab::Utils.to_boolean(ENV['ENABLE_BULLET'].to_s)
-end
-
-if defined?(Bullet) && (bullet_enabled? || Rails.env.development?)
+if Gitlab::Bullet.configure_bullet?
   Rails.application.configure do
     config.after_initialize do
       Bullet.enable = true
 
-      Bullet.bullet_logger = bullet_enabled?
-      Bullet.console = bullet_enabled?
+      if Gitlab::Bullet.extra_logging_enabled?
+        Bullet.bullet_logger = true
+        Bullet.console = true
+      end
 
       Bullet.raise = Rails.env.test?
+
+      Bullet.stacktrace_excludes = Gitlab::Bullet::Exclusions.new.execute
     end
   end
 end
diff --git a/config/initializers/graphql.rb b/config/initializers/graphql.rb
index f1bc289f1f0..52c26e756a5 100644
--- a/config/initializers/graphql.rb
+++ b/config/initializers/graphql.rb
@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
 GraphQL::ObjectType.accepts_definitions(authorize: GraphQL::Define.assign_metadata_key(:authorize))
-GraphQL::Field.accepts_definitions(authorize: GraphQL::Define.assign_metadata_key(:authorize))
 
 GraphQL::Schema::Object.accepts_definition(:authorize)
-GraphQL::Schema::Field.accepts_definition(:authorize)
diff --git a/config/initializers/hangouts_chat_http_override.rb b/config/initializers/hangouts_chat_http_override.rb
index 4fd886697e4..edb31ed53f1 100644
--- a/config/initializers/hangouts_chat_http_override.rb
+++ b/config/initializers/hangouts_chat_http_override.rb
@@ -17,9 +17,8 @@ module HangoutsChat
             headers: { 'Content-Type' => 'application/json' },
             parse: nil # disables automatic response parsing
           )
-          net_http_response = httparty_response.response
+          httparty_response.response
           # The rest of the integration expects a Net::HTTP response
-          net_http_response
         end
       end
 
diff --git a/config/initializers/json_validator_patch.rb b/config/initializers/json_validator_patch.rb
deleted file mode 100644
index cb4158045ee..00000000000
--- a/config/initializers/json_validator_patch.rb
+++ /dev/null
@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-# This patches https://github.com/ruby-json-schema/json-schema/blob/765e6d8fdbfdaca1a42fa743f4621e757f9f6a03/lib/json-schema/validator.rb
-# to address https://github.com/ruby-json-schema/json-schema/issues/148.
-require 'json-schema'
-
-module JSON
-  class Validator
-    def initialize_data(data)
-      if @options[:parse_data]
-        if @options[:json]
-          data = self.class.parse(data)
-        elsif @options[:uri]
-          json_uri = Util::URI.normalized_uri(data)
-          data = self.class.parse(custom_open(json_uri))
-        elsif data.is_a?(String)
-          begin
-            data = self.class.parse(data)
-          rescue JSON::Schema::JsonParseError
-            # Silently discard the error - use the data as-is
-          end
-        end
-      end
-
-      JSON::Schema.stringify(data)
-    end
-  end
-end
diff --git a/config/initializers/kramdown_patch.rb b/config/initializers/kramdown_patch.rb
deleted file mode 100644
index 5cb769cec24..00000000000
--- a/config/initializers/kramdown_patch.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-#
-# This pulls in https://github.com/gettalong/kramdown/pull/708 for kramdown v2.3.0.
-# Remove this file when that pull request is merged and released.
-require 'kramdown/converter'
-require 'kramdown/converter/syntax_highlighter/rouge'
-
-module Kramdown::Converter::SyntaxHighlighter
-  module Rouge
-    def self.formatter_class(opts = {})
-      case formatter = opts[:formatter]
-      when Class
-        formatter
-      when /\A[[:upper:]][[:alnum:]_]*\z/
-        ::Rouge::Formatters.const_get(formatter, false)
-      else
-        # Available in Rouge 2.0 or later
-        ::Rouge::Formatters::HTMLLegacy
-      end
-    rescue NameError
-      # Fallback to Rouge 1.x
-      ::Rouge::Formatters::HTML
-    end
-  end
-end
diff --git a/config/initializers/pages_storage_check.rb b/config/initializers/pages_storage_check.rb
new file mode 100644
index 00000000000..8e0e0464ed6
--- /dev/null
+++ b/config/initializers/pages_storage_check.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# This is to make sure at least one storage strategy for Pages is enabled.
+
+pages = Settings.pages
+
+return unless pages['enabled'] && pages['local_store']
+
+local_store_enabled = Gitlab::Utils.to_boolean(pages['local_store']['enabled'])
+object_store_enabled = Gitlab::Utils.to_boolean(pages['object_store']['enabled'])
+
+if !local_store_enabled && !object_store_enabled
+  raise "Please enable at least one of the two Pages storage strategy (local_store or object_store) in your config/gitlab.yml."
+end
diff --git a/config/initializers/postgres_cte_as_materialized.rb b/config/initializers/postgres_cte_as_materialized.rb
new file mode 100644
index 00000000000..85b3361e25e
--- /dev/null
+++ b/config/initializers/postgres_cte_as_materialized.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# This patch adds support for AS MATERIALIZED in Arel, see Gitlab::Database::AsWithMaterialized for more info
+module Arel
+  module Visitors
+    class Arel::Visitors::PostgreSQL
+      def visit_Gitlab_Database_AsWithMaterialized(obj, collector) # rubocop:disable Naming/MethodName
+        collector = visit obj.left, collector
+        collector << " AS#{obj.expr} "
+        visit obj.right, collector
+      end
+    end
+  end
+end
diff --git a/config/initializers/postgresql_cte.rb b/config/initializers/postgresql_cte.rb
index 1ea0b4cfb58..6a9af7b4868 100644
--- a/config/initializers/postgresql_cte.rb
+++ b/config/initializers/postgresql_cte.rb
@@ -121,6 +121,8 @@ module ActiveRecord
           end
         when Arel::Nodes::As
           with_value
+        when Gitlab::Database::AsWithMaterialized
+          with_value
         end
       end
 
diff --git a/config/initializers/puma_client_tempfile_patch.rb b/config/initializers/puma_client_tempfile_patch.rb
new file mode 100644
index 00000000000..e1faa21804f
--- /dev/null
+++ b/config/initializers/puma_client_tempfile_patch.rb
@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+if Gitlab::Runtime.puma?
+  raise "Remove this monkey patch: #{__FILE__}" unless Puma::Const::VERSION == '5.1.1'
+
+  if ENV['GITLAB_TEMPFILE_IMMEDIATE_UNLINK'] == '1'
+    # This is copied from https://github.com/puma/puma/blob/v5.1.1/lib/puma/client.rb,
+    # with two additions: both times we create a temporary file, we immediately
+    # call `#unlink`. This means that if the process gets terminated without being
+    # able to clean up itself, the temporary file will not linger on the file
+    # system. We will try to get this patch accepted upstream if it works for us
+    # (we just need to check if the temporary file responds to `#unlink` as that
+    # won't work on Windows, for instance).
+    module Puma
+      class Client
+        private
+
+        def setup_body
+          @body_read_start = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+
+          if @env[HTTP_EXPECT] == CONTINUE
+            # TODO allow a hook here to check the headers before
+            # going forward
+            @io << HTTP_11_100
+            @io.flush
+          end
+
+          @read_header = false
+
+          body = @parser.body
+
+          te = @env[TRANSFER_ENCODING2]
+
+          if te
+            if te.include?(",")
+              te.split(",").each do |part|
+                if CHUNKED.casecmp(part.strip) == 0 # rubocop:disable Metrics/BlockNesting
+                  return setup_chunked_body(body)
+                end
+              end
+            elsif CHUNKED.casecmp(te) == 0
+              return setup_chunked_body(body)
+            end
+          end
+
+          @chunked_body = false
+
+          cl = @env[CONTENT_LENGTH]
+
+          unless cl
+            @buffer = body.empty? ? nil : body
+            @body = EmptyBody
+            set_ready
+            return true
+          end
+
+          remain = cl.to_i - body.bytesize
+
+          if remain <= 0
+            @body = StringIO.new(body)
+            @buffer = nil
+            set_ready
+            return true
+          end
+
+          if remain > MAX_BODY
+            @body = Tempfile.new(Const::PUMA_TMP_BASE)
+            @body.binmode
+            @body.unlink # This is the changed part
+            @tempfile = @body
+          else
+            # The body[0,0] trick is to get an empty string in the same
+            # encoding as body.
+            @body = StringIO.new body[0,0] # rubocop:disable Layout/SpaceAfterComma
+          end
+
+          @body.write body
+
+          @body_remain = remain
+
+          return false # rubocop:disable Style/RedundantReturn
+        end
+
+        def setup_chunked_body(body)
+          @chunked_body = true
+          @partial_part_left = 0
+          @prev_chunk = ""
+
+          @body = Tempfile.new(Const::PUMA_TMP_BASE)
+          @body.binmode
+          @body.unlink # This is the changed part
+          @tempfile = @body
+          @chunked_content_length = 0
+
+          if decode_chunk(body)
+            @env[CONTENT_LENGTH] = @chunked_content_length
+            return true # rubocop:disable Style/RedundantReturn
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/config/initializers/query_limiting.rb b/config/initializers/query_limiting.rb
index 66aefc97c6a..7dfa27c395a 100644
--- a/config/initializers/query_limiting.rb
+++ b/config/initializers/query_limiting.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-if Gitlab::QueryLimiting.enable?
+if Gitlab::QueryLimiting.enabled_for_env?
   require_dependency 'gitlab/query_limiting/active_support_subscriber'
   require_dependency 'gitlab/query_limiting/transaction'
   require_dependency 'gitlab/query_limiting/middleware'
diff --git a/config/initializers/smtp_settings.rb.sample b/config/initializers/smtp_settings.rb.sample
index bd37080b1c8..4a50c29143d 100644
--- a/config/initializers/smtp_settings.rb.sample
+++ b/config/initializers/smtp_settings.rb.sample
@@ -5,7 +5,7 @@
 #
 # For full list of options and their values see http://api.rubyonrails.org/classes/ActionMailer/Base.html
 #
-# If you change this file in a Merge Request, please also create a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
+# If you change this file in a merge request, please also create a merge request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
 
 if Rails.env.production?
   Rails.application.config.action_mailer.delivery_method = :smtp
@@ -22,3 +22,28 @@ if Rails.env.production?
     openssl_verify_mode: 'peer' # See ActionMailer documentation for other possible options
   }
 end
+
+# To use an SMTP connection pool, uncomment the following section:
+#
+# require 'mail/smtp_pool'
+#
+# ActionMailer::Base.add_delivery_method :smtp_pool, Mail::SMTPPool
+#
+# if Rails.env.production?
+#   Rails.application.config.action_mailer.delivery_method = :smtp_pool
+#
+#   ActionMailer::Base.delivery_method = :smtp_pool
+#   ActionMailer::Base.smtp_pool_settings = {
+#     pool: Mail::SMTPPool.create_pool(
+#       pool_size: Gitlab::Runtime.max_threads,
+#       address: "email.server.com",
+#       port: 465,
+#       user_name: "smtp",
+#       password: "123456",
+#       domain: "gitlab.company.com",
+#       authentication: :login,
+#       enable_starttls_auto: true,
+#       openssl_verify_mode: 'peer' # See ActionMailer documentation for other possible options
+#     )
+#   }
+# end
diff --git a/config/initializers/trusted_proxies.rb b/config/initializers/trusted_proxies.rb
index 79e4b831c5e..f03561b5617 100644
--- a/config/initializers/trusted_proxies.rb
+++ b/config/initializers/trusted_proxies.rb
@@ -17,7 +17,7 @@ end
 
 gitlab_trusted_proxies = Array(Gitlab.config.gitlab.trusted_proxies).map do |proxy|
   IPAddr.new(proxy)
-                         rescue IPAddr::InvalidAddressError
+rescue IPAddr::InvalidAddressError
 end.compact
 
 Rails.application.config.action_dispatch.trusted_proxies = (