Add latest changes from gitlab-org/gitlab@13-5-stable-eev13.5.0-rc42

13 files changed, 131 insertions, 96 deletions

diff --git a/config/initializers/0_license.rb b/config/initializers/0_license.rb
index e7b46a14630..ce3103be2e4 100644
--- a/config/initializers/0_license.rb
+++ b/config/initializers/0_license.rb
@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
 Gitlab.ee do
-  public_key_file = File.read(Rails.root.join(".license_encryption_key.pub"))
+  prefix = ENV['GITLAB_LICENSE_MODE'] == 'test' ? 'test_' : ''
+  public_key_file = File.read(Rails.root.join(".#{prefix}license_encryption_key.pub"))
   public_key = OpenSSL::PKey::RSA.new(public_key_file)
   Gitlab::License.encryption_key = public_key
 rescue
diff --git a/config/initializers/0_marginalia.rb b/config/initializers/0_marginalia.rb
index a697f67dbf2..5c6cf7752c4 100644
--- a/config/initializers/0_marginalia.rb
+++ b/config/initializers/0_marginalia.rb
@@ -21,4 +21,4 @@ Gitlab::Marginalia.set_application_name
 
 Gitlab::Marginalia.enable_sidekiq_instrumentation
 
-Gitlab::Marginalia.set_feature_cache
+Gitlab::Marginalia.set_enabled_from_feature_flag
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 15c9fd427ff..affbc85d5a9 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -297,6 +297,10 @@ Settings.pages['external_http'] ||= false unless Settings.pages['external_http']
 Settings.pages['external_https'] ||= false unless Settings.pages['external_https'].present?
 Settings.pages['artifacts_server'] ||= Settings.pages['enabled'] if Settings.pages['artifacts_server'].nil?
 Settings.pages['secret_file'] ||= Rails.root.join('.gitlab_pages_secret')
+# We want pages zip archives to be stored on the same directory as old pages hierarchical structure
+# this will allow us to easier migrate existing instances with NFS
+Settings.pages['storage_path']      = Settings.pages['path']
+Settings.pages['object_store']      = ObjectStoreSettings.legacy_parse(Settings.pages['object_store'])
 
 #
 # Geo
@@ -412,6 +416,9 @@ Settings.cron_jobs['pipeline_schedule_worker']['job_class'] = 'PipelineScheduleW
 Settings.cron_jobs['expire_build_artifacts_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['expire_build_artifacts_worker']['cron'] ||= '50 * * * *'
 Settings.cron_jobs['expire_build_artifacts_worker']['job_class'] = 'ExpireBuildArtifactsWorker'
+Settings.cron_jobs['ci_schedule_delete_objects_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['ci_schedule_delete_objects_worker']['cron'] ||= '*/16 * * * *'
+Settings.cron_jobs['ci_schedule_delete_objects_worker']['job_class'] = 'Ci::ScheduleDeleteObjectsCronWorker'
 Settings.cron_jobs['environments_auto_stop_cron_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['environments_auto_stop_cron_worker']['cron'] ||= '24 * * * *'
 Settings.cron_jobs['environments_auto_stop_cron_worker']['job_class'] = 'Environments::AutoStopCronWorker'
@@ -520,8 +527,14 @@ Settings.cron_jobs['ci_platform_metrics_update_cron_worker']['job_class'] = 'CiP
 Settings.cron_jobs['analytics_instance_statistics_count_job_trigger_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['analytics_instance_statistics_count_job_trigger_worker']['cron'] ||= '50 23 */1 * *'
 Settings.cron_jobs['analytics_instance_statistics_count_job_trigger_worker']['job_class'] ||= 'Analytics::InstanceStatistics::CountJobTriggerWorker'
+Settings.cron_jobs['member_invitation_reminder_emails_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['member_invitation_reminder_emails_worker']['cron'] ||= '0 0 * * *'
+Settings.cron_jobs['member_invitation_reminder_emails_worker']['job_class'] = 'MemberInvitationReminderEmailsWorker'
 
 Gitlab.ee do
+  Settings.cron_jobs['active_user_count_threshold_worker'] ||= Settingslogic.new({})
+  Settings.cron_jobs['active_user_count_threshold_worker']['cron'] ||= '0 12 * * *'
+  Settings.cron_jobs['active_user_count_threshold_worker']['job_class'] = 'ActiveUserCountThresholdWorker'
   Settings.cron_jobs['adjourned_group_deletion_worker'] ||= Settingslogic.new({})
   Settings.cron_jobs['adjourned_group_deletion_worker']['cron'] ||= '0 3 * * *'
   Settings.cron_jobs['adjourned_group_deletion_worker']['job_class'] = 'AdjournedGroupDeletionWorker'
@@ -561,6 +574,9 @@ Gitlab.ee do
   Settings.cron_jobs['historical_data_worker'] ||= Settingslogic.new({})
   Settings.cron_jobs['historical_data_worker']['cron'] ||= '0 12 * * *'
   Settings.cron_jobs['historical_data_worker']['job_class'] = 'HistoricalDataWorker'
+  Settings.cron_jobs['incident_sla_exceeded_check_worker'] ||= Settingslogic.new({})
+  Settings.cron_jobs['incident_sla_exceeded_check_worker']['cron'] ||= '*/2 * * * *'
+  Settings.cron_jobs['incident_sla_exceeded_check_worker']['job_class'] = 'IncidentManagement::IncidentSlaExceededCheckWorker'
   Settings.cron_jobs['import_software_licenses_worker'] ||= Settingslogic.new({})
   Settings.cron_jobs['import_software_licenses_worker']['cron'] ||= '0 3 * * 0'
   Settings.cron_jobs['import_software_licenses_worker']['job_class'] = 'ImportSoftwareLicensesWorker'
@@ -727,6 +743,7 @@ Gitlab.ee do
   Settings['kerberos'] ||= Settingslogic.new({})
   Settings.kerberos['enabled'] = false if Settings.kerberos['enabled'].nil?
   Settings.kerberos['keytab'] = nil if Settings.kerberos['keytab'].blank? # nil means use default keytab
+  Settings.kerberos['simple_ldap_linking_allowed_realms'] = [] if Settings.kerberos['simple_ldap_linking_allowed_realms'].blank?
   Settings.kerberos['service_principal_name'] = nil if Settings.kerberos['service_principal_name'].blank? # nil means any SPN in keytab
   Settings.kerberos['use_dedicated_port'] = false if Settings.kerberos['use_dedicated_port'].nil?
   Settings.kerberos['https'] = Settings.gitlab.https if Settings.kerberos['https'].nil?
@@ -750,6 +767,13 @@ Gitlab.ee do
 end
 
 #
+# FortiAuthenticator
+#
+Settings['forti_authenticator'] ||= Settingslogic.new({})
+Settings.forti_authenticator['enabled'] = false if Settings.forti_authenticator['enabled'].nil?
+Settings.forti_authenticator['port'] = 443 if Settings.forti_authenticator['port'].to_i == 0
+
+#
 # Extra customization
 #
 Settings['extra'] ||= Settingslogic.new({})
@@ -774,10 +798,15 @@ Settings['gitaly'] ||= Settingslogic.new({})
 # Webpack settings
 #
 Settings['webpack'] ||= Settingslogic.new({})
+Settings.webpack['config_file'] ||= 'config/webpack.config.js'
+Settings.webpack['output_dir']  ||= 'public/assets/webpack'
+Settings.webpack['public_path'] ||= 'assets/webpack'
+Settings.webpack['manifest_filename'] ||= 'manifest.json'
 Settings.webpack['dev_server'] ||= Settingslogic.new({})
 Settings.webpack.dev_server['enabled'] ||= false
 Settings.webpack.dev_server['host']    ||= 'localhost'
 Settings.webpack.dev_server['port']    ||= 3808
+Settings.webpack.dev_server['https']   ||= false
 
 #
 # Monitoring settings
diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb
index d5d8587f1c8..dbaebc83658 100644
--- a/config/initializers/7_prometheus_metrics.rb
+++ b/config/initializers/7_prometheus_metrics.rb
@@ -69,7 +69,9 @@ if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled?
 
     Gitlab::Metrics.gauge(:deployments, 'GitLab Version', {}, :max).set({ version: Gitlab::VERSION }, 1)
 
-    Gitlab::Metrics::RequestsRackMiddleware.initialize_http_request_duration_seconds
+    unless Gitlab::Runtime.sidekiq?
+      Gitlab::Metrics::RequestsRackMiddleware.initialize_http_request_duration_seconds
+    end
   rescue IOError => e
     Gitlab::ErrorTracking.track_exception(e)
     Gitlab::Metrics.error_detected!
diff --git a/config/initializers/database_config.rb b/config/initializers/database_config.rb
index cccd4335a7d..a91f67224d7 100644
--- a/config/initializers/database_config.rb
+++ b/config/initializers/database_config.rb
@@ -20,21 +20,12 @@ Gitlab.ee do
   end
 end
 
-# Because of the way Ruby on Rails manages database connections, it is
-# important that we have at least as many connections as we have
-# threads. While there is a 'pool' setting in database.yml, it is not
-# very practical because you need to maintain it in tandem with the
-# number of application threads. Because of this we override the number
-# of allowed connections in the database connection pool based on the
-# configured number of application threads.
+# We configure the database connection pool size automatically based on the
+# configured concurrency. We also add some headroom, to make sure we don't run
+# out of connections when more threads besides the 'user-facing' ones are
+# running.
 #
-# Gitlab::Runtime.max_threads is the number of "user facing" application
-# threads the process has been configured with. We also have auxiliary
-# threads that use database connections. Because it is not practical to
-# keep an accurate count of the number auxiliary threads as the
-# application evolves over time, we just add a fixed headroom to the
-# number of user-facing threads. It is OK if this number is too large
-# because connections are instantiated lazily.
+# Read more about this in doc/development/database/client_side_connection_pool.md
 
 headroom = (ENV["DB_POOL_HEADROOM"].presence || 10).to_i
 calculated_pool_size = Gitlab::Runtime.max_threads + headroom
diff --git a/config/initializers/forbid_sidekiq_in_transactions.rb b/config/initializers/forbid_sidekiq_in_transactions.rb
index 6bcd4dbd52f..f505fb5843a 100644
--- a/config/initializers/forbid_sidekiq_in_transactions.rb
+++ b/config/initializers/forbid_sidekiq_in_transactions.rb
@@ -46,7 +46,7 @@ end
 module ActiveRecord
   class Base
     module SkipTransactionCheckAfterCommit
-      def committed!(*)
+      def committed!(*args, **kwargs)
         Sidekiq::Worker.skipping_transaction_check { super }
       end
     end
diff --git a/config/initializers/lograge.rb b/config/initializers/lograge.rb
index e3601a9538e..0ea0adf86bc 100644
--- a/config/initializers/lograge.rb
+++ b/config/initializers/lograge.rb
@@ -1,30 +1,40 @@
 # Only use Lograge for Rails
 unless Gitlab::Runtime.sidekiq?
-  filename = File.join(Rails.root, 'log', "#{Rails.env}_json.log")
+  Rails.application.reloader.to_prepare do
+    filename = File.join(Rails.root, 'log', "#{Rails.env}_json.log")
+    db_counter = Gitlab::Metrics::Subscribers::ActiveRecord
 
-  Rails.application.configure do
-    config.lograge.enabled = true
-    # Store the lograge JSON files in a separate file
-    config.lograge.keep_original_rails_log = Gitlab::Utils.to_boolean(ENV.fetch('UNSTRUCTURED_RAILS_LOG', 'true'))
-    # Don't use the Logstash formatter since this requires logstash-event, an
-    # unmaintained gem that monkey patches `Time`
-    config.lograge.formatter = Lograge::Formatters::Json.new
-    config.lograge.logger = ActiveSupport::Logger.new(filename)
-    config.lograge.before_format = lambda do |data, payload|
-      data.delete(:error)
-      data[:db_duration_s] = Gitlab::Utils.ms_to_round_sec(data.delete(:db)) if data[:db]
-      data[:view_duration_s] = Gitlab::Utils.ms_to_round_sec(data.delete(:view)) if data[:view]
-      data[:duration_s] = Gitlab::Utils.ms_to_round_sec(data.delete(:duration)) if data[:duration]
-      data.merge!(::Gitlab::Metrics::Subscribers::ActiveRecord.db_counter_payload)
+    Rails.application.configure do
+      config.lograge.enabled = true
+      # Store the lograge JSON files in a separate file
+      config.lograge.keep_original_rails_log = Gitlab::Utils.to_boolean(ENV.fetch('UNSTRUCTURED_RAILS_LOG', 'true'))
+      # Don't use the Logstash formatter since this requires logstash-event, an
+      # unmaintained gem that monkey patches `Time`
+      config.lograge.formatter = Lograge::Formatters::Json.new
+      config.lograge.logger = ActiveSupport::Logger.new(filename)
+      config.lograge.before_format = lambda do |data, payload|
+        data.delete(:error)
+        data[:db_duration_s] = Gitlab::Utils.ms_to_round_sec(data.delete(:db)) if data[:db]
+        data[:view_duration_s] = Gitlab::Utils.ms_to_round_sec(data.delete(:view)) if data[:view]
+        data[:duration_s] = Gitlab::Utils.ms_to_round_sec(data.delete(:duration)) if data[:duration]
+        data.merge!(db_counter.db_counter_payload)
 
-      data
-    end
+        # Remove empty hashes to prevent type mismatches
+        # These are set to empty hashes in Lograge's ActionCable subscriber
+        # https://github.com/roidrage/lograge/blob/v0.11.2/lib/lograge/log_subscribers/action_cable.rb#L14-L16
+        %i(method path format).each do |key|
+          data[key] = nil if data[key] == {}
+        end
+
+        data
+      end
 
-    # This isn't a user-reachable controller; we use it to check for a
-    # valid CSRF token in the API
-    config.lograge.ignore_actions = ['Gitlab::RequestForgeryProtection::Controller#index']
+      # This isn't a user-reachable controller; we use it to check for a
+      # valid CSRF token in the API
+      config.lograge.ignore_actions = ['Gitlab::RequestForgeryProtection::Controller#index']
 
-    # Add request parameters to log output
-    config.lograge.custom_options = Gitlab::Lograge::CustomOptions
+      # Add request parameters to log output
+      config.lograge.custom_options = Gitlab::Lograge::CustomOptions
+    end
   end
 end
diff --git a/config/initializers/rails_host_authorization_gitpod.rb b/config/initializers/rails_host_authorization_gitpod.rb
new file mode 100644
index 00000000000..0c1822bc91a
--- /dev/null
+++ b/config/initializers/rails_host_authorization_gitpod.rb
@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+if Rails.env.development? && ENV['GITPOD_WORKSPACE_ID'].present?
+  gitpod_host = URI(`gp url 3000`.strip).host
+  Rails.application.config.hosts += [gitpod_host]
+end
diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
index a33c28090e2..72e2b94fe07 100644
--- a/config/initializers/sidekiq.rb
+++ b/config/initializers/sidekiq.rb
@@ -3,13 +3,13 @@
 def enable_reliable_fetch?
   return true unless Feature::FlipperFeature.table_exists?
 
-  Feature.enabled?(:gitlab_sidekiq_reliable_fetcher, default_enabled: true)
+  Feature.enabled?(:gitlab_sidekiq_reliable_fetcher, type: :ops, default_enabled: true)
 end
 
 def enable_semi_reliable_fetch_mode?
   return true unless Feature::FlipperFeature.table_exists?
 
-  Feature.enabled?(:gitlab_sidekiq_enable_semi_reliable_fetcher, default_enabled: true)
+  Feature.enabled?(:gitlab_sidekiq_enable_semi_reliable_fetcher, type: :ops, default_enabled: true)
 end
 
 # Custom Queues configuration
diff --git a/config/initializers/sprockets.rb b/config/initializers/sprockets.rb
new file mode 100644
index 00000000000..a20b7dc75e9
--- /dev/null
+++ b/config/initializers/sprockets.rb
@@ -0,0 +1 @@
+Sprockets.register_compressor 'application/javascript', :terser, Terser::Compressor
diff --git a/config/initializers/stackprof.rb b/config/initializers/stackprof.rb
index 797efdb9bbd..4c4d241f065 100644
--- a/config/initializers/stackprof.rb
+++ b/config/initializers/stackprof.rb
@@ -2,14 +2,18 @@
 
 # trigger stackprof by sending a SIGUSR2 signal
 #
-# default settings:
-# * collect raw samples
-# * sample at 100hz (every 10k microseconds)
-# * timeout profile after 30 seconds
-# * write to $TMPDIR/stackprof.$PID.$RAND.profile
+# Docs: https://docs.gitlab.com/ee/development/performance.html#production
 
 module Gitlab
   class StackProf
+    DEFAULT_FILE_PREFIX = Dir.tmpdir
+    DEFAULT_TIMEOUT_SEC = 30
+    DEFAULT_MODE = :cpu
+    # Sample interval as a frequency in microseconds (~100hz); appropriate for CPU profiles
+    DEFAULT_INTERVAL_US = 10_000
+    # Sample interval in event occurrences (n = every nth event); appropriate for allocation profiles
+    DEFAULT_INTERVAL_EVENTS = 1_000
+
     # this is a workaround for sidekiq, which defines its own SIGUSR2 handler.
     # by defering to the sidekiq startup event, we get to set up our own
     # handler late enough.
@@ -32,11 +36,7 @@ module Gitlab
     end
 
     def self.on_worker_start
-      Gitlab::AppJsonLogger.info(
-        event: "stackprof",
-        message: "listening on SIGUSR2 signal",
-        pid: Process.pid
-      )
+      log_event('listening for SIGUSR2 signal')
 
       # create a pipe in order to propagate signal out of the signal handler
       # see also: https://cr.yp.to/docs/selfpipe.html
@@ -55,43 +55,46 @@ module Gitlab
       # a given interval (by default 30 seconds), avoiding unbounded memory
       # growth from a profile that was started and never stopped.
       t = Thread.new do
-        timeout_s = ENV['STACKPROF_TIMEOUT_S']&.to_i || 30
+        timeout_s = ENV['STACKPROF_TIMEOUT_S']&.to_i || DEFAULT_TIMEOUT_SEC
         current_timeout_s = nil
         loop do
-          got_value = IO.select([read], nil, nil, current_timeout_s)
-          read.getbyte if got_value
+          read.getbyte if IO.select([read], nil, nil, current_timeout_s)
 
           if ::StackProf.running?
-            stackprof_file_prefix = ENV['STACKPROF_FILE_PREFIX'] || Dir.tmpdir
+            stackprof_file_prefix = ENV['STACKPROF_FILE_PREFIX'] || DEFAULT_FILE_PREFIX
             stackprof_out_file = "#{stackprof_file_prefix}/stackprof.#{Process.pid}.#{SecureRandom.hex(6)}.profile"
 
-            Gitlab::AppJsonLogger.info(
-              event: "stackprof",
-              message: "stopping profile",
-              output_filename: stackprof_out_file,
-              pid: Process.pid,
-              timeout_s: timeout_s,
-              timed_out: got_value.nil?
+            log_event(
+              'stopping profile',
+              profile_filename: stackprof_out_file,
+              profile_timeout_s: timeout_s
             )
 
             ::StackProf.stop
             ::StackProf.results(stackprof_out_file)
             current_timeout_s = nil
           else
-            Gitlab::AppJsonLogger.info(
-              event: "stackprof",
-              message: "starting profile",
-              pid: Process.pid
+            mode = ENV['STACKPROF_MODE']&.to_sym || DEFAULT_MODE
+            interval = ENV['STACKPROF_INTERVAL']&.to_i
+            interval ||= (mode == :object ? DEFAULT_INTERVAL_EVENTS : DEFAULT_INTERVAL_US)
+
+            log_event(
+              'starting profile',
+              profile_mode: mode,
+              profile_interval: interval,
+              profile_timeout: timeout_s
             )
 
             ::StackProf.start(
-              mode: :cpu,
+              mode: mode,
               raw: Gitlab::Utils.to_boolean(ENV['STACKPROF_RAW'] || 'true'),
-              interval: ENV['STACKPROF_INTERVAL_US']&.to_i || 10_000
+              interval: interval
             )
             current_timeout_s = timeout_s
           end
         end
+      rescue => e
+        log_event("stackprof failed: #{e}")
       end
       t.abort_on_exception = true
 
@@ -121,6 +124,14 @@ module Gitlab
         write.write('.')
       end
     end
+
+    def self.log_event(event, labels = {})
+      Gitlab::AppJsonLogger.info({
+        event: 'stackprof',
+        message: event,
+        pid: Process.pid
+      }.merge(labels.compact))
+    end
   end
 end
 
diff --git a/config/initializers/static_files.rb b/config/initializers/static_files.rb
index e02f0868e9f..4e19fec084a 100644
--- a/config/initializers/static_files.rb
+++ b/config/initializers/static_files.rb
@@ -15,32 +15,14 @@ if app.config.public_file_server.enabled
 
   # If webpack-dev-server is configured, proxy webpack's public directory
   # instead of looking for static assets
-  dev_server = Gitlab.config.webpack.dev_server
-
-  if dev_server.enabled
-    settings = {
-      enabled: true,
-      host: dev_server.host,
-      port: dev_server.port,
-      manifest_host: dev_server.host,
-      manifest_port: dev_server.port
-    }
-
-    if Rails.env.development?
-      settings.merge!(
-        host: Gitlab.config.gitlab.host,
-        port: Gitlab.config.gitlab.port,
-        https: false
-      )
-      app.config.middleware.insert_before(
-        Gitlab::Middleware::Static,
-        Gitlab::Webpack::DevServerMiddleware,
-        proxy_path: app.config.webpack.public_path,
-        proxy_host: dev_server.host,
-        proxy_port: dev_server.port
-      )
-    end
-
-    app.config.webpack.dev_server.merge!(settings)
+  if Gitlab.config.webpack.dev_server.enabled && Rails.env.development?
+    app.config.middleware.insert_before(
+      Gitlab::Middleware::Static,
+      Gitlab::Webpack::DevServerMiddleware,
+      proxy_path: Gitlab.config.webpack.public_path,
+      proxy_host: Gitlab.config.webpack.dev_server.host,
+      proxy_port: Gitlab.config.webpack.dev_server.port,
+      proxy_https: Gitlab.config.webpack.dev_server.https
+    )
   end
 end
diff --git a/config/initializers/webauthn.rb b/config/initializers/webauthn.rb
index 8dc5dfd56ed..1f37e7c84c3 100644
--- a/config/initializers/webauthn.rb
+++ b/config/initializers/webauthn.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 WebAuthn.configure do |config|
   # This value needs to match `window.location.origin` evaluated by
   # the User Agent during registration and authentication ceremonies.