Merge branch 'an-multithreading' into 'master'

Puma in GDK and rack server lifecycle event abstractions

Closes #52762

See merge request gitlab-org/gitlab-ce!22372

9 files changed, 200 insertions, 54 deletions

diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb
index 146c4b1e024..8052880cc3d 100644
--- a/config/initializers/7_prometheus_metrics.rb
+++ b/config/initializers/7_prometheus_metrics.rb
@@ -26,9 +26,25 @@ Sidekiq.configure_server do |config|
 end
 
 if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled?
-  unless Sidekiq.server?
-    Gitlab::Metrics::Samplers::UnicornSampler.initialize_instance(Settings.monitoring.unicorn_sampler_interval).start
+  Gitlab::Cluster::LifecycleEvents.on_worker_start do
+    defined?(::Prometheus::Client.reinitialize_on_pid_change) && Prometheus::Client.reinitialize_on_pid_change
+
+    unless Sidekiq.server?
+      Gitlab::Metrics::Samplers::UnicornSampler.initialize_instance(Settings.monitoring.unicorn_sampler_interval).start
+    end
+
+    Gitlab::Metrics::Samplers::RubySampler.initialize_instance(Settings.monitoring.ruby_sampler_interval).start
   end
+end
 
-  Gitlab::Metrics::Samplers::RubySampler.initialize_instance(Settings.monitoring.ruby_sampler_interval).start
+Gitlab::Cluster::LifecycleEvents.on_master_restart do
+  # The following is necessary to ensure stale Prometheus metrics don't
+  # accumulate over time. It needs to be done in this hook as opposed to
+  # inside an init script to ensure metrics files aren't deleted after new
+  # unicorn workers start after a SIGUSR2 is received.
+  prometheus_multiproc_dir = ENV['prometheus_multiproc_dir']
+  if prometheus_multiproc_dir
+    old_metrics = Dir[File.join(prometheus_multiproc_dir, '*.db')]
+    FileUtils.rm_rf(old_metrics)
+  end
 end
diff --git a/config/initializers/8_metrics.rb b/config/initializers/8_metrics.rb
index eccf82ab8dc..c8d261d415e 100644
--- a/config/initializers/8_metrics.rb
+++ b/config/initializers/8_metrics.rb
@@ -158,7 +158,9 @@ if Gitlab::Metrics.enabled? && !Rails.env.test?
 
   GC::Profiler.enable
 
-  Gitlab::Metrics::Samplers::InfluxSampler.initialize_instance.start
+  Gitlab::Cluster::LifecycleEvents.on_worker_start do
+    Gitlab::Metrics::Samplers::InfluxSampler.initialize_instance.start
+  end
 
   module TrackNewRedisConnections
     def connect(*args)
diff --git a/config/initializers/active_record_lifecycle.rb b/config/initializers/active_record_lifecycle.rb
new file mode 100644
index 00000000000..7fa37121efc
--- /dev/null
+++ b/config/initializers/active_record_lifecycle.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# Don't handle sidekiq configuration as it
+# has its own special active record configuration here
+if defined?(ActiveRecord::Base) && !Sidekiq.server?
+  Gitlab::Cluster::LifecycleEvents.on_worker_start do
+    ActiveSupport.on_load(:active_record) do
+      ActiveRecord::Base.establish_connection
+
+      Rails.logger.debug("ActiveRecord connection established")
+    end
+  end
+end
+
+if defined?(ActiveRecord::Base)
+  Gitlab::Cluster::LifecycleEvents.on_before_fork do
+    # the following is highly recommended for Rails + "preload_app true"
+    # as there's no need for the master process to hold a connection
+    ActiveRecord::Base.connection.disconnect!
+
+    Rails.logger.debug("ActiveRecord connection disconnected")
+  end
+end
diff --git a/config/initializers/macos.rb b/config/initializers/macos.rb
new file mode 100644
index 00000000000..f410af6ed47
--- /dev/null
+++ b/config/initializers/macos.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+if /darwin/ =~ RUBY_PLATFORM
+  Gitlab::Cluster::LifecycleEvents.on_before_fork do
+    require 'fiddle'
+
+    # Dynamically load Foundation.framework, ~implicitly~ initialising
+    # the Objective-C runtime before any forking happens in Unicorn
+    #
+    # From https://bugs.ruby-lang.org/issues/14009
+    Fiddle.dlopen '/System/Library/Frameworks/Foundation.framework/Foundation'
+  end
+end
diff --git a/config/initializers/rbtrace.rb b/config/initializers/rbtrace.rb
new file mode 100644
index 00000000000..6a1b71bf4bd
--- /dev/null
+++ b/config/initializers/rbtrace.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+if ENV['ENABLE_RBTRACE']
+  Gitlab::Cluster::LifecycleEvents.on_worker_start do
+    # Unicorn clears out signals before it forks, so rbtrace won't work
+    # unless it is enabled after the fork.
+    require 'rbtrace'
+  end
+end
diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
index bc6b7aed6aa..565efc858d1 100644
--- a/config/initializers/sidekiq.rb
+++ b/config/initializers/sidekiq.rb
@@ -14,8 +14,6 @@ Sidekiq.default_worker_options = { retry: 3 }
 enable_json_logs = Gitlab.config.sidekiq.log_format == 'json'
 
 Sidekiq.configure_server do |config|
-  require 'rbtrace' if ENV['ENABLE_RBTRACE']
-
   config.redis = queues_config_hash
 
   config.server_middleware do |chain|
diff --git a/config/puma.example.development.rb b/config/puma.example.development.rb
new file mode 100644
index 00000000000..490c940077a
--- /dev/null
+++ b/config/puma.example.development.rb
@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+# -----------------------------------------------------------------------
+# This file is used by the GDK to generate a default config/puma.rb file
+# Note that `/home/git` will be substituted for the actual GDK root
+# directory when this file is generated
+# -----------------------------------------------------------------------
+
+# Load "path" as a rackup file.
+#
+# The default is "config.ru".
+#
+rackup 'config.ru'
+pidfile '/home/git/gitlab/tmp/pids/puma.pid'
+state_path '/home/git/gitlab/tmp/pids/puma.state'
+
+stdout_redirect '/home/git/gitlab/log/puma.stdout.log',
+  '/home/git/gitlab/log/puma.stderr.log',
+  true
+
+# Configure "min" to be the minimum number of threads to use to answer
+# requests and "max" the maximum.
+#
+# The default is "0, 16".
+#
+threads 1, 4
+
+# By default, workers accept all requests and queue them to pass to handlers.
+# When false, workers accept the number of simultaneous requests configured.
+#
+# Queueing requests generally improves performance, but can cause deadlocks if
+# the app is waiting on a request to itself. See https://github.com/puma/puma/issues/612
+#
+# When set to false this may require a reverse proxy to handle slow clients and
+# queue requests before they reach puma. This is due to disabling HTTP keepalive
+queue_requests false
+
+# Bind the server to "url". "tcp://", "unix://" and "ssl://" are the only
+# accepted protocols.
+bind 'unix:///home/git/gitlab.socket'
+
+workers 2
+
+require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events"
+require_relative "/home/git/gitlab/lib/gitlab/cluster/puma_worker_killer_initializer"
+
+on_restart do
+  # Signal application hooks that we're about to restart
+  Gitlab::Cluster::LifecycleEvents.do_master_restart
+end
+
+before_fork do
+  # Signal to the puma killer
+  Gitlab::Cluster::PumaWorkerKillerInitializer.start @config.options unless ENV['DISABLE_PUMA_WORKER_KILLER']
+
+  # Signal application hooks that we're about to fork
+  Gitlab::Cluster::LifecycleEvents.do_before_fork
+end
+
+Gitlab::Cluster::LifecycleEvents.set_puma_options @config.options
+on_worker_boot do
+  # Signal application hooks of worker start
+  Gitlab::Cluster::LifecycleEvents.do_worker_start
+end
+
+# Preload the application before starting the workers; this conflicts with
+# phased restart feature. (off by default)
+
+preload_app!
+
+tag 'gitlab-puma-worker'
+
+# Verifies that all workers have checked in to the master process within
+# the given timeout. If not the worker process will be restarted. Default
+# value is 60 seconds.
+#
+worker_timeout 60
diff --git a/config/unicorn.rb.example b/config/unicorn.rb.example
index e06cce3e97a..4637eb8bc6e 100644
--- a/config/unicorn.rb.example
+++ b/config/unicorn.rb.example
@@ -81,22 +81,16 @@ preload_app true
 # fast LAN.
 check_client_connection false
 
+require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events"
+
 before_exec do |server|
-  # The following is necessary to ensure stale Prometheus metrics don't
-  # accumulate over time. It needs to be done in this hook as opposed to
-  # inside an init script to ensure metrics files aren't deleted after new
-  # unicorn workers start after a SIGUSR2 is received.
-  if ENV['prometheus_multiproc_dir']
-    old_metrics = Dir[File.join(ENV['prometheus_multiproc_dir'], '*.db')]
-    FileUtils.rm_rf(old_metrics)
-  end
+  # Signal application hooks that we're about to restart
+  Gitlab::Cluster::LifecycleEvents.do_master_restart
 end
 
 before_fork do |server, worker|
-  # the following is highly recommended for Rails + "preload_app true"
-  # as there's no need for the master process to hold a connection
-  defined?(ActiveRecord::Base) &&
-    ActiveRecord::Base.connection.disconnect!
+  # Signal application hooks that we're about to fork
+  Gitlab::Cluster::LifecycleEvents.do_before_fork
 
   # The following is only recommended for memory/DB-constrained
   # installations.  It is not needed if your system can house
@@ -124,25 +118,10 @@ before_fork do |server, worker|
 end
 
 after_fork do |server, worker|
-  # Unicorn clears out signals before it forks, so rbtrace won't work
-  # unless it is enabled after the fork.
-  require 'rbtrace' if ENV['ENABLE_RBTRACE']
+  # Signal application hooks of worker start
+  Gitlab::Cluster::LifecycleEvents.do_worker_start
 
   # per-process listener ports for debugging/admin/migrations
   # addr = "127.0.0.1:#{9293 + worker.nr}"
   # server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true)
-
-  # the following is *required* for Rails + "preload_app true",
-  defined?(ActiveRecord::Base) &&
-    ActiveRecord::Base.establish_connection
-
-  # reset prometheus client, this will cause any opened metrics files to be closed
-  defined?(::Prometheus::Client.reinitialize_on_pid_change) &&
-    Prometheus::Client.reinitialize_on_pid_change
-
-  # if preload_app is true, then you may also want to check and
-  # restart any other shared sockets/descriptors such as Memcached,
-  # and Redis.  TokyoCabinet file handles are safe to reuse
-  # between any number of forked children (assuming your kernel
-  # correctly implements pread()/pwrite() system calls)
 end
diff --git a/config/unicorn.rb.example.development b/config/unicorn.rb.example.development
index f31df66015a..f7541bb9d55 100644
--- a/config/unicorn.rb.example.development
+++ b/config/unicorn.rb.example.development
@@ -1,32 +1,61 @@
+# frozen_string_literal: true
+
+# -------------------------------------------------------------------------
+# This file is used by the GDK to generate a default config/unicorn.rb file
+# Note that `/home/git` will be substituted for the actual GDK root
+# directory when this file is generated
+# -------------------------------------------------------------------------
+
 worker_processes 2
 timeout 60
 
+listen '/home/git/gitlab.socket'
+
 preload_app true
 check_client_connection false
 
+require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events"
+
+before_exec do |server|
+  # Signal application hooks that we're about to restart
+  Gitlab::Cluster::LifecycleEvents.do_master_restart
+end
+
 before_fork do |server, worker|
-  # the following is highly recommended for Rails + "preload_app true"
-  # as there's no need for the master process to hold a connection
-  defined?(ActiveRecord::Base) &&
-    ActiveRecord::Base.connection.disconnect!
-
-  if /darwin/ =~ RUBY_PLATFORM
-    require 'fiddle'
-
-    # Dynamically load Foundation.framework, ~implicitly~ initialising
-    # the Objective-C runtime before any forking happens in Unicorn
-    #
-    # From https://bugs.ruby-lang.org/issues/14009
-    Fiddle.dlopen '/System/Library/Frameworks/Foundation.framework/Foundation'
+  # Signal application hooks that we're about to fork
+  Gitlab::Cluster::LifecycleEvents.do_before_fork
+
+  # The following is only recommended for memory/DB-constrained
+  # installations.  It is not needed if your system can house
+  # twice as many worker_processes as you have configured.
+  #
+  # This allows a new master process to incrementally
+  # phase out the old master process with SIGTTOU to avoid a
+  # thundering herd (especially in the "preload_app false" case)
+  # when doing a transparent upgrade.  The last worker spawned
+  # will then kill off the old master process with a SIGQUIT.
+  old_pid = "#{server.config[:pid]}.oldbin"
+  if old_pid != server.pid
+    begin
+      sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
+      Process.kill(sig, File.read(old_pid).to_i)
+    rescue Errno::ENOENT, Errno::ESRCH
+    end
   end
+  #
+  # Throttle the master from forking too quickly by sleeping.  Due
+  # to the implementation of standard Unix signal handlers, this
+  # helps (but does not completely) prevent identical, repeated signals
+  # from being lost when the receiving process is busy.
+  # sleep 1
 end
 
 after_fork do |server, worker|
-  # Unicorn clears out signals before it forks, so rbtrace won't work
-  # unless it is enabled after the fork.
-  require 'rbtrace' if ENV['ENABLE_RBTRACE']
+  # Signal application hooks of worker start
+  Gitlab::Cluster::LifecycleEvents.do_worker_start
 
-  # the following is *required* for Rails + "preload_app true",
-  defined?(ActiveRecord::Base) &&
-    ActiveRecord::Base.establish_connection
+  # per-process listener ports for debugging/admin/migrations
+  # addr = "127.0.0.1:#{9293 + worker.nr}"
+  # server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true)
 end
+