Add latest changes from gitlab-org/gitlab@master

6 files changed, 43 insertions, 61 deletions

diff --git a/lib/api/ci/secure_files.rb b/lib/api/ci/secure_files.rb
index 6bf12eaec2f..6483abcc74e 100644
--- a/lib/api/ci/secure_files.rb
+++ b/lib/api/ci/secure_files.rb
@@ -7,7 +7,6 @@ module API
 
       before do
         authenticate!
-        feature_flag_enabled?
         authorize! :read_secure_files, user_project
       end
 
@@ -113,10 +112,6 @@ module API
       end
 
       helpers do
-        def feature_flag_enabled?
-          service_unavailable! unless Feature.enabled?(:ci_secure_files, user_project)
-        end
-
         def read_only_feature_flag_enabled?
           service_unavailable! if Feature.enabled?(:ci_secure_files_read_only, user_project, type: :ops)
         end
diff --git a/lib/api/terraform/state.rb b/lib/api/terraform/state.rb
index e751d2dd0e5..3c45f17d423 100644
--- a/lib/api/terraform/state.rb
+++ b/lib/api/terraform/state.rb
@@ -90,7 +90,8 @@ module API
             ]
             failure [
               { code: 403, message: 'Forbidden' },
-              { code: 422, message: 'Validation failure' }
+              { code: 422, message: 'Validation failure' },
+              { code: 413, message: 'Request Entity Too Large' }
             ]
             tags %w[terraform_state]
           end
@@ -101,6 +102,9 @@ module API
             data = request.body.read
             no_content! if data.empty?
 
+            max_state_size = Gitlab::CurrentSettings.max_terraform_state_size_bytes
+            file_too_large! if max_state_size > 0 && data.size > max_state_size
+
             remote_state_handler.handle_with_lock do |state|
               state.update_file!(CarrierWaveStringFile.new(data), version: params[:serial], build: current_authenticated_job)
             end
diff --git a/lib/gitlab/memory/watchdog/event_reporter.rb b/lib/gitlab/memory/watchdog/event_reporter.rb
index 4d37a5e14fd..c37426cb660 100644
--- a/lib/gitlab/memory/watchdog/event_reporter.rb
+++ b/lib/gitlab/memory/watchdog/event_reporter.rb
@@ -10,6 +10,7 @@ module Gitlab
 
         def initialize(logger: Gitlab::AppLogger)
           @logger = logger
+          init_prometheus_metrics
         end
 
         def started(labels = {})
@@ -21,13 +22,13 @@ module Gitlab
         end
 
         def threshold_violated(monitor_name)
-          counter_violations.increment(reason: monitor_name)
+          @counter_violations.increment(reason: monitor_name)
         end
 
         def strikes_exceeded(monitor_name, labels = {})
           logger.warn(log_labels(labels))
 
-          counter_violations_handled.increment(reason: monitor_name)
+          @counter_violations_handled.increment(reason: monitor_name)
         end
 
         private
@@ -48,24 +49,18 @@ module Gitlab
           ::Prometheus::PidProvider.worker_id
         end
 
-        def counter_violations
-          strong_memoize("counter_violations") do
-            ::Gitlab::Metrics.counter(
-              :gitlab_memwd_violations_total,
-              'Total number of times a Ruby process violated a memory threshold',
-              { pid: worker_id }
-            )
-          end
-        end
-
-        def counter_violations_handled
-          strong_memoize("counter_violations_handled") do
-            ::Gitlab::Metrics.counter(
-              :gitlab_memwd_violations_handled_total,
-              'Total number of times Ruby process memory violations were handled',
-              { pid: worker_id }
-            )
-          end
+        def init_prometheus_metrics
+          default_labels = { pid: worker_id }
+          @counter_violations = Gitlab::Metrics.counter(
+            :gitlab_memwd_violations_total,
+            'Total number of times a Ruby process violated a memory threshold',
+            default_labels
+          )
+          @counter_violations_handled = Gitlab::Metrics.counter(
+            :gitlab_memwd_violations_handled_total,
+            'Total number of times Ruby process memory violations were handled',
+            default_labels
+          )
         end
       end
     end
diff --git a/lib/gitlab/memory/watchdog/sidekiq_event_reporter.rb b/lib/gitlab/memory/watchdog/sidekiq_event_reporter.rb
index db94edd0992..473ed1b8094 100644
--- a/lib/gitlab/memory/watchdog/sidekiq_event_reporter.rb
+++ b/lib/gitlab/memory/watchdog/sidekiq_event_reporter.rb
@@ -10,6 +10,7 @@ module Gitlab
 
         def initialize(logger: ::Sidekiq.logger)
           @event_reporter = EventReporter.new(logger: logger)
+          @sidekiq_daemon_monitor = Gitlab::SidekiqDaemon::Monitor.instance
           init_prometheus_metrics
         end
 
@@ -26,16 +27,12 @@ module Gitlab
         attr_reader :event_reporter
 
         def fetch_running_jobs
-          running_jobs = []
-          Gitlab::SidekiqDaemon::Monitor.instance.with_running_jobs do |jobs|
-            running_jobs = jobs.map do |jid, job|
-              {
-                jid: jid,
-                worker_class: job[:worker_class].name
-              }
-            end
+          @sidekiq_daemon_monitor.jobs.map do |jid, job|
+            {
+              jid: jid,
+              worker_class: job[:worker_class].name
+            }
           end
-          running_jobs
         end
 
         def increment_worker_counters(running_jobs)
diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb
index c535e103420..4bf9fd8470a 100644
--- a/lib/gitlab/sidekiq_daemon/memory_killer.rb
+++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb
@@ -35,6 +35,7 @@ module Gitlab
 
         @enabled = true
         @metrics = init_metrics
+        @sidekiq_daemon_monitor = Gitlab::SidekiqDaemon::Monitor.instance
       end
 
       private
@@ -193,17 +194,12 @@ module Gitlab
       end
 
       def fetch_running_jobs
-        jobs = []
-        Gitlab::SidekiqDaemon::Monitor.instance.jobs_mutex.synchronize do
-          jobs = Gitlab::SidekiqDaemon::Monitor.instance.jobs.map do |jid, job|
-            {
-              jid: jid,
-              worker_class: job[:worker_class].name
-            }
-          end
+        @sidekiq_daemon_monitor.jobs.map do |jid, job|
+          {
+            jid: jid,
+            worker_class: job[:worker_class].name
+          }
         end
-
-        jobs
       end
 
       def out_of_range_description(rss, hard_limit, soft_limit, deadline_exceeded)
@@ -269,10 +265,8 @@ module Gitlab
       end
 
       def rss_increase_by_jobs
-        Gitlab::SidekiqDaemon::Monitor.instance.jobs_mutex.synchronize do
-          Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job|
-            rss_increase_by_job(job)
-          end
+        @sidekiq_daemon_monitor.jobs.sum do |_, job|
+          rss_increase_by_job(job)
         end
       end
 
@@ -297,7 +291,7 @@ module Gitlab
       end
 
       def any_jobs?
-        Gitlab::SidekiqDaemon::Monitor.instance.jobs.any?
+        @sidekiq_daemon_monitor.jobs.any?
       end
     end
   end
diff --git a/lib/gitlab/sidekiq_daemon/monitor.rb b/lib/gitlab/sidekiq_daemon/monitor.rb
index 2079683a2c1..125402c1e4b 100644
--- a/lib/gitlab/sidekiq_daemon/monitor.rb
+++ b/lib/gitlab/sidekiq_daemon/monitor.rb
@@ -15,9 +15,6 @@ module Gitlab
       # that should not be caught by application
       CancelledError = Class.new(Exception) # rubocop:disable Lint/InheritException
 
-      attr_reader :jobs
-      attr_reader :jobs_mutex
-
       def initialize
         super
 
@@ -31,8 +28,8 @@ module Gitlab
       end
 
       def within_job(worker_class, jid, queue)
-        jobs_mutex.synchronize do
-          jobs[jid] = { worker_class: worker_class, thread: Thread.current, started_at: Gitlab::Metrics::System.monotonic_time }
+        @jobs_mutex.synchronize do
+          @jobs[jid] = { worker_class: worker_class, thread: Thread.current, started_at: Gitlab::Metrics::System.monotonic_time }
         end
 
         if cancelled?(jid)
@@ -48,8 +45,8 @@ module Gitlab
 
         yield
       ensure
-        jobs_mutex.synchronize do
-          jobs.delete(jid)
+        @jobs_mutex.synchronize do
+          @jobs.delete(jid)
         end
       end
 
@@ -65,9 +62,9 @@ module Gitlab
         end
       end
 
-      def with_running_jobs
+      def jobs
         @jobs_mutex.synchronize do
-          yield @jobs.dup
+          @jobs.dup
         end
       end
 
@@ -172,14 +169,14 @@ module Gitlab
       # This is why it passes thread in block,
       # to ensure that we do process this thread
       def find_thread_unsafe(jid)
-        jobs.dig(jid, :thread)
+        @jobs.dig(jid, :thread)
       end
 
       def find_thread_with_lock(jid)
         # don't try to lock if we cannot find the thread
         return unless find_thread_unsafe(jid)
 
-        jobs_mutex.synchronize do
+        @jobs_mutex.synchronize do
           find_thread_unsafe(jid).tap do |thread|
             yield(thread) if thread
           end