Add latest changes from gitlab-org/gitlab@master

3 files changed, 63 insertions, 40 deletions

diff --git a/lib/gitlab/memory/diagnostic_reports_logger.rb b/lib/gitlab/memory/diagnostic_reports_logger.rb
new file mode 100644
index 00000000000..cc5b719fa19
--- /dev/null
+++ b/lib/gitlab/memory/diagnostic_reports_logger.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module Gitlab
+  module Memory
+    class DiagnosticReportsLogger < ::Logger
+      def format_message(severity, timestamp, progname, message)
+        data = {}
+        data[:severity] = severity
+        data[:time] = timestamp.utc.iso8601(3)
+
+        data.merge!(message)
+
+        "#{JSON.generate(data)}\n" # rubocop:disable Gitlab/Json
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/memory/reports_uploader.rb b/lib/gitlab/memory/reports_uploader.rb
index 846a9e5fcae..76c3e0862e2 100644
--- a/lib/gitlab/memory/reports_uploader.rb
+++ b/lib/gitlab/memory/reports_uploader.rb
@@ -1,35 +1,52 @@
 # frozen_string_literal: true
 
+require_relative '../metrics/system'
+
 module Gitlab
   module Memory
     class ReportsUploader
-      # This is no-op currently, it will only write logs.
-      # The uploader implementation will be done in the next MR(s). For more details, check:
-      # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/97155#note_1099244930
+      def initialize(gcs_key:, gcs_project:, gcs_bucket:, logger:)
+        @gcs_bucket = gcs_bucket
+        @fog = Fog::Storage::Google.new(google_project: gcs_project, google_json_key_location: gcs_key)
+        @logger = logger
+      end
+
       def upload(path)
         log_upload_requested(path)
+        start_monotonic_time = Gitlab::Metrics::System.monotonic_time
 
-        false # nothing is uploaded in the current implementation
+        File.open(path.to_s) { |file| fog.put_object(gcs_bucket, File.basename(path), file) }
+
+        duration_s = Gitlab::Metrics::System.monotonic_time - start_monotonic_time
+        log_upload_success(path, duration_s)
+      rescue StandardError, Errno::ENOENT => error
+        log_exception(error)
       end
 
       private
 
+      attr_reader :gcs_bucket, :fog, :logger
+
       def log_upload_requested(path)
-        Gitlab::AppLogger.info(log_labels.merge(perf_report_status: 'upload requested', perf_report_path: path))
+        logger.info(log_labels.merge(perf_report_status: 'upload requested', perf_report_path: path))
+      end
+
+      def log_upload_success(path, duration_s)
+        logger.info(log_labels.merge(perf_report_status: 'upload success', perf_report_path: path,
+                                     duration_s: duration_s))
+      end
+
+      def log_exception(error)
+        logger.error(log_labels.merge(perf_report_status: "error", error: error.message))
       end
 
       def log_labels
         {
           message: "Diagnostic reports",
           class: self.class.name,
-          pid: $$,
-          worker_id: worker_id
+          pid: $$
         }
       end
-
-      def worker_id
-        ::Prometheus::PidProvider.worker_id
-      end
     end
   end
 end
diff --git a/lib/gitlab/memory/upload_and_cleanup_reports.rb b/lib/gitlab/memory/upload_and_cleanup_reports.rb
index b14f24fe13c..27d94df478c 100644
--- a/lib/gitlab/memory/upload_and_cleanup_reports.rb
+++ b/lib/gitlab/memory/upload_and_cleanup_reports.rb
@@ -6,32 +6,26 @@ module Gitlab
       DEFAULT_SLEEP_TIME_SECONDS = 900 # 15 minutes
 
       def initialize(
-        sleep_time_seconds: ENV['GITLAB_DIAGNOSTIC_REPORTS_UPLOADER_SLEEP_S']&.to_i || DEFAULT_SLEEP_TIME_SECONDS,
-        reports_path: ENV["GITLAB_DIAGNOSTIC_REPORTS_PATH"])
+        uploader:,
+        reports_path:,
+        logger:,
+        sleep_time_seconds: ENV['GITLAB_DIAGNOSTIC_REPORTS_UPLOADER_SLEEP_S']&.to_i || DEFAULT_SLEEP_TIME_SECONDS)
 
-        @sleep_time_seconds = sleep_time_seconds
+        @uploader = uploader
         @reports_path = reports_path
-
-        unless @reports_path.present?
-          log_error_reports_path_missing
-          return
-        end
-
-        @uploader = ReportsUploader.new
-
+        @sleep_time_seconds = sleep_time_seconds
         @alive = true
+        @logger = logger
       end
 
-      attr_reader :sleep_time_seconds, :reports_path, :uploader, :alive
+      attr_reader :uploader, :reports_path, :sleep_time_seconds, :logger
 
       def call
         log_started
 
-        while alive
+        loop do
           sleep(sleep_time_seconds)
 
-          next unless Feature.enabled?(:gitlab_diagnostic_reports_uploader, type: :ops)
-
           files_to_process.each { |path| upload_and_cleanup!(path) }
         end
       end
@@ -39,9 +33,11 @@ module Gitlab
       private
 
       def upload_and_cleanup!(path)
-        cleanup!(path) if uploader.upload(path)
-      rescue StandardError => error
+        uploader.upload(path)
+      rescue StandardError, Errno::ENOENT => error
         log_exception(error)
+      ensure
+        cleanup!(path)
       end
 
       def cleanup!(path)
@@ -56,30 +52,21 @@ module Gitlab
           .select { |path| File.file?(path) }
       end
 
-      def log_error_reports_path_missing
-        Gitlab::AppLogger.error(log_labels.merge(perf_report_status: "path is not configured"))
-      end
-
       def log_started
-        Gitlab::AppLogger.info(log_labels.merge(perf_report_status: "started"))
+        logger.info(log_labels.merge(perf_report_status: "started"))
       end
 
       def log_exception(error)
-        Gitlab::ErrorTracking.log_exception(error, log_labels)
+        logger.error(log_labels.merge(perf_report_status: "error", error: error.message))
       end
 
       def log_labels
         {
           message: "Diagnostic reports",
           class: self.class.name,
-          pid: $$,
-          worker_id: worker_id
+          pid: $$
         }
       end
-
-      def worker_id
-        ::Prometheus::PidProvider.worker_id
-      end
     end
   end
 end