Add latest changes from gitlab-org/gitlab@master

7 files changed, 146 insertions, 8 deletions

diff --git a/lib/api/entities/bulk_imports/entity.rb b/lib/api/entities/bulk_imports/entity.rb
index 7e9b9973e15..ba828fefdeb 100644
--- a/lib/api/entities/bulk_imports/entity.rb
+++ b/lib/api/entities/bulk_imports/entity.rb
@@ -25,6 +25,7 @@ module API
         expose :failures, using: EntityFailure, documentation: { is_array: true }
         expose :migrate_projects, documentation: { type: 'boolean', example: true }
         expose :has_failures, documentation: { type: 'boolean', example: false }
+        expose :checksums, as: :stats, documentation: { type: 'object' }
       end
     end
   end
diff --git a/lib/bulk_imports/object_counter.rb b/lib/bulk_imports/object_counter.rb
new file mode 100644
index 00000000000..750e9865a23
--- /dev/null
+++ b/lib/bulk_imports/object_counter.rb
@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module BulkImports
+  class ObjectCounter
+    SOURCE_COUNTER = :source
+    FETCHED_COUNTER = :fetched
+    IMPORTED_COUNTER = :imported
+    COUNTER_TYPES = [SOURCE_COUNTER, FETCHED_COUNTER, IMPORTED_COUNTER].freeze
+    CACHE_KEY = 'bulk_imports/object_counter/%{tracker_id}'
+
+    class << self
+      def increment(tracker, counter_type, value = 1)
+        return unless valid_input?(counter_type, value)
+
+        Gitlab::Cache::Import::Caching.hash_increment(counter_key(tracker), counter_type, value)
+      end
+
+      def set(tracker, counter_type, value = 1)
+        return unless valid_input?(counter_type, value)
+
+        Gitlab::Cache::Import::Caching.hash_add(counter_key(tracker), counter_type, value)
+      end
+
+      def summary(tracker)
+        object_counters = Gitlab::Cache::Import::Caching.values_from_hash(counter_key(tracker))
+
+        return unless object_counters.is_a?(Hash)
+        return if object_counters.empty?
+
+        empty_response.merge(object_counters.symbolize_keys.transform_values(&:to_i))
+      end
+
+      # Commits counters from redis to the database
+      def persist!(tracker)
+        counters = summary(tracker)
+
+        return unless counters
+
+        tracker.update!(
+          source_objects_count: counters[SOURCE_COUNTER],
+          fetched_objects_count: counters[FETCHED_COUNTER],
+          imported_objects_count: counters[IMPORTED_COUNTER]
+        )
+      end
+
+      private
+
+      def counter_key(tracker)
+        Kernel.format(CACHE_KEY, tracker_id: tracker.id)
+      end
+
+      def valid_input?(counter_type, value)
+        return false unless value.is_a?(Integer)
+        return false if value <= 0
+        return false unless COUNTER_TYPES.include?(counter_type)
+
+        true
+      end
+
+      def empty_response
+        {
+          SOURCE_COUNTER => 0,
+          FETCHED_COUNTER => 0,
+          IMPORTED_COUNTER => 0
+        }
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb
index 7b5e1e68459..b1d5b55459b 100644
--- a/lib/bulk_imports/pipeline/runner.rb
+++ b/lib/bulk_imports/pipeline/runner.rb
@@ -12,6 +12,7 @@ module BulkImports
 
         info(message: 'Pipeline started')
 
+        set_source_objects_counter
         extracted_data = extracted_data_from
 
         if extracted_data
@@ -21,6 +22,8 @@ module BulkImports
             raw_entry = entry.dup
             next if already_processed?(raw_entry, index)
 
+            increment_fetched_objects_counter
+
             transformers.each do |transformer|
               entry = run_pipeline_step(:transformer, transformer.class.name) do
                 transformer.transform(context, entry)
@@ -29,6 +32,8 @@ module BulkImports
 
             run_pipeline_step(:loader, loader.class.name, entry) do
               loader.load(context, entry)
+
+              increment_imported_objects_counter
             end
 
             save_processed_entry(raw_entry, index)
@@ -196,6 +201,21 @@ module BulkImports
         context.entity.touch
         context.bulk_import.touch
       end
+
+      def set_source_objects_counter
+        # Export status is cached for 24h and read from Redis at this point
+        export_status = ExportStatus.new(tracker, tracker.importing_relation)
+
+        ObjectCounter.set(tracker, ObjectCounter::SOURCE_COUNTER, export_status.total_objects_count)
+      end
+
+      def increment_fetched_objects_counter
+        ObjectCounter.increment(tracker, ObjectCounter::FETCHED_COUNTER)
+      end
+
+      def increment_imported_objects_counter
+        ObjectCounter.increment(tracker, ObjectCounter::IMPORTED_COUNTER)
+      end
     end
   end
 end
diff --git a/lib/cloud_connector/config.rb b/lib/cloud_connector/config.rb
new file mode 100644
index 00000000000..5f14690da4d
--- /dev/null
+++ b/lib/cloud_connector/config.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module CloudConnector
+  module Config
+    extend self
+
+    def base_url
+      Gitlab.config.cloud_connector.base_url
+    end
+  end
+end
diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb
index f3251d47e7a..bc1671ff650 100644
--- a/lib/gitlab/cache/import/caching.rb
+++ b/lib/gitlab/cache/import/caching.rb
@@ -239,6 +239,25 @@ module Gitlab
           end
         end
 
+        # Increments value of a field in a hash
+        #
+        # raw_key - The key of the hash to add to.
+        # field - The field to increment.
+        # value - The field value to add to the hash.
+        # timeout - The new timeout of the key.
+        def self.hash_increment(raw_key, field, value, timeout: TIMEOUT)
+          return if value.to_i <= 0
+
+          key = cache_key_for(raw_key)
+
+          with_redis do |redis|
+            redis.multi do |m|
+              m.hincrby(key, field, value.to_i)
+              m.expire(key, timeout)
+            end
+          end
+        end
+
         # Adds a value to a list.
         #
         # raw_key - The key of the list to add to.
diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb
index c94deea0dfb..1790ce36819 100644
--- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb
+++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb
@@ -181,7 +181,14 @@ module Gitlab
           spend_time_message(time_spent, time_spent_date, true)
         end
 
-        params '<time(1h30m | -1h30m)> <date(YYYY-MM-DD)>'
+        params do
+          base_params = 'time(1h30m | -1h30m) <date(YYYY-MM-DD)>'
+          if Feature.enabled?(:timelog_categories, quick_action_target.project)
+            "#{base_params} <[timecategory:category-name]>"
+          else
+            base_params
+          end
+        end
         types Issue, MergeRequest
         condition do
           quick_action_target.supports_time_tracking? &&
@@ -190,12 +197,13 @@ module Gitlab
         parse_params do |raw_time_date|
           Gitlab::QuickActions::SpendTimeAndDateSeparator.new(raw_time_date).execute
         end
-        command :spend, :spent, :spend_time do |time_spent, time_spent_date|
+        command :spend, :spent, :spend_time do |time_spent, time_spent_date, category|
           if time_spent
             @updates[:spend_time] = {
               duration: time_spent,
               user_id: current_user.id,
-              spent_at: time_spent_date
+              spent_at: time_spent_date,
+              category: category
             }
           end
         end
diff --git a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb
index a8dd4aa17c5..2fd0c8c161b 100644
--- a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb
+++ b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb
@@ -12,6 +12,7 @@ module Gitlab
     # in other cases return nil
     class SpendTimeAndDateSeparator
       DATE_REGEX = %r{(\d{2,4}[/\-.]\d{1,2}[/\-.]\d{1,2})}
+      CATEGORY_REGEX = %r{\[timecategory:(.*)\]}
 
       def initialize(spend_command_arg)
         @spend_arg = spend_command_arg
@@ -19,20 +20,23 @@ module Gitlab
 
       def execute
         return if @spend_arg.blank?
-        return [get_time, DateTime.current] unless date_present?
-        return unless valid_date?
+        return if date_present? && !valid_date?
 
-        [get_time, get_date]
+        [time_spent, spent_at, category]
       end
 
       private
 
-      def get_time
+      def time_spent
         raw_time = @spend_arg.gsub(DATE_REGEX, '')
+        raw_time = raw_time.gsub(CATEGORY_REGEX, '')
+
         Gitlab::TimeTrackingFormatter.parse(raw_time)
       end
 
-      def get_date
+      def spent_at
+        return DateTime.current unless date_present?
+
         string_date = @spend_arg.match(DATE_REGEX)[0]
         Date.parse(string_date)
       end
@@ -55,6 +59,12 @@ module Gitlab
       def date_past_or_today?(date)
         date&.past? || date&.today?
       end
+
+      def category
+        return unless @spend_arg.match?(CATEGORY_REGEX)
+
+        @spend_arg.match(CATEGORY_REGEX)[1]
+      end
     end
   end
 end