Add latest changes from gitlab-org/gitlab@13-12-stable-eev13.12.0-rc42

21 files changed, 121 insertions, 72 deletions

diff --git a/lib/gitlab/analytics/cycle_analytics/average.rb b/lib/gitlab/analytics/cycle_analytics/average.rb
index a449b71b165..7140d31d536 100644
--- a/lib/gitlab/analytics/cycle_analytics/average.rb
+++ b/lib/gitlab/analytics/cycle_analytics/average.rb
@@ -7,9 +7,10 @@ module Gitlab
         include Gitlab::Utils::StrongMemoize
         include StageQueryHelpers
 
-        def initialize(stage:, query:)
+        def initialize(stage:, query:, params: {})
           @stage = stage
           @query = query
+          @params = params
         end
 
         def seconds
@@ -22,7 +23,7 @@ module Gitlab
 
         private
 
-        attr_reader :stage
+        attr_reader :stage, :params
 
         # rubocop: disable CodeReuse/ActiveRecord
         def select_average
diff --git a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb
index 4dec71b35e8..c7987d63153 100644
--- a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb
+++ b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb
@@ -5,6 +5,7 @@ module Gitlab
     module CycleAnalytics
       class BaseQueryBuilder
         include Gitlab::CycleAnalytics::MetricsTables
+        include StageQueryHelpers
 
         delegate :subject_class, to: :stage
 
@@ -13,17 +14,19 @@ module Gitlab
           Issue.to_s => IssuesFinder
         }.freeze
 
+        DEFAULT_END_EVENT_FILTER = :finished
+
         def initialize(stage:, params: {})
           @stage = stage
           @params = build_finder_params(params)
+          @params[:state] = :opened if in_progress?
         end
 
         # rubocop: disable CodeReuse/ActiveRecord
         def build
           query = finder.execute
           query = stage.start_event.apply_query_customization(query)
-          query = stage.end_event.apply_query_customization(query)
-          query.where(duration_condition)
+          apply_end_event_query_customization(query)
         end
         # rubocop: enable CodeReuse/ActiveRecord
 
@@ -46,6 +49,7 @@ module Gitlab
         def build_finder_params(params)
           {}.tap do |finder_params|
             finder_params[:current_user] = params[:current_user]
+            finder_params[:end_event_filter] = params[:end_event_filter] || DEFAULT_END_EVENT_FILTER
 
             add_parent_model_params!(finder_params)
             add_time_range_params!(finder_params, params[:from], params[:to])
@@ -62,9 +66,20 @@ module Gitlab
           finder_params[:created_after] = from || 30.days.ago
           finder_params[:created_before] = to if to
         end
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        def apply_end_event_query_customization(query)
+          if in_progress?
+            stage.end_event.apply_negated_query_customization(query)
+          else
+            query = stage.end_event.apply_query_customization(query)
+            query.where(duration_condition)
+          end
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
       end
     end
   end
 end
 
-Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder.prepend_if_ee('EE::Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder')
+Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder')
diff --git a/lib/gitlab/analytics/cycle_analytics/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/data_collector.rb
index 10a008a76d5..56179533ffb 100644
--- a/lib/gitlab/analytics/cycle_analytics/data_collector.rb
+++ b/lib/gitlab/analytics/cycle_analytics/data_collector.rb
@@ -12,6 +12,8 @@ module Gitlab
       class DataCollector
         include Gitlab::Utils::StrongMemoize
 
+        MAX_COUNT = 1001
+
         delegate :serialized_records, to: :records_fetcher
 
         def initialize(stage:, params: {})
@@ -27,13 +29,19 @@ module Gitlab
 
         def median
           strong_memoize(:median) do
-            Median.new(stage: stage, query: query)
+            Median.new(stage: stage, query: query, params: params)
           end
         end
 
         def average
           strong_memoize(:average) do
-            Average.new(stage: stage, query: query)
+            Average.new(stage: stage, query: query, params: params)
+          end
+        end
+
+        def count
+          strong_memoize(:count) do
+            limit_count
           end
         end
 
@@ -44,9 +52,16 @@ module Gitlab
         def query
           BaseQueryBuilder.new(stage: stage, params: params).build
         end
+
+        # Limiting the maximum number of records so the COUNT(*) query stays efficient for large groups.
+        # COUNT = 1001, show 1000+ on the UI
+        # COUNT < 1001, show the actual number on the UI
+        def limit_count
+          query.limit(MAX_COUNT).count
+        end
       end
     end
   end
 end
 
-Gitlab::Analytics::CycleAnalytics::DataCollector.prepend_if_ee('EE::Gitlab::Analytics::CycleAnalytics::DataCollector')
+Gitlab::Analytics::CycleAnalytics::DataCollector.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::DataCollector')
diff --git a/lib/gitlab/analytics/cycle_analytics/median.rb b/lib/gitlab/analytics/cycle_analytics/median.rb
index 6c0450ac9e5..5775d0324c6 100644
--- a/lib/gitlab/analytics/cycle_analytics/median.rb
+++ b/lib/gitlab/analytics/cycle_analytics/median.rb
@@ -6,9 +6,10 @@ module Gitlab
       class Median
         include StageQueryHelpers
 
-        def initialize(stage:, query:)
+        def initialize(stage:, query:, params: {})
           @stage = stage
           @query = query
+          @params = params
         end
 
         # rubocop: disable CodeReuse/ActiveRecord
@@ -26,7 +27,7 @@ module Gitlab
 
         private
 
-        attr_reader :stage
+        attr_reader :stage, :params
 
         def percentile_cont
           percentile_cont_ordering = Arel::Nodes::UnaryOperation.new(Arel::Nodes::SqlLiteral.new('ORDER BY'), duration)
diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
index b4752ed9e5b..9a37a41ff81 100644
--- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
+++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
@@ -124,7 +124,7 @@ module Gitlab
         def time_columns
           [
             stage.start_event.timestamp_projection.as('start_event_timestamp'),
-            stage.end_event.timestamp_projection.as('end_event_timestamp'),
+            end_event_timestamp_projection.as('end_event_timestamp'),
             round_duration_to_seconds.as('total_time')
           ]
         end
@@ -133,4 +133,4 @@ module Gitlab
   end
 end
 
-Gitlab::Analytics::CycleAnalytics::RecordsFetcher.prepend_if_ee('EE::Gitlab::Analytics::CycleAnalytics::RecordsFetcher')
+Gitlab::Analytics::CycleAnalytics::RecordsFetcher.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::RecordsFetcher')
diff --git a/lib/gitlab/analytics/cycle_analytics/sorting.rb b/lib/gitlab/analytics/cycle_analytics/sorting.rb
index 828879d466d..c399bac423b 100644
--- a/lib/gitlab/analytics/cycle_analytics/sorting.rb
+++ b/lib/gitlab/analytics/cycle_analytics/sorting.rb
@@ -4,23 +4,35 @@ module Gitlab
   module Analytics
     module CycleAnalytics
       class Sorting
+        include StageQueryHelpers
+
+        def initialize(stage:, query:, params: {})
+          @stage = stage
+          @query = query
+          @params = params
+        end
+
         # rubocop: disable CodeReuse/ActiveRecord
-        SORTING_OPTIONS = {
-          end_event: {
-            asc: -> (query, stage) { query.reorder(stage.end_event.timestamp_projection.asc) },
-            desc: -> (query, stage) { query.reorder(stage.end_event.timestamp_projection.desc) }
-          }.freeze,
-          duration: {
-            asc: -> (query, stage) { query.reorder(Arel::Nodes::Subtraction.new(stage.end_event.timestamp_projection, stage.start_event.timestamp_projection).asc) },
-            desc: -> (query, stage) { query.reorder(Arel::Nodes::Subtraction.new(stage.end_event.timestamp_projection, stage.start_event.timestamp_projection).desc) }
-          }.freeze
-        }.freeze
-        # rubocop: enable CodeReuse/ActiveRecord,
+        def apply(sort, direction)
+          sorting_options = {
+            end_event: {
+              asc: -> { query.reorder(end_event_timestamp_projection.asc) },
+              desc: -> { query.reorder(end_event_timestamp_projection.desc) }
+            },
+            duration: {
+              asc: -> { query.reorder(duration.asc) },
+              desc: -> { query.reorder(duration.desc) }
+            }
+          }
 
-        def self.apply(query, stage, sort, direction)
-          sort_lambda = SORTING_OPTIONS.dig(sort, direction) || SORTING_OPTIONS.dig(:end_event, :desc)
-          sort_lambda.call(query, stage)
+          sort_lambda = sorting_options.dig(sort, direction) || sorting_options.dig(:end_event, :desc)
+          sort_lambda.call
         end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        private
+
+        attr_reader :stage, :query, :params
       end
     end
   end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events.rb b/lib/gitlab/analytics/cycle_analytics/stage_events.rb
index 02b1024b8b3..b7a11bc0418 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events.rb
@@ -85,4 +85,4 @@ module Gitlab
   end
 end
 
-Gitlab::Analytics::CycleAnalytics::StageEvents.prepend_if_ee('::EE::Gitlab::Analytics::CycleAnalytics::StageEvents')
+Gitlab::Analytics::CycleAnalytics::StageEvents.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::StageEvents')
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb
index 4bb225b63f1..8e87245e62b 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb
@@ -17,11 +17,6 @@ module Gitlab
             MergeRequest
           end
 
-          def timestamp_projection
-            Arel::Nodes::NamedFunction.new('COALESCE', column_list)
-          end
-
-          override :column_list
           def column_list
             [
               issue_metrics_table[:first_mentioned_in_commit_at],
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb
index a159580b7bd..30b457b667c 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb
@@ -17,8 +17,8 @@ module Gitlab
             Issue
           end
 
-          def timestamp_projection
-            issue_table[:created_at]
+          def column_list
+            [issue_table[:created_at]]
           end
         end
       end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb
index 3e93e60e686..4ca3c19051e 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb
@@ -17,13 +17,8 @@ module Gitlab
             Issue
           end
 
-          def timestamp_projection
-            mr_metrics_table[:first_deployed_to_production_at]
-          end
-
-          override :column_list
           def column_list
-            [timestamp_projection]
+            [mr_metrics_table[:first_deployed_to_production_at]]
           end
 
           # rubocop: disable CodeReuse/ActiveRecord
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb
index a3b7fa16daf..aa509e8c4d2 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb
@@ -17,8 +17,8 @@ module Gitlab
             Issue
           end
 
-          def timestamp_projection
-            issue_metrics_table[:first_mentioned_in_commit_at]
+          def column_list
+            [issue_metrics_table[:first_mentioned_in_commit_at]]
           end
         end
       end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb
index 7c1f4436c93..284d8534b96 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb
@@ -17,11 +17,6 @@ module Gitlab
             Issue
           end
 
-          def timestamp_projection
-            Arel::Nodes::NamedFunction.new('COALESCE', column_list)
-          end
-
-          override :column_list
           def column_list
             [
               issue_metrics_table[:first_associated_with_milestone_at],
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb
index 013e068e479..31249ae2036 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb
@@ -17,8 +17,8 @@ module Gitlab
             MergeRequest
           end
 
-          def timestamp_projection
-            mr_table[:created_at]
+          def column_list
+            [mr_table[:created_at]]
           end
         end
       end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb
index 654d0befbc3..4c0e9b61e64 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb
@@ -17,8 +17,8 @@ module Gitlab
             MergeRequest
           end
 
-          def timestamp_projection
-            mr_metrics_table[:first_deployed_to_production_at]
+          def column_list
+            [mr_metrics_table[:first_deployed_to_production_at]]
           end
 
           # rubocop: disable CodeReuse/ActiveRecord
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb
index a0b1c12756f..178fe03d7db 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb
@@ -17,8 +17,8 @@ module Gitlab
             MergeRequest
           end
 
-          def timestamp_projection
-            mr_metrics_table[:latest_build_finished_at]
+          def column_list
+            [mr_metrics_table[:latest_build_finished_at]]
           end
         end
       end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb
index da3b5cdfaa4..95e59cd29a6 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb
@@ -17,8 +17,8 @@ module Gitlab
             MergeRequest
           end
 
-          def timestamp_projection
-            mr_metrics_table[:latest_build_started_at]
+          def column_list
+            [mr_metrics_table[:latest_build_started_at]]
           end
         end
       end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb
index e67a6f7eea6..00ac2e7d56c 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb
@@ -17,8 +17,8 @@ module Gitlab
             MergeRequest
           end
 
-          def timestamp_projection
-            mr_metrics_table[:merged_at]
+          def column_list
+            [mr_metrics_table[:merged_at]]
           end
         end
       end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb
index fe477490648..fd30ab5277d 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb
@@ -11,7 +11,12 @@ module Gitlab
           end
           # rubocop: enable CodeReuse/ActiveRecord
 
-          override :column_list
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_negated_query_customization(query)
+            super.joins(:metrics)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+
           def column_list
             [timestamp_projection]
           end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
index bddc326de71..9b4cbc9090c 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
@@ -17,11 +17,6 @@ module Gitlab
             Issue
           end
 
-          def timestamp_projection
-            Arel::Nodes::NamedFunction.new('COALESCE', column_list)
-          end
-
-          override :column_list
           def column_list
             [
               issue_metrics_table[:first_associated_with_milestone_at],
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb
index cfc9300a710..530e53f9d10 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb
@@ -34,14 +34,16 @@ module Gitlab
           # Each StageEvent must expose a timestamp or a timestamp like expression in order to build a range query.
           # Example: get me all the Issue records between start event end end event
           def timestamp_projection
-            raise NotImplementedError
+            columns = column_list
+
+            columns.one? ? columns.first : Arel::Nodes::NamedFunction.new('COALESCE', columns)
           end
 
           # List of columns that are referenced in the `timestamp_projection` expression
           # Example timestamp projection: COALESCE(issue_metrics.created_at, issue_metrics.updated_at)
           # Expected column list: issue_metrics.created_at, issue_metrics.updated_at
           def column_list
-            []
+            raise NotImplementedError
           end
 
           # Optionally a StageEvent may apply additional filtering or join other tables on the base query.
@@ -49,6 +51,12 @@ module Gitlab
             query
           end
 
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_negated_query_customization(query)
+            query.where(timestamp_projection.eq(nil))
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+
           def self.label_based?
             false
           end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
index 777a8278e6e..11fe1dde12f 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
@@ -18,22 +18,30 @@ module Gitlab
 
         def duration
           Arel::Nodes::Subtraction.new(
-            stage.end_event.timestamp_projection,
+            end_event_timestamp_projection,
             stage.start_event.timestamp_projection
           )
         end
 
+        def end_event_timestamp_projection
+          if in_progress?
+            Arel::Nodes::NamedFunction.new('TO_TIMESTAMP', [Time.current.to_i])
+          else
+            stage.end_event.timestamp_projection
+          end
+        end
+
         # rubocop: disable CodeReuse/ActiveRecord
         def order_by(query, sort, direction, extra_columns_to_select = [:id])
-          ordered_query = Gitlab::Analytics::CycleAnalytics::Sorting.apply(query, stage, sort, direction)
+          ordered_query = Gitlab::Analytics::CycleAnalytics::Sorting.new(stage: stage, query: query, params: params).apply(sort, direction)
 
           # When filtering for more than one label, postgres requires the columns in ORDER BY to be present in the GROUP BY clause
           if requires_grouping?
-            column_list = [
-              *extra_columns_to_select,
-              *stage.end_event.column_list,
-              *stage.start_event.column_list
-            ]
+            column_list = [].tap do |array|
+              array.concat(extra_columns_to_select)
+              array.concat(stage.end_event.column_list) unless in_progress?
+              array.concat(stage.start_event.column_list)
+            end
 
             ordered_query = ordered_query.group(column_list)
           end
@@ -45,6 +53,10 @@ module Gitlab
         def requires_grouping?
           Array(params[:label_name]).size > 1
         end
+
+        def in_progress?
+          params[:end_event_filter] == :in_progress
+        end
       end
     end
   end