Add latest changes from gitlab-org/gitlab@master

6 files changed, 270 insertions, 3 deletions

diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb
new file mode 100644
index 00000000000..ee459f2790a
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb
@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module Aggregated
+        # rubocop: disable CodeReuse/ActiveRecord
+        class BaseQueryBuilder
+          include StageQueryHelpers
+
+          MODEL_CLASSES = {
+            MergeRequest.to_s => ::Analytics::CycleAnalytics::MergeRequestStageEvent,
+            Issue.to_s => ::Analytics::CycleAnalytics::IssueStageEvent
+          }.freeze
+
+          # Allowed params:
+          # * from - stage end date filter start date
+          # * to - stage end date filter to date
+          # * author_username
+          # * milestone_title
+          # * label_name (array)
+          # * assignee_username (array)
+          # * project_ids (array)
+          def initialize(stage:, params: {})
+            @stage = stage
+            @params = params
+            @root_ancestor = stage.parent.root_ancestor
+            @stage_event_model = MODEL_CLASSES.fetch(stage.subject_class.to_s)
+          end
+
+          def build
+            query = base_query
+            query = filter_by_stage_parent(query)
+            query = filter_author(query)
+            query = filter_milestone_ids(query)
+            query = filter_label_names(query)
+            filter_assignees(query)
+          end
+
+          def filter_author(query)
+            return query if params[:author_username].blank?
+
+            user = User.by_username(params[:author_username]).first
+
+            return query.none if user.blank?
+
+            query.authored(user)
+          end
+
+          def filter_milestone_ids(query)
+            return query if params[:milestone_title].blank?
+
+            milestone = MilestonesFinder
+              .new(group_ids: root_ancestor.self_and_descendant_ids, project_ids: root_ancestor.all_projects.select(:id), title: params[:milestone_title])
+              .execute
+              .first
+
+            return query.none if milestone.blank?
+
+            query.with_milestone_id(milestone.id)
+          end
+
+          def filter_label_names(query)
+            return query if params[:label_name].blank?
+
+            all_label_ids = Issuables::LabelFilter
+              .new(group: root_ancestor, project: nil, params: { label_name: params[:label_name] })
+              .find_label_ids(params[:label_name])
+
+            return query.none if params[:label_name].size != all_label_ids.size
+
+            all_label_ids.each do |label_ids|
+              relation = LabelLink
+                .where(target_type: stage.subject_class.name)
+                .where(LabelLink.arel_table['target_id'].eq(query.model.arel_table[query.model.issuable_id_column]))
+
+              relation = relation.where(label_id: label_ids)
+
+              query = query.where(relation.arel.exists)
+            end
+
+            query
+          end
+
+          def filter_assignees(query)
+            return query if params[:assignee_username].blank?
+
+            Issuables::AssigneeFilter
+              .new(params: { assignee_username: params[:assignee_username] })
+              .filter(query)
+          end
+
+          def filter_by_stage_parent(query)
+            query.by_project_id(stage.parent_id)
+          end
+
+          def base_query
+            query = stage_event_model
+              .by_stage_event_hash_id(stage.stage_event_hash_id)
+
+            from = params[:from] || 30.days.ago
+            if in_progress?
+              query = query
+                .end_event_is_not_happened_yet
+                .opened_state
+                .start_event_timestamp_after(from)
+              query = query.start_event_timestamp_before(params[:to]) if params[:to]
+            else
+              query = query.end_event_timestamp_after(from)
+              query = query.end_event_timestamp_before(params[:to]) if params[:to]
+            end
+
+            query
+          end
+
+          private
+
+          attr_reader :stage, :params, :root_ancestor, :stage_event_model
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+      end
+    end
+  end
+end
+Gitlab::Analytics::CycleAnalytics::Aggregated::BaseQueryBuilder.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::Aggregated::BaseQueryBuilder')
diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb
new file mode 100644
index 00000000000..ab3ae93f5ff
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/aggregated/data_collector.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module Aggregated
+        # Arguments:
+        #   stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::GroupStage
+        #   params:
+        #     current_user: an instance of User
+        #     from: DateTime
+        #     to: DateTime
+        class DataCollector
+          include Gitlab::Utils::StrongMemoize
+
+          MAX_COUNT = 10001
+
+          delegate :serialized_records, to: :records_fetcher
+
+          def initialize(stage:, params: {})
+            @stage = stage
+            @params = params
+          end
+
+          def median
+            strong_memoize(:median) { Median.new(stage: stage, query: query, params: params) }
+          end
+
+          def count
+            strong_memoize(:count) { limit_count }
+          end
+
+          private
+
+          attr_reader :stage, :params
+
+          def query
+            BaseQueryBuilder.new(stage: stage, params: params).build
+          end
+
+          def limit_count
+            query.limit(MAX_COUNT).count
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/median.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/median.rb
new file mode 100644
index 00000000000..181ee20948b
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/aggregated/median.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module Aggregated
+        class Median
+          include StageQueryHelpers
+
+          def initialize(stage:, query:, params:)
+            @stage = stage
+            @query = query
+            @params = params
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def seconds
+            @query = @query.select(median_duration_in_seconds.as('median')).reorder(nil)
+            result = @query.take || {}
+
+            result['median'] || nil
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+
+          def days
+            seconds ? seconds.fdiv(1.day) : nil
+          end
+
+          private
+
+          attr_reader :stage, :query, :params
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb
new file mode 100644
index 00000000000..f23d1832df9
--- /dev/null
+++ b/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module Aggregated
+        module StageQueryHelpers
+          def percentile_cont
+            percentile_cont_ordering = Arel::Nodes::UnaryOperation.new(Arel::Nodes::SqlLiteral.new('ORDER BY'), duration)
+            Arel::Nodes::NamedFunction.new(
+              'percentile_cont(0.5) WITHIN GROUP',
+              [percentile_cont_ordering]
+            )
+          end
+
+          def duration
+            if in_progress?
+              Arel::Nodes::Subtraction.new(
+                Arel::Nodes::NamedFunction.new('TO_TIMESTAMP', [Time.current.to_i]),
+                query.model.arel_table[:start_event_timestamp]
+              )
+            else
+              Arel::Nodes::Subtraction.new(
+                query.model.arel_table[:end_event_timestamp],
+                query.model.arel_table[:start_event_timestamp]
+              )
+            end
+          end
+
+          def median_duration_in_seconds
+            Arel::Nodes::Extract.new(percentile_cont, :epoch)
+          end
+
+          def in_progress?
+            params[:end_event_filter] == :in_progress
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/analytics/cycle_analytics/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/data_collector.rb
index 56179533ffb..a26df55bd0a 100644
--- a/lib/gitlab/analytics/cycle_analytics/data_collector.rb
+++ b/lib/gitlab/analytics/cycle_analytics/data_collector.rb
@@ -29,7 +29,11 @@ module Gitlab
 
         def median
           strong_memoize(:median) do
-            Median.new(stage: stage, query: query, params: params)
+            if use_aggregated_data_collector?
+              aggregated_data_collector.median
+            else
+              Median.new(stage: stage, query: query, params: params)
+            end
           end
         end
 
@@ -41,7 +45,11 @@ module Gitlab
 
         def count
           strong_memoize(:count) do
-            limit_count
+            if use_aggregated_data_collector?
+              aggregated_data_collector.count
+            else
+              limit_count
+            end
           end
         end
 
@@ -59,6 +67,14 @@ module Gitlab
         def limit_count
           query.limit(MAX_COUNT).count
         end
+
+        def aggregated_data_collector
+          @aggregated_data_collector ||= Aggregated::DataCollector.new(stage: stage, params: params)
+        end
+
+        def use_aggregated_data_collector?
+          params.fetch(:use_aggregated_data_collector, false)
+        end
       end
     end
   end
diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb
index 94e20762368..bc9d94ef09c 100644
--- a/lib/gitlab/analytics/cycle_analytics/request_params.rb
+++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb
@@ -79,7 +79,8 @@ module Gitlab
             sort: sort&.to_sym,
             direction: direction&.to_sym,
             page: page,
-            end_event_filter: end_event_filter.to_sym
+            end_event_filter: end_event_filter.to_sym,
+            use_aggregated_data_collector: Feature.enabled?(:use_vsa_aggregated_tables, group || project, default_enabled: :yaml)
           }.merge(attributes.symbolize_keys.slice(*FINDER_PARAM_NAMES))
         end