Merge branch 'feature/cycle-analytics-events' into 'master'

Cycle Analytics: Events per stage

Adds list of events to each stage:

- Issue: list of issues created in the last XX days, that have been labeled or added to a milestone.
- Plan: list of commits that reference for the fist time an issue from the last stage.
- Code: list of MR created in this stage
- Test: List of unique builds triggered by the commits.
- Review: List of MR merged
- Staging: List of deployed builds
- Production: list of issues with the time from idea to production

Fixes #23449 

- [x] [CHANGELOG](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CHANGELOG) entry added
- [ ] [Documentation created/updated](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/development/doc_styleguide.md)
- Tests
  - [x] Added for this feature/bug
  - [x] All builds are passing
- [x] Conform by the [merge request performance guides](http://docs.gitlab.com/ce/development/merge_request_performance_guidelines.html)
- [x] Conform by the [style guides](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md#style-guides)
- [x] Branch has no merge conflicts with `master` (if it does - rebase it please)
- [ ] [Squashed related commits together](https://git-scm.com/book/en/Git-Tools-Rewriting-History#Squashing-Commits)

See merge request !6859

16 files changed, 228 insertions, 67 deletions

diff --git a/app/controllers/concerns/cycle_analytics_params.rb b/app/controllers/concerns/cycle_analytics_params.rb
new file mode 100644
index 00000000000..2aaf8f2b451
--- /dev/null
+++ b/app/controllers/concerns/cycle_analytics_params.rb
@@ -0,0 +1,7 @@
+module CycleAnalyticsParams
+  extend ActiveSupport::Concern
+
+  def start_date(params)
+    params[:start_date] == '30' ? 30.days.ago : 90.days.ago
+  end
+end
diff --git a/app/controllers/projects/cycle_analytics/events_controller.rb b/app/controllers/projects/cycle_analytics/events_controller.rb
new file mode 100644
index 00000000000..13b3eec761f
--- /dev/null
+++ b/app/controllers/projects/cycle_analytics/events_controller.rb
@@ -0,0 +1,65 @@
+module Projects
+  module CycleAnalytics
+    class EventsController < Projects::ApplicationController
+      include CycleAnalyticsParams
+    
+      before_action :authorize_read_cycle_analytics!
+      before_action :authorize_read_build!, only: [:test, :staging]
+      before_action :authorize_read_issue!, only: [:issue, :production]
+      before_action :authorize_read_merge_request!, only: [:code, :review]
+
+      def issue
+        render_events(events.issue_events)
+      end
+    
+      def plan
+        render_events(events.plan_events)
+      end
+    
+      def code
+        render_events(events.code_events)
+      end
+    
+      def test
+        options[:branch] = events_params[:branch_name]
+    
+        render_events(events.test_events)
+      end
+    
+      def review
+        render_events(events.review_events)
+      end
+    
+      def staging
+        render_events(events.staging_events)
+      end
+    
+      def production
+        render_events(events.production_events)
+      end
+    
+      private
+    
+      def render_events(events_list)
+        respond_to do |format|
+          format.html
+          format.json { render json: { events: events_list } }
+        end
+      end
+    
+      def events
+        @events ||= Gitlab::CycleAnalytics::Events.new(project: project, options: options)
+      end
+    
+      def options
+        @options ||= { from: start_date(events_params), current_user: current_user }
+      end
+    
+      def events_params
+        return {} unless params[:events].present?
+    
+        params[:events].slice(:start_date, :branch_name)
+      end
+    end
+  end
+end
diff --git a/app/controllers/projects/cycle_analytics_controller.rb b/app/controllers/projects/cycle_analytics_controller.rb
index 16a7b1fc6e2..96eb75a0547 100644
--- a/app/controllers/projects/cycle_analytics_controller.rb
+++ b/app/controllers/projects/cycle_analytics_controller.rb
@@ -1,11 +1,12 @@
 class Projects::CycleAnalyticsController < Projects::ApplicationController
   include ActionView::Helpers::DateHelper
   include ActionView::Helpers::TextHelper
+  include CycleAnalyticsParams
 
   before_action :authorize_read_cycle_analytics!
 
   def show
-    @cycle_analytics = CycleAnalytics.new(@project, from: parse_start_date)
+    @cycle_analytics = ::CycleAnalytics.new(@project, from: start_date(cycle_analytics_params))
 
     respond_to do |format|
       format.html
@@ -15,14 +16,6 @@ class Projects::CycleAnalyticsController < Projects::ApplicationController
 
   private
 
-  def parse_start_date
-    case cycle_analytics_params[:start_date]
-    when '30' then 30.days.ago
-    when '90' then 90.days.ago
-    else 90.days.ago
-    end
-  end
-
   def cycle_analytics_params
     return {} unless params[:cycle_analytics].present?
 
diff --git a/app/models/cycle_analytics.rb b/app/models/cycle_analytics.rb
index 8ed4a56b19b..314a1ce9b63 100644
--- a/app/models/cycle_analytics.rb
+++ b/app/models/cycle_analytics.rb
@@ -1,12 +1,8 @@
 class CycleAnalytics
-  include Gitlab::Database::Median
-  include Gitlab::Database::DateTime
-
-  DEPLOYMENT_METRIC_STAGES = %i[production staging]
-
   def initialize(project, from:)
     @project = project
     @from = from
+    @fetcher = Gitlab::CycleAnalytics::MetricsFetcher.new(project: project, from: from, branch: nil)
   end
 
   def summary
@@ -14,90 +10,46 @@ class CycleAnalytics
   end
 
   def issue
-    calculate_metric(:issue,
+    @fetcher.calculate_metric(:issue,
                      Issue.arel_table[:created_at],
                      [Issue::Metrics.arel_table[:first_associated_with_milestone_at],
                       Issue::Metrics.arel_table[:first_added_to_board_at]])
   end
 
   def plan
-    calculate_metric(:plan,
+    @fetcher.calculate_metric(:plan,
                      [Issue::Metrics.arel_table[:first_associated_with_milestone_at],
                       Issue::Metrics.arel_table[:first_added_to_board_at]],
                      Issue::Metrics.arel_table[:first_mentioned_in_commit_at])
   end
 
   def code
-    calculate_metric(:code,
+    @fetcher.calculate_metric(:code,
                      Issue::Metrics.arel_table[:first_mentioned_in_commit_at],
                      MergeRequest.arel_table[:created_at])
   end
 
   def test
-    calculate_metric(:test,
+    @fetcher.calculate_metric(:test,
                      MergeRequest::Metrics.arel_table[:latest_build_started_at],
                      MergeRequest::Metrics.arel_table[:latest_build_finished_at])
   end
 
   def review
-    calculate_metric(:review,
+    @fetcher.calculate_metric(:review,
                      MergeRequest.arel_table[:created_at],
                      MergeRequest::Metrics.arel_table[:merged_at])
   end
 
   def staging
-    calculate_metric(:staging,
+    @fetcher.calculate_metric(:staging,
                      MergeRequest::Metrics.arel_table[:merged_at],
                      MergeRequest::Metrics.arel_table[:first_deployed_to_production_at])
   end
 
   def production
-    calculate_metric(:production,
+    @fetcher.calculate_metric(:production,
                      Issue.arel_table[:created_at],
                      MergeRequest::Metrics.arel_table[:first_deployed_to_production_at])
   end
-
-  private
-
-  def calculate_metric(name, start_time_attrs, end_time_attrs)
-    cte_table = Arel::Table.new("cte_table_for_#{name}")
-
-    # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time).
-    # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time).
-    # We compute the (end_time - start_time) interval, and give it an alias based on the current
-    # cycle analytics stage.
-    interval_query = Arel::Nodes::As.new(
-      cte_table,
-      subtract_datetimes(base_query_for(name), end_time_attrs, start_time_attrs, name.to_s))
-
-    median_datetime(cte_table, interval_query, name)
-  end
-
-  # Join table with a row for every <issue,merge_request> pair (where the merge request
-  # closes the given issue) with issue and merge request metrics included. The metrics
-  # are loaded with an inner join, so issues / merge requests without metrics are
-  # automatically excluded.
-  def base_query_for(name)
-    arel_table = MergeRequestsClosingIssues.arel_table
-
-    # Load issues
-    query = arel_table.join(Issue.arel_table).on(Issue.arel_table[:id].eq(arel_table[:issue_id])).
-            join(Issue::Metrics.arel_table).on(Issue.arel_table[:id].eq(Issue::Metrics.arel_table[:issue_id])).
-            where(Issue.arel_table[:project_id].eq(@project.id)).
-            where(Issue.arel_table[:deleted_at].eq(nil)).
-            where(Issue.arel_table[:created_at].gteq(@from))
-
-    # Load merge_requests
-    query = query.join(MergeRequest.arel_table, Arel::Nodes::OuterJoin).
-            on(MergeRequest.arel_table[:id].eq(arel_table[:merge_request_id])).
-            join(MergeRequest::Metrics.arel_table).
-            on(MergeRequest.arel_table[:id].eq(MergeRequest::Metrics.arel_table[:merge_request_id]))
-
-    if DEPLOYMENT_METRIC_STAGES.include?(name)
-      # Limit to merge requests that have been deployed to production after `@from`
-      query.where(MergeRequest::Metrics.arel_table[:first_deployed_to_production_at].gteq(@from))
-    end
-
-    query
-  end
 end
diff --git a/app/models/merge_request/metrics.rb b/app/models/merge_request/metrics.rb
index 99c49a020c9..cdc408738be 100644
--- a/app/models/merge_request/metrics.rb
+++ b/app/models/merge_request/metrics.rb
@@ -1,5 +1,6 @@
 class MergeRequest::Metrics < ActiveRecord::Base
   belongs_to :merge_request
+  belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :pipeline_id
 
   def record!
     if merge_request.merged? && self.merged_at.blank?
diff --git a/app/serializers/analytics_build_entity.rb b/app/serializers/analytics_build_entity.rb
new file mode 100644
index 00000000000..5fdf2bbf7c3
--- /dev/null
+++ b/app/serializers/analytics_build_entity.rb
@@ -0,0 +1,40 @@
+class AnalyticsBuildEntity < Grape::Entity
+  include RequestAwareEntity
+  include EntityDateHelper
+
+  expose :name
+  expose :id
+  expose :ref, as: :branch
+  expose :short_sha
+  expose :author, using: UserEntity
+
+  expose :started_at, as: :date do |build|
+    interval_in_words(build[:started_at])
+  end
+
+  expose :duration, as: :total_time do |build|
+    distance_of_time_as_hash(build[:duration].to_f)
+  end
+
+  expose :branch do
+    expose :ref, as: :name
+
+    expose :url do |build|
+      url_to(:namespace_project_tree, build, build.ref)
+    end
+  end
+
+  expose :url do |build|
+    url_to(:namespace_project_build, build)
+  end
+
+  expose :commit_url do |build|
+    url_to(:namespace_project_commit, build, build.sha)
+  end
+
+  private
+
+  def url_to(route, build, id = nil)
+    public_send("#{route}_url", build.project.namespace, build.project, id || build)
+  end
+end
diff --git a/app/serializers/analytics_build_serializer.rb b/app/serializers/analytics_build_serializer.rb
new file mode 100644
index 00000000000..f172d67d356
--- /dev/null
+++ b/app/serializers/analytics_build_serializer.rb
@@ -0,0 +1,3 @@
+class AnalyticsBuildSerializer < BaseSerializer
+  entity AnalyticsBuildEntity
+end
diff --git a/app/serializers/analytics_commit_entity.rb b/app/serializers/analytics_commit_entity.rb
new file mode 100644
index 00000000000..402cecbfd08
--- /dev/null
+++ b/app/serializers/analytics_commit_entity.rb
@@ -0,0 +1,13 @@
+class AnalyticsCommitEntity < CommitEntity
+  include EntityDateHelper
+
+  expose :short_id, as: :short_sha
+
+  expose :total_time do |commit|
+    distance_of_time_as_hash(request.total_time.to_f)
+  end
+
+  unexpose :author_name
+  unexpose :author_email
+  unexpose :message
+end
diff --git a/app/serializers/analytics_commit_serializer.rb b/app/serializers/analytics_commit_serializer.rb
new file mode 100644
index 00000000000..cdbfecf2b70
--- /dev/null
+++ b/app/serializers/analytics_commit_serializer.rb
@@ -0,0 +1,3 @@
+class AnalyticsCommitSerializer < BaseSerializer
+  entity AnalyticsCommitEntity
+end
diff --git a/app/serializers/analytics_generic_serializer.rb b/app/serializers/analytics_generic_serializer.rb
new file mode 100644
index 00000000000..9f4859e8410
--- /dev/null
+++ b/app/serializers/analytics_generic_serializer.rb
@@ -0,0 +1,7 @@
+class AnalyticsGenericSerializer < BaseSerializer
+  def represent(resource, opts = {})
+    resource.symbolize_keys!
+
+    super(resource, opts)
+  end
+end
diff --git a/app/serializers/analytics_issue_entity.rb b/app/serializers/analytics_issue_entity.rb
new file mode 100644
index 00000000000..44c50f18613
--- /dev/null
+++ b/app/serializers/analytics_issue_entity.rb
@@ -0,0 +1,29 @@
+class AnalyticsIssueEntity < Grape::Entity
+  include RequestAwareEntity
+  include EntityDateHelper
+
+  expose :title
+  expose :author, using: UserEntity
+
+  expose :iid do |object|
+    object[:iid].to_s
+  end
+
+  expose :total_time do |object|
+    distance_of_time_as_hash(object[:total_time].to_f)
+  end
+
+  expose(:created_at) do |object|
+    interval_in_words(object[:created_at])
+  end
+
+  expose :url do |object|
+    url_to(:namespace_project_issue, id: object[:iid].to_s)
+  end
+
+  private
+
+  def url_to(route, id)
+    public_send("#{route}_url", request.project.namespace, request.project, id)
+  end
+end
diff --git a/app/serializers/analytics_issue_serializer.rb b/app/serializers/analytics_issue_serializer.rb
new file mode 100644
index 00000000000..4fb3e8f1bb4
--- /dev/null
+++ b/app/serializers/analytics_issue_serializer.rb
@@ -0,0 +1,3 @@
+class AnalyticsIssueSerializer < AnalyticsGenericSerializer
+  entity AnalyticsIssueEntity
+end
diff --git a/app/serializers/analytics_merge_request_entity.rb b/app/serializers/analytics_merge_request_entity.rb
new file mode 100644
index 00000000000..888265eaa38
--- /dev/null
+++ b/app/serializers/analytics_merge_request_entity.rb
@@ -0,0 +1,7 @@
+class AnalyticsMergeRequestEntity < AnalyticsIssueEntity
+  expose :state
+
+  expose :url do |object|
+    url_to(:namespace_project_merge_request, id: object[:iid].to_s)
+  end
+end
diff --git a/app/serializers/analytics_merge_request_serializer.rb b/app/serializers/analytics_merge_request_serializer.rb
new file mode 100644
index 00000000000..4622a1dd855
--- /dev/null
+++ b/app/serializers/analytics_merge_request_serializer.rb
@@ -0,0 +1,3 @@
+class AnalyticsMergeRequestSerializer < AnalyticsGenericSerializer
+  entity AnalyticsMergeRequestEntity
+end
diff --git a/app/serializers/entity_date_helper.rb b/app/serializers/entity_date_helper.rb
new file mode 100644
index 00000000000..b333b3344c3
--- /dev/null
+++ b/app/serializers/entity_date_helper.rb
@@ -0,0 +1,35 @@
+module EntityDateHelper
+  include ActionView::Helpers::DateHelper
+
+  def interval_in_words(diff)
+    "#{distance_of_time_in_words(diff.to_f)} ago"
+  end
+
+  # Converts seconds into a hash such as:
+  # { days: 1, hours: 3, mins: 42, seconds: 40 }
+  #
+  # It returns 0 seconds for zero or negative numbers
+  # It rounds to nearest time unit and does not return zero
+  # i.e { min: 1 } instead of { mins: 1, seconds: 0 }
+  def distance_of_time_as_hash(diff)
+    diff = diff.abs.floor
+
+    return { seconds: 0 } if diff == 0
+
+    mins = (diff / 60).floor
+    seconds = diff % 60
+    hours = (mins / 60).floor
+    mins = mins % 60
+    days = (hours / 24).floor
+    hours = hours % 24
+
+    duration_hash = {}
+
+    duration_hash[:days] = days if days > 0
+    duration_hash[:hours] = hours if hours > 0
+    duration_hash[:mins] = mins if mins > 0
+    duration_hash[:seconds] = seconds if seconds > 0
+
+    duration_hash
+  end
+end
diff --git a/app/workers/pipeline_metrics_worker.rb b/app/workers/pipeline_metrics_worker.rb
index 34f6ef161fb..070943f1ecc 100644
--- a/app/workers/pipeline_metrics_worker.rb
+++ b/app/workers/pipeline_metrics_worker.rb
@@ -12,11 +12,11 @@ class PipelineMetricsWorker
   private
 
   def update_metrics_for_active_pipeline(pipeline)
-    metrics(pipeline).update_all(latest_build_started_at: pipeline.started_at, latest_build_finished_at: nil)
+    metrics(pipeline).update_all(latest_build_started_at: pipeline.started_at, latest_build_finished_at: nil, pipeline_id: pipeline.id)
   end
 
   def update_metrics_for_succeeded_pipeline(pipeline)
-    metrics(pipeline).update_all(latest_build_started_at: pipeline.started_at, latest_build_finished_at: pipeline.finished_at)
+    metrics(pipeline).update_all(latest_build_started_at: pipeline.started_at, latest_build_finished_at: pipeline.finished_at, pipeline_id: pipeline.id)
   end
 
   def metrics(pipeline)