diff options
author | Timothy Andrew <mail@timothyandrew.net> | 2016-09-15 11:59:36 +0300 |
---|---|---|
committer | Timothy Andrew <mail@timothyandrew.net> | 2016-09-15 12:23:02 +0300 |
commit | ba25e2f1ac61b47940f939a2d9f1d0ad417e1de2 (patch) | |
tree | e30ce100687fde731bb738c95a8ccd12033f9c42 /app/models/cycle_analytics | |
parent | 798b17a35311d60fe18440bfc53dba3aadd7b099 (diff) |
Improve performance of the cycle analytics page.
1. These changes bring down page load time for 100 issues from more than
a minute to about 1.5 seconds.
2. This entire commit is composed of these types of performance
enhancements:
- Cache relevant data in `IssueMetrics` wherever possible.
- Cache relevant data in `MergeRequestMetrics` wherever possible.
- Preload metrics
3. Given these improvements, we now only need to make 4 SQL calls:
- Load all issues
- Load all merge requests
- Load all metrics for the issues
- Load all metrics for the merge requests
4. A list of all the data points that are now being pre-calculated:
a. The first time an issue is mentioned in a commit
- In `GitPushService`, find all issues mentioned by the given commit
using `ReferenceExtractor`. Set the `first_mentioned_in_commit_at`
flag for each of them.
- There seems to be a (pre-existing) bug here - files (and
therefore commits) created using the Web CI don't have
cross-references created, and issues are not closed even when
the commit title is "Fixes #xx".
b. The first time a merge request is deployed to production
When a `Deployment` is created, find all merge requests that
were merged in before the deployment, and set the
`first_deployed_to_production_at` flag for each of them.
c. The start / end time for a merge request pipeline
Hook into the `Pipeline` state machine. When the `status` moves to
`running`, find the merge requests whose tip commit matches the
pipeline, and record the `latest_build_started_at` time for each
of them. When the `status` moves to `success`, record the
`latest_build_finished_at` time.
d. The merge requests that close an issue
- This was a big cause of the performance problems we were having
with Cycle Analytics. We need to use `ReferenceExtractor` to make
this calculation, which is slow when we have to run it on a large
number of merge requests.
- When a merge request is created, updated, or refreshed, find the
issues it closes, and create an instance of
`MergeRequestsClosingIssues`, which acts as a join model between
merge requests and issues.
- If a `MergeRequestsClosingIssues` instance links a merge request
and an issue, that issue closes that merge request.
5. The `Queries` module was changed into a class, so we can cache the
results of `issues` and `merge_requests_closing_issues` across
various cycle analytics stages.
6. The code added in this commit is untested. Tests will be added in the
next commit.
Diffstat (limited to 'app/models/cycle_analytics')
-rw-r--r-- | app/models/cycle_analytics/queries.rb | 151 |
1 files changed, 55 insertions, 96 deletions
diff --git a/app/models/cycle_analytics/queries.rb b/app/models/cycle_analytics/queries.rb index 861460f1de4..7bdf1fb6290 100644 --- a/app/models/cycle_analytics/queries.rb +++ b/app/models/cycle_analytics/queries.rb @@ -1,121 +1,80 @@ class CycleAnalytics - module Queries - class << self - def issues(project, created_after:) - project.issues.where("created_at >= ?", created_after).preload(:metrics, :system_notes).map { |issue| { issue: issue } } - end - - def merge_requests_closing_issues(project, options = {}) - issues(project, options).map do |data_point| - merge_requests = data_point[:issue].closed_by_merge_requests(nil, check_if_open: false) - merge_requests.map { |merge_request| { issue: data_point[:issue], merge_request: merge_request } } - end.flatten - end + class Queries + def initialize(project) + @project = project + end - def issue_first_associated_with_milestone_at - lambda do |data_point| - issue = data_point[:issue] - issue.metrics.first_associated_with_milestone_at if issue.metrics.present? + def issues(options = {}) + @issues_data ||= + begin + issues_query(options).preload(:metrics).map { |issue| { issue: issue } } end - end + end - def issue_first_added_to_list_label_at - lambda do |data_point| - issue = data_point[:issue] - issue.metrics.first_added_to_board_at if issue.metrics.present? - end - end + def merge_requests_closing_issues(options = {}) + @merge_requests_closing_issues_data ||= + begin + merge_requests_closing_issues = MergeRequestsClosingIssues.where(issue: issues_query(options)).preload(issue: [:metrics], merge_request: [:metrics]) - def issue_first_mentioned_in_commit_at - lambda do |data_point| - issue = data_point[:issue] - commits_mentioning_issue = issue.system_notes.map { |note| note.all_references.commits }.flatten - commits_mentioning_issue.map(&:committed_date).min if commits_mentioning_issue.present? + merge_requests_closing_issues.map do |record| + { issue: record.issue, merge_request: record.merge_request } + end end - end + end - def merge_request_first_closed_at - lambda do |data_point| - merge_request = data_point[:merge_request] - merge_request.metrics.first_closed_at if merge_request.metrics.present? - end + def issue_first_associated_with_milestone_at + lambda do |data_point| + issue = data_point[:issue] + issue.metrics.first_associated_with_milestone_at if issue.metrics.present? end + end - def merge_request_merged_at - lambda do |data_point| - merge_request = data_point[:merge_request] - merge_request.metrics.merged_at if merge_request.metrics.present? - end + def issue_first_added_to_list_label_at + lambda do |data_point| + issue = data_point[:issue] + issue.metrics.first_added_to_board_at if issue.metrics.present? end + end - def merge_request_build_started_at - lambda do |data_point| - merge_request = data_point[:merge_request] - tip = merge_request.commits.first - return unless tip - - pipeline = Ci::Pipeline.success.find_by_sha(tip.sha) - pipeline.started_at if pipeline - end + def issue_first_mentioned_in_commit_at + lambda do |data_point| + issue = data_point[:issue] + issue.metrics.first_mentioned_in_commit_at if issue.metrics.present? end + end - def merge_request_build_finished_at - lambda do |data_point| - merge_request = data_point[:merge_request] - tip = merge_request.commits.first - return unless tip - - pipeline = Ci::Pipeline.success.find_by_sha(tip.sha) - pipeline.finished_at if pipeline - end + def merge_request_merged_at + lambda do |data_point| + merge_request = data_point[:merge_request] + merge_request.metrics.merged_at if merge_request.metrics.present? end + end - def merge_request_deployed_to_any_environment_at - lambda do |data_point| - merge_request = data_point[:merge_request] - if merge_request.metrics.present? - deployments = Deployment.where(ref: merge_request.target_branch).where("created_at > ?", merge_request.metrics.merged_at) - deployment = deployments.order(:created_at).first - deployment.created_at if deployment - end - end + def merge_request_build_started_at + lambda do |data_point| + merge_request = data_point[:merge_request] + merge_request.metrics.latest_build_started_at if merge_request.metrics.present? end + end - def merge_request_deployed_to_production_at - lambda do |data_point| - merge_request = data_point[:merge_request] - if merge_request.metrics.present? - # The first production deploy to the target branch that occurs after the merge request has been merged in. - # TODO: Does this need to account for reverts? - deployments = Deployment.joins(:environment).where(ref: merge_request.target_branch, "environments.name" => "production"). - where("deployments.created_at > ?", merge_request.metrics.merged_at) - deployment = deployments.order(:created_at).first - deployment.created_at if deployment - end - end + def merge_request_build_finished_at + lambda do |data_point| + merge_request = data_point[:merge_request] + merge_request.metrics.latest_build_finished_at if merge_request.metrics.present? end + end - def issue_closing_merge_request_opened_at - lambda do |data_point| - issue = data_point[:issue] - merge_requests = issue.closed_by_merge_requests(nil, check_if_open: false) - merge_requests.map(&:created_at).min if merge_requests.present? - end + def merge_request_deployed_to_production_at + lambda do |data_point| + merge_request = data_point[:merge_request] + merge_request.metrics.first_deployed_to_production_at if merge_request.metrics.present? end + end - def merge_request_wip_flag_first_removed_at - lambda do |data_point| - merge_request = data_point[:merge_request] - merge_request.metrics.wip_flag_first_removed_at if merge_request.metrics.present? - end - end + private - def merge_request_first_assigned_to_user_other_than_author_at - lambda do |data_point| - merge_request = data_point[:merge_request] - merge_request.metrics.first_assigned_to_user_other_than_author if merge_request.metrics.present? - end - end + def issues_query(created_after:) + @project.issues.where("created_at >= ?", created_after) end end end |