Cache merged and closed events data in merge_request_metrics table

3 files changed, 110 insertions, 0 deletions

diff --git a/db/migrate/20171127151038_add_events_related_columns_to_merge_request_metrics.rb b/db/migrate/20171127151038_add_events_related_columns_to_merge_request_metrics.rb
new file mode 100644
index 00000000000..18af697cf88
--- /dev/null
+++ b/db/migrate/20171127151038_add_events_related_columns_to_merge_request_metrics.rb
@@ -0,0 +1,37 @@
+class AddEventsRelatedColumnsToMergeRequestMetrics < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    change_table :merge_request_metrics do |t|
+      t.references :merged_by, references: :users
+      t.references :latest_closed_by, references: :users
+    end
+
+    add_column :merge_request_metrics, :latest_closed_at, :datetime_with_timezone
+
+    add_concurrent_foreign_key :merge_request_metrics, :users,
+                               column: :merged_by_id,
+                               on_delete: :nullify
+
+    add_concurrent_foreign_key :merge_request_metrics, :users,
+                               column: :latest_closed_by_id,
+                               on_delete: :nullify
+  end
+
+  def down
+    if foreign_keys_for(:merge_request_metrics, :merged_by_id).any?
+      remove_foreign_key :merge_request_metrics, column: :merged_by_id
+    end
+
+    if foreign_keys_for(:merge_request_metrics, :latest_closed_by_id).any?
+      remove_foreign_key :merge_request_metrics, column: :latest_closed_by_id
+    end
+
+    remove_columns :merge_request_metrics,
+      :merged_by_id, :latest_closed_by_id, :latest_closed_at
+  end
+end
diff --git a/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb b/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb
new file mode 100644
index 00000000000..547cc68e10e
--- /dev/null
+++ b/db/post_migrate/20171128214150_schedule_populate_merge_request_metrics_with_events_data.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+# rubocop:disable GitlabSecurity/SqlInjection
+
+class SchedulePopulateMergeRequestMetricsWithEventsData < ActiveRecord::Migration
+  DOWNTIME = false
+  BATCH_SIZE = 10_000
+  MIGRATION = 'PopulateMergeRequestMetricsWithEventsData'
+
+  disable_ddl_transaction!
+
+  class MergeRequest < ActiveRecord::Base
+    self.table_name = 'merge_requests'
+
+    include ::EachBatch
+  end
+
+  def up
+    merge_requests = MergeRequest.where("id IN (#{updatable_merge_requests_union_sql})").reorder(:id)
+
+    say 'Scheduling `PopulateMergeRequestMetricsWithEventsData` jobs'
+    # It will update around 4_000_000 records in batches of 10_000 merge
+    # requests (running between 10 minutes) and should take around 66 hours to complete.
+    # Apparently, production PostgreSQL is able to vacuum 10k-20k dead_tuples by
+    # minute, and at maximum, each of these jobs should UPDATE 20k records.
+    #
+    # More information about the updates in `PopulateMergeRequestMetricsWithEventsData` class.
+    #
+    merge_requests.each_batch(of: BATCH_SIZE) do |relation, index|
+      range = relation.pluck('MIN(id)', 'MAX(id)').first
+
+      BackgroundMigrationWorker.perform_in(index * 10.minutes, MIGRATION, range)
+    end
+  end
+
+  def down
+    execute "update merge_request_metrics set latest_closed_at = null"
+    execute "update merge_request_metrics set latest_closed_by_id = null"
+    execute "update merge_request_metrics set merged_by_id = null"
+  end
+
+  private
+
+  # On staging:
+  # Planning time: 0.682 ms
+  # Execution time: 22033.158 ms
+  #
+  def updatable_merge_requests_union_sql
+    metrics_not_exists_clause =
+      'NOT EXISTS (SELECT 1 FROM merge_request_metrics WHERE merge_request_metrics.merge_request_id = merge_requests.id)'
+
+    without_metrics_data = <<-SQL.strip_heredoc
+      merge_request_metrics.merged_by_id IS NULL OR
+      merge_request_metrics.latest_closed_by_id IS NULL OR
+      merge_request_metrics.latest_closed_at IS NULL
+    SQL
+
+    mrs_without_metrics_record = MergeRequest
+      .where(metrics_not_exists_clause)
+      .select(:id)
+
+    mrs_without_events_data = MergeRequest
+      .joins('INNER JOIN merge_request_metrics ON merge_requests.id = merge_request_metrics.merge_request_id')
+      .where(without_metrics_data)
+      .select(:id)
+
+    Gitlab::SQL::Union.new([mrs_without_metrics_record, mrs_without_events_data]).to_sql
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 42715d5e1e8..9d3b5db117e 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -1056,6 +1056,9 @@ ActiveRecord::Schema.define(version: 20171220191323) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.integer "pipeline_id"
+    t.integer "merged_by_id"
+    t.integer "latest_closed_by_id"
+    t.datetime_with_timezone "latest_closed_at"
   end
 
   add_index "merge_request_metrics", ["first_deployed_to_production_at"], name: "index_merge_request_metrics_on_first_deployed_to_production_at", using: :btree
@@ -1995,6 +1998,8 @@ ActiveRecord::Schema.define(version: 20171220191323) do
   add_foreign_key "merge_request_diffs", "merge_requests", name: "fk_8483f3258f", on_delete: :cascade
   add_foreign_key "merge_request_metrics", "ci_pipelines", column: "pipeline_id", on_delete: :cascade
   add_foreign_key "merge_request_metrics", "merge_requests", on_delete: :cascade
+  add_foreign_key "merge_request_metrics", "users", column: "latest_closed_by_id", name: "fk_ae440388cc", on_delete: :nullify
+  add_foreign_key "merge_request_metrics", "users", column: "merged_by_id", name: "fk_7f28d925f3", on_delete: :nullify
   add_foreign_key "merge_requests", "ci_pipelines", column: "head_pipeline_id", name: "fk_fd82eae0b9", on_delete: :nullify
   add_foreign_key "merge_requests", "merge_request_diffs", column: "latest_merge_request_diff_id", name: "fk_06067f5644", on_delete: :nullify
   add_foreign_key "merge_requests", "milestones", name: "fk_6a5165a692", on_delete: :nullify