Merge branch 'split-events-into-push-events' into 'master'

Use a separate table for storing push events

See merge request !12463

5 files changed, 206 insertions, 1 deletions

diff --git a/db/migrate/20170608152747_prepare_events_table_for_push_events_migration.rb b/db/migrate/20170608152747_prepare_events_table_for_push_events_migration.rb
new file mode 100644
index 00000000000..f4f03bbabaf
--- /dev/null
+++ b/db/migrate/20170608152747_prepare_events_table_for_push_events_migration.rb
@@ -0,0 +1,51 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class PrepareEventsTableForPushEventsMigration < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    # The order of these columns is deliberate and results in the following
+    # columns and sizes:
+    #
+    # * id (4 bytes)
+    # * project_id (4 bytes)
+    # * author_id (4 bytes)
+    # * target_id (4 bytes)
+    # * created_at (8 bytes)
+    # * updated_at (8 bytes)
+    # * action (2 bytes)
+    # * target_type (variable)
+    #
+    # Unfortunately we can't make the "id" column a bigint/bigserial as Rails 4
+    # does not support this properly.
+    create_table :events_for_migration do |t|
+      t.references :project,
+                   index: true,
+                   foreign_key: { on_delete: :cascade }
+
+      t.integer :author_id, index: true, null: false
+      t.integer :target_id
+
+      t.timestamps_with_timezone null: false
+
+      t.integer :action, null: false, limit: 2, index: true
+      t.string :target_type
+
+      t.index %i[target_type target_id]
+    end
+
+    # t.references doesn't like it when the column name doesn't make the table
+    # name so we have to add the foreign key separately.
+    add_concurrent_foreign_key(:events_for_migration, :users, column: :author_id)
+  end
+
+  def down
+    drop_table :events_for_migration
+  end
+end
diff --git a/db/migrate/20170608152748_create_push_event_payloads_tables.rb b/db/migrate/20170608152748_create_push_event_payloads_tables.rb
new file mode 100644
index 00000000000..6c55ad1f2f7
--- /dev/null
+++ b/db/migrate/20170608152748_create_push_event_payloads_tables.rb
@@ -0,0 +1,46 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CreatePushEventPayloadsTables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    create_table :push_event_payloads, id: false do |t|
+      t.bigint :commit_count, null: false
+
+      t.integer :event_id, null: false
+      t.integer :action, null: false, limit: 2
+      t.integer :ref_type, null: false, limit: 2
+
+      t.binary :commit_from
+      t.binary :commit_to
+
+      t.text :ref
+      t.string :commit_title, limit: 70
+
+      t.index :event_id, unique: true
+    end
+
+    # We're adding a foreign key to the _shadow_ table, and this is deliberate.
+    # By using the shadow table we don't have to recreate/revalidate this
+    # foreign key after swapping the "events_for_migration" and "events" tables.
+    #
+    # The "events_for_migration" table has a foreign key to "projects.id"
+    # ensuring that project removals also remove events from the shadow table
+    # (and thus also from this table).
+    add_concurrent_foreign_key(
+      :push_event_payloads,
+      :events_for_migration,
+      column: :event_id
+    )
+  end
+
+  def down
+    drop_table :push_event_payloads
+  end
+end
diff --git a/db/migrate/20170727123534_add_index_on_events_project_id_id.rb b/db/migrate/20170727123534_add_index_on_events_project_id_id.rb
new file mode 100644
index 00000000000..1c4aaaf9dd6
--- /dev/null
+++ b/db/migrate/20170727123534_add_index_on_events_project_id_id.rb
@@ -0,0 +1,37 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddIndexOnEventsProjectIdId < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  COLUMNS = %i[project_id id].freeze
+  TABLES = %i[events events_for_migration].freeze
+
+  disable_ddl_transaction!
+
+  def up
+    TABLES.each do |table|
+      add_concurrent_index(table, COLUMNS) unless index_exists?(table, COLUMNS)
+
+      # We remove the index _after_ adding the new one since MySQL doesn't let
+      # you remove an index when a foreign key exists for the same column.
+      if index_exists?(table, :project_id)
+        remove_concurrent_index(table, :project_id)
+      end
+    end
+  end
+
+  def down
+    TABLES.each do |table|
+      unless index_exists?(table, :project_id)
+        add_concurrent_index(table, :project_id)
+      end
+
+      unless index_exists?(table, COLUMNS)
+        remove_concurrent_index(table, COLUMNS)
+      end
+    end
+  end
+end
diff --git a/db/post_migrate/20170627101016_schedule_event_migrations.rb b/db/post_migrate/20170627101016_schedule_event_migrations.rb
new file mode 100644
index 00000000000..1f34375ff0d
--- /dev/null
+++ b/db/post_migrate/20170627101016_schedule_event_migrations.rb
@@ -0,0 +1,40 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class ScheduleEventMigrations < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  BUFFER_SIZE = 1000
+
+  disable_ddl_transaction!
+
+  class Event < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'events'
+  end
+
+  def up
+    jobs = []
+
+    Event.each_batch(of: 1000) do |relation|
+      min, max = relation.pluck('MIN(id), MAX(id)').first
+
+      if jobs.length == BUFFER_SIZE
+        # We push multiple jobs at a time to reduce the time spent in
+        # Sidekiq/Redis operations. We're using this buffer based approach so we
+        # don't need to run additional queries for every range.
+        BackgroundMigrationWorker.perform_bulk(jobs)
+        jobs.clear
+      end
+
+      jobs << ['MigrateEventsToPushEventPayloads', [min, max]]
+    end
+
+    BackgroundMigrationWorker.perform_bulk(jobs) unless jobs.empty?
+  end
+
+  def down
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index ec92b185be2..65afe37a21c 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -530,10 +530,25 @@ ActiveRecord::Schema.define(version: 20170809142252) do
   add_index "events", ["action"], name: "index_events_on_action", using: :btree
   add_index "events", ["author_id"], name: "index_events_on_author_id", using: :btree
   add_index "events", ["created_at"], name: "index_events_on_created_at", using: :btree
-  add_index "events", ["project_id"], name: "index_events_on_project_id", using: :btree
+  add_index "events", ["project_id", "id"], name: "index_events_on_project_id_and_id", using: :btree
   add_index "events", ["target_id"], name: "index_events_on_target_id", using: :btree
   add_index "events", ["target_type"], name: "index_events_on_target_type", using: :btree
 
+  create_table "events_for_migration", force: :cascade do |t|
+    t.integer "project_id"
+    t.integer "author_id", null: false
+    t.integer "target_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.integer "action", limit: 2, null: false
+    t.string "target_type"
+  end
+
+  add_index "events_for_migration", ["action"], name: "index_events_for_migration_on_action", using: :btree
+  add_index "events_for_migration", ["author_id"], name: "index_events_for_migration_on_author_id", using: :btree
+  add_index "events_for_migration", ["project_id", "id"], name: "index_events_for_migration_on_project_id_and_id", using: :btree
+  add_index "events_for_migration", ["target_type", "target_id"], name: "index_events_for_migration_on_target_type_and_target_id", using: :btree
+
   create_table "feature_gates", force: :cascade do |t|
     t.string "feature_key", null: false
     t.string "key", null: false
@@ -1254,6 +1269,19 @@ ActiveRecord::Schema.define(version: 20170809142252) do
 
   add_index "protected_tags", ["project_id"], name: "index_protected_tags_on_project_id", using: :btree
 
+  create_table "push_event_payloads", id: false, force: :cascade do |t|
+    t.integer "commit_count", limit: 8, null: false
+    t.integer "event_id", null: false
+    t.integer "action", limit: 2, null: false
+    t.integer "ref_type", limit: 2, null: false
+    t.binary "commit_from"
+    t.binary "commit_to"
+    t.text "ref"
+    t.string "commit_title", limit: 70
+  end
+
+  add_index "push_event_payloads", ["event_id"], name: "index_push_event_payloads_on_event_id", unique: true, using: :btree
+
   create_table "redirect_routes", force: :cascade do |t|
     t.integer "source_id", null: false
     t.string "source_type", null: false
@@ -1654,6 +1682,8 @@ ActiveRecord::Schema.define(version: 20170809142252) do
   add_foreign_key "deployments", "projects", name: "fk_b9a3851b82", on_delete: :cascade
   add_foreign_key "environments", "projects", name: "fk_d1c8c1da6a", on_delete: :cascade
   add_foreign_key "events", "projects", name: "fk_0434b48643", on_delete: :cascade
+  add_foreign_key "events_for_migration", "projects", on_delete: :cascade
+  add_foreign_key "events_for_migration", "users", column: "author_id", name: "fk_edfd187b6f", on_delete: :cascade
   add_foreign_key "forked_project_links", "projects", column: "forked_to_project_id", name: "fk_434510edb0", on_delete: :cascade
   add_foreign_key "gpg_keys", "users", on_delete: :cascade
   add_foreign_key "gpg_signatures", "gpg_keys", on_delete: :nullify
@@ -1696,6 +1726,7 @@ ActiveRecord::Schema.define(version: 20170809142252) do
   add_foreign_key "protected_tag_create_access_levels", "protected_tags"
   add_foreign_key "protected_tag_create_access_levels", "users"
   add_foreign_key "protected_tags", "projects", name: "fk_8e4af87648", on_delete: :cascade
+  add_foreign_key "push_event_payloads", "events_for_migration", column: "event_id", name: "fk_36c74129da", on_delete: :cascade
   add_foreign_key "releases", "projects", name: "fk_47fe2a0596", on_delete: :cascade
   add_foreign_key "services", "projects", name: "fk_71cce407f9", on_delete: :cascade
   add_foreign_key "snippets", "projects", name: "fk_be41fd4bb7", on_delete: :cascade