Precalculate trending projects

This commit introduces a Sidekiq worker that precalculates the list of
trending projects on a daily basis. The resulting set is stored in a
database table that is then queried by Project.trending.

This setup means that Unicorn workers no longer _may_ have to calculate
the list of trending projects. Furthermore it supports filtering without
any complex caching mechanisms.

The data in the "trending_projects" table is inserted in the same order
as the project ranking. This means that getting the projects in the
correct order is simply a matter of:

    SELECT projects.*
    FROM projects
    INNER JOIN trending_projects ON trending_projects.project_id = projects.id
    ORDER BY trending_projects.id ASC;

Such a query will only take a few milliseconds at most (as measured on
GitLab.com), opposed to a few seconds for the query used for calculating
the project ranks.

The migration in this commit does not require downtime and takes care of
populating an initial list of trending projects.

2 files changed, 46 insertions, 1 deletions

diff --git a/db/migrate/20161007133303_precalculate_trending_projects.rb b/db/migrate/20161007133303_precalculate_trending_projects.rb
new file mode 100644
index 00000000000..b324cd94268
--- /dev/null
+++ b/db/migrate/20161007133303_precalculate_trending_projects.rb
@@ -0,0 +1,38 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class PrecalculateTrendingProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    create_table :trending_projects do |t|
+      t.references :project, index: true, foreign_key: { on_delete: :cascade }, null: false
+    end
+
+    timestamp = connection.quote(1.month.ago)
+
+    # We're hardcoding the visibility level (public) here so that if it ever
+    # changes this query doesn't suddenly use the new value (which may break
+    # later migrations).
+    visibility = 20
+
+    execute <<-EOF.strip_heredoc
+      INSERT INTO trending_projects (project_id)
+      SELECT project_id
+      FROM notes
+      INNER JOIN projects ON projects.id = notes.project_id
+      WHERE notes.created_at >= #{timestamp}
+      AND notes.system IS FALSE
+      AND projects.visibility_level = #{visibility}
+      GROUP BY project_id
+      ORDER BY count(*) DESC
+      LIMIT 100;
+    EOF
+  end
+
+  def down
+    drop_table :trending_projects
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 56da70b3c02..c5ddf9eee32 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20160926145521) do
+ActiveRecord::Schema.define(version: 20161007133303) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -1070,6 +1070,12 @@ ActiveRecord::Schema.define(version: 20160926145521) do
   add_index "todos", ["target_type", "target_id"], name: "index_todos_on_target_type_and_target_id", using: :btree
   add_index "todos", ["user_id"], name: "index_todos_on_user_id", using: :btree
 
+  create_table "trending_projects", force: :cascade do |t|
+    t.integer "project_id", null: false
+  end
+
+  add_index "trending_projects", ["project_id"], name: "index_trending_projects_on_project_id", using: :btree
+
   create_table "u2f_registrations", force: :cascade do |t|
     t.text     "certificate"
     t.string   "key_handle"
@@ -1212,5 +1218,6 @@ ActiveRecord::Schema.define(version: 20160926145521) do
   add_foreign_key "personal_access_tokens", "users"
   add_foreign_key "protected_branch_merge_access_levels", "protected_branches"
   add_foreign_key "protected_branch_push_access_levels", "protected_branches"
+  add_foreign_key "trending_projects", "projects", on_delete: :cascade
   add_foreign_key "u2f_registrations", "users"
 end