Add more storage statistics

This adds counters for build artifacts and LFS objects, and moves
the preexisting repository_size and commit_count from the projects
table into a new project_statistics table.

The counters are displayed in the administration area for projects
and groups, and also available through the API for admins (on */all)
and normal users (on */owned)

The statistics are updated through ProjectCacheWorker, which can now
do more granular updates with the new :statistics argument.

3 files changed, 57 insertions, 2 deletions

diff --git a/db/migrate/20161201155511_create_project_statistics.rb b/db/migrate/20161201155511_create_project_statistics.rb
new file mode 100644
index 00000000000..26e6d3623eb
--- /dev/null
+++ b/db/migrate/20161201155511_create_project_statistics.rb
@@ -0,0 +1,20 @@
+class CreateProjectStatistics < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    # use bigint columns to support values >2GB
+    counter_column = { limit: 8, null: false, default: 0 }
+
+    create_table :project_statistics do |t|
+      t.references :project, null: false, index: { unique: true }, foreign_key: { on_delete: :cascade }
+      t.references :namespace, null: false, index: true
+      t.integer :commit_count, counter_column
+      t.integer :storage_size, counter_column
+      t.integer :repository_size, counter_column
+      t.integer :lfs_objects_size, counter_column
+      t.integer :build_artifacts_size, counter_column
+    end
+  end
+end
diff --git a/db/migrate/20161201160452_migrate_project_statistics.rb b/db/migrate/20161201160452_migrate_project_statistics.rb
new file mode 100644
index 00000000000..3ae3f2c159b
--- /dev/null
+++ b/db/migrate/20161201160452_migrate_project_statistics.rb
@@ -0,0 +1,23 @@
+class MigrateProjectStatistics < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = true
+  DOWNTIME_REASON = 'Removes two columns from the projects table'
+
+  def up
+    # convert repository_size in float (megabytes) to integer (bytes),
+    # initialize total storage_size with repository_size
+    execute <<-EOF
+      INSERT INTO project_statistics (project_id, namespace_id, commit_count, storage_size, repository_size)
+        SELECT id, namespace_id, commit_count, (repository_size * 1024 * 1024), (repository_size * 1024 * 1024) FROM projects
+    EOF
+
+    remove_column :projects, :repository_size
+    remove_column :projects, :commit_count
+  end
+
+  def down
+    add_column_with_default :projects, :repository_size, :float, default: 0.0
+    add_column_with_default :projects, :commit_count, :integer, default: 0
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 13a847827cc..95a4f7e8ee0 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -901,6 +901,19 @@ ActiveRecord::Schema.define(version: 20161220141214) do
 
   add_index "project_import_data", ["project_id"], name: "index_project_import_data_on_project_id", using: :btree
 
+  create_table "project_statistics", force: :cascade do |t|
+    t.integer "project_id", null: false
+    t.integer "namespace_id", null: false
+    t.integer "commit_count", limit: 8, default: 0, null: false
+    t.integer "storage_size", limit: 8, default: 0, null: false
+    t.integer "repository_size", limit: 8, default: 0, null: false
+    t.integer "lfs_objects_size", limit: 8, default: 0, null: false
+    t.integer "build_artifacts_size", limit: 8, default: 0, null: false
+  end
+
+  add_index "project_statistics", ["namespace_id"], name: "index_project_statistics_on_namespace_id", using: :btree
+  add_index "project_statistics", ["project_id"], name: "index_project_statistics_on_project_id", unique: true, using: :btree
+
   create_table "projects", force: :cascade do |t|
     t.string "name"
     t.string "path"
@@ -915,11 +928,9 @@ ActiveRecord::Schema.define(version: 20161220141214) do
     t.boolean "archived", default: false, null: false
     t.string "avatar"
     t.string "import_status"
-    t.float "repository_size", default: 0.0
     t.integer "star_count", default: 0, null: false
     t.string "import_type"
     t.string "import_source"
-    t.integer "commit_count", default: 0
     t.text "import_error"
     t.integer "ci_id"
     t.boolean "shared_runners_enabled", default: true, null: false
@@ -1288,6 +1299,7 @@ ActiveRecord::Schema.define(version: 20161220141214) do
   add_foreign_key "personal_access_tokens", "users"
   add_foreign_key "project_authorizations", "projects", on_delete: :cascade
   add_foreign_key "project_authorizations", "users", on_delete: :cascade
+  add_foreign_key "project_statistics", "projects", on_delete: :cascade
   add_foreign_key "protected_branch_merge_access_levels", "protected_branches"
   add_foreign_key "protected_branch_push_access_levels", "protected_branches"
   add_foreign_key "subscriptions", "projects", on_delete: :cascade