diff options
author | Markus Koller <markus-koller@gmx.ch> | 2016-11-22 19:58:10 +0300 |
---|---|---|
committer | Markus Koller <markus-koller@gmx.ch> | 2016-12-21 18:39:49 +0300 |
commit | 3ef4f74b1acc9399db320b53dffc592542de0126 (patch) | |
tree | edc99011c4ed64ec50e457d3e59c1149fbb3a5ce | |
parent | 6fd58ee48dcfbca49c609c45004d6c25035af2eb (diff) |
Add more storage statistics
This adds counters for build artifacts and LFS objects, and moves
the preexisting repository_size and commit_count from the projects
table into a new project_statistics table.
The counters are displayed in the administration area for projects
and groups, and also available through the API for admins (on */all)
and normal users (on */owned)
The statistics are updated through ProjectCacheWorker, which can now
do more granular updates with the new :statistics argument.
49 files changed, 748 insertions, 121 deletions
diff --git a/app/controllers/admin/groups_controller.rb b/app/controllers/admin/groups_controller.rb index 1e3d194e9f9..61a3a03182a 100644 --- a/app/controllers/admin/groups_controller.rb +++ b/app/controllers/admin/groups_controller.rb @@ -1,17 +1,18 @@ class Admin::GroupsController < Admin::ApplicationController - before_action :group, only: [:edit, :show, :update, :destroy, :project_update, :members_update] + before_action :group, only: [:edit, :update, :destroy, :project_update, :members_update] def index - @groups = Group.all + @groups = Group.with_statistics @groups = @groups.sort(@sort = params[:sort]) @groups = @groups.search(params[:name]) if params[:name].present? @groups = @groups.page(params[:page]) end def show + @group = Group.with_statistics.find_by_full_path(params[:id]) @members = @group.members.order("access_level DESC").page(params[:members_page]) @requesters = AccessRequestsFinder.new(@group).execute(current_user) - @projects = @group.projects.page(params[:projects_page]) + @projects = @group.projects.with_statistics.page(params[:projects_page]) end def new diff --git a/app/controllers/admin/projects_controller.rb b/app/controllers/admin/projects_controller.rb index 1d963bdf7d5..b09ae423096 100644 --- a/app/controllers/admin/projects_controller.rb +++ b/app/controllers/admin/projects_controller.rb @@ -3,7 +3,7 @@ class Admin::ProjectsController < Admin::ApplicationController before_action :group, only: [:show, :transfer] def index - @projects = Project.all + @projects = Project.with_statistics @projects = @projects.in_namespace(params[:namespace_id]) if params[:namespace_id].present? @projects = @projects.where(visibility_level: params[:visibility_level]) if params[:visibility_level].present? @projects = @projects.with_push if params[:with_push].present? diff --git a/app/controllers/groups_controller.rb b/app/controllers/groups_controller.rb index efe9c001bcf..01c8fa2739f 100644 --- a/app/controllers/groups_controller.rb +++ b/app/controllers/groups_controller.rb @@ -75,7 +75,7 @@ class GroupsController < Groups::ApplicationController end def projects - @projects = @group.projects.page(params[:page]) + @projects = @group.projects.with_statistics.page(params[:page]) end def update diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index 9c9e38c4ed7..fc9eccd2942 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -246,11 +246,6 @@ module ProjectsHelper end end - def repository_size(project = @project) - size_in_bytes = project.repository_size * 1.megabyte - number_to_human_size(size_in_bytes, delimiter: ',', precision: 2) - end - def default_url_to_repo(project = @project) case default_clone_protocol when 'ssh' diff --git a/app/helpers/sorting_helper.rb b/app/helpers/sorting_helper.rb index f03c4627050..ff787fb4131 100644 --- a/app/helpers/sorting_helper.rb +++ b/app/helpers/sorting_helper.rb @@ -11,6 +11,7 @@ module SortingHelper sort_value_due_date_soon => sort_title_due_date_soon, sort_value_due_date_later => sort_title_due_date_later, sort_value_largest_repo => sort_title_largest_repo, + sort_value_largest_group => sort_title_largest_group, sort_value_recently_signin => sort_title_recently_signin, sort_value_oldest_signin => sort_title_oldest_signin, sort_value_downvotes => sort_title_downvotes, @@ -92,6 +93,10 @@ module SortingHelper 'Largest repository' end + def sort_title_largest_group + 'Largest group' + end + def sort_title_recently_signin 'Recent sign in' end @@ -193,7 +198,11 @@ module SortingHelper end def sort_value_largest_repo - 'repository_size_desc' + 'storage_size_desc' + end + + def sort_value_largest_group + 'storage_size_desc' end def sort_value_recently_signin diff --git a/app/helpers/storage_helper.rb b/app/helpers/storage_helper.rb new file mode 100644 index 00000000000..e19c67a37ca --- /dev/null +++ b/app/helpers/storage_helper.rb @@ -0,0 +1,7 @@ +module StorageHelper + def storage_counter(size_in_bytes) + precision = size_in_bytes < 1.megabyte ? 0 : 1 + + number_to_human_size(size_in_bytes, delimiter: ',', precision: precision, significant: false) + end +end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index cb76cdf5981..27042798741 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -43,6 +43,8 @@ module Ci before_destroy { project } after_create :execute_hooks + after_save :update_project_statistics, if: :artifacts_size_changed? + after_destroy :update_project_statistics class << self def first_pending @@ -584,5 +586,9 @@ module Ci Ci::MaskSecret.mask!(trace, token) trace end + + def update_project_statistics + ProjectCacheWorker.perform_async(project_id, [], [:build_artifacts_size]) + end end end diff --git a/app/models/group.rb b/app/models/group.rb index ac8a82c8c1e..85696ad9747 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -48,7 +48,13 @@ class Group < Namespace end def sort(method) - order_by(method) + if method == 'storage_size_desc' + # storage_size is a virtual column so we need to + # pass a string to avoid AR adding the table name + reorder('storage_size DESC, namespaces.id DESC') + else + order_by(method) + end end def reference_prefix diff --git a/app/models/lfs_objects_project.rb b/app/models/lfs_objects_project.rb index 0fd5f089db9..007eed5600a 100644 --- a/app/models/lfs_objects_project.rb +++ b/app/models/lfs_objects_project.rb @@ -5,4 +5,13 @@ class LfsObjectsProject < ActiveRecord::Base validates :lfs_object_id, presence: true validates :lfs_object_id, uniqueness: { scope: [:project_id], message: "already exists in project" } validates :project_id, presence: true + + after_create :update_project_statistics + after_destroy :update_project_statistics + + private + + def update_project_statistics + ProjectCacheWorker.perform_async(project_id, [], [:lfs_objects_size]) + end end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index b52f08c7081..d41833de66f 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -9,6 +9,7 @@ class Namespace < ActiveRecord::Base cache_markdown_field :description, pipeline: :description has_many :projects, dependent: :destroy + has_many :project_statistics belongs_to :owner, class_name: "User" belongs_to :parent, class_name: "Namespace" @@ -38,6 +39,18 @@ class Namespace < ActiveRecord::Base scope :root, -> { where('type IS NULL') } + scope :with_statistics, -> do + joins('LEFT JOIN project_statistics ps ON ps.namespace_id = namespaces.id') + .group('namespaces.id') + .select( + 'namespaces.*', + 'COALESCE(SUM(ps.storage_size), 0) AS storage_size', + 'COALESCE(SUM(ps.repository_size), 0) AS repository_size', + 'COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size', + 'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size', + ) + end + class << self def by_path(path) find_by('lower(path) = :value', value: path.downcase) diff --git a/app/models/project.rb b/app/models/project.rb index 26fa20f856d..19bbe65b01d 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -44,6 +44,7 @@ class Project < ActiveRecord::Base after_create :ensure_dir_exist after_create :create_project_feature, unless: :project_feature after_save :ensure_dir_exist, if: :namespace_id_changed? + after_save :update_project_statistics, if: :namespace_id_changed? # set last_activity_at to the same as created_at after_create :set_last_activity_at @@ -151,6 +152,7 @@ class Project < ActiveRecord::Base has_one :import_data, dependent: :destroy, class_name: "ProjectImportData" has_one :project_feature, dependent: :destroy + has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete has_many :commit_statuses, dependent: :destroy, foreign_key: :gl_project_id has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline', foreign_key: :gl_project_id @@ -220,6 +222,7 @@ class Project < ActiveRecord::Base scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) } scope :with_project_feature, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id') } + scope :with_statistics, -> { includes(:statistics) } # "enabled" here means "not disabled". It includes private features! scope :with_feature_enabled, ->(feature) { @@ -332,8 +335,10 @@ class Project < ActiveRecord::Base end def sort(method) - if method == 'repository_size_desc' - reorder(repository_size: :desc, id: :desc) + if method == 'storage_size_desc' + # storage_size is a joined column so we need to + # pass a string to avoid AR adding the table name + reorder('project_statistics.storage_size DESC, projects.id DESC') else order_by(method) end @@ -1036,14 +1041,6 @@ class Project < ActiveRecord::Base forked? && project == forked_from_project end - def update_repository_size - update_attribute(:repository_size, repository.size) - end - - def update_commit_count - update_attribute(:commit_count, repository.commit_count) - end - def forks_count forks.count end @@ -1322,4 +1319,9 @@ class Project < ActiveRecord::Base def full_path_changed? path_changed? || namespace_id_changed? end + + def update_project_statistics + stats = statistics || build_statistics + stats.update(namespace_id: namespace_id) + end end diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb new file mode 100644 index 00000000000..2270ac75071 --- /dev/null +++ b/app/models/project_statistics.rb @@ -0,0 +1,43 @@ +class ProjectStatistics < ActiveRecord::Base + belongs_to :project + belongs_to :namespace + + before_save :update_storage_size + + STORAGE_COLUMNS = [:repository_size, :lfs_objects_size, :build_artifacts_size] + STATISTICS_COLUMNS = [:commit_count] + STORAGE_COLUMNS + + def total_repository_size + repository_size + lfs_objects_size + end + + def refresh!(only: nil) + STATISTICS_COLUMNS.each do |column, generator| + if only.blank? || only.include?(column) + public_send("update_#{column}") + end + end + + save! + end + + def update_commit_count + self.commit_count = project.repository.commit_count + end + + def update_repository_size + self.repository_size = project.repository.size + end + + def update_lfs_objects_size + self.lfs_objects_size = project.lfs_objects.sum(:size) + end + + def update_build_artifacts_size + self.build_artifacts_size = project.builds.sum(:artifacts_size) + end + + def update_storage_size + self.storage_size = STORAGE_COLUMNS.sum(&method(:read_attribute)) + end +end diff --git a/app/services/git_push_service.rb b/app/services/git_push_service.rb index 185556c12cc..f603036cf03 100644 --- a/app/services/git_push_service.rb +++ b/app/services/git_push_service.rb @@ -74,7 +74,7 @@ class GitPushService < BaseService types = [] end - ProjectCacheWorker.perform_async(@project.id, types) + ProjectCacheWorker.perform_async(@project.id, types, [:commit_count, :repository_size]) end protected diff --git a/app/services/git_tag_push_service.rb b/app/services/git_tag_push_service.rb index 20a4445bddf..96432837481 100644 --- a/app/services/git_tag_push_service.rb +++ b/app/services/git_tag_push_service.rb @@ -12,7 +12,7 @@ class GitTagPushService < BaseService project.execute_hooks(@push_data.dup, :tag_push_hooks) project.execute_services(@push_data.dup, :tag_push_hooks) Ci::CreatePipelineService.new(project, current_user, @push_data).execute - ProjectCacheWorker.perform_async(project.id) + ProjectCacheWorker.perform_async(project.id, [], [:commit_count, :repository_size]) true end diff --git a/app/views/admin/groups/_group.html.haml b/app/views/admin/groups/_group.html.haml index cf28f92853e..6fc212119c4 100644 --- a/app/views/admin/groups/_group.html.haml +++ b/app/views/admin/groups/_group.html.haml @@ -5,6 +5,9 @@ = link_to 'Edit', admin_group_edit_path(group), id: "edit_#{dom_id(group)}", class: 'btn' = link_to 'Delete', [:admin, group], data: { confirm: "Are you sure you want to remove #{group.name}?" }, method: :delete, class: 'btn btn-remove' .stats + %span.badge + = storage_counter(group.storage_size) + %span = icon('bookmark') = number_with_delimiter(group.projects.count) diff --git a/app/views/admin/groups/index.html.haml b/app/views/admin/groups/index.html.haml index 794f910a61f..07775247cfd 100644 --- a/app/views/admin/groups/index.html.haml +++ b/app/views/admin/groups/index.html.haml @@ -27,6 +27,8 @@ = sort_title_recently_updated = link_to admin_groups_path(sort: sort_value_oldest_updated, name: project_name) do = sort_title_oldest_updated + = link_to admin_groups_path(sort: sort_value_largest_group, name: project_name) do + = sort_title_largest_group = link_to new_admin_group_path, class: "btn btn-new" do New Group %ul.content-list diff --git a/app/views/admin/groups/show.html.haml b/app/views/admin/groups/show.html.haml index 7b0175af214..ab9c79f6add 100644 --- a/app/views/admin/groups/show.html.haml +++ b/app/views/admin/groups/show.html.haml @@ -39,6 +39,18 @@ = @group.created_at.to_s(:medium) %li + %span.light Storage: + %strong= storage_counter(@group.storage_size) + ( + = storage_counter(@group.repository_size) + repositories, + = storage_counter(@group.build_artifacts_size) + build artifacts, + = storage_counter(@group.lfs_objects_size) + LFS + ) + + %li %span.light Group Git LFS status: %strong = group_lfs_status(@group) @@ -55,8 +67,8 @@ %li %strong = link_to project.name_with_namespace, [:admin, project.namespace.becomes(Namespace), project] - %span.label.label-gray - = repository_size(project) + %span.badge + = storage_counter(project.statistics.storage_size) %span.pull-right.light %span.monospace= project.path_with_namespace + ".git" .panel-footer @@ -73,8 +85,8 @@ %li %strong = link_to project.name_with_namespace, [:admin, project.namespace.becomes(Namespace), project] - %span.label.label-gray - = repository_size(project) + %span.badge + = storage_counter(project.statistics.storage_size) %span.pull-right.light %span.monospace= project.path_with_namespace + ".git" diff --git a/app/views/admin/projects/index.html.haml b/app/views/admin/projects/index.html.haml index 8bc7dc7dd51..2e6f03fcde0 100644 --- a/app/views/admin/projects/index.html.haml +++ b/app/views/admin/projects/index.html.haml @@ -69,8 +69,8 @@ .controls - if project.archived %span.label.label-warning archived - %span.label.label-gray - = repository_size(project) + %span.badge + = storage_counter(project.statistics.storage_size) = link_to 'Edit', edit_namespace_project_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn" = link_to 'Delete', [project.namespace.becomes(Namespace), project], data: { confirm: remove_project_message(project) }, method: :delete, class: "btn btn-remove" .title diff --git a/app/views/admin/projects/show.html.haml b/app/views/admin/projects/show.html.haml index 6c7c3c48604..2967da6e692 100644 --- a/app/views/admin/projects/show.html.haml +++ b/app/views/admin/projects/show.html.haml @@ -65,9 +65,16 @@ = @project.repository.path_to_repo %li - %span.light Size - %strong - = repository_size(@project) + %span.light Storage: + %strong= storage_counter(@project.statistics.storage_size) + ( + = storage_counter(@project.statistics.repository_size) + repository, + = storage_counter(@project.statistics.build_artifacts_size) + build artifacts, + = storage_counter(@project.statistics.lfs_objects_size) + LFS + ) %li %span.light last commit: diff --git a/app/views/groups/projects.html.haml b/app/views/groups/projects.html.haml index 33fee334d93..2e7e5e5c309 100644 --- a/app/views/groups/projects.html.haml +++ b/app/views/groups/projects.html.haml @@ -18,8 +18,8 @@ .pull-right - if project.archived %span.label.label-warning archived - %span.label.label-gray - = repository_size(project) + %span.badge + = storage_counter(project.statistics.storage_size) = link_to 'Members', namespace_project_project_members_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn btn-sm" = link_to 'Edit', edit_namespace_project_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn btn-sm" = link_to 'Remove', project, data: { confirm: remove_project_message(project)}, method: :delete, class: "btn btn-sm btn-remove" diff --git a/app/views/projects/show.html.haml b/app/views/projects/show.html.haml index 8a214e1de58..eb31fe430b6 100644 --- a/app/views/projects/show.html.haml +++ b/app/views/projects/show.html.haml @@ -17,10 +17,10 @@ %ul.nav %li = link_to project_files_path(@project) do - Files (#{repository_size}) + Files (#{storage_counter(@project.statistics.total_repository_size)}) %li = link_to namespace_project_commits_path(@project.namespace, @project, current_ref) do - #{'Commit'.pluralize(@project.commit_count)} (#{number_with_delimiter(@project.commit_count)}) + #{'Commit'.pluralize(@project.statistics.commit_count)} (#{number_with_delimiter(@project.statistics.commit_count)}) %li = link_to namespace_project_branches_path(@project.namespace, @project) do #{'Branch'.pluralize(@repository.branch_count)} (#{number_with_delimiter(@repository.branch_count)}) diff --git a/app/workers/project_cache_worker.rb b/app/workers/project_cache_worker.rb index 27d7e652721..8ff9d07860f 100644 --- a/app/workers/project_cache_worker.rb +++ b/app/workers/project_cache_worker.rb @@ -6,26 +6,27 @@ class ProjectCacheWorker LEASE_TIMEOUT = 15.minutes.to_i # project_id - The ID of the project for which to flush the cache. - # refresh - An Array containing extra types of data to refresh such as - # `:readme` to flush the README and `:changelog` to flush the - # CHANGELOG. - def perform(project_id, refresh = []) + # files - An Array containing extra types of files to refresh such as + # `:readme` to flush the README and `:changelog` to flush the + # CHANGELOG. + # statistics - An Array containing columns from ProjectStatistics to + # refresh, if empty all columns will be refreshed + def perform(project_id, files = [], statistics = []) project = Project.find_by(id: project_id) return unless project && project.repository.exists? - update_repository_size(project) - project.update_commit_count + update_statistics(project, statistics.map(&:to_sym)) - project.repository.refresh_method_caches(refresh.map(&:to_sym)) + project.repository.refresh_method_caches(files.map(&:to_sym)) end - def update_repository_size(project) - return unless try_obtain_lease_for(project.id, :update_repository_size) + def update_statistics(project, statistics = []) + return unless try_obtain_lease_for(project.id, :update_statistics) - Rails.logger.info("Updating repository size for project #{project.id}") + Rails.logger.info("Updating statistics for project #{project.id}") - project.update_repository_size + project.statistics.refresh!(only: statistics) end private diff --git a/changelogs/unreleased/feature-more-storage-statistics.yml b/changelogs/unreleased/feature-more-storage-statistics.yml new file mode 100644 index 00000000000..824fd36dc34 --- /dev/null +++ b/changelogs/unreleased/feature-more-storage-statistics.yml @@ -0,0 +1,4 @@ +--- +title: Add more storage statistics +merge_request: 7754 +author: Markus Koller diff --git a/config/initializers/inflections.rb b/config/initializers/inflections.rb index 3d1a41a4652..d4197da3fa9 100644 --- a/config/initializers/inflections.rb +++ b/config/initializers/inflections.rb @@ -10,5 +10,5 @@ # end # ActiveSupport::Inflector.inflections do |inflect| - inflect.uncountable %w(award_emoji) + inflect.uncountable %w(award_emoji project_statistics) end diff --git a/db/migrate/20161201155511_create_project_statistics.rb b/db/migrate/20161201155511_create_project_statistics.rb new file mode 100644 index 00000000000..26e6d3623eb --- /dev/null +++ b/db/migrate/20161201155511_create_project_statistics.rb @@ -0,0 +1,20 @@ +class CreateProjectStatistics < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + # use bigint columns to support values >2GB + counter_column = { limit: 8, null: false, default: 0 } + + create_table :project_statistics do |t| + t.references :project, null: false, index: { unique: true }, foreign_key: { on_delete: :cascade } + t.references :namespace, null: false, index: true + t.integer :commit_count, counter_column + t.integer :storage_size, counter_column + t.integer :repository_size, counter_column + t.integer :lfs_objects_size, counter_column + t.integer :build_artifacts_size, counter_column + end + end +end diff --git a/db/migrate/20161201160452_migrate_project_statistics.rb b/db/migrate/20161201160452_migrate_project_statistics.rb new file mode 100644 index 00000000000..3ae3f2c159b --- /dev/null +++ b/db/migrate/20161201160452_migrate_project_statistics.rb @@ -0,0 +1,23 @@ +class MigrateProjectStatistics < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = true + DOWNTIME_REASON = 'Removes two columns from the projects table' + + def up + # convert repository_size in float (megabytes) to integer (bytes), + # initialize total storage_size with repository_size + execute <<-EOF + INSERT INTO project_statistics (project_id, namespace_id, commit_count, storage_size, repository_size) + SELECT id, namespace_id, commit_count, (repository_size * 1024 * 1024), (repository_size * 1024 * 1024) FROM projects + EOF + + remove_column :projects, :repository_size + remove_column :projects, :commit_count + end + + def down + add_column_with_default :projects, :repository_size, :float, default: 0.0 + add_column_with_default :projects, :commit_count, :integer, default: 0 + end +end diff --git a/db/schema.rb b/db/schema.rb index 13a847827cc..95a4f7e8ee0 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -901,6 +901,19 @@ ActiveRecord::Schema.define(version: 20161220141214) do add_index "project_import_data", ["project_id"], name: "index_project_import_data_on_project_id", using: :btree + create_table "project_statistics", force: :cascade do |t| + t.integer "project_id", null: false + t.integer "namespace_id", null: false + t.integer "commit_count", limit: 8, default: 0, null: false + t.integer "storage_size", limit: 8, default: 0, null: false + t.integer "repository_size", limit: 8, default: 0, null: false + t.integer "lfs_objects_size", limit: 8, default: 0, null: false + t.integer "build_artifacts_size", limit: 8, default: 0, null: false + end + + add_index "project_statistics", ["namespace_id"], name: "index_project_statistics_on_namespace_id", using: :btree + add_index "project_statistics", ["project_id"], name: "index_project_statistics_on_project_id", unique: true, using: :btree + create_table "projects", force: :cascade do |t| t.string "name" t.string "path" @@ -915,11 +928,9 @@ ActiveRecord::Schema.define(version: 20161220141214) do t.boolean "archived", default: false, null: false t.string "avatar" t.string "import_status" - t.float "repository_size", default: 0.0 t.integer "star_count", default: 0, null: false t.string "import_type" t.string "import_source" - t.integer "commit_count", default: 0 t.text "import_error" t.integer "ci_id" t.boolean "shared_runners_enabled", default: true, null: false @@ -1288,6 +1299,7 @@ ActiveRecord::Schema.define(version: 20161220141214) do add_foreign_key "personal_access_tokens", "users" add_foreign_key "project_authorizations", "projects", on_delete: :cascade add_foreign_key "project_authorizations", "users", on_delete: :cascade + add_foreign_key "project_statistics", "projects", on_delete: :cascade add_foreign_key "protected_branch_merge_access_levels", "protected_branches" add_foreign_key "protected_branch_push_access_levels", "protected_branches" add_foreign_key "subscriptions", "projects", on_delete: :cascade diff --git a/doc/administration/build_artifacts.md b/doc/administration/build_artifacts.md index 3ba8387c7f0..cca422892ec 100644 --- a/doc/administration/build_artifacts.md +++ b/doc/administration/build_artifacts.md @@ -88,3 +88,9 @@ artifacts through the [Admin area settings](../user/admin_area/settings/continuo [reconfigure gitlab]: restart_gitlab.md "How to restart GitLab" [restart gitlab]: restart_gitlab.md "How to restart GitLab" + +## Storage statistics + +You can see the total storage used for build artifacts on groups and projects +in the administration area, as well as through the [groups](../api/groups.md) +and [projects APIs](../api/projects.md). diff --git a/doc/api/groups.md b/doc/api/groups.md index 134d7bda22f..bc737bff8ee 100644 --- a/doc/api/groups.md +++ b/doc/api/groups.md @@ -13,6 +13,7 @@ Parameters: | `search` | string | no | Return list of authorized groups matching the search criteria | | `order_by` | string | no | Order groups by `name` or `path`. Default is `name` | | `sort` | string | no | Order groups in `asc` or `desc` order. Default is `asc` | +| `statistics` | boolean | no | Include group statistics (admins only) | ``` GET /groups @@ -31,7 +32,6 @@ GET /groups You can search for groups by name or path, see below. -======= ## List owned groups Get a list of groups which are owned by the authenticated user. @@ -40,6 +40,12 @@ Get a list of groups which are owned by the authenticated user. GET /groups/owned ``` +Parameters: + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `statistics` | boolean | no | Include group statistics | + ## List a group's projects Get a list of projects in this group. diff --git a/doc/api/projects.md b/doc/api/projects.md index edffad555a5..122075bbd11 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -307,6 +307,8 @@ Parameters: | `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` | | `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` | | `search` | string | no | Return list of authorized projects matching the search criteria | +| `simple` | boolean | no | Return only the ID, URL, name, and path of each project | +| `statistics` | boolean | no | Include project statistics | ### List starred projects @@ -325,6 +327,7 @@ Parameters: | `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` | | `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` | | `search` | string | no | Return list of authorized projects matching the search criteria | +| `simple` | boolean | no | Return only the ID, URL, name, and path of each project | ### List ALL projects @@ -343,6 +346,7 @@ Parameters: | `order_by` | string | no | Return projects ordered by `id`, `name`, `path`, `created_at`, `updated_at`, or `last_activity_at` fields. Default is `created_at` | | `sort` | string | no | Return projects sorted in `asc` or `desc` order. Default is `desc` | | `search` | string | no | Return list of authorized projects matching the search criteria | +| `statistics` | boolean | no | Include project statistics | ### Get single project diff --git a/doc/workflow/lfs/lfs_administration.md b/doc/workflow/lfs/lfs_administration.md index b3c73e947f0..5f6a718135d 100644 --- a/doc/workflow/lfs/lfs_administration.md +++ b/doc/workflow/lfs/lfs_administration.md @@ -40,6 +40,12 @@ In `config/gitlab.yml`: storage_path: /mnt/storage/lfs-objects ``` +## Storage statistics + +You can see the total storage used for LFS objects on groups and projects +in the administration area, as well as through the [groups](../api/groups.md) +and [projects APIs](../api/projects.md). + ## Known limitations * Currently, storing GitLab Git LFS objects on a non-local storage (like S3 buckets) @@ -47,3 +53,5 @@ In `config/gitlab.yml`: * Currently, removing LFS objects from GitLab Git LFS storage is not supported * LFS authentications via SSH was added with GitLab 8.12 * Only compatible with the GitLFS client versions 1.1.0 and up, or 1.0.2. +* The storage statistics currently count each LFS object multiple times for + every project linking to it diff --git a/lib/api/entities.rb b/lib/api/entities.rb index c1e42fb7d47..9f15c08f472 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -101,6 +101,16 @@ module API expose :only_allow_merge_if_build_succeeds expose :request_access_enabled expose :only_allow_merge_if_all_discussions_are_resolved + + expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics + end + + class ProjectStatistics < Grape::Entity + expose :commit_count + expose :storage_size + expose :repository_size + expose :lfs_objects_size + expose :build_artifacts_size end class Member < UserBasic @@ -127,6 +137,15 @@ module API expose :avatar_url expose :web_url expose :request_access_enabled + + expose :statistics, if: :statistics do + with_options format_with: -> (value) { value.to_i } do + expose :storage_size + expose :repository_size + expose :lfs_objects_size + expose :build_artifacts_size + end + end end class GroupDetail < Group diff --git a/lib/api/groups.rb b/lib/api/groups.rb index a9ae2977dc5..e04d2e40fb6 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -11,6 +11,20 @@ module API optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' end + + params :statistics_params do + optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' + end + + def present_groups(groups, options = {}) + options = options.reverse_merge( + with: Entities::Group, + current_user: current_user, + ) + + groups = groups.with_statistics if options[:statistics] + present paginate(groups), options + end end resource :groups do @@ -18,6 +32,7 @@ module API success Entities::Group end params do + use :statistics_params optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list' optional :all_available, type: Boolean, desc: 'Show all group that you have access to' optional :search, type: String, desc: 'Search for a specific group' @@ -38,7 +53,7 @@ module API groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? groups = groups.reorder(params[:order_by] => params[:sort]) - present paginate(groups), with: Entities::Group, current_user: current_user + present_groups groups, statistics: params[:statistics] && current_user.is_admin? end desc 'Get list of owned groups for authenticated user' do @@ -46,10 +61,10 @@ module API end params do use :pagination + use :statistics_params end get '/owned' do - groups = current_user.owned_groups - present paginate(groups), with: Entities::Group, current_user: current_user + present_groups current_user.owned_groups, statistics: params[:statistics] end desc 'Create a group. Available only for users who can create groups.' do diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 4be659fc20b..fe00c83bff3 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -248,7 +248,7 @@ module API rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500) end - # Projects helpers + # project helpers def filter_projects(projects) if params[:search].present? diff --git a/lib/api/projects.rb b/lib/api/projects.rb index f5609d878f8..3be14e8eb76 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -40,6 +40,15 @@ module API resource :projects do helpers do + params :collection_params do + use :sort_params + use :filter_params + use :pagination + + optional :simple, type: Boolean, default: false, + desc: 'Return only the ID, URL, name, and path of each project' + end + params :sort_params do optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at], default: 'created_at', desc: 'Return projects ordered by field' @@ -52,97 +61,94 @@ module API optional :visibility, type: String, values: %w[public internal private], desc: 'Limit by visibility' optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria' - use :sort_params + end + + params :statistics_params do + optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' end params :create_params do optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.' optional :import_url, type: String, desc: 'URL from which the project is imported' end + + def present_projects(projects, options = {}) + options = options.reverse_merge( + with: Entities::Project, + current_user: current_user, + simple: params[:simple], + ) + + projects = filter_projects(projects) + projects = projects.with_statistics if options[:statistics] + options[:with] = Entities::BasicProjectDetails if options[:simple] + + present paginate(projects), options + end end desc 'Get a list of visible projects for authenticated user' do success Entities::BasicProjectDetails end params do - optional :simple, type: Boolean, default: false, - desc: 'Return only the ID, URL, name, and path of each project' - use :filter_params - use :pagination + use :collection_params end get '/visible' do - projects = ProjectsFinder.new.execute(current_user) - projects = filter_projects(projects) - entity = params[:simple] || !current_user ? Entities::BasicProjectDetails : Entities::ProjectWithAccess - - present paginate(projects), with: entity, current_user: current_user + entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails + present_projects ProjectsFinder.new.execute(current_user), with: entity end desc 'Get a projects list for authenticated user' do success Entities::BasicProjectDetails end params do - optional :simple, type: Boolean, default: false, - desc: 'Return only the ID, URL, name, and path of each project' - use :filter_params - use :pagination + use :collection_params end get do authenticate! - projects = current_user.authorized_projects - projects = filter_projects(projects) - entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess - - present paginate(projects), with: entity, current_user: current_user + present_projects current_user.authorized_projects, + with: Entities::ProjectWithAccess end desc 'Get an owned projects list for authenticated user' do success Entities::BasicProjectDetails end params do - use :filter_params - use :pagination + use :collection_params + use :statistics_params end get '/owned' do authenticate! - projects = current_user.owned_projects - projects = filter_projects(projects) - - present paginate(projects), with: Entities::ProjectWithAccess, current_user: current_user + present_projects current_user.owned_projects, + with: Entities::ProjectWithAccess, + statistics: params[:statistics] end desc 'Gets starred project for the authenticated user' do success Entities::BasicProjectDetails end params do - use :filter_params - use :pagination + use :collection_params end get '/starred' do authenticate! - projects = current_user.viewable_starred_projects - projects = filter_projects(projects) - - present paginate(projects), with: Entities::Project, current_user: current_user + present_projects current_user.viewable_starred_projects end desc 'Get all projects for admin user' do success Entities::BasicProjectDetails end params do - use :filter_params - use :pagination + use :collection_params + use :statistics_params end get '/all' do authenticated_as_admin! - projects = Project.all - projects = filter_projects(projects) - - present paginate(projects), with: Entities::ProjectWithAccess, current_user: current_user + present_projects Project.all, with: Entities::ProjectWithAccess, statistics: params[:statistics] end desc 'Search for projects the current user has access to' do diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index dbdd4e977e8..a2eca74a3c8 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -63,8 +63,7 @@ namespace :gitlab do if project.persisted? puts " * Created #{project.name} (#{repo_path})".color(:green) - project.update_repository_size - project.update_commit_count + ProjectCacheWorker.perform(project.id) else puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) puts " Errors: #{project.errors.messages}".color(:red) diff --git a/spec/factories/lfs_objects.rb b/spec/factories/lfs_objects.rb index a81645acd2b..477fab9e964 100644 --- a/spec/factories/lfs_objects.rb +++ b/spec/factories/lfs_objects.rb @@ -2,7 +2,7 @@ include ActionDispatch::TestProcess FactoryGirl.define do factory :lfs_object do - oid "b68143e6463773b1b6c6fd009a76c32aeec041faff32ba2ed42fd7f708a17f80" + sequence(:oid) { |n| "b68143e6463773b1b6c6fd009a76c32aeec041faff32ba2ed42fd7f708a%05x" % n } size 499013 end diff --git a/spec/factories/project_statistics.rb b/spec/factories/project_statistics.rb new file mode 100644 index 00000000000..72d43096216 --- /dev/null +++ b/spec/factories/project_statistics.rb @@ -0,0 +1,6 @@ +FactoryGirl.define do + factory :project_statistics do + project { create :project } + namespace { project.namespace } + end +end diff --git a/spec/helpers/storage_helper_spec.rb b/spec/helpers/storage_helper_spec.rb new file mode 100644 index 00000000000..4627a1e1872 --- /dev/null +++ b/spec/helpers/storage_helper_spec.rb @@ -0,0 +1,21 @@ +require 'spec_helper' + +describe StorageHelper do + describe '#storage_counter' do + it 'formats bytes to one decimal place' do + expect(helper.storage_counter(1.23.megabytes)).to eq '1.2 MB' + end + + it 'does not add decimals for sizes < 1 MB' do + expect(helper.storage_counter(23.5.kilobytes)).to eq '24 KB' + end + + it 'does not add decimals for zeroes' do + expect(helper.storage_counter(2.megabytes)).to eq '2 MB' + end + + it 'uses commas as thousands separator' do + expect(helper.storage_counter(100_000_000_000_000_000)).to eq '90,949.5 TB' + end + end +end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index f420d71dee2..ceed9c942c1 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -192,6 +192,7 @@ project: - authorized_users - project_authorizations - route +- statistics award_emoji: - awardable - user diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index a7e90c8a381..7e1d1126b97 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -85,4 +85,30 @@ describe Ci::Build, models: true do it { expect(build.trace_file_path).to eq(build.old_path_to_trace) } end end + + describe '#update_project_statistics' do + let!(:build) { create(:ci_build, artifacts_size: 23) } + + it 'updates project statistics when the artifact size changes' do + expect(ProjectCacheWorker).to receive(:perform_async) + .with(build.project_id, [], [:build_artifacts_size]) + + build.artifacts_size = 42 + build.save! + end + + it 'does not update project statistics when the artifact size stays the same' do + expect(ProjectCacheWorker).not_to receive(:perform_async) + + build.name = 'changed' + build.save! + end + + it 'updates project statistics when the build is destroyed' do + expect(ProjectCacheWorker).to receive(:perform_async) + .with(build.project_id, [], [:build_artifacts_size]) + + build.destroy + end + end end diff --git a/spec/models/lfs_objects_project_spec.rb b/spec/models/lfs_objects_project_spec.rb new file mode 100644 index 00000000000..7bc278e350f --- /dev/null +++ b/spec/models/lfs_objects_project_spec.rb @@ -0,0 +1,36 @@ +require 'spec_helper' + +describe LfsObjectsProject, models: true do + subject { create(:lfs_objects_project, project: project) } + let(:project) { create(:empty_project) } + + describe 'associations' do + it { is_expected.to belong_to(:project) } + it { is_expected.to belong_to(:lfs_object) } + end + + describe 'validation' do + it { is_expected.to validate_presence_of(:lfs_object_id) } + it { is_expected.to validate_uniqueness_of(:lfs_object_id).scoped_to(:project_id).with_message("already exists in project") } + + it { is_expected.to validate_presence_of(:project_id) } + end + + describe '#update_project_statistics' do + it 'updates project statistics when the object is added' do + expect(ProjectCacheWorker).to receive(:perform_async) + .with(project.id, [], [:lfs_objects_size]) + + subject.save! + end + + it 'updates project statistics when the object is removed' do + subject.save! + + expect(ProjectCacheWorker).to receive(:perform_async) + .with(project.id, [], [:lfs_objects_size]) + + subject.destroy + end + end +end diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index 9fd06bb6b23..600538ff5f4 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -4,6 +4,7 @@ describe Namespace, models: true do let!(:namespace) { create(:namespace) } it { is_expected.to have_many :projects } + it { is_expected.to have_many :project_statistics } it { is_expected.to validate_presence_of(:name) } it { is_expected.to validate_uniqueness_of(:name).scoped_to(:parent_id) } @@ -57,6 +58,50 @@ describe Namespace, models: true do end end + describe '.with_statistics' do + let(:namespace) { create :namespace } + + let(:project1) do + create(:empty_project, + namespace: namespace, + statistics: build(:project_statistics, + storage_size: 606, + repository_size: 101, + lfs_objects_size: 202, + build_artifacts_size: 303)) + end + + let(:project2) do + create(:empty_project, + namespace: namespace, + statistics: build(:project_statistics, + storage_size: 60, + repository_size: 10, + lfs_objects_size: 20, + build_artifacts_size: 30)) + end + + it "sums all project storage counters in the namespace" do + project1 + project2 + statistics = Namespace.with_statistics.find(namespace.id) + + expect(statistics.storage_size).to eq 666 + expect(statistics.repository_size).to eq 111 + expect(statistics.lfs_objects_size).to eq 222 + expect(statistics.build_artifacts_size).to eq 333 + end + + it "correctly handles namespaces without projects" do + statistics = Namespace.with_statistics.find(namespace.id) + + expect(statistics.storage_size).to eq 0 + expect(statistics.repository_size).to eq 0 + expect(statistics.lfs_objects_size).to eq 0 + expect(statistics.build_artifacts_size).to eq 0 + end + end + describe '#move_dir' do before do @namespace = create :namespace diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 88d5d14f855..fb225eb7625 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -49,6 +49,7 @@ describe Project, models: true do it { is_expected.to have_one(:gitlab_issue_tracker_service).dependent(:destroy) } it { is_expected.to have_one(:external_wiki_service).dependent(:destroy) } it { is_expected.to have_one(:project_feature).dependent(:destroy) } + it { is_expected.to have_one(:statistics).class_name('ProjectStatistics').dependent(:delete) } it { is_expected.to have_one(:import_data).class_name('ProjectImportData').dependent(:destroy) } it { is_expected.to have_one(:last_event).class_name('Event') } it { is_expected.to have_one(:forked_from_project).through(:forked_project_link) } @@ -1729,6 +1730,26 @@ describe Project, models: true do end end + describe '#update_project_statistics' do + let(:project) { create(:empty_project) } + + it "is called after creation" do + expect(project.statistics).to be_a ProjectStatistics + expect(project.statistics).to be_persisted + end + + it "copies the namespace_id" do + expect(project.statistics.namespace_id).to eq project.namespace_id + end + + it "updates the namespace_id when changed" do + namespace = create(:namespace) + project.update(namespace: namespace) + + expect(project.statistics.namespace_id).to eq namespace.id + end + end + def enable_lfs allow(Gitlab.config.lfs).to receive(:enabled).and_return(true) end diff --git a/spec/models/project_statistics_spec.rb b/spec/models/project_statistics_spec.rb new file mode 100644 index 00000000000..77403cc9eb0 --- /dev/null +++ b/spec/models/project_statistics_spec.rb @@ -0,0 +1,160 @@ +require 'rails_helper' + +describe ProjectStatistics, models: true do + let(:project) { create :empty_project } + let(:statistics) { project.statistics } + + describe 'constants' do + describe 'STORAGE_COLUMNS' do + it 'is an array of symbols' do + expect(described_class::STORAGE_COLUMNS).to be_kind_of Array + expect(described_class::STORAGE_COLUMNS.map(&:class).uniq).to eq [Symbol] + end + end + + describe 'STATISTICS_COLUMNS' do + it 'is an array of symbols' do + expect(described_class::STATISTICS_COLUMNS).to be_kind_of Array + expect(described_class::STATISTICS_COLUMNS.map(&:class).uniq).to eq [Symbol] + end + + it 'includes all storage columns' do + expect(described_class::STATISTICS_COLUMNS & described_class::STORAGE_COLUMNS).to eq described_class::STORAGE_COLUMNS + end + end + end + + describe 'associations' do + it { is_expected.to belong_to(:project) } + it { is_expected.to belong_to(:namespace) } + end + + describe 'statistics columns' do + it "support values up to 8 exabytes" do + statistics.update!( + commit_count: 8.exabytes - 1, + repository_size: 2.exabytes, + lfs_objects_size: 2.exabytes, + build_artifacts_size: 4.exabytes - 1, + ) + + statistics.reload + + expect(statistics.commit_count).to eq(8.exabytes - 1) + expect(statistics.repository_size).to eq(2.exabytes) + expect(statistics.lfs_objects_size).to eq(2.exabytes) + expect(statistics.build_artifacts_size).to eq(4.exabytes - 1) + expect(statistics.storage_size).to eq(8.exabytes - 1) + end + end + + describe '#total_repository_size' do + it "sums repository and LFS object size" do + statistics.repository_size = 2 + statistics.lfs_objects_size = 3 + statistics.build_artifacts_size = 4 + + expect(statistics.total_repository_size).to eq 5 + end + end + + describe '#refresh!' do + before do + allow(statistics).to receive(:update_commit_count) + allow(statistics).to receive(:update_repository_size) + allow(statistics).to receive(:update_lfs_objects_size) + allow(statistics).to receive(:update_build_artifacts_size) + allow(statistics).to receive(:update_storage_size) + end + + context "without arguments" do + before do + statistics.refresh! + end + + it "sums all counters" do + expect(statistics).to have_received(:update_commit_count) + expect(statistics).to have_received(:update_repository_size) + expect(statistics).to have_received(:update_lfs_objects_size) + expect(statistics).to have_received(:update_build_artifacts_size) + end + end + + context "when passing an only: argument" do + before do + statistics.refresh! only: [:lfs_objects_size] + end + + it "only updates the given columns" do + expect(statistics).to have_received(:update_lfs_objects_size) + expect(statistics).not_to have_received(:update_commit_count) + expect(statistics).not_to have_received(:update_repository_size) + expect(statistics).not_to have_received(:update_build_artifacts_size) + end + end + end + + describe '#update_commit_count' do + before do + allow(project.repository).to receive(:commit_count).and_return(23) + statistics.update_commit_count + end + + it "stores the number of commits in the repository" do + expect(statistics.commit_count).to eq 23 + end + end + + describe '#update_repository_size' do + before do + allow(project.repository).to receive(:size).and_return(12.megabytes) + statistics.update_repository_size + end + + it "stores the size of the repository" do + expect(statistics.repository_size).to eq 12.megabytes + end + end + + describe '#update_lfs_objects_size' do + let!(:lfs_object1) { create(:lfs_object, size: 23.megabytes) } + let!(:lfs_object2) { create(:lfs_object, size: 34.megabytes) } + let!(:lfs_objects_project1) { create(:lfs_objects_project, project: project, lfs_object: lfs_object1) } + let!(:lfs_objects_project2) { create(:lfs_objects_project, project: project, lfs_object: lfs_object2) } + + before do + statistics.update_lfs_objects_size + end + + it "stores the size of related LFS objects" do + expect(statistics.lfs_objects_size).to eq 57.megabytes + end + end + + describe '#update_build_artifacts_size' do + let!(:pipeline) { create(:ci_pipeline, project: project) } + let!(:build1) { create(:ci_build, pipeline: pipeline, artifacts_size: 45.megabytes) } + let!(:build2) { create(:ci_build, pipeline: pipeline, artifacts_size: 56.megabytes) } + + before do + statistics.update_build_artifacts_size + end + + it "stores the size of related build artifacts" do + expect(statistics.build_artifacts_size).to eq 101.megabytes + end + end + + describe '#update_storage_size' do + it "sums all storage counters" do + statistics.update!( + repository_size: 2, + lfs_objects_size: 3, + ) + + statistics.reload + + expect(statistics.storage_size).to eq 5 + end + end +end diff --git a/spec/requests/api/groups_spec.rb b/spec/requests/api/groups_spec.rb index cdeb965b413..0e8d6faea27 100644 --- a/spec/requests/api/groups_spec.rb +++ b/spec/requests/api/groups_spec.rb @@ -35,6 +35,14 @@ describe API::Groups, api: true do expect(json_response.length).to eq(1) expect(json_response.first['name']).to eq(group1.name) end + + it "does not include statistics" do + get api("/groups", user1), statistics: true + + expect(response).to have_http_status(200) + expect(json_response).to be_an Array + expect(json_response.first).not_to include 'statistics' + end end context "when authenticated as admin" do @@ -44,6 +52,31 @@ describe API::Groups, api: true do expect(json_response).to be_an Array expect(json_response.length).to eq(2) end + + it "does not include statistics by default" do + get api("/groups", admin) + + expect(response).to have_http_status(200) + expect(json_response).to be_an Array + expect(json_response.first).not_to include('statistics') + end + + it "includes statistics if requested" do + attributes = { + storage_size: 702, + repository_size: 123, + lfs_objects_size: 234, + build_artifacts_size: 345, + } + + project1.statistics.update!(attributes) + + get api("/groups", admin), statistics: true + + expect(response).to have_http_status(200) + expect(json_response).to be_an Array + expect(json_response.first['statistics']).to eq attributes.stringify_keys + end end context "when using skip_groups in request" do diff --git a/spec/requests/api/projects_spec.rb b/spec/requests/api/projects_spec.rb index a626b41845f..f5788d15f93 100644 --- a/spec/requests/api/projects_spec.rb +++ b/spec/requests/api/projects_spec.rb @@ -49,7 +49,7 @@ describe API::Projects, api: true do end end - context 'when authenticated' do + context 'when authenticated as regular user' do it 'returns an array of projects' do get api('/projects', user) expect(response).to have_http_status(200) @@ -172,6 +172,22 @@ describe API::Projects, api: true do end end end + + it "does not include statistics by default" do + get api('/projects/all', admin) + + expect(response).to have_http_status(200) + expect(json_response).to be_an Array + expect(json_response.first).not_to include('statistics') + end + + it "includes statistics if requested" do + get api('/projects/all', admin), statistics: true + + expect(response).to have_http_status(200) + expect(json_response).to be_an Array + expect(json_response.first).to include 'statistics' + end end end @@ -196,6 +212,32 @@ describe API::Projects, api: true do expect(json_response.first['name']).to eq(project4.name) expect(json_response.first['owner']['username']).to eq(user4.username) end + + it "does not include statistics by default" do + get api('/projects/owned', user4) + + expect(response).to have_http_status(200) + expect(json_response).to be_an Array + expect(json_response.first).not_to include('statistics') + end + + it "includes statistics if requested" do + attributes = { + commit_count: 23, + storage_size: 702, + repository_size: 123, + lfs_objects_size: 234, + build_artifacts_size: 345, + } + + project4.statistics.update!(attributes) + + get api('/projects/owned', user4), statistics: true + + expect(response).to have_http_status(200) + expect(json_response).to be_an Array + expect(json_response.first['statistics']).to eq attributes.stringify_keys + end end end diff --git a/spec/services/git_push_service_spec.rb b/spec/services/git_push_service_spec.rb index e7624e70725..6ed08accb6d 100644 --- a/spec/services/git_push_service_spec.rb +++ b/spec/services/git_push_service_spec.rb @@ -583,7 +583,7 @@ describe GitPushService, services: true do service.push_commits = [commit] expect(ProjectCacheWorker).to receive(:perform_async). - with(project.id, %i(readme)) + with(project.id, %i(readme), %i(commit_count repository_size)) service.update_caches end @@ -596,7 +596,7 @@ describe GitPushService, services: true do it 'does not flush any conditional caches' do expect(ProjectCacheWorker).to receive(:perform_async). - with(project.id, []). + with(project.id, [], %i(commit_count repository_size)). and_call_original service.update_caches diff --git a/spec/workers/project_cache_worker_spec.rb b/spec/workers/project_cache_worker_spec.rb index 855c28b584e..f4f63b57a5f 100644 --- a/spec/workers/project_cache_worker_spec.rb +++ b/spec/workers/project_cache_worker_spec.rb @@ -1,8 +1,9 @@ require 'spec_helper' describe ProjectCacheWorker do - let(:project) { create(:project) } let(:worker) { described_class.new } + let(:project) { create(:project) } + let(:statistics) { project.statistics } describe '#perform' do before do @@ -12,7 +13,7 @@ describe ProjectCacheWorker do context 'with a non-existing project' do it 'does nothing' do - expect(worker).not_to receive(:update_repository_size) + expect(worker).not_to receive(:update_statistics) worker.perform(-1) end @@ -22,24 +23,19 @@ describe ProjectCacheWorker do it 'does nothing' do allow_any_instance_of(Repository).to receive(:exists?).and_return(false) - expect(worker).not_to receive(:update_repository_size) + expect(worker).not_to receive(:update_statistics) worker.perform(project.id) end end context 'with an existing project' do - it 'updates the repository size' do - expect(worker).to receive(:update_repository_size).and_call_original - - worker.perform(project.id) - end - - it 'updates the commit count' do - expect_any_instance_of(Project).to receive(:update_commit_count). - and_call_original + it 'updates the project statistics' do + expect(worker).to receive(:update_statistics) + .with(kind_of(Project), %i(repository_size)) + .and_call_original - worker.perform(project.id) + worker.perform(project.id, [], %w(repository_size)) end it 'refreshes the method caches' do @@ -47,33 +43,35 @@ describe ProjectCacheWorker do with(%i(readme)). and_call_original - worker.perform(project.id, %i(readme)) + worker.perform(project.id, %w(readme)) end end end - describe '#update_repository_size' do + describe '#update_statistics' do context 'when a lease could not be obtained' do it 'does not update the repository size' do allow(worker).to receive(:try_obtain_lease_for). - with(project.id, :update_repository_size). + with(project.id, :update_statistics). and_return(false) - expect(project).not_to receive(:update_repository_size) + expect(statistics).not_to receive(:refresh!) - worker.update_repository_size(project) + worker.update_statistics(project) end end context 'when a lease could be obtained' do - it 'updates the repository size' do + it 'updates the project statistics' do allow(worker).to receive(:try_obtain_lease_for). - with(project.id, :update_repository_size). + with(project.id, :update_statistics). and_return(true) - expect(project).to receive(:update_repository_size).and_call_original + expect(statistics).to receive(:refresh!) + .with(only: %i(repository_size)) + .and_call_original - worker.update_repository_size(project) + worker.update_statistics(project, %i(repository_size)) end end end |