Includes logic to persist namespace statistics

- Add two new ActiveRecord models:
  - RootNamespaceStoragestatistics will persist root namespace statistics
  - NamespaceAggregationSchedule will save information when a new update
to the namespace statistics needs to be scheduled
- Inject into UpdateProjectStatistics concern a new callback that will
call an async job to insert a new row onto NamespaceAggregationSchedule
table
- When a new row is inserted a new job is scheduled. This job will
update call an specific service to update the statistics and after that
it will delete thee aggregated scheduled row
- The RefresherServices makes heavy use of arel to build composable
queries to update Namespace::RootStorageStatistics attributes.
- Add an extra worker to traverse pending rows on
NAmespace::AggregationSchedule table and schedule a worker for each one
of this rows.
- Add an extra worker to traverse pending rows on
NAmespace::AggregationSchedule table and schedule a worker for each one
of this rows

4 files changed, 102 insertions, 0 deletions

diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index e55962b629e..3d34bfc05c7 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -26,6 +26,7 @@
 - cronjob:issue_due_scheduler
 - cronjob:prune_web_hook_logs
 - cronjob:schedule_migrate_external_diffs
+- cronjob:namespaces_prune_aggregation_schedules
 
 - gcp_cluster:cluster_install_app
 - gcp_cluster:cluster_patch_app
@@ -101,6 +102,9 @@
 - todos_destroyer:todos_destroyer_project_private
 - todos_destroyer:todos_destroyer_private_features
 
+- update_namespace_statistics:namespaces_schedule_aggregation
+- update_namespace_statistics:namespaces_root_statistics
+
 - object_pool:object_pool_create
 - object_pool:object_pool_schedule_join
 - object_pool:object_pool_join
diff --git a/app/workers/namespaces/prune_aggregation_schedules_worker.rb b/app/workers/namespaces/prune_aggregation_schedules_worker.rb
new file mode 100644
index 00000000000..4e40feee702
--- /dev/null
+++ b/app/workers/namespaces/prune_aggregation_schedules_worker.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Namespaces
+  class PruneAggregationSchedulesWorker
+    include ApplicationWorker
+    include CronjobQueue
+
+    # Worker to prune pending rows on Namespace::AggregationSchedule
+    # It's scheduled to run once a day at 1:05am.
+    def perform
+      aggregation_schedules.find_each do |aggregation_schedule|
+        aggregation_schedule.schedule_root_storage_statistics
+      end
+    end
+
+    private
+
+    def aggregation_schedules
+      Namespace::AggregationSchedule.all
+    end
+  end
+end
diff --git a/app/workers/namespaces/root_statistics_worker.rb b/app/workers/namespaces/root_statistics_worker.rb
new file mode 100644
index 00000000000..48876825564
--- /dev/null
+++ b/app/workers/namespaces/root_statistics_worker.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Namespaces
+  class RootStatisticsWorker
+    include ApplicationWorker
+
+    queue_namespace :update_namespace_statistics
+
+    def perform(namespace_id)
+      namespace = Namespace.find(namespace_id)
+
+      return unless update_statistics_enabled_for?(namespace) && namespace.aggregation_scheduled?
+
+      Namespaces::StatisticsRefresherService.new.execute(namespace)
+
+      namespace.aggregation_schedule.destroy
+    rescue ::Namespaces::StatisticsRefresherService::RefresherError, ActiveRecord::RecordNotFound => ex
+      log_error(namespace.full_path, ex.message) if namespace
+    end
+
+    private
+
+    def log_error(namespace_path, error_message)
+      Gitlab::SidekiqLogger.error("Namespace statistics can't be updated for #{namespace_path}: #{error_message}")
+    end
+
+    def update_statistics_enabled_for?(namespace)
+      Feature.enabled?(:update_statistics_namespace, namespace)
+    end
+  end
+end
diff --git a/app/workers/namespaces/schedule_aggregation_worker.rb b/app/workers/namespaces/schedule_aggregation_worker.rb
new file mode 100644
index 00000000000..a4594b84b13
--- /dev/null
+++ b/app/workers/namespaces/schedule_aggregation_worker.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Namespaces
+  class ScheduleAggregationWorker
+    include ApplicationWorker
+
+    queue_namespace :update_namespace_statistics
+
+    def perform(namespace_id)
+      return unless aggregation_schedules_table_exists?
+
+      namespace = Namespace.find(namespace_id)
+      root_ancestor = namespace.root_ancestor
+
+      return unless update_statistics_enabled_for?(root_ancestor) && !root_ancestor.aggregation_scheduled?
+
+      Namespace::AggregationSchedule.safe_find_or_create_by!(namespace_id: root_ancestor.id)
+    rescue ActiveRecord::RecordNotFound
+      log_error(namespace_id)
+    end
+
+    private
+
+    # On db/post_migrate/20180529152628_schedule_to_archive_legacy_traces.rb
+    # traces are archived through build.trace.archive, which in consequence
+    # calls UpdateProjectStatistics#schedule_namespace_statistics_worker.
+    #
+    # The migration and specs fails since NamespaceAggregationSchedule table
+    # does not exist at that point.
+    # https://gitlab.com/gitlab-org/gitlab-ce/issues/50712
+    def aggregation_schedules_table_exists?
+      return true unless Rails.env.test?
+
+      Namespace::AggregationSchedule.table_exists?
+    end
+
+    def log_error(root_ancestor_id)
+      Gitlab::SidekiqLogger.error("Namespace can't be scheduled for aggregation: #{root_ancestor_id} does not exist")
+    end
+
+    def update_statistics_enabled_for?(root_ancestor)
+      Feature.enabled?(:update_statistics_namespace, root_ancestor)
+    end
+  end
+end