diff options
author | Mayra Cabrera <mcabrera@gitlab.com> | 2019-07-02 17:44:39 +0300 |
---|---|---|
committer | Kamil TrzciĆski <ayufan@ayufan.eu> | 2019-07-02 17:44:39 +0300 |
commit | dfdfa913ba9cb74beb7adad0352c5efadec84494 (patch) | |
tree | 16d730e52e00d6f921087ec6531ab463447d09f8 /app/workers | |
parent | e07ebe66af957c46e7c69329b3ab561bb539351b (diff) |
Includes logic to persist namespace statistics
- Add two new ActiveRecord models:
- RootNamespaceStoragestatistics will persist root namespace statistics
- NamespaceAggregationSchedule will save information when a new update
to the namespace statistics needs to be scheduled
- Inject into UpdateProjectStatistics concern a new callback that will
call an async job to insert a new row onto NamespaceAggregationSchedule
table
- When a new row is inserted a new job is scheduled. This job will
update call an specific service to update the statistics and after that
it will delete thee aggregated scheduled row
- The RefresherServices makes heavy use of arel to build composable
queries to update Namespace::RootStorageStatistics attributes.
- Add an extra worker to traverse pending rows on
NAmespace::AggregationSchedule table and schedule a worker for each one
of this rows.
- Add an extra worker to traverse pending rows on
NAmespace::AggregationSchedule table and schedule a worker for each one
of this rows
Diffstat (limited to 'app/workers')
-rw-r--r-- | app/workers/all_queues.yml | 4 | ||||
-rw-r--r-- | app/workers/namespaces/prune_aggregation_schedules_worker.rb | 22 | ||||
-rw-r--r-- | app/workers/namespaces/root_statistics_worker.rb | 31 | ||||
-rw-r--r-- | app/workers/namespaces/schedule_aggregation_worker.rb | 45 |
4 files changed, 102 insertions, 0 deletions
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index e55962b629e..3d34bfc05c7 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -26,6 +26,7 @@ - cronjob:issue_due_scheduler - cronjob:prune_web_hook_logs - cronjob:schedule_migrate_external_diffs +- cronjob:namespaces_prune_aggregation_schedules - gcp_cluster:cluster_install_app - gcp_cluster:cluster_patch_app @@ -101,6 +102,9 @@ - todos_destroyer:todos_destroyer_project_private - todos_destroyer:todos_destroyer_private_features +- update_namespace_statistics:namespaces_schedule_aggregation +- update_namespace_statistics:namespaces_root_statistics + - object_pool:object_pool_create - object_pool:object_pool_schedule_join - object_pool:object_pool_join diff --git a/app/workers/namespaces/prune_aggregation_schedules_worker.rb b/app/workers/namespaces/prune_aggregation_schedules_worker.rb new file mode 100644 index 00000000000..4e40feee702 --- /dev/null +++ b/app/workers/namespaces/prune_aggregation_schedules_worker.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Namespaces + class PruneAggregationSchedulesWorker + include ApplicationWorker + include CronjobQueue + + # Worker to prune pending rows on Namespace::AggregationSchedule + # It's scheduled to run once a day at 1:05am. + def perform + aggregation_schedules.find_each do |aggregation_schedule| + aggregation_schedule.schedule_root_storage_statistics + end + end + + private + + def aggregation_schedules + Namespace::AggregationSchedule.all + end + end +end diff --git a/app/workers/namespaces/root_statistics_worker.rb b/app/workers/namespaces/root_statistics_worker.rb new file mode 100644 index 00000000000..48876825564 --- /dev/null +++ b/app/workers/namespaces/root_statistics_worker.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Namespaces + class RootStatisticsWorker + include ApplicationWorker + + queue_namespace :update_namespace_statistics + + def perform(namespace_id) + namespace = Namespace.find(namespace_id) + + return unless update_statistics_enabled_for?(namespace) && namespace.aggregation_scheduled? + + Namespaces::StatisticsRefresherService.new.execute(namespace) + + namespace.aggregation_schedule.destroy + rescue ::Namespaces::StatisticsRefresherService::RefresherError, ActiveRecord::RecordNotFound => ex + log_error(namespace.full_path, ex.message) if namespace + end + + private + + def log_error(namespace_path, error_message) + Gitlab::SidekiqLogger.error("Namespace statistics can't be updated for #{namespace_path}: #{error_message}") + end + + def update_statistics_enabled_for?(namespace) + Feature.enabled?(:update_statistics_namespace, namespace) + end + end +end diff --git a/app/workers/namespaces/schedule_aggregation_worker.rb b/app/workers/namespaces/schedule_aggregation_worker.rb new file mode 100644 index 00000000000..a4594b84b13 --- /dev/null +++ b/app/workers/namespaces/schedule_aggregation_worker.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Namespaces + class ScheduleAggregationWorker + include ApplicationWorker + + queue_namespace :update_namespace_statistics + + def perform(namespace_id) + return unless aggregation_schedules_table_exists? + + namespace = Namespace.find(namespace_id) + root_ancestor = namespace.root_ancestor + + return unless update_statistics_enabled_for?(root_ancestor) && !root_ancestor.aggregation_scheduled? + + Namespace::AggregationSchedule.safe_find_or_create_by!(namespace_id: root_ancestor.id) + rescue ActiveRecord::RecordNotFound + log_error(namespace_id) + end + + private + + # On db/post_migrate/20180529152628_schedule_to_archive_legacy_traces.rb + # traces are archived through build.trace.archive, which in consequence + # calls UpdateProjectStatistics#schedule_namespace_statistics_worker. + # + # The migration and specs fails since NamespaceAggregationSchedule table + # does not exist at that point. + # https://gitlab.com/gitlab-org/gitlab-ce/issues/50712 + def aggregation_schedules_table_exists? + return true unless Rails.env.test? + + Namespace::AggregationSchedule.table_exists? + end + + def log_error(root_ancestor_id) + Gitlab::SidekiqLogger.error("Namespace can't be scheduled for aggregation: #{root_ancestor_id} does not exist") + end + + def update_statistics_enabled_for?(root_ancestor) + Feature.enabled?(:update_statistics_namespace, root_ancestor) + end + end +end |