diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-06-01 09:08:03 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-06-01 09:08:03 +0300 |
commit | 8f535d9082bb0f10f2ef6b9133dbc77691ace4db (patch) | |
tree | 5e8df6512d9c5a5fee8e499b10e5b0817799d4aa /db | |
parent | 4346c3ae00a177d8afe73a898514b0edeb43e07d (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'db')
3 files changed, 179 insertions, 105 deletions
diff --git a/db/post_migrate/20230202211434_migrate_redis_slot_keys.rb b/db/post_migrate/20230202211434_migrate_redis_slot_keys.rb index e1d3787cbb3..126555ae391 100644 --- a/db/post_migrate/20230202211434_migrate_redis_slot_keys.rb +++ b/db/post_migrate/20230202211434_migrate_redis_slot_keys.rb @@ -4,110 +4,10 @@ class MigrateRedisSlotKeys < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up - BackupHLLRedisCounter.known_events.each do |event| - if event[:aggregation].to_sym == :daily - migrate_daily_aggregated(event) - else - migrate_weekly_aggregated(event) - end - end + # deleted as contained a bug end def down # no-op end - - private - - def migrate_daily_aggregated(event) - days_back = BackupHLLRedisCounter::DEFAULT_DAILY_KEY_EXPIRY_LENGTH - start_date = Date.today - days_back - 1.day - end_date = Date.today + 1.day - - (start_date..end_date).each do |date| - rename_key(event, date) - end - end - - def migrate_weekly_aggregated(event) - weeks_back = BackupHLLRedisCounter::DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH - start_date = (Date.today - weeks_back).beginning_of_week - 1.day - end_date = Date.today.end_of_week + 1.day - - (start_date..end_date).each { |date| rename_key(event, date) } - end - - def rename_key(event, date) - old_key = old_redis_key(event, date) - new_key = BackupHLLRedisCounter.redis_key(event, date) - - # cannot simply rename due to different slots - Gitlab::Redis::SharedState.with do |r| - break unless r.exists?(old_key) - - Gitlab::Redis::HLL.add( - key: new_key, - value: r.pfcount(old_key), - expiry: r.ttl(old_key) - ) - end - end - - def old_redis_key(event, time) - name_with_slot = if event[:redis_slot].present? - event[:name].to_s.gsub(event[:redis_slot], "{#{event[:redis_slot]}}") - else - "{#{event[:name]}}" - end - - BackupHLLRedisCounter.apply_time_aggregation(name_with_slot, time, event) - end - - # :nocov: Existing backed up class # rubocop:disable Gitlab/NoCodeCoverageComment - module BackupHLLRedisCounter - DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks - DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days - REDIS_SLOT = 'hll_counters' - - KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__) - ALLOWED_AGGREGATIONS = %i[daily weekly].freeze - - class << self - def known_events - @known_events ||= load_events(KNOWN_EVENTS_PATH) - end - - def load_events(wildcard) - Dir[wildcard].each_with_object([]) do |path, events| - events.push(*load_yaml_from_path(path)) - end - end - - def load_yaml_from_path(path) - YAML.safe_load(File.read(path))&.map(&:with_indifferent_access) - end - - def known_events_names - known_events.map { |event| event[:name] } # rubocop:disable Rails/Pluck - end - - def redis_key(event, time, context = '') - key = "{#{REDIS_SLOT}}_#{event[:name]}" - key = apply_time_aggregation(key, time, event) - key = "#{context}_#{key}" if context.present? - key - end - - def apply_time_aggregation(key, time, event) - if event[:aggregation].to_sym == :daily - year_day = time.strftime('%G-%j') - "#{year_day}-#{key}" - else - year_week = time.strftime('%G-%V') - "#{key}-#{year_week}" - end - end - end - end - # :nocov: # rubocop:disable Gitlab/NoCodeCoverageComment end diff --git a/db/post_migrate/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation.rb b/db/post_migrate/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation.rb index b4468c851ae..59bff26f964 100644 --- a/db/post_migrate/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation.rb +++ b/db/post_migrate/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation.rb @@ -3,6 +3,93 @@ class MigrateDailyRedisHllEventsToWeeklyAggregation < Gitlab::Database::Migration[2.1] disable_ddl_transaction! + DAILY_EVENTS = + %w[g_edit_by_web_ide + g_edit_by_sfe + g_edit_by_snippet_ide + g_edit_by_live_preview + wiki_action + design_action + project_action + git_write_action + merge_request_action + i_source_code_code_intelligence + g_project_management_issue_title_changed + g_project_management_issue_description_changed + g_project_management_issue_assignee_changed + g_project_management_issue_made_confidential + g_project_management_issue_made_visible + g_project_management_issue_created + g_project_management_issue_closed + g_project_management_issue_reopened + g_project_management_issue_label_changed + g_project_management_issue_milestone_changed + g_project_management_issue_cross_referenced + g_project_management_issue_moved + g_project_management_issue_related + g_project_management_issue_unrelated + g_project_management_issue_marked_as_duplicate + g_project_management_issue_locked + g_project_management_issue_unlocked + g_project_management_issue_designs_added + g_project_management_issue_designs_modified + g_project_management_issue_designs_removed + g_project_management_issue_due_date_changed + g_project_management_issue_design_comments_removed + g_project_management_issue_time_estimate_changed + g_project_management_issue_time_spent_changed + g_project_management_issue_comment_added + g_project_management_issue_comment_edited + g_project_management_issue_comment_removed + g_project_management_issue_cloned + g_geo_proxied_requests + approval_project_rule_created + g_project_management_issue_added_to_epic + g_project_management_issue_changed_epic + g_project_management_issue_health_status_changed + g_project_management_issue_iteration_changed + g_project_management_issue_removed_from_epic + g_project_management_issue_weight_changed + g_geo_proxied_requests + g_project_management_users_creating_epic_boards + g_project_management_users_viewing_epic_boards + g_project_management_users_updating_epic_board_names + g_project_management_epic_created + project_management_users_unchecking_epic_task + project_management_users_checking_epic_task + g_project_management_users_updating_epic_titles + g_project_management_users_updating_epic_descriptions + g_project_management_users_creating_epic_notes + g_project_management_users_updating_epic_notes + g_project_management_users_destroying_epic_notes + g_project_management_users_awarding_epic_emoji + g_project_management_users_removing_epic_emoji + g_project_management_users_setting_epic_start_date_as_fixed + g_project_management_users_updating_fixed_epic_start_date + g_project_management_users_setting_epic_start_date_as_inherited + g_project_management_users_setting_epic_due_date_as_fixed + g_project_management_users_updating_fixed_epic_due_date + g_project_management_users_setting_epic_due_date_as_inherited + g_project_management_epic_issue_added + g_project_management_epic_issue_removed + g_project_management_epic_issue_moved_from_project + g_project_management_users_updating_epic_parent + g_project_management_epic_closed + g_project_management_epic_reopened + g_project_management_issue_promoted_to_epic + g_project_management_users_setting_epic_confidential + g_project_management_users_setting_epic_visible + g_project_management_epic_users_changing_labels + g_project_management_epic_destroyed + g_project_management_epic_cross_referenced + g_project_management_users_epic_issue_added_from_epic + g_project_management_epic_related_added + g_project_management_epic_related_removed + g_project_management_epic_blocking_added + g_project_management_epic_blocking_removed + g_project_management_epic_blocked_added + g_project_management_epic_blocked_removed].freeze + def up days_back = 29.days start_date = Date.today - days_back - 1.day @@ -10,7 +97,7 @@ class MigrateDailyRedisHllEventsToWeeklyAggregation < Gitlab::Database::Migratio keys = {} Gitlab::UsageDataCounters::HLLRedisCounter.known_events.each do |event| - next unless event[:aggregation].to_sym == :daily + next unless DAILY_EVENTS.include?(event[:name].to_s) (start_date..end_date).each do |date| daily_key = redis_key(event, date, :daily) diff --git a/db/post_migrate/20230328111013_re_migrate_redis_slot_keys.rb b/db/post_migrate/20230328111013_re_migrate_redis_slot_keys.rb index f09888bd821..71ab958b601 100644 --- a/db/post_migrate/20230328111013_re_migrate_redis_slot_keys.rb +++ b/db/post_migrate/20230328111013_re_migrate_redis_slot_keys.rb @@ -3,9 +3,96 @@ class ReMigrateRedisSlotKeys < Gitlab::Database::Migration[2.1] disable_ddl_transaction! + DAILY_EVENTS = + %w[g_edit_by_web_ide + g_edit_by_sfe + g_edit_by_snippet_ide + g_edit_by_live_preview + wiki_action + design_action + project_action + git_write_action + merge_request_action + i_source_code_code_intelligence + g_project_management_issue_title_changed + g_project_management_issue_description_changed + g_project_management_issue_assignee_changed + g_project_management_issue_made_confidential + g_project_management_issue_made_visible + g_project_management_issue_created + g_project_management_issue_closed + g_project_management_issue_reopened + g_project_management_issue_label_changed + g_project_management_issue_milestone_changed + g_project_management_issue_cross_referenced + g_project_management_issue_moved + g_project_management_issue_related + g_project_management_issue_unrelated + g_project_management_issue_marked_as_duplicate + g_project_management_issue_locked + g_project_management_issue_unlocked + g_project_management_issue_designs_added + g_project_management_issue_designs_modified + g_project_management_issue_designs_removed + g_project_management_issue_due_date_changed + g_project_management_issue_design_comments_removed + g_project_management_issue_time_estimate_changed + g_project_management_issue_time_spent_changed + g_project_management_issue_comment_added + g_project_management_issue_comment_edited + g_project_management_issue_comment_removed + g_project_management_issue_cloned + g_geo_proxied_requests + approval_project_rule_created + g_project_management_issue_added_to_epic + g_project_management_issue_changed_epic + g_project_management_issue_health_status_changed + g_project_management_issue_iteration_changed + g_project_management_issue_removed_from_epic + g_project_management_issue_weight_changed + g_geo_proxied_requests + g_project_management_users_creating_epic_boards + g_project_management_users_viewing_epic_boards + g_project_management_users_updating_epic_board_names + g_project_management_epic_created + project_management_users_unchecking_epic_task + project_management_users_checking_epic_task + g_project_management_users_updating_epic_titles + g_project_management_users_updating_epic_descriptions + g_project_management_users_creating_epic_notes + g_project_management_users_updating_epic_notes + g_project_management_users_destroying_epic_notes + g_project_management_users_awarding_epic_emoji + g_project_management_users_removing_epic_emoji + g_project_management_users_setting_epic_start_date_as_fixed + g_project_management_users_updating_fixed_epic_start_date + g_project_management_users_setting_epic_start_date_as_inherited + g_project_management_users_setting_epic_due_date_as_fixed + g_project_management_users_updating_fixed_epic_due_date + g_project_management_users_setting_epic_due_date_as_inherited + g_project_management_epic_issue_added + g_project_management_epic_issue_removed + g_project_management_epic_issue_moved_from_project + g_project_management_users_updating_epic_parent + g_project_management_epic_closed + g_project_management_epic_reopened + g_project_management_issue_promoted_to_epic + g_project_management_users_setting_epic_confidential + g_project_management_users_setting_epic_visible + g_project_management_epic_users_changing_labels + g_project_management_epic_destroyed + g_project_management_epic_cross_referenced + g_project_management_users_epic_issue_added_from_epic + g_project_management_epic_related_added + g_project_management_epic_related_removed + g_project_management_epic_blocking_added + g_project_management_epic_blocking_removed + g_project_management_epic_blocked_added + g_project_management_epic_blocked_removed].freeze + def up Gitlab::UsageDataCounters::HLLRedisCounter.known_events.each do |event| - if event[:aggregation].to_sym == :daily + if DAILY_EVENTS.include?(event[:name].to_s) migrate_daily_aggregated(event) else migrate_weekly_aggregated(event) @@ -20,7 +107,7 @@ class ReMigrateRedisSlotKeys < Gitlab::Database::Migration[2.1] private def migrate_daily_aggregated(event) - days_back = Gitlab::UsageDataCounters::HLLRedisCounter::DEFAULT_DAILY_KEY_EXPIRY_LENGTH + days_back = 29.days start_date = Date.today - days_back - 1.day end_date = Date.today + 1.day @@ -77,7 +164,7 @@ class ReMigrateRedisSlotKeys < Gitlab::Database::Migration[2.1] end def apply_time_aggregation(key, time, event) - if event[:aggregation].to_sym == :daily + if DAILY_EVENTS.include?(event[:name].to_s) year_day = time.strftime('%G-%j') "#{year_day}-#{key}" else |