1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
# frozen_string_literal: true
class MigrateDailyRedisHllEventsToWeeklyAggregation < Gitlab::Database::Migration[2.1]
disable_ddl_transaction!
# Due to the feature flag `use_metric_definitions_for_events_list`, this makes a `SELECT` call
# so this migration is now considered a DML (data manipulation) migration.
# For the time being, we need to specify only `main` database because it is now technically retriving DB data.
# Should be removed with <issue link>
restrict_gitlab_migration gitlab_schema: :gitlab_main
DAILY_EVENTS =
%w[g_edit_by_web_ide
g_edit_by_sfe
g_edit_by_snippet_ide
g_edit_by_live_preview
wiki_action
design_action
project_action
git_write_action
merge_request_action
i_source_code_code_intelligence
g_project_management_issue_title_changed
g_project_management_issue_description_changed
g_project_management_issue_assignee_changed
g_project_management_issue_made_confidential
g_project_management_issue_made_visible
g_project_management_issue_created
g_project_management_issue_closed
g_project_management_issue_reopened
g_project_management_issue_label_changed
g_project_management_issue_milestone_changed
g_project_management_issue_cross_referenced
g_project_management_issue_moved
g_project_management_issue_related
g_project_management_issue_unrelated
g_project_management_issue_marked_as_duplicate
g_project_management_issue_locked
g_project_management_issue_unlocked
g_project_management_issue_designs_added
g_project_management_issue_designs_modified
g_project_management_issue_designs_removed
g_project_management_issue_due_date_changed
g_project_management_issue_design_comments_removed
g_project_management_issue_time_estimate_changed
g_project_management_issue_time_spent_changed
g_project_management_issue_comment_added
g_project_management_issue_comment_edited
g_project_management_issue_comment_removed
g_project_management_issue_cloned
g_geo_proxied_requests
approval_project_rule_created
g_project_management_issue_added_to_epic
g_project_management_issue_changed_epic
g_project_management_issue_health_status_changed
g_project_management_issue_iteration_changed
g_project_management_issue_removed_from_epic
g_project_management_issue_weight_changed
g_geo_proxied_requests
g_project_management_users_creating_epic_boards
g_project_management_users_viewing_epic_boards
g_project_management_users_updating_epic_board_names
g_project_management_epic_created
project_management_users_unchecking_epic_task
project_management_users_checking_epic_task
g_project_management_users_updating_epic_titles
g_project_management_users_updating_epic_descriptions
g_project_management_users_creating_epic_notes
g_project_management_users_updating_epic_notes
g_project_management_users_destroying_epic_notes
g_project_management_users_awarding_epic_emoji
g_project_management_users_removing_epic_emoji
g_project_management_users_setting_epic_start_date_as_fixed
g_project_management_users_updating_fixed_epic_start_date
g_project_management_users_setting_epic_start_date_as_inherited
g_project_management_users_setting_epic_due_date_as_fixed
g_project_management_users_updating_fixed_epic_due_date
g_project_management_users_setting_epic_due_date_as_inherited
g_project_management_epic_issue_added
g_project_management_epic_issue_removed
g_project_management_epic_issue_moved_from_project
g_project_management_users_updating_epic_parent
g_project_management_epic_closed
g_project_management_epic_reopened
g_project_management_issue_promoted_to_epic
g_project_management_users_setting_epic_confidential
g_project_management_users_setting_epic_visible
g_project_management_epic_users_changing_labels
g_project_management_epic_destroyed
g_project_management_epic_cross_referenced
g_project_management_users_epic_issue_added_from_epic
g_project_management_epic_related_added
g_project_management_epic_related_removed
g_project_management_epic_blocking_added
g_project_management_epic_blocking_removed
g_project_management_epic_blocked_added
g_project_management_epic_blocked_removed].freeze
def up
days_back = 29.days
start_date = Date.today - days_back - 1.day
end_date = Date.today + 1.day
keys = {}
Gitlab::UsageDataCounters::HLLRedisCounter.known_events.each do |event|
next unless DAILY_EVENTS.include?(event[:name].to_s)
(start_date..end_date).each do |date|
daily_key = redis_key(event, date, :daily)
weekly_key = redis_key(event, date, :weekly)
keys.key?(weekly_key) ? keys[weekly_key] << daily_key : keys[weekly_key] = [daily_key]
end
end
keys.each do |weekly_key, daily_keys|
Gitlab::Redis::SharedState.with do |redis|
redis.pfmerge(weekly_key, *daily_keys)
redis.expire(weekly_key, 6.weeks)
end
end
end
def down
# no-op
end
# can't set daily key in HLLRedisCounter anymore, so need to duplicate logic here
def redis_key(event, time, aggregation)
key = "{hll_counters}_#{event[:name]}"
if aggregation.to_sym == :daily
year_day = time.strftime('%G-%j')
"#{year_day}-#{key}"
else
year_week = time.strftime('%G-%V')
"#{key}-#{year_week}"
end
end
end
|
