1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
|
# frozen_string_literal: true
class ReMigrateRedisSlotKeys < Gitlab::Database::Migration[2.1]
disable_ddl_transaction!
DAILY_EVENTS =
%w[g_edit_by_web_ide
g_edit_by_sfe
g_edit_by_snippet_ide
g_edit_by_live_preview
wiki_action
design_action
project_action
git_write_action
merge_request_action
i_source_code_code_intelligence
g_project_management_issue_title_changed
g_project_management_issue_description_changed
g_project_management_issue_assignee_changed
g_project_management_issue_made_confidential
g_project_management_issue_made_visible
g_project_management_issue_created
g_project_management_issue_closed
g_project_management_issue_reopened
g_project_management_issue_label_changed
g_project_management_issue_milestone_changed
g_project_management_issue_cross_referenced
g_project_management_issue_moved
g_project_management_issue_related
g_project_management_issue_unrelated
g_project_management_issue_marked_as_duplicate
g_project_management_issue_locked
g_project_management_issue_unlocked
g_project_management_issue_designs_added
g_project_management_issue_designs_modified
g_project_management_issue_designs_removed
g_project_management_issue_due_date_changed
g_project_management_issue_design_comments_removed
g_project_management_issue_time_estimate_changed
g_project_management_issue_time_spent_changed
g_project_management_issue_comment_added
g_project_management_issue_comment_edited
g_project_management_issue_comment_removed
g_project_management_issue_cloned
g_geo_proxied_requests
approval_project_rule_created
g_project_management_issue_added_to_epic
g_project_management_issue_changed_epic
g_project_management_issue_health_status_changed
g_project_management_issue_iteration_changed
g_project_management_issue_removed_from_epic
g_project_management_issue_weight_changed
g_geo_proxied_requests
g_project_management_users_creating_epic_boards
g_project_management_users_viewing_epic_boards
g_project_management_users_updating_epic_board_names
g_project_management_epic_created
project_management_users_unchecking_epic_task
project_management_users_checking_epic_task
g_project_management_users_updating_epic_titles
g_project_management_users_updating_epic_descriptions
g_project_management_users_creating_epic_notes
g_project_management_users_updating_epic_notes
g_project_management_users_destroying_epic_notes
g_project_management_users_awarding_epic_emoji
g_project_management_users_removing_epic_emoji
g_project_management_users_setting_epic_start_date_as_fixed
g_project_management_users_updating_fixed_epic_start_date
g_project_management_users_setting_epic_start_date_as_inherited
g_project_management_users_setting_epic_due_date_as_fixed
g_project_management_users_updating_fixed_epic_due_date
g_project_management_users_setting_epic_due_date_as_inherited
g_project_management_epic_issue_added
g_project_management_epic_issue_removed
g_project_management_epic_issue_moved_from_project
g_project_management_users_updating_epic_parent
g_project_management_epic_closed
g_project_management_epic_reopened
g_project_management_issue_promoted_to_epic
g_project_management_users_setting_epic_confidential
g_project_management_users_setting_epic_visible
g_project_management_epic_users_changing_labels
g_project_management_epic_destroyed
g_project_management_epic_cross_referenced
g_project_management_users_epic_issue_added_from_epic
g_project_management_epic_related_added
g_project_management_epic_related_removed
g_project_management_epic_blocking_added
g_project_management_epic_blocking_removed
g_project_management_epic_blocked_added
g_project_management_epic_blocked_removed].freeze
def up
Gitlab::UsageDataCounters::HLLRedisCounter.known_events.each do |event|
if DAILY_EVENTS.include?(event[:name].to_s)
migrate_daily_aggregated(event)
else
migrate_weekly_aggregated(event)
end
end
end
def down
# no-op
end
private
def migrate_daily_aggregated(event)
days_back = 29.days
start_date = Date.today - days_back - 1.day
end_date = Date.today + 1.day
(start_date..end_date).each do |date|
rename_key(event, date)
end
end
def migrate_weekly_aggregated(event)
weeks_back = Gitlab::UsageDataCounters::HLLRedisCounter::DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH
start_date = (Date.today - weeks_back).beginning_of_week - 1.day
end_date = Date.today.end_of_week + 1.day
(start_date..end_date).step(7).each { |date| rename_key(event, date) }
end
def rename_key(event, date)
old_key = old_redis_key(event, date)
new_key = new_redis_key(event, date)
# cannot simply rename due to different slots
Gitlab::Redis::SharedState.with do |redis|
hll_blob = redis.get(old_key)
break unless hll_blob
temp_key = new_key + "_#{Time.current.to_i}"
ttl = redis.ttl(old_key)
ttl = ttl > 0 ? ttl : Gitlab::UsageDataCounters::HLLRedisCounter.send(:expiry, event)
redis.multi do |multi|
multi.set(temp_key, hll_blob, ex: 1.day.to_i)
multi.pfmerge(new_key, new_key, temp_key)
multi.expire(new_key, ttl)
end
redis.del(temp_key)
end
end
def old_redis_key(event, time)
name_with_slot = if event[:redis_slot].present?
event[:name].to_s.gsub(event[:redis_slot], "{#{event[:redis_slot]}}")
else
"{#{event[:name]}}"
end
apply_time_aggregation(name_with_slot, time, event)
end
def new_redis_key(event, time)
key = "{hll_counters}_#{event[:name]}"
apply_time_aggregation(key, time, event)
end
def apply_time_aggregation(key, time, event)
if DAILY_EVENTS.include?(event[:name].to_s)
year_day = time.strftime('%G-%j')
"#{year_day}-#{key}"
else
year_week = time.strftime('%G-%V')
"#{key}-#{year_week}"
end
end
end
|
