1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
|
# frozen_string_literal: true
class MigrateRedisSlotKeys < Gitlab::Database::Migration[2.1]
disable_ddl_transaction!
def up
BackupHLLRedisCounter.known_events.each do |event|
if event[:aggregation].to_sym == :daily
migrate_daily_aggregated(event)
else
migrate_weekly_aggregated(event)
end
end
end
def down
# no-op
end
private
def migrate_daily_aggregated(event)
days_back = BackupHLLRedisCounter::DEFAULT_DAILY_KEY_EXPIRY_LENGTH
start_date = Date.today - days_back - 1.day
end_date = Date.today + 1.day
(start_date..end_date).each do |date|
rename_key(event, date)
end
end
def migrate_weekly_aggregated(event)
weeks_back = BackupHLLRedisCounter::DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH
start_date = (Date.today - weeks_back).beginning_of_week - 1.day
end_date = Date.today.end_of_week + 1.day
(start_date..end_date).each { |date| rename_key(event, date) }
end
def rename_key(event, date)
old_key = old_redis_key(event, date)
new_key = BackupHLLRedisCounter.redis_key(event, date)
# cannot simply rename due to different slots
Gitlab::Redis::SharedState.with do |r|
break unless r.exists?(old_key)
Gitlab::Redis::HLL.add(
key: new_key,
value: r.pfcount(old_key),
expiry: r.ttl(old_key)
)
end
end
def old_redis_key(event, time)
name_with_slot = if event[:redis_slot].present?
event[:name].to_s.gsub(event[:redis_slot], "{#{event[:redis_slot]}}")
else
"{#{event[:name]}}"
end
BackupHLLRedisCounter.apply_time_aggregation(name_with_slot, time, event)
end
# :nocov: Existing backed up class # rubocop:disable Gitlab/NoCodeCoverageComment
module BackupHLLRedisCounter
DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks
DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days
REDIS_SLOT = 'hll_counters'
KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__)
ALLOWED_AGGREGATIONS = %i[daily weekly].freeze
class << self
def known_events
@known_events ||= load_events(KNOWN_EVENTS_PATH)
end
def load_events(wildcard)
Dir[wildcard].each_with_object([]) do |path, events|
events.push(*load_yaml_from_path(path))
end
end
def load_yaml_from_path(path)
YAML.safe_load(File.read(path))&.map(&:with_indifferent_access)
end
def known_events_names
known_events.map { |event| event[:name] } # rubocop:disable Rails/Pluck
end
def redis_key(event, time, context = '')
key = "{#{REDIS_SLOT}}_#{event[:name]}"
key = apply_time_aggregation(key, time, event)
key = "#{context}_#{key}" if context.present?
key
end
def apply_time_aggregation(key, time, event)
if event[:aggregation].to_sym == :daily
year_day = time.strftime('%G-%j')
"#{year_day}-#{key}"
else
year_week = time.strftime('%G-%V')
"#{key}-#{year_week}"
end
end
end
end
# :nocov: # rubocop:disable Gitlab/NoCodeCoverageComment
end
|
