1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
|
# frozen_string_literal: true
require 'spec_helper'
RSpec.shared_examples 'validate dictionary' do |objects, directory_path, required_fields|
context 'for each object' do
let(:directory_path) { directory_path }
let(:metadata_allowed_fields) do
required_fields + %i[
feature_categories
classes
description
introduced_by_url
milestone
gitlab_schema
schema_inconsistencies
sharding_key
desired_sharding_key
allow_cross_joins
allow_cross_transactions
allow_cross_foreign_keys
]
end
let(:metadata) do
objects.each_with_object({}) do |object_name, hash|
next unless File.exist?(object_metadata_file_path(object_name))
hash[object_name] ||= load_object_metadata(required_fields, object_name)
end
end
# This list is used to provide temporary exceptions for feature categories
# that are transitioning and not yet in the feature_categories.yml file
# any additions here should be accompanied by a link to an issue link
let(:valid_feature_categories) do
[
'jihu' # https://gitlab.com/gitlab-org/database-team/team-tasks/-/issues/192
]
end
let(:all_feature_categories) do
YAML.load_file(Rails.root.join('config/feature_categories.yml')) + valid_feature_categories
end
let(:objects_without_metadata) do
objects.reject { |t| metadata.has_key?(t) }
end
let(:objects_without_valid_metadata) do
metadata.select { |_, t| t.has_key?(:error) }.keys
end
let(:objects_with_disallowed_fields) do
metadata.select { |_, t| t.has_key?(:disallowed_fields) }.keys
end
let(:objects_with_missing_required_fields) do
metadata.select { |_, t| t.has_key?(:missing_required_fields) }.keys
end
let(:objects_with_invalid_feature_category) do
metadata.select { |_, t| t.has_key?(:invalid_feature_category) }.keys
end
it 'has a metadata file' do
expect(objects_without_metadata).to be_empty, multiline_error(
'Missing metadata files',
objects_without_metadata.map { |t| " #{object_metadata_file(t)}" }
)
end
it 'has a valid metadata file' do
expect(objects_without_valid_metadata).to be_empty, object_metadata_errors(
'Table metadata files with errors',
:error,
objects_without_valid_metadata
)
end
it 'has a valid feature category' do
message = <<~TEXT.chomp
Please use a category from https://about.gitlab.com/handbook/product/categories/#categories-a-z
Table metadata files with an invalid feature category
TEXT
expect(objects_with_invalid_feature_category).to be_empty, object_metadata_errors(
message,
:invalid_feature_category,
objects_with_invalid_feature_category
)
end
it 'has a valid metadata file with allowed fields' do
expect(objects_with_disallowed_fields).to be_empty, object_metadata_errors(
'Table metadata files with disallowed fields',
:disallowed_fields,
objects_with_disallowed_fields
)
end
it 'has a valid metadata file without missing fields' do
expect(objects_with_missing_required_fields).to be_empty, object_metadata_errors(
'Table metadata files with missing fields',
:missing_required_fields,
objects_with_missing_required_fields
)
end
end
private
def object_metadata_file(object_name)
File.join(directory_path, "#{object_name}.yml")
end
def object_metadata_file_path(object_name)
Rails.root.join(object_metadata_file(object_name))
end
def invalid_feature_categories(object_feature_categories)
return false unless object_feature_categories.present?
object_feature_categories - all_feature_categories
end
def load_object_metadata(required_fields, object_name)
result = {}
begin
result[:metadata] = YAML.safe_load(File.read(object_metadata_file_path(object_name))).deep_symbolize_keys
disallowed_fields = (result[:metadata].keys - metadata_allowed_fields)
result[:disallowed_fields] = "fields not allowed: #{disallowed_fields.join(', ')}" unless disallowed_fields.empty?
missing_required_fields = (required_fields - result[:metadata].reject { |_, v| v.blank? }.keys)
unless missing_required_fields.empty?
result[:missing_required_fields] = "missing required fields: #{missing_required_fields.join(', ')}"
end
if required_fields.include?(:feature_categories)
object_feature_categories = result.dig(:metadata, :feature_categories)
if (invalid = invalid_feature_categories(object_feature_categories)).any?
result[:invalid_feature_category] = "invalid feature category: #{invalid.join(', ')}"
end
end
rescue Psych::SyntaxError => ex
result[:error] = ex.message
end
result
end
# rubocop:disable Naming/HeredocDelimiterNaming
def object_metadata_errors(title, field, objects)
lines = objects.map do |object_name|
<<~EOM
#{object_metadata_file(object_name)}
#{metadata[object_name][field]}
EOM
end
multiline_error(title, lines)
end
def multiline_error(title, lines)
<<~EOM
#{title}:
#{lines.join("\n")}
EOM
end
# rubocop:enable Naming/HeredocDelimiterNaming
end
RSpec.describe 'Views documentation', feature_category: :database do
excluded = %w[geo jh]
database_base_models = Gitlab::Database.database_base_models.reject { |k, _| k.in?(excluded) }
views = database_base_models.flat_map { |_, m| m.connection.views }.sort.uniq
directory_path = File.join('db', 'docs', 'views')
required_fields = %i[feature_categories view_name gitlab_schema]
include_examples 'validate dictionary', views, directory_path, required_fields
end
RSpec.describe 'Tables documentation', feature_category: :database do
excluded = %w[geo jh]
database_base_models = Gitlab::Database.database_base_models.reject { |k, _| k.in?(excluded) }
tables = database_base_models.flat_map { |_, m| m.connection.tables }.sort.uniq
directory_path = File.join('db', 'docs')
required_fields = %i[feature_categories table_name gitlab_schema milestone]
include_examples 'validate dictionary', tables, directory_path, required_fields
end
RSpec.describe 'Deleted tables documentation', feature_category: :database do
directory_path = File.join('db', 'docs', 'deleted_tables')
tables = Dir.glob(File.join(directory_path, '*.yml')).map { |f| File.basename(f, '.yml') }.sort.uniq
required_fields = %i[table_name gitlab_schema removed_by_url removed_in_milestone]
include_examples 'validate dictionary', tables, directory_path, required_fields
end
RSpec.describe 'Deleted views documentation', feature_category: :database do
directory_path = File.join('db', 'docs', 'deleted_views')
views = Dir.glob(File.join(directory_path, '*.yml')).map { |f| File.basename(f, '.yml') }.sort.uniq
required_fields = %i[view_name gitlab_schema removed_by_url removed_in_milestone]
include_examples 'validate dictionary', views, directory_path, required_fields
end
|