1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
|
# frozen_string_literal: true
require 'active_support/inflector'
module QA
module Service
module ClusterProvider
class Gcloud < Base
def validate_dependencies
find_executable('gcloud') || raise("You must first install `gcloud` executable to run these tests.")
end
def initialize(rbac:)
super(rbac: rbac)
@attempts = 0
@available_regions = %w(
asia-east1 asia-east2
asia-northeast1 asia-south1
asia-southeast1 australia-southeast1
europe-west1 europe-west2 europe-west4
northamerica-northeast1 southamerica-east1
us-central1 us-east1 us-east4
us-west1 us-west2
)
end
def setup
login_if_not_already_logged_in
create_cluster
install_helm
end
def teardown
delete_cluster
end
def connect
login_if_not_already_logged_in
shell <<~CMD.tr("\n", ' ')
gcloud container clusters get-credentials
--region #{Runtime::Env.workspaces_cluster_region}
#{Runtime::Env.workspaces_cluster_name}
CMD
end
def install_kubernetes_agent(agent_token:, kas_address:, agent_name: "gitlab-agent")
cmd_str = <<~CMD.tr("\n", ' ')
helm repo add gitlab https://charts.gitlab.io &&
helm repo update &&
helm upgrade --install gitlab-agent gitlab/gitlab-agent
--namespace "#{agent_name}"
--create-namespace
--set image.tag=#{Runtime::Env.gitlab_agentk_version}
--set config.token=#{agent_token}
--set config.kasAddress=#{kas_address}
--set config.kasHeaders="{Cookie: gitlab_canary=#{target_canary?}}"
CMD
shell(cmd_str, mask_secrets: [agent_token])
end
def uninstall_kubernetes_agent(agent_name: "gitlab-agent")
shell <<~CMD.tr("\n", ' ')
helm uninstall gitlab-agent \
--namespace "#{agent_name}"
CMD
end
def install_ngnix_ingress
shell <<~CMD.tr("\n", ' ')
helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx &&
helm repo update &&
helm install \
ingress-nginx ingress-nginx/ingress-nginx \
--namespace ingress-nginx \
--create-namespace \
--version 4.3.0
CMD
end
def install_gitlab_workspaces_proxy
cmd_str = <<~CMD.tr("\n", ' ')
helm repo add gitlab-workspaces-proxy \
https://gitlab.com/api/v4/projects/gitlab-org%2fremote-development%2fgitlab-workspaces-proxy/packages/helm/devel &&
helm repo update &&
helm upgrade --install gitlab-workspaces-proxy \
gitlab-workspaces-proxy/gitlab-workspaces-proxy \
--version 0.1.6 \
--namespace=gitlab-workspaces \
--create-namespace \
--set="auth.client_id=#{Runtime::Env.workspaces_oauth_app_id}" \
--set="auth.client_secret=#{Runtime::Env.workspaces_oauth_app_secret}" \
--set="auth.host=#{Runtime::Env.gitlab_url}" \
--set="auth.redirect_uri=https://#{Runtime::Env.workspaces_proxy_domain}/auth/callback" \
--set="auth.signing_key=#{Runtime::Env.workspaces_oauth_signing_key}" \
--set="ingress.host.workspaceDomain=#{Runtime::Env.workspaces_proxy_domain}" \
--set="ingress.host.wildcardDomain=*.#{Runtime::Env.workspaces_proxy_domain}" \
--set="ingress.tls.workspaceDomainCert=$(cat #{Runtime::Env.workspaces_domain_cert})" \
--set="ingress.tls.workspaceDomainKey=$(cat #{Runtime::Env.workspaces_domain_key})" \
--set="ingress.tls.wildcardDomainCert=$(cat #{Runtime::Env.workspaces_wildcard_cert})" \
--set="ingress.tls.wildcardDomainKey=$(cat #{Runtime::Env.workspaces_wildcard_key})" \
--set="ingress.className=nginx"
CMD
shell(cmd_str, mask_secrets: [Runtime::Env.workspaces_oauth_app_secret, Runtime::Env.workspaces_oauth_signing_key])
end
def update_dns(load_balancer_ip)
shell <<~CMD.tr("\n", ' ')
gcloud dns record-sets update #{Runtime::Env.workspaces_proxy_domain} \
--rrdatas=#{load_balancer_ip} \
--ttl=300 \
--type=A \
--zone=gitlabqa-dev
CMD
shell <<~CMD.tr("\n", ' ')
gcloud dns record-sets update "*.#{Runtime::Env.workspaces_proxy_domain}" \
--rrdatas=#{load_balancer_ip} \
--ttl=300 \
--type=A \
--zone=gitlabqa-dev
CMD
end
private
def install_helm
shell <<~CMD.tr("\n", ' ')
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 &&
chmod 700 get_helm.sh &&
DESIRED_VERSION=v3.7.0 ./get_helm.sh
CMD
end
def target_canary?
Runtime::Env.qa_cookies.to_s.include?("gitlab_canary=true")
end
def login_if_not_already_logged_in
if Runtime::Env.has_gcloud_credentials?
attempt_login_with_env_vars
else
account = `gcloud auth list --filter=status:ACTIVE --format="value(account)"`
if account.empty?
raise "Failed to login to gcloud. No credentials provided in environment and no credentials found locally."
else
QA::Runtime::Logger.debug("gcloud account found. Using: #{account} for creating K8s cluster.")
end
end
end
def attempt_login_with_env_vars
QA::Runtime::Logger.debug("Logging in with GCLOUD_ACCOUNT_EMAIL and GCLOUD_ACCOUNT_KEY.")
gcloud_account_key = Tempfile.new('gcloud-account-key')
gcloud_account_key.write(Runtime::Env.gcloud_account_key)
gcloud_account_key.close
gcloud_account_email = Runtime::Env.gcloud_account_email
shell("gcloud auth activate-service-account #{gcloud_account_email} --key-file #{gcloud_account_key.path}")
ensure
gcloud_account_key && gcloud_account_key.unlink
end
def auth_options
"--enable-legacy-authorization" unless rbac
end
def create_cluster
@region = get_region
shell <<~CMD.tr("\n", ' ')
gcloud container clusters
create #{cluster_name}
#{auth_options}
--region #{@region}
--disk-size 15GB
--num-nodes #{Runtime::Env.gcloud_num_nodes}
&& gcloud container clusters
get-credentials
--region #{@region}
#{cluster_name}
CMD
rescue QA::Service::Shellout::CommandError
@attempts += 1
retry unless @attempts > 1
raise $!, "Tried and failed to provision the cluster #{@attempts} #{"time".pluralize(@attempts)}.", $!.backtrace
end
def delete_cluster
shell <<~CMD.tr("\n", ' ')
gcloud container clusters delete
--region #{@region}
#{cluster_name}
--quiet --async
CMD
end
def get_region
Runtime::Env.gcloud_region || @available_regions.delete(@available_regions.sample)
end
end
end
end
end
|
