diff options
23 files changed, 417 insertions, 5 deletions
diff --git a/app/models/clusters/concerns.rb b/app/models/clusters/concerns.rb new file mode 100644 index 00000000000..cd09863bcfc --- /dev/null +++ b/app/models/clusters/concerns.rb @@ -0,0 +1,4 @@ +module Clusters + module Concerns + end +end diff --git a/app/models/clusters/concerns/app_status.rb b/app/models/clusters/concerns/app_status.rb new file mode 100644 index 00000000000..f6b817e9ce7 --- /dev/null +++ b/app/models/clusters/concerns/app_status.rb @@ -0,0 +1,33 @@ +module Clusters + module Concerns + module AppStatus + extend ActiveSupport::Concern + + included do + state_machine :status, initial: :scheduled do + state :errored, value: -1 + state :scheduled, value: 0 + state :installing, value: 1 + state :installed, value: 2 + + event :make_installing do + transition any - [:installing] => :installing + end + + event :make_installed do + transition any - [:installed] => :installed + end + + event :make_errored do + transition any - [:errored] => :errored + end + + before_transition any => [:errored] do |app_status, transition| + status_reason = transition.args.first + app_status.status_reason = status_reason if status_reason + end + end + end + end + end +end diff --git a/app/models/clusters/kubernetes.rb b/app/models/clusters/kubernetes.rb new file mode 100644 index 00000000000..b68e2ae401e --- /dev/null +++ b/app/models/clusters/kubernetes.rb @@ -0,0 +1,16 @@ +module Clusters + module Kubernetes + def self.table_name_prefix + 'clusters_kubernetes_' + end + + def self.app(app_name) + case app_name + when HelmApp::NAME + HelmApp + else + raise ArgumentError, "Unknown app #{app_name}" + end + end + end +end diff --git a/app/models/clusters/kubernetes/helm_app.rb b/app/models/clusters/kubernetes/helm_app.rb new file mode 100644 index 00000000000..32c9e13a469 --- /dev/null +++ b/app/models/clusters/kubernetes/helm_app.rb @@ -0,0 +1,18 @@ +module Clusters + module Kubernetes + class HelmApp < ActiveRecord::Base + NAME = 'helm'.freeze + + include ::Clusters::Concerns::AppStatus + belongs_to :kubernetes_service, class_name: 'KubernetesService', foreign_key: :service_id + + default_value_for :version, Gitlab::Clusters::Helm::HELM_VERSION + + alias_method :cluster, :kubernetes_service + + def name + NAME + end + end + end +end diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb index 5c0b3338a62..b4654e8d1ea 100644 --- a/app/models/project_services/kubernetes_service.rb +++ b/app/models/project_services/kubernetes_service.rb @@ -3,6 +3,8 @@ class KubernetesService < DeploymentService include Gitlab::Kubernetes include ReactiveCaching + has_one :helm_app, class_name: 'Clusters::Kubernetes::HelmApp', foreign_key: :service_id + self.reactive_cache_key = ->(service) { [service.class.model_name.singular, service.project_id] } # Namespace defaults to the project path, but can be overridden in case that @@ -136,6 +138,10 @@ class KubernetesService < DeploymentService { pods: read_pods } end + def helm + Gitlab::Clusters::Helm.new(build_kubeclient!) + end + TEMPLATE_PLACEHOLDER = 'Kubernetes namespace'.freeze private diff --git a/app/serializers/cluster_entity.rb b/app/serializers/cluster_entity.rb index 08a113c4d8a..84ce34afb32 100644 --- a/app/serializers/cluster_entity.rb +++ b/app/serializers/cluster_entity.rb @@ -3,4 +3,16 @@ class ClusterEntity < Grape::Entity expose :status_name, as: :status expose :status_reason + expose :applications do |cluster, options| + if cluster.created? + { + helm: { status: 'installed' }, + ingress: { status: 'error', status_reason: 'Missing namespace' }, + runner: { status: 'installing' }, + prometheus: { status: 'installable' } + } + else + {} + end + end end diff --git a/app/serializers/cluster_serializer.rb b/app/serializers/cluster_serializer.rb index 2c87202a105..2e13c1501e7 100644 --- a/app/serializers/cluster_serializer.rb +++ b/app/serializers/cluster_serializer.rb @@ -2,6 +2,6 @@ class ClusterSerializer < BaseSerializer entity ClusterEntity def represent_status(resource) - represent(resource, { only: [:status, :status_reason] }) + represent(resource, { only: [:status, :status_reason, :applications] }) end end diff --git a/app/services/clusters/base_helm_service.rb b/app/services/clusters/base_helm_service.rb new file mode 100644 index 00000000000..b8ed52bf376 --- /dev/null +++ b/app/services/clusters/base_helm_service.rb @@ -0,0 +1,17 @@ +module Clusters + class BaseHelmService + attr_accessor :app + + def initialize(app) + @app = app + end + + protected + + def helm + return @helm if defined?(@helm) + + @helm = @app.cluster.helm + end + end +end diff --git a/app/services/clusters/fetch_app_installation_status_service.rb b/app/services/clusters/fetch_app_installation_status_service.rb new file mode 100644 index 00000000000..e21aa49bb43 --- /dev/null +++ b/app/services/clusters/fetch_app_installation_status_service.rb @@ -0,0 +1,13 @@ +module Clusters + class FetchAppInstallationStatusService < BaseHelmService + def execute + return unless app.installing? + + phase = helm.installation_status(app) + log = helm.installation_log(app) if phase == 'Failed' + yield(phase, log) if block_given? + rescue KubeException => ke + app.make_errored!("Kubernetes error: #{ke.message}") unless app.errored? + end + end +end diff --git a/app/services/clusters/finalize_app_installation_service.rb b/app/services/clusters/finalize_app_installation_service.rb new file mode 100644 index 00000000000..c921747febc --- /dev/null +++ b/app/services/clusters/finalize_app_installation_service.rb @@ -0,0 +1,15 @@ +module Clusters + class FinalizeAppInstallationService < BaseHelmService + def execute + helm.delete_installation_pod!(app) + + app.make_errored!('Installation aborted') if aborted? + end + + private + + def aborted? + app.installing? || app.scheduled? + end + end +end diff --git a/app/services/clusters/install_app_service.rb b/app/services/clusters/install_app_service.rb new file mode 100644 index 00000000000..dd8556108d4 --- /dev/null +++ b/app/services/clusters/install_app_service.rb @@ -0,0 +1,23 @@ +module Clusters + class InstallAppService < BaseHelmService + def execute + return unless app.scheduled? + + begin + helm.install(app) + if app.make_installing + ClusterWaitForAppInstallationWorker.perform_in( + ClusterWaitForAppInstallationWorker::INITIAL_INTERVAL, app.name, app.id) + else + app.make_errored!("Failed to update app record; #{app.errors}") + end + + rescue KubeException => ke + app.make_errored!("Kubernetes error: #{ke.message}") + rescue StandardError => e + Rails.logger.warn(e.message) + app.make_errored!("Can't start installation process") + end + end + end +end diff --git a/app/services/clusters/install_tiller_service.rb b/app/services/clusters/install_tiller_service.rb new file mode 100644 index 00000000000..ac77a7ea3c2 --- /dev/null +++ b/app/services/clusters/install_tiller_service.rb @@ -0,0 +1,24 @@ +module Clusters + class InstallTillerService < BaseService + def execute + ensure_namespace + install + end + + private + + def kubernetes_service + return @kubernetes_service if defined?(@kubernetes_service) + + @kubernetes_service = project&.kubernetes_service + end + + def ensure_namespace + kubernetes_service&.ensure_namespace! + end + + def install + kubernetes_service&.helm_client&.init! + end + end +end diff --git a/app/workers/cluster_install_app_worker.rb b/app/workers/cluster_install_app_worker.rb new file mode 100644 index 00000000000..4993b2b7349 --- /dev/null +++ b/app/workers/cluster_install_app_worker.rb @@ -0,0 +1,11 @@ +class ClusterInstallAppWorker + include Sidekiq::Worker + include ClusterQueue + include ClusterApp + + def perform(app_name, app_id) + find_app(app_name, app_id) do |app| + Clusters::InstallAppService.new(app).execute + end + end +end diff --git a/app/workers/cluster_wait_for_app_installation_worker.rb b/app/workers/cluster_wait_for_app_installation_worker.rb new file mode 100644 index 00000000000..21149cf2d19 --- /dev/null +++ b/app/workers/cluster_wait_for_app_installation_worker.rb @@ -0,0 +1,33 @@ +class ClusterWaitForAppInstallationWorker + include Sidekiq::Worker + include ClusterQueue + include ClusterApp + + INITIAL_INTERVAL = 30.seconds + EAGER_INTERVAL = 10.seconds + TIMEOUT = 20.minutes + + def perform(app_name, app_id) + find_app(app_name, app_id) do |app| + Clusters::FetchAppInstallationStatusService.new(app).execute do |phase, log| + case phase + when 'Succeeded' + if app.make_installed + Clusters::FinalizeAppInstallationService.new(app).execute + else + app.make_errored!("Failed to update app record; #{app.errors}") + end + when 'Failed' + app.make_errored!(log || 'Installation silently failed') + Clusters::FinalizeAppInstallationService.new(app).execute + else + if Time.now.utc - app.updated_at.to_time.utc > TIMEOUT + app.make_errored!('App installation timeouted') + else + ClusterWaitForAppInstallationWorker.perform_in(EAGER_INTERVAL, app.name, app.id) + end + end + end + end + end +end diff --git a/app/workers/concerns/cluster_app.rb b/app/workers/concerns/cluster_app.rb new file mode 100644 index 00000000000..2170f8be6f6 --- /dev/null +++ b/app/workers/concerns/cluster_app.rb @@ -0,0 +1,10 @@ +module ClusterApp + extend ActiveSupport::Concern + + included do + def find_app(app_name, id) + app = Clusters::Kubernetes.app(app_name).find(id) + yield(app) if block_given? + end + end +end diff --git a/db/migrate/20171031100710_create_clusters_kubernetes_helm_apps.rb b/db/migrate/20171031100710_create_clusters_kubernetes_helm_apps.rb new file mode 100644 index 00000000000..93611bf8a12 --- /dev/null +++ b/db/migrate/20171031100710_create_clusters_kubernetes_helm_apps.rb @@ -0,0 +1,14 @@ +class CreateClustersKubernetesHelmApps < ActiveRecord::Migration + def change + create_table :clusters_kubernetes_helm_apps do |t| + t.integer :status, null: false + + t.datetime_with_timezone :created_at, null: false + t.datetime_with_timezone :updated_at, null: false + + t.references :service, index: true, null: false, foreign_key: { on_delete: :cascade } + t.string :version, null: false + t.text :status_reason + end + end +end diff --git a/lib/gitlab/clusters/helm.rb b/lib/gitlab/clusters/helm.rb new file mode 100644 index 00000000000..9c75fe2be96 --- /dev/null +++ b/lib/gitlab/clusters/helm.rb @@ -0,0 +1,104 @@ +module Gitlab + module Clusters + class Helm + Error = Class.new(StandardError) + HELM_VERSION = '2.7.0'.freeze + NAMESPACE = 'gitlab-managed-apps'.freeze + COMMAND_SCRIPT = <<-EOS.freeze + set -eo pipefail + apk add -U ca-certificates openssl >/dev/null + wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null + mv /tmp/linux-amd64/helm /usr/bin/ + helm init ${HELM_INIT_OPTS} >/dev/null + [[ -z "${HELM_COMMAND+x}" ]] || helm ${HELM_COMMAND} >/dev/null + EOS + + def initialize(kubeclient) + @kubeclient = kubeclient + end + + def init! + ensure_namespace! + @kubeclient.create_pod(pod_resource(OpenStruct.new(name: 'helm'))) + end + + def install(app) + ensure_namespace! + @kubeclient.create_pod(pod_resource(app)) + end + + ## + # Returns Pod phase + # + # https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + # + # values: "Pending", "Running", "Succeeded", "Failed", "Unknown" + # + def installation_status(app) + @kubeclient.get_pod(pod_name(app), NAMESPACE).status.phase + end + + def installation_log(app) + @kubeclient.get_pod_log(pod_name(app), NAMESPACE).body + end + + def delete_installation_pod!(app) + @kubeclient.delete_pod(pod_name(app), NAMESPACE) + end + + private + + def pod_name(app) + "install-#{app.name}" + end + + def pod_resource(app) + labels = { 'gitlab.org/action': 'install', 'gitlab.org/application': app.name } + metadata = { name: pod_name(app), namespace: NAMESPACE, labels: labels } + container = { + name: 'helm', + image: 'alpine:3.6', + env: generate_pod_env(app), + command: %w(/bin/sh), + args: %w(-c $(COMMAND_SCRIPT)) + } + spec = { containers: [container], restartPolicy: 'Never' } + + ::Kubeclient::Resource.new(metadata: metadata, spec: spec) + end + + def generate_pod_env(app) + env = { + HELM_VERSION: HELM_VERSION, + TILLER_NAMESPACE: NAMESPACE, + COMMAND_SCRIPT: COMMAND_SCRIPT + } + + if app.name != 'helm' + env[:HELM_INIT_OPTS] = '--client-only' + env[:HELM_COMMAND] = helm_install_comand(app) + end + + env.map { |key, value| { name: key, value: value } } + end + + def helm_install_comand(app) + "install #{app.chart} --name #{app.name} --namespace #{NAMESPACE}" + end + + def ensure_namespace! + begin + @kubeclient.get_namespace(NAMESPACE) + rescue KubeException => ke + raise ke unless ke.error_code == 404 + + namespace_resource = ::Kubeclient::Resource.new + namespace_resource.metadata = {} + namespace_resource.metadata.name = NAMESPACE + + @kubeclient.create_namespace(namespace_resource) + end + end + end + end +end diff --git a/spec/fixtures/api/schemas/cluster_status.json b/spec/fixtures/api/schemas/cluster_status.json index 1f255a17881..451ea50f0f9 100644 --- a/spec/fixtures/api/schemas/cluster_status.json +++ b/spec/fixtures/api/schemas/cluster_status.json @@ -5,7 +5,38 @@ ], "properties" : { "status": { "type": "string" }, - "status_reason": { "type": ["string", "null"] } + "status_reason": { "type": ["string", "null"] }, + "applications": { "$ref": "#/definitions/applications" } }, - "additionalProperties": false + "additionalProperties": false, + "definitions": { + "applications": { + "type": "object", + "additionalProperties": false, + "properties" : { + "helm": { "$ref": "#/definitions/app_status" }, + "runner": { "$ref": "#/definitions/app_status" }, + "ingress": { "$ref": "#/definitions/app_status" }, + "prometheus": { "$ref": "#/definitions/app_status" } + } + }, + "app_status": { + "type": "object", + "additionalProperties": false, + "properties" : { + "status": { + "type": { + "enum": [ + "installable", + "installing", + "installed", + "error" + ] + } + }, + "status_reason": { "type": ["string", "null"] } + }, + "required" : [ "status" ] + } + } } diff --git a/spec/models/clusters/kubernetes/helm_app_spec.rb b/spec/models/clusters/kubernetes/helm_app_spec.rb new file mode 100644 index 00000000000..27a1561ce6c --- /dev/null +++ b/spec/models/clusters/kubernetes/helm_app_spec.rb @@ -0,0 +1,14 @@ +require 'rails_helper' +require_relative '../kubernetes_spec' + +RSpec.describe Clusters::Kubernetes::HelmApp, type: :model do + it_behaves_like 'a registered kubernetes app' + + it { is_expected.to belong_to(:kubernetes_service) } + + describe '#cluster' do + it 'is an alias to #kubernetes_service' do + expect(subject.method(:cluster).original_name).to eq(:kubernetes_service) + end + end +end diff --git a/spec/models/clusters/kubernetes_spec.rb b/spec/models/clusters/kubernetes_spec.rb new file mode 100644 index 00000000000..5876f08250f --- /dev/null +++ b/spec/models/clusters/kubernetes_spec.rb @@ -0,0 +1,9 @@ +require 'rails_helper' + +RSpec.shared_examples 'a registered kubernetes app' do + let(:name) { described_class::NAME } + + it 'can be retrieved with Clusters::Kubernetes.app' do + expect(Clusters::Kubernetes.app(name)).to eq(described_class) + end +end diff --git a/spec/models/project_services/kubernetes_service_spec.rb b/spec/models/project_services/kubernetes_service_spec.rb index 7617e1f89b1..6d9fc28bf58 100644 --- a/spec/models/project_services/kubernetes_service_spec.rb +++ b/spec/models/project_services/kubernetes_service_spec.rb @@ -7,8 +7,9 @@ describe KubernetesService, :use_clean_rails_memory_store_caching do let(:project) { build_stubbed(:kubernetes_project) } let(:service) { project.kubernetes_service } - describe "Associations" do + describe 'Associations' do it { is_expected.to belong_to :project } + it { is_expected.to have_one(:helm_app) } end describe 'Validations' do diff --git a/spec/serializers/cluster_entity_spec.rb b/spec/serializers/cluster_entity_spec.rb index 7b132a1b84d..abfc3731fb2 100644 --- a/spec/serializers/cluster_entity_spec.rb +++ b/spec/serializers/cluster_entity_spec.rb @@ -34,5 +34,9 @@ describe ClusterEntity do expect(subject[:status_reason]).to be_nil end end + + it 'contains applications' do + expect(subject[:applications]).to eq({}) + end end end diff --git a/spec/serializers/cluster_serializer_spec.rb b/spec/serializers/cluster_serializer_spec.rb index e5da92a451e..04d8728303c 100644 --- a/spec/serializers/cluster_serializer_spec.rb +++ b/spec/serializers/cluster_serializer_spec.rb @@ -17,7 +17,7 @@ describe ClusterSerializer do let(:cluster) { create(:cluster, provider_type: :user) } it 'serializes only status' do - expect(subject.keys).to contain_exactly(:status, :status_reason) + expect(subject.keys).to contain_exactly(:status, :status_reason, :applications) end end end |