Merge branch '40418-migrate-existing-data-from-kubernetesservice-to-clusters-platforms-kubernetes' into 'master'

Migrate existing data from KubernetesService to Clusters::Platforms::Kubernetes

Closes #40418

See merge request gitlab-org/gitlab-ce!15589

(cherry picked from commit 4b92efd90cedaa0aff218d11fdce279701128bea)

5b2ca1c6 Migrate existing data from KubernetesService to Clusters::Platforms::Kubernetes
e4745492 Add test. Disable KubernetesService when migrated
b9fbfe5a Fix unmanaged_kubernetes_service scope for multiple clusters
40c6af54 Fix migration file typos and reorder Table definition
27111e29 Restructure spec
4dc14576 Fix comments
8e6ffe35 Fix test
7eeada80 Add env_scope tests
f083739e Add logic to swtich environment_scope by the situation
9b7719b6 Use explicit namespace for avoiding reference from application code
665972e2 Avoid quotes in ActiveRecord query
c8059881 Opitmize migration process by using both unmanaged_kubernetes_service and…
b8a275d3 Use bulk_insert instead of AR create
acfb8464 Fix static anylysy
8bc3221f Fix query to look for proper unmanaged kubernetes service
2d3c7d29 Use batch update for Service deactivation
1c404c91 Add a new test for emptified params
183dbdc8 Revert bulk_insert and bring back AR insert(one by one)
54d20d1b Add changelog
290c2248 Fix change log
58d074e0 Fix StaticSnalysys
df658c7b Disable STI of ActiveRecord. Refactoring specs.
c425ff75 Fix static analysys
67327952 Add memoization for properties

3 files changed, 468 insertions, 0 deletions

diff --git a/changelogs/unreleased/40418-migrate-existing-data-from-kubernetesservice-to-clusters-platforms-kubernetes.yml b/changelogs/unreleased/40418-migrate-existing-data-from-kubernetesservice-to-clusters-platforms-kubernetes.yml
new file mode 100644
index 00000000000..5e158d831a6
--- /dev/null
+++ b/changelogs/unreleased/40418-migrate-existing-data-from-kubernetesservice-to-clusters-platforms-kubernetes.yml
@@ -0,0 +1,5 @@
+---
+title: Migrate existing data from KubernetesService to Clusters::Platforms::Kubernetes
+merge_request: 15589
+author:
+type: changed
diff --git a/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb
new file mode 100644
index 00000000000..11b581e4b57
--- /dev/null
+++ b/db/post_migrate/20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb
@@ -0,0 +1,151 @@
+class MigrateKubernetesServiceToNewClustersArchitectures < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  DEFAULT_KUBERNETES_SERVICE_CLUSTER_NAME = 'KubernetesService'.freeze
+
+  disable_ddl_transaction!
+
+  class Project < ActiveRecord::Base
+    self.table_name = 'projects'
+
+    has_many :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::ClustersProject'
+    has_many :clusters, through: :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster'
+    has_many :services, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Service'
+    has_one :kubernetes_service, -> { where(category: 'deployment', type: 'KubernetesService') }, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Service', inverse_of: :project, foreign_key: :project_id
+  end
+
+  class Cluster < ActiveRecord::Base
+    self.table_name = 'clusters'
+
+    has_many :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::ClustersProject'
+    has_many :projects, through: :cluster_projects, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project'
+    has_one :platform_kubernetes, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::PlatformsKubernetes'
+
+    accepts_nested_attributes_for :platform_kubernetes
+
+    enum platform_type: {
+      kubernetes: 1
+    }
+
+    enum provider_type: {
+      user: 0,
+      gcp: 1
+    }
+  end
+
+  class ClustersProject < ActiveRecord::Base
+    self.table_name = 'cluster_projects'
+
+    belongs_to :cluster, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster'
+    belongs_to :project, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project'
+  end
+
+  class PlatformsKubernetes < ActiveRecord::Base
+    self.table_name = 'cluster_platforms_kubernetes'
+
+    belongs_to :cluster, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Cluster'
+
+    attr_encrypted :token,
+      mode: :per_attribute_iv,
+      key: Gitlab::Application.secrets.db_key_base,
+      algorithm: 'aes-256-cbc'
+  end
+
+  class Service < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'services'
+    self.inheritance_column = :_type_disabled # Disable STI, otherwise KubernetesModel will be looked up
+
+    belongs_to :project, class_name: 'MigrateKubernetesServiceToNewClustersArchitectures::Project', foreign_key: :project_id
+
+    scope :unmanaged_kubernetes_service, -> do
+      joins('LEFT JOIN projects ON projects.id = services.project_id')
+      .joins('LEFT JOIN cluster_projects ON cluster_projects.project_id = projects.id')
+      .joins('LEFT JOIN cluster_platforms_kubernetes ON cluster_platforms_kubernetes.cluster_id = cluster_projects.cluster_id')
+      .where(category: 'deployment', type: 'KubernetesService', template: false)
+      .where("services.properties LIKE '%api_url%'")
+      .where("(services.properties NOT LIKE CONCAT('%', cluster_platforms_kubernetes.api_url, '%')) OR cluster_platforms_kubernetes.api_url IS NULL")
+      .group(:id)
+      .order(id: :asc)
+    end
+
+    scope :kubernetes_service_without_template, -> do
+      where(category: 'deployment', type: 'KubernetesService', template: false)
+    end
+
+    def api_url
+      parsed_properties['api_url']
+    end
+
+    def ca_pem
+      parsed_properties['ca_pem']
+    end
+
+    def namespace
+      parsed_properties['namespace']
+    end
+
+    def token
+      parsed_properties['token']
+    end
+
+    private
+
+    def parsed_properties
+      @parsed_properties ||= JSON.parse(self.properties)
+    end
+  end
+
+  def find_dedicated_environement_scope(project)
+    environment_scopes = project.clusters.map(&:environment_scope)
+
+    return '*' if environment_scopes.exclude?('*') # KubernetesService should be added as a default cluster (environment_scope: '*') at first place
+    return 'migrated/*' if environment_scopes.exclude?('migrated/*') # If it's conflicted, the KubernetesService added as a migrated cluster
+
+    unique_iid = 0
+
+    # If it's still conflicted, finding an unique environment scope incrementaly
+    loop do
+      candidate = "migrated#{unique_iid}/*"
+      return candidate if environment_scopes.exclude?(candidate)
+
+      unique_iid += 1
+    end
+  end
+
+  def up
+    ActiveRecord::Base.transaction do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service
+        .unmanaged_kubernetes_service.find_each(batch_size: 1) do |kubernetes_service|
+        MigrateKubernetesServiceToNewClustersArchitectures::Cluster.create(
+          enabled: kubernetes_service.active,
+          user_id: nil, # KubernetesService doesn't have
+          name: DEFAULT_KUBERNETES_SERVICE_CLUSTER_NAME,
+          provider_type: MigrateKubernetesServiceToNewClustersArchitectures::Cluster.provider_types[:user],
+          platform_type: MigrateKubernetesServiceToNewClustersArchitectures::Cluster.platform_types[:kubernetes],
+          projects: [kubernetes_service.project],
+          environment_scope: find_dedicated_environement_scope(kubernetes_service.project),
+          platform_kubernetes_attributes: {
+            api_url: kubernetes_service.api_url,
+            ca_cert: kubernetes_service.ca_pem,
+            namespace: kubernetes_service.namespace,
+            username: nil, # KubernetesService doesn't have
+            encrypted_password: nil, # KubernetesService doesn't have
+            encrypted_password_iv: nil, # KubernetesService doesn't have
+            token: kubernetes_service.token # encrypted_token and encrypted_token_iv
+          } )
+      end
+    end
+
+    MigrateKubernetesServiceToNewClustersArchitectures::Service
+      .kubernetes_service_without_template.each_batch(of: 100) do |kubernetes_service|
+      kubernetes_service.update_all(active: false)
+    end
+  end
+
+  def down
+    # noop
+  end
+end
diff --git a/spec/migrations/migrate_kubernetes_service_to_new_clusters_architectures_spec.rb b/spec/migrations/migrate_kubernetes_service_to_new_clusters_architectures_spec.rb
new file mode 100644
index 00000000000..df0015b6dd3
--- /dev/null
+++ b/spec/migrations/migrate_kubernetes_service_to_new_clusters_architectures_spec.rb
@@ -0,0 +1,312 @@
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20171124104327_migrate_kubernetes_service_to_new_clusters_architectures.rb')
+
+describe MigrateKubernetesServiceToNewClustersArchitectures, :migration do
+  context 'when unique KubernetesService exists' do
+    shared_examples 'KubernetesService migration' do
+      let(:sample_num) { 2 }
+
+      let(:projects) do
+        (1..sample_num).each_with_object([]) do |n, array|
+          array << MigrateKubernetesServiceToNewClustersArchitectures::Project.create!
+        end
+      end
+
+      let!(:kubernetes_services) do
+        projects.map do |project|
+          MigrateKubernetesServiceToNewClustersArchitectures::Service.create!(
+            project: project,
+            active: active,
+            category: 'deployment',
+            type: 'KubernetesService',
+            properties: "{\"namespace\":\"prod\",\"api_url\":\"https://kubernetes#{project.id}.com\",\"ca_pem\":\"ca_pem#{project.id}\",\"token\":\"token#{project.id}\"}")
+        end
+      end
+
+      it 'migrates the KubernetesService to Platform::Kubernetes' do
+        expect { migrate! }.to change { MigrateKubernetesServiceToNewClustersArchitectures::Cluster.count }.by(sample_num)
+
+        projects.each do |project|
+          project.clusters.last.tap do |cluster|
+            expect(cluster.enabled).to eq(active)
+            expect(cluster.platform_kubernetes.api_url).to eq(project.kubernetes_service.api_url)
+            expect(cluster.platform_kubernetes.ca_cert).to eq(project.kubernetes_service.ca_pem)
+            expect(cluster.platform_kubernetes.token).to eq(project.kubernetes_service.token)
+            expect(project.kubernetes_service).not_to be_active
+          end
+        end
+      end
+    end
+
+    context 'when KubernetesService is active' do
+      let(:active) { true }
+
+      it_behaves_like 'KubernetesService migration'
+    end
+  end
+
+  context 'when unique KubernetesService spawned from Service Template' do
+    let(:sample_num) { 2 }
+
+    let(:projects) do
+      (1..sample_num).each_with_object([]) do |n, array|
+        array << MigrateKubernetesServiceToNewClustersArchitectures::Project.create!
+      end
+    end
+
+    let!(:kubernetes_service_template) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service.create!(
+        template: true,
+        category: 'deployment',
+        type: 'KubernetesService',
+        properties: "{\"namespace\":\"prod\",\"api_url\":\"https://sample.kubernetes.com\",\"ca_pem\":\"ca_pem-sample\",\"token\":\"token-sample\"}")
+    end
+
+    let!(:kubernetes_services) do
+      projects.map do |project|
+        MigrateKubernetesServiceToNewClustersArchitectures::Service.create!(
+          project: project,
+          category: 'deployment',
+          type: 'KubernetesService',
+          properties: "{\"namespace\":\"prod\",\"api_url\":\"#{kubernetes_service_template.api_url}\",\"ca_pem\":\"#{kubernetes_service_template.ca_pem}\",\"token\":\"#{kubernetes_service_template.token}\"}")
+      end
+    end
+
+    it 'migrates the KubernetesService to Platform::Kubernetes without template' do
+      expect { migrate! }.to change { MigrateKubernetesServiceToNewClustersArchitectures::Cluster.count }.by(sample_num)
+
+      projects.each do |project|
+        project.clusters.last.tap do |cluster|
+          expect(cluster.platform_kubernetes.api_url).to eq(project.kubernetes_service.api_url)
+          expect(cluster.platform_kubernetes.ca_cert).to eq(project.kubernetes_service.ca_pem)
+          expect(cluster.platform_kubernetes.token).to eq(project.kubernetes_service.token)
+          expect(project.kubernetes_service).not_to be_active
+        end
+      end
+    end
+  end
+
+  context 'when managed KubernetesService exists' do
+    let(:project) { MigrateKubernetesServiceToNewClustersArchitectures::Project.create! }
+
+    let(:cluster) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Cluster.create!(
+        projects: [project],
+        name: 'sample-cluster',
+        platform_type: :kubernetes,
+        provider_type: :user,
+        platform_kubernetes_attributes: {
+          api_url: 'https://sample.kubernetes.com',
+          ca_cert: 'ca_pem-sample',
+          token: 'token-sample'
+        } )
+    end
+
+    let!(:kubernetes_service) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service.create!(
+        project: project,
+        active: cluster.enabled,
+        category: 'deployment',
+        type: 'KubernetesService',
+        properties: "{\"api_url\":\"#{cluster.platform_kubernetes.api_url}\"}")
+    end
+
+    it 'does not migrate the KubernetesService and disables the kubernetes_service' do # Because the corresponding Platform::Kubernetes already exists
+      expect { migrate! }.not_to change { MigrateKubernetesServiceToNewClustersArchitectures::Cluster.count }
+
+      kubernetes_service.reload
+      expect(kubernetes_service).not_to be_active
+    end
+  end
+
+  context 'when production cluster has already been existed' do # i.e. There are no environment_scope conflicts
+    let(:project) { MigrateKubernetesServiceToNewClustersArchitectures::Project.create! }
+
+    let(:cluster) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Cluster.create!(
+        projects: [project],
+        name: 'sample-cluster',
+        platform_type: :kubernetes,
+        provider_type: :user,
+        environment_scope: 'production/*',
+        platform_kubernetes_attributes: {
+          api_url: 'https://sample.kubernetes.com',
+          ca_cert: 'ca_pem-sample',
+          token: 'token-sample'
+        } )
+    end
+
+    let!(:kubernetes_service) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service.create!(
+        project: project,
+        active: true,
+        category: 'deployment',
+        type: 'KubernetesService',
+        properties: "{\"api_url\":\"https://debug.kube.com\"}")
+    end
+
+    it 'migrates the KubernetesService to Platform::Kubernetes' do
+      expect { migrate! }.to change { MigrateKubernetesServiceToNewClustersArchitectures::Cluster.count }.by(1)
+
+      kubernetes_service.reload
+      project.clusters.last.tap do |cluster|
+        expect(cluster.environment_scope).to eq('*')
+        expect(cluster.platform_kubernetes.api_url).to eq(kubernetes_service.api_url)
+        expect(cluster.platform_kubernetes.ca_cert).to eq(kubernetes_service.ca_pem)
+        expect(cluster.platform_kubernetes.token).to eq(kubernetes_service.token)
+        expect(kubernetes_service).not_to be_active
+      end
+    end
+  end
+
+  context 'when default cluster has already been existed' do
+    let(:project) { MigrateKubernetesServiceToNewClustersArchitectures::Project.create! }
+
+    let!(:cluster) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Cluster.create!(
+        projects: [project],
+        name: 'sample-cluster',
+        platform_type: :kubernetes,
+        provider_type: :user,
+        environment_scope: '*',
+        platform_kubernetes_attributes: {
+          api_url: 'https://sample.kubernetes.com',
+          ca_cert: 'ca_pem-sample',
+          token: 'token-sample'
+        } )
+    end
+
+    let!(:kubernetes_service) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service.create!(
+        project: project,
+        active: true,
+        category: 'deployment',
+        type: 'KubernetesService',
+        properties: "{\"api_url\":\"https://debug.kube.com\"}")
+    end
+
+    it 'migrates the KubernetesService to Platform::Kubernetes with dedicated environment_scope' do # Because environment_scope is duplicated
+      expect { migrate! }.to change { MigrateKubernetesServiceToNewClustersArchitectures::Cluster.count }.by(1)
+
+      kubernetes_service.reload
+      project.clusters.last.tap do |cluster|
+        expect(cluster.environment_scope).to eq('migrated/*')
+        expect(cluster.platform_kubernetes.api_url).to eq(kubernetes_service.api_url)
+        expect(cluster.platform_kubernetes.ca_cert).to eq(kubernetes_service.ca_pem)
+        expect(cluster.platform_kubernetes.token).to eq(kubernetes_service.token)
+        expect(kubernetes_service).not_to be_active
+      end
+    end
+  end
+
+  context 'when default cluster and migrated cluster has already been existed' do
+    let(:project) { MigrateKubernetesServiceToNewClustersArchitectures::Project.create! }
+
+    let!(:cluster) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Cluster.create!(
+        projects: [project],
+        name: 'sample-cluster',
+        platform_type: :kubernetes,
+        provider_type: :user,
+        environment_scope: '*',
+        platform_kubernetes_attributes: {
+          api_url: 'https://sample.kubernetes.com',
+          ca_cert: 'ca_pem-sample',
+          token: 'token-sample'
+        } )
+    end
+
+    let!(:migrated_cluster) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Cluster.create!(
+        projects: [project],
+        name: 'sample-cluster',
+        platform_type: :kubernetes,
+        provider_type: :user,
+        environment_scope: 'migrated/*',
+        platform_kubernetes_attributes: {
+          api_url: 'https://sample.kubernetes.com',
+          ca_cert: 'ca_pem-sample',
+          token: 'token-sample'
+        } )
+    end
+
+    let!(:kubernetes_service) do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service.create!(
+        project: project,
+        active: true,
+        category: 'deployment',
+        type: 'KubernetesService',
+        properties: "{\"api_url\":\"https://debug.kube.com\"}")
+    end
+
+    it 'migrates the KubernetesService to Platform::Kubernetes with dedicated environment_scope' do # Because environment_scope is duplicated
+      expect { migrate! }.to change { MigrateKubernetesServiceToNewClustersArchitectures::Cluster.count }.by(1)
+
+      kubernetes_service.reload
+      project.clusters.last.tap do |cluster|
+        expect(cluster.environment_scope).to eq('migrated0/*')
+        expect(cluster.platform_kubernetes.api_url).to eq(kubernetes_service.api_url)
+        expect(cluster.platform_kubernetes.ca_cert).to eq(kubernetes_service.ca_pem)
+        expect(cluster.platform_kubernetes.token).to eq(kubernetes_service.token)
+        expect(kubernetes_service).not_to be_active
+      end
+    end
+  end
+
+  context 'when KubernetesService has nullified parameters' do
+    let(:project) { MigrateKubernetesServiceToNewClustersArchitectures::Project.create! }
+
+    before do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service.create!(
+        project: project,
+        active: false,
+        category: 'deployment',
+        type: 'KubernetesService',
+        properties: "{}")
+    end
+
+    it 'does not migrate the KubernetesService and disables the kubernetes_service' do
+      expect { migrate! }.not_to change { MigrateKubernetesServiceToNewClustersArchitectures::Cluster.count }
+
+      expect(project.kubernetes_service).not_to be_active
+    end
+  end
+
+  # Platforms::Kubernetes validates `token` reagdless of the activeness,
+  # whereas KubernetesService validates `token` if only it's activated
+  # However, in this migration file, there are no validations because of the re-defined model class
+  # therefore, we should safely add this raw to Platform::Kubernetes
+  context 'when KubernetesService has empty token' do
+    let(:project) { MigrateKubernetesServiceToNewClustersArchitectures::Project.create! }
+
+    before do
+      MigrateKubernetesServiceToNewClustersArchitectures::Service.create!(
+        project: project,
+        active: false,
+        category: 'deployment',
+        type: 'KubernetesService',
+        properties: "{\"namespace\":\"prod\",\"api_url\":\"http://111.111.111.111\",\"ca_pem\":\"a\",\"token\":\"\"}")
+    end
+
+    it 'does not migrate the KubernetesService and disables the kubernetes_service' do
+      expect { migrate! }.to change { MigrateKubernetesServiceToNewClustersArchitectures::Cluster.count }.by(1)
+
+      project.clusters.last.tap do |cluster|
+        expect(cluster.environment_scope).to eq('*')
+        expect(cluster.platform_kubernetes.namespace).to eq('prod')
+        expect(cluster.platform_kubernetes.api_url).to eq('http://111.111.111.111')
+        expect(cluster.platform_kubernetes.ca_cert).to eq('a')
+        expect(cluster.platform_kubernetes.token).to be_empty
+        expect(project.kubernetes_service).not_to be_active
+      end
+    end
+  end
+
+  context 'when KubernetesService does not exist' do
+    let!(:project) { MigrateKubernetesServiceToNewClustersArchitectures::Project.create! }
+
+    it 'does not migrate the KubernetesService' do
+      expect { migrate! }.not_to change { MigrateKubernetesServiceToNewClustersArchitectures::Cluster.count }
+    end
+  end
+end