Add latest changes from gitlab-org/gitlab@13-3-stable-ee

10 files changed, 204 insertions, 114 deletions

diff --git a/lib/gitlab/kubernetes/cilium_network_policy.rb b/lib/gitlab/kubernetes/cilium_network_policy.rb
new file mode 100644
index 00000000000..55afd2b586e
--- /dev/null
+++ b/lib/gitlab/kubernetes/cilium_network_policy.rb
@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    class CiliumNetworkPolicy
+      include NetworkPolicyCommon
+      extend ::Gitlab::Utils::Override
+
+      API_VERSION = "cilium.io/v2"
+      KIND = 'CiliumNetworkPolicy'
+
+      def initialize(name:, namespace:, selector:, ingress:, resource_version:, labels: nil, creation_timestamp: nil, egress: nil)
+        @name = name
+        @namespace = namespace
+        @labels = labels
+        @creation_timestamp = creation_timestamp
+        @selector = selector
+        @resource_version = resource_version
+        @ingress = ingress
+        @egress = egress
+      end
+
+      def generate
+        ::Kubeclient::Resource.new.tap do |resource|
+          resource.kind = KIND
+          resource.apiVersion = API_VERSION
+          resource.metadata = metadata
+          resource.spec = spec
+        end
+      end
+
+      def self.from_yaml(manifest)
+        return unless manifest
+
+        policy = YAML.safe_load(manifest, symbolize_names: true)
+        return if !policy[:metadata] || !policy[:spec]
+
+        metadata = policy[:metadata]
+        spec = policy[:spec]
+        self.new(
+          name: metadata[:name],
+          namespace: metadata[:namespace],
+          resource_version: metadata[:resourceVersion],
+          labels: metadata[:labels],
+          selector: spec[:endpointSelector],
+          ingress: spec[:ingress],
+          egress: spec[:egress]
+        )
+      rescue Psych::SyntaxError, Psych::DisallowedClass
+        nil
+      end
+
+      def self.from_resource(resource)
+        return unless resource
+        return if !resource[:metadata] || !resource[:spec]
+
+        metadata = resource[:metadata]
+        spec = resource[:spec].to_h
+        self.new(
+          name: metadata[:name],
+          namespace: metadata[:namespace],
+          resource_version: metadata[:resourceVersion],
+          labels: metadata[:labels]&.to_h,
+          creation_timestamp: metadata[:creationTimestamp],
+          selector: spec[:endpointSelector],
+          ingress: spec[:ingress],
+          egress: spec[:egress]
+        )
+      end
+
+      private
+
+      attr_reader :name, :namespace, :labels, :creation_timestamp, :resource_version, :ingress, :egress
+
+      def selector
+        @selector ||= {}
+      end
+
+      override :spec
+      def spec
+        {
+          endpointSelector: selector,
+          ingress: ingress,
+          egress: egress
+        }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb
index f27ad05599e..49d2969f7f3 100644
--- a/lib/gitlab/kubernetes/helm/base_command.rb
+++ b/lib/gitlab/kubernetes/helm/base_command.rb
@@ -6,21 +6,16 @@ module Gitlab
       class BaseCommand
         attr_reader :name, :files
 
-        def initialize(rbac:, name:, files:, local_tiller_enabled:)
+        def initialize(rbac:, name:, files:)
           @rbac = rbac
           @name = name
           @files = files
-          @local_tiller_enabled = local_tiller_enabled
         end
 
         def rbac?
           @rbac
         end
 
-        def local_tiller_enabled?
-          @local_tiller_enabled
-        end
-
         def pod_resource
           pod_service_account_name = rbac? ? service_account_name : nil
 
diff --git a/lib/gitlab/kubernetes/helm/client_command.rb b/lib/gitlab/kubernetes/helm/client_command.rb
index 24458e1b4b3..a9e93c0c90e 100644
--- a/lib/gitlab/kubernetes/helm/client_command.rb
+++ b/lib/gitlab/kubernetes/helm/client_command.rb
@@ -5,30 +5,11 @@ module Gitlab
     module Helm
       module ClientCommand
         def init_command
-          if local_tiller_enabled?
-            <<~HEREDOC.chomp
+          <<~SHELL.chomp
             export HELM_HOST="localhost:44134"
             tiller -listen ${HELM_HOST} -alsologtostderr &
             helm init --client-only
-            HEREDOC
-          else
-            # Here we are always upgrading to the latest version of Tiller when
-            # installing an app. We ensure the helm version stored in the
-            # database is correct by also updating this after transition to
-            # :installed,:updated in Clusters::Concerns::ApplicationStatus
-            'helm init --upgrade'
-          end
-        end
-
-        def wait_for_tiller_command
-          return if local_tiller_enabled?
-
-          helm_check = ['helm', 'version', *optional_tls_flags].shelljoin
-          # This is necessary to give Tiller time to restart after upgrade.
-          # Ideally we'd be able to use --wait but cannot because of
-          # https://github.com/helm/helm/issues/4855
-
-          "for i in $(seq 1 30); do #{helm_check} && s=0 && break || s=$?; sleep 1s; echo \"Retrying ($i)...\"; done; (exit $s)"
+          SHELL
         end
 
         def repository_command
@@ -37,12 +18,6 @@ module Gitlab
 
         private
 
-        def tls_flags_if_remote_tiller
-          return [] if local_tiller_enabled?
-
-          optional_tls_flags
-        end
-
         def repository_update_command
           'helm repo update'
         end
diff --git a/lib/gitlab/kubernetes/helm/delete_command.rb b/lib/gitlab/kubernetes/helm/delete_command.rb
index 3bb41d09994..f8b9601bc98 100644
--- a/lib/gitlab/kubernetes/helm/delete_command.rb
+++ b/lib/gitlab/kubernetes/helm/delete_command.rb
@@ -17,7 +17,6 @@ module Gitlab
         def generate_script
           super + [
             init_command,
-            wait_for_tiller_command,
             predelete,
             delete_command,
             postdelete
@@ -29,9 +28,7 @@ module Gitlab
         end
 
         def delete_command
-          command = ['helm', 'delete', '--purge', name] + tls_flags_if_remote_tiller
-
-          command.shelljoin
+          ['helm', 'delete', '--purge', name].shelljoin
         end
       end
     end
diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb
index cf6d993cad4..d166842fce6 100644
--- a/lib/gitlab/kubernetes/helm/install_command.rb
+++ b/lib/gitlab/kubernetes/helm/install_command.rb
@@ -21,7 +21,6 @@ module Gitlab
         def generate_script
           super + [
             init_command,
-            wait_for_tiller_command,
             repository_command,
             repository_update_command,
             preinstall,
@@ -39,7 +38,6 @@ module Gitlab
             install_flag +
             rollback_support_flag +
             reset_values_flag +
-            tls_flags_if_remote_tiller +
             optional_version_flag +
             rbac_create_flag +
             namespace_flag +
diff --git a/lib/gitlab/kubernetes/helm/patch_command.rb b/lib/gitlab/kubernetes/helm/patch_command.rb
index 1a5fab116bd..a33dbdac134 100644
--- a/lib/gitlab/kubernetes/helm/patch_command.rb
+++ b/lib/gitlab/kubernetes/helm/patch_command.rb
@@ -26,7 +26,6 @@ module Gitlab
         def generate_script
           super + [
             init_command,
-            wait_for_tiller_command,
             repository_command,
             repository_update_command,
             upgrade_command
@@ -38,7 +37,6 @@ module Gitlab
         def upgrade_command
           command = ['helm', 'upgrade', name, chart] +
             reuse_values_flag +
-            tls_flags_if_remote_tiller +
             version_flag +
             namespace_flag +
             value_flag
diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb
index 2110d586d30..9e3cf58bb1e 100644
--- a/lib/gitlab/kubernetes/kube_client.rb
+++ b/lib/gitlab/kubernetes/kube_client.rb
@@ -21,7 +21,8 @@ module Gitlab
         istio: { group: 'apis/networking.istio.io', version: 'v1alpha3' },
         knative: { group: 'apis/serving.knative.dev', version: 'v1alpha1' },
         metrics: { group: 'apis/metrics.k8s.io', version: 'v1beta1' },
-        networking: { group: 'apis/networking.k8s.io', version: 'v1' }
+        networking: { group: 'apis/networking.k8s.io', version: 'v1' },
+        cilium_networking: { group: 'apis/cilium.io', version: 'v2' }
       }.freeze
 
       SUPPORTED_API_GROUPS.each do |name, params|
@@ -95,6 +96,14 @@ module Gitlab
         :delete_network_policy,
         to: :networking_client
 
+      # CiliumNetworkPolicy methods delegate to the apis/cilium.io api
+      # group client
+      delegate :create_cilium_network_policy,
+        :get_cilium_network_policies,
+        :update_cilium_network_policy,
+        :delete_cilium_network_policy,
+        to: :cilium_networking_client
+
       attr_reader :api_prefix, :kubeclient_options
 
       DEFAULT_KUBECLIENT_OPTIONS = {
@@ -107,15 +116,15 @@ module Gitlab
       def self.graceful_request(cluster_id)
         { status: :connected, response: yield }
       rescue *Gitlab::Kubernetes::Errors::CONNECTION
-        { status: :unreachable }
+        { status: :unreachable, connection_error: :connection_error }
       rescue *Gitlab::Kubernetes::Errors::AUTHENTICATION
-        { status: :authentication_failure }
+        { status: :authentication_failure, connection_error: :authentication_error }
       rescue Kubeclient::HttpError => e
-        { status: kubeclient_error_status(e.message) }
+        { status: kubeclient_error_status(e.message), connection_error: :http_error }
       rescue => e
         Gitlab::ErrorTracking.track_exception(e, cluster_id: cluster_id)
 
-        { status: :unknown_failure }
+        { status: :unknown_failure, connection_error: :unknown_error }
       end
 
       # KubeClient uses the same error class
diff --git a/lib/gitlab/kubernetes/network_policy.rb b/lib/gitlab/kubernetes/network_policy.rb
index dc13a614551..28810dc4453 100644
--- a/lib/gitlab/kubernetes/network_policy.rb
+++ b/lib/gitlab/kubernetes/network_policy.rb
@@ -3,19 +3,27 @@
 module Gitlab
   module Kubernetes
     class NetworkPolicy
-      DISABLED_BY_LABEL = :'network-policy.gitlab.com/disabled_by'
+      include NetworkPolicyCommon
+      extend ::Gitlab::Utils::Override
 
-      def initialize(name:, namespace:, pod_selector:, ingress:, labels: nil, creation_timestamp: nil, policy_types: ["Ingress"], egress: nil)
+      def initialize(name:, namespace:, selector:, ingress:, labels: nil, creation_timestamp: nil, policy_types: ["Ingress"], egress: nil)
         @name = name
         @namespace = namespace
         @labels = labels
         @creation_timestamp = creation_timestamp
-        @pod_selector = pod_selector
+        @selector = selector
         @policy_types = policy_types
         @ingress = ingress
         @egress = egress
       end
 
+      def generate
+        ::Kubeclient::Resource.new.tap do |resource|
+          resource.metadata = metadata
+          resource.spec = spec
+        end
+      end
+
       def self.from_yaml(manifest)
         return unless manifest
 
@@ -28,7 +36,7 @@ module Gitlab
           name: metadata[:name],
           namespace: metadata[:namespace],
           labels: metadata[:labels],
-          pod_selector: spec[:podSelector],
+          selector: spec[:podSelector],
           policy_types: spec[:policyTypes],
           ingress: spec[:ingress],
           egress: spec[:egress]
@@ -48,81 +56,30 @@ module Gitlab
           namespace: metadata[:namespace],
           labels: metadata[:labels]&.to_h,
           creation_timestamp: metadata[:creationTimestamp],
-          pod_selector: spec[:podSelector],
+          selector: spec[:podSelector],
           policy_types: spec[:policyTypes],
           ingress: spec[:ingress],
           egress: spec[:egress]
         )
       end
 
-      def generate
-        ::Kubeclient::Resource.new.tap do |resource|
-          resource.metadata = metadata
-          resource.spec = spec
-        end
-      end
-
-      def as_json(opts = nil)
-        {
-          name: name,
-          namespace: namespace,
-          creation_timestamp: creation_timestamp,
-          manifest: manifest,
-          is_autodevops: autodevops?,
-          is_enabled: enabled?
-        }
-      end
-
-      def autodevops?
-        return false unless labels
-
-        !labels[:chart].nil? && labels[:chart].start_with?('auto-deploy-app-')
-      end
-
-      # podSelector selects pods that should be targeted by this
-      # policy. We can narrow selection by requiring this policy to
-      # match our custom labels. Since DISABLED_BY label will not be
-      # on any pod a policy will be effectively disabled.
-      def enabled?
-        return true unless pod_selector&.key?(:matchLabels)
-
-        !pod_selector[:matchLabels]&.key?(DISABLED_BY_LABEL)
-      end
-
-      def enable
-        return if enabled?
-
-        pod_selector[:matchLabels].delete(DISABLED_BY_LABEL)
-      end
-
-      def disable
-        @pod_selector ||= {}
-        pod_selector[:matchLabels] ||= {}
-        pod_selector[:matchLabels].merge!(DISABLED_BY_LABEL => 'gitlab')
-      end
-
       private
 
-      attr_reader :name, :namespace, :labels, :creation_timestamp, :pod_selector, :policy_types, :ingress, :egress
+      attr_reader :name, :namespace, :labels, :creation_timestamp, :policy_types, :ingress, :egress
 
-      def metadata
-        meta = { name: name, namespace: namespace }
-        meta[:labels] = labels if labels
-        meta
+      def selector
+        @selector ||= {}
       end
 
+      override :spec
       def spec
         {
-          podSelector: pod_selector,
+          podSelector: selector,
           policyTypes: policy_types,
           ingress: ingress,
           egress: egress
         }
       end
-
-      def manifest
-        YAML.dump({ metadata: metadata, spec: spec }.deep_stringify_keys)
-      end
     end
   end
 end
diff --git a/lib/gitlab/kubernetes/network_policy_common.rb b/lib/gitlab/kubernetes/network_policy_common.rb
new file mode 100644
index 00000000000..3b6e46d21ef
--- /dev/null
+++ b/lib/gitlab/kubernetes/network_policy_common.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    module NetworkPolicyCommon
+      DISABLED_BY_LABEL = :'network-policy.gitlab.com/disabled_by'
+
+      def as_json(opts = nil)
+        {
+          name: name,
+          namespace: namespace,
+          creation_timestamp: creation_timestamp,
+          manifest: manifest,
+          is_autodevops: autodevops?,
+          is_enabled: enabled?
+        }
+      end
+
+      def autodevops?
+        return false unless labels
+
+        !labels[:chart].nil? && labels[:chart].start_with?('auto-deploy-app-')
+      end
+
+      # selector selects pods that should be targeted by this
+      # policy. It can represent podSelector, nodeSelector or
+      # endpointSelector  We can narrow selection by requiring
+      # this policy to match our custom labels. Since DISABLED_BY
+      # label will not be on any pod a policy will be effectively disabled.
+      def enabled?
+        return true unless selector&.key?(:matchLabels)
+
+        !selector[:matchLabels]&.key?(DISABLED_BY_LABEL)
+      end
+
+      def enable
+        return if enabled?
+
+        selector[:matchLabels].delete(DISABLED_BY_LABEL)
+      end
+
+      def disable
+        selector[:matchLabels] ||= {}
+        selector[:matchLabels].merge!(DISABLED_BY_LABEL => 'gitlab')
+      end
+
+      private
+
+      def metadata
+        meta = { name: name, namespace: namespace }
+        meta[:labels] = labels if labels
+        meta[:resourceVersion] = resource_version if defined?(resource_version)
+        meta
+      end
+
+      def spec
+        raise NotImplementedError
+      end
+
+      def manifest
+        YAML.dump({ metadata: metadata, spec: spec }.deep_stringify_keys)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/kubernetes/node.rb b/lib/gitlab/kubernetes/node.rb
index bd765ef3852..d516bdde6f6 100644
--- a/lib/gitlab/kubernetes/node.rb
+++ b/lib/gitlab/kubernetes/node.rb
@@ -8,22 +8,29 @@ module Gitlab
       end
 
       def all
-        nodes.map do |node|
-          attributes = node(node)
-          attributes.merge(node_metrics(node))
-        end
+        {
+          nodes: metadata.presence,
+          node_connection_error: nodes_from_cluster[:connection_error],
+          metrics_connection_error: nodes_metrics_from_cluster[:connection_error]
+        }.compact
       end
 
       private
 
       attr_reader :cluster
 
+      def metadata
+        nodes.map do |node|
+          base_data(node).merge(node_metrics(node))
+        end
+      end
+
       def nodes_from_cluster
-        graceful_request { cluster.kubeclient.get_nodes }
+        @nodes_from_cluster ||= graceful_request { cluster.kubeclient.get_nodes }
       end
 
       def nodes_metrics_from_cluster
-        graceful_request { cluster.kubeclient.metrics_client.get_nodes }
+        @nodes_metrics_from_cluster ||= graceful_request { cluster.kubeclient.metrics_client.get_nodes }
       end
 
       def nodes
@@ -44,7 +51,7 @@ module Gitlab
         ::Gitlab::Kubernetes::KubeClient.graceful_request(cluster.id, &block)
       end
 
-      def node(node)
+      def base_data(node)
         {
           'metadata' => {
             'name' => node.metadata.name