Merge branch '10-4-stable-prepare-rc7' into '10-4-stable'

Prepare 10.4 RC7 release

See merge request gitlab-org/gitlab-ce!16519

9 files changed, 145 insertions, 13 deletions

diff --git a/lib/api/internal.rb b/lib/api/internal.rb
index 79b302aae70..8bf53939751 100644
--- a/lib/api/internal.rb
+++ b/lib/api/internal.rb
@@ -82,6 +82,18 @@ module API
       end
 
       #
+      # Get a ssh key using the fingerprint
+      #
+      get "/authorized_keys" do
+        fingerprint = params.fetch(:fingerprint) do
+          Gitlab::InsecureKeyFingerprint.new(params.fetch(:key)).fingerprint
+        end
+        key = Key.find_by(fingerprint: fingerprint)
+        not_found!("Key") if key.nil?
+        present key, with: Entities::SSHKey
+      end
+
+      #
       # Discover user by ssh key or user id
       #
       get "/discover" do
diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb
index a116ab3c9bd..9c205514b3a 100644
--- a/lib/api/jobs.rb
+++ b/lib/api/jobs.rb
@@ -38,6 +38,7 @@ module API
         builds = user_project.builds.order('id DESC')
         builds = filter_builds(builds, params[:scope])
 
+        builds = builds.preload(:user, :job_artifacts_archive, :runner, pipeline: :project)
         present paginate(builds), with: Entities::Job
       end
 
diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb
index fa0bef39602..ac76fece931 100644
--- a/lib/api/v3/builds.rb
+++ b/lib/api/v3/builds.rb
@@ -36,6 +36,7 @@ module API
           builds = user_project.builds.order('id DESC')
           builds = filter_builds(builds, params[:scope])
 
+          builds = builds.preload(:user, :job_artifacts_archive, :runner, pipeline: :project)
           present paginate(builds), with: ::API::V3::Entities::Build
         end
 
diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb
index e7a1ec8457d..072d24e5a11 100644
--- a/lib/banzai/filter/wiki_link_filter/rewriter.rb
+++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb
@@ -9,6 +9,10 @@ module Banzai
         end
 
         def apply_rules
+          # Special case: relative URLs beginning with `/uploads/` refer to
+          # user-uploaded files and will be handled elsewhere.
+          return @uri.to_s if @uri.relative? && @uri.path.starts_with?('/uploads/')
+
           apply_file_link_rules!
           apply_hierarchical_link_rules!
           apply_relative_link_rules!
diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb
index 4974205b8fd..f8b2e7e0e21 100644
--- a/lib/gitlab/git/rev_list.rb
+++ b/lib/gitlab/git/rev_list.rb
@@ -95,7 +95,7 @@ module Gitlab
         object_output.map do |output_line|
           sha, path = output_line.split(' ', 2)
 
-          next if require_path && path.blank?
+          next if require_path && path.to_s.empty?
 
           sha
         end.reject(&:nil?)
diff --git a/lib/gitlab/insecure_key_fingerprint.rb b/lib/gitlab/insecure_key_fingerprint.rb
new file mode 100644
index 00000000000..f85b6e9197f
--- /dev/null
+++ b/lib/gitlab/insecure_key_fingerprint.rb
@@ -0,0 +1,23 @@
+module Gitlab
+  #
+  # Calculates the fingerprint of a given key without using
+  # openssh key validations. For this reason, only use
+  # for calculating the fingerprint to find the key with it.
+  #
+  # DO NOT use it for checking the validity of a ssh key.
+  #
+  class InsecureKeyFingerprint
+    attr_accessor :key
+
+    #
+    # Gets the base64 encoded string representing a rsa or dsa key
+    #
+    def initialize(key_base64)
+      @key = key_base64
+    end
+
+    def fingerprint
+      OpenSSL::Digest::MD5.hexdigest(Base64.decode64(@key)).scan(/../).join(':')
+    end
+  end
+end
diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb
index 8d8c441a4b1..bf6981035f4 100644
--- a/lib/gitlab/kubernetes/helm/install_command.rb
+++ b/lib/gitlab/kubernetes/helm/install_command.rb
@@ -36,7 +36,11 @@ module Gitlab
         def complete_command(namespace_name)
           return unless chart
 
-          "helm install #{chart} --name #{name} --namespace #{namespace_name} >/dev/null"
+          if chart_values_file
+            "helm install #{chart} --name #{name} --namespace #{namespace_name} -f /data/helm/#{name}/config/values.yaml >/dev/null"
+          else
+            "helm install #{chart} --name #{name} --namespace #{namespace_name} >/dev/null"
+          end
         end
 
         def install_dps_command
diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb
index 233f6bf6227..2a2319d8cfe 100644
--- a/lib/gitlab/kubernetes/helm/pod.rb
+++ b/lib/gitlab/kubernetes/helm/pod.rb
@@ -10,9 +10,10 @@ module Gitlab
 
         def generate
           spec = { containers: [container_specification], restartPolicy: 'Never' }
+
           if command.chart_values_file
-            generate_config_map
-            spec['volumes'] = volumes_specification
+            create_config_map
+            spec[:volumes] = volumes_specification
           end
           ::Kubeclient::Resource.new(metadata: metadata, spec: spec)
         end
@@ -34,19 +35,39 @@ module Gitlab
         end
 
         def labels
-          { 'gitlab.org/action': 'install', 'gitlab.org/application': command.name }
+          {
+            'gitlab.org/action': 'install',
+            'gitlab.org/application': command.name
+          }
         end
 
         def metadata
-          { name: command.pod_name, namespace: namespace_name, labels: labels }
+          {
+            name: command.pod_name,
+            namespace: namespace_name,
+            labels: labels
+          }
         end
 
         def volume_mounts_specification
-          [{ name: 'config-volume', mountPath: '/etc/config' }]
+          [
+            {
+              name: 'configuration-volume',
+              mountPath: "/data/helm/#{command.name}/config"
+            }
+          ]
         end
 
         def volumes_specification
-          [{ name: 'config-volume', configMap: { name: 'values-config' } }]
+          [
+            {
+              name: 'configuration-volume',
+              configMap: {
+                name: 'values-content-configuration',
+                items: [{ key: 'values', path: 'values.yaml' }]
+              }
+            }
+          ]
         end
 
         def generate_pod_env(command)
@@ -57,10 +78,10 @@ module Gitlab
           }.map { |key, value| { name: key, value: value } }
         end
 
-        def generate_config_map
+        def create_config_map
           resource = ::Kubeclient::Resource.new
-          resource.metadata = { name: 'values-config', namespace: namespace_name }
-          resource.data = YAML.load_file(command.chart_values_file)
+          resource.metadata = { name: 'values-content-configuration', namespace: namespace_name, labels: { name: 'values-content-configuration' } }
+          resource.data = { values: File.read(command.chart_values_file) }
           kubeclient.create_config_map(resource)
         end
       end
diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb
index a8a4ec996c4..392f66c99d3 100644
--- a/lib/gitlab/shell.rb
+++ b/lib/gitlab/shell.rb
@@ -183,6 +183,8 @@ module Gitlab
     #   add_key("key-42", "sha-rsa ...")
     #
     def add_key(key_id, key_content)
+      return unless self.authorized_keys_enabled?
+
       gitlab_shell_fast_execute([gitlab_shell_keys_path,
                                  'add-key', key_id, self.class.strip_key(key_content)])
     end
@@ -192,6 +194,8 @@ module Gitlab
     # Ex.
     #   batch_add_keys { |adder| adder.add_key("key-42", "sha-rsa ...") }
     def batch_add_keys(&block)
+      return unless self.authorized_keys_enabled?
+
       IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys batch-add-keys), 'w') do |io|
         yield(KeyAdder.new(io))
       end
@@ -202,10 +206,11 @@ module Gitlab
     # Ex.
     #   remove_key("key-342", "sha-rsa ...")
     #
-    def remove_key(key_id, key_content)
+    def remove_key(key_id, key_content = nil)
+      return unless self.authorized_keys_enabled?
+
       args = [gitlab_shell_keys_path, 'rm-key', key_id]
       args << key_content if key_content
-
       gitlab_shell_fast_execute(args)
     end
 
@@ -215,9 +220,62 @@ module Gitlab
     #   remove_all_keys
     #
     def remove_all_keys
+      return unless self.authorized_keys_enabled?
+
       gitlab_shell_fast_execute([gitlab_shell_keys_path, 'clear'])
     end
 
+    # Remove ssh keys from gitlab shell that are not in the DB
+    #
+    # Ex.
+    #   remove_keys_not_found_in_db
+    #
+    def remove_keys_not_found_in_db
+      return unless self.authorized_keys_enabled?
+
+      Rails.logger.info("Removing keys not found in DB")
+
+      batch_read_key_ids do |ids_in_file|
+        ids_in_file.uniq!
+        keys_in_db = Key.where(id: ids_in_file)
+
+        next unless ids_in_file.size > keys_in_db.count # optimization
+
+        ids_to_remove = ids_in_file - keys_in_db.pluck(:id)
+        ids_to_remove.each do |id|
+          Rails.logger.info("Removing key-#{id} not found in DB")
+          remove_key("key-#{id}")
+        end
+      end
+    end
+
+    # Iterate over all ssh key IDs from gitlab shell, in batches
+    #
+    # Ex.
+    #   batch_read_key_ids { |batch| keys = Key.where(id: batch) }
+    #
+    def batch_read_key_ids(batch_size: 100, &block)
+      return unless self.authorized_keys_enabled?
+
+      list_key_ids do |key_id_stream|
+        key_id_stream.lazy.each_slice(batch_size) do |lines|
+          key_ids = lines.map { |l| l.chomp.to_i }
+          yield(key_ids)
+        end
+      end
+    end
+
+    # Stream all ssh key IDs from gitlab shell, separated by newlines
+    #
+    # Ex.
+    #   list_key_ids
+    #
+    def list_key_ids(&block)
+      return unless self.authorized_keys_enabled?
+
+      IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys list-key-ids), &block)
+    end
+
     # Add empty directory for storing repositories
     #
     # Ex.
@@ -333,6 +391,14 @@ module Gitlab
       File.join(gitlab_shell_path, 'bin', 'gitlab-keys')
     end
 
+    def authorized_keys_enabled?
+      # Return true if nil to ensure the authorized_keys methods work while
+      # fixing the authorized_keys file during migration.
+      return true if Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled.nil?
+
+      Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled
+    end
+
     private
 
     def gitlab_projects(shard_path, disk_path)