Add latest changes from gitlab-org/gitlab@master

2 files changed, 80 insertions, 32 deletions

diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb
index bec4c7fc4d4..fbde1778a89 100644
--- a/lib/gitlab/github_import/user_finder.rb
+++ b/lib/gitlab/github_import/user_finder.rb
@@ -130,20 +130,32 @@ module Gitlab
         email = read_email_from_cache(username)
 
         if email.blank? && !email_fetched_for_project?(username)
-          # If an ETAG is available, make an API call with the ETAG.
-          # Only make a rate-limited API call if the ETAG is not available and the email is nil.
-          etag = read_etag_from_cache(username)
-          email = fetch_email_from_github(username, etag: etag) || email
-
-          cache_email!(username, email)
-          cache_etag!(username) if email.blank? && etag.nil?
-
-          # If a non-blank email is cached, we don't need the ETAG or project check caches.
-          # Otherwise, indicate that the project has been checked.
-          if email.present?
-            clear_caches!(username)
-          else
-            set_project_as_checked!(username)
+          feature_flag_in_lock(lease_key, ttl: 3.minutes, sleep_sec: 1.second, retries: 30) do |retried|
+            # when retried, check the cache again as the other process that had the lease may have fetched the email
+            if retried
+              email = read_email_from_cache(username)
+
+              # early return if the other process fetched a non-empty email. If the email is empty, we'll attempt to
+              # fetch it again in the lines below, but using the ETAG cached by the other process which won't count to
+              # the rate limit.
+              next email if email.present?
+            end
+
+            # If an ETAG is available, make an API call with the ETAG.
+            # Only make a rate-limited API call if the ETAG is not available and the email is nil.
+            etag = read_etag_from_cache(username)
+            email = fetch_email_from_github(username, etag: etag) || email
+
+            cache_email!(username, email)
+            cache_etag!(username) if email.blank? && etag.nil?
+
+            # If a non-blank email is cached, we don't need the ETAG or project check caches.
+            # Otherwise, indicate that the project has been checked.
+            if email.present?
+              clear_caches!(username)
+            else
+              set_project_as_checked!(username)
+            end
           end
         end
 
@@ -240,20 +252,11 @@ module Gitlab
       end
 
       def fetch_email_from_github(username, etag: nil)
-        in_lock(lease_key, ttl: 3.minutes, sleep_sec: 1.second, retries: 30) do |retried|
-          # when retried, check the cache again as the other process that had the lease may have fetched the email
-          if retried
-            email = read_email_from_cache(username)
-
-            next email if email.present?
-          end
+        log(EMAIL_API_CALL_LOGGING_MESSAGE[etag.present?], username: username)
 
-          log(EMAIL_API_CALL_LOGGING_MESSAGE[etag.present?], username: username)
-
-          # Only make a rate-limited API call if the ETAG is not available })
-          user = client.user(username, { headers: { 'If-None-Match' => etag }.compact })
-          user[:email] || '' if user
-        end
+        # Only make a rate-limited API call if the ETAG is not available })
+        user = client.user(username, { headers: { 'If-None-Match' => etag }.compact })
+        user[:email] || '' if user
       end
 
       # Caches the email associated to the username
@@ -303,6 +306,14 @@ module Gitlab
           message: message
         )
       end
+
+      def feature_flag_in_lock(lease_key, ttl: 3.minutes, sleep_sec: 1.second, retries: 30)
+        return yield(false) if Feature.disabled?(:github_import_lock_user_finder, project.creator)
+
+        in_lock(lease_key, ttl: ttl, sleep_sec: sleep_sec, retries: retries) do |retried|
+          yield(retried)
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/patch/database_config.rb b/lib/gitlab/patch/database_config.rb
index 8a7566f6e0e..33be63a31c0 100644
--- a/lib/gitlab/patch/database_config.rb
+++ b/lib/gitlab/patch/database_config.rb
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../popen'
+
 # The purpose of this code is to set the migrations path
 # for the Geo tracking database and the embedding database.
 module Gitlab
@@ -7,25 +9,60 @@ module Gitlab
     module DatabaseConfig
       extend ActiveSupport::Concern
 
+      CommandExecutionError = Class.new(StandardError)
+
       def database_configuration
         super.to_h do |env, configs|
+          parsed_config = parse_extra_config(configs)
+
           if Gitlab.ee?
             ee_databases = %w[embedding geo]
 
             ee_databases.each do |ee_db_name|
-              next unless configs.key?(ee_db_name)
+              next unless parsed_config.key?(ee_db_name)
 
-              migrations_paths = Array(configs[ee_db_name]['migrations_paths'])
+              migrations_paths = Array(parsed_config[ee_db_name]['migrations_paths'])
               migrations_paths << File.join('ee', 'db', ee_db_name, 'migrate') if migrations_paths.empty?
               migrations_paths << File.join('ee', 'db', ee_db_name, 'post_migrate') unless ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS']
 
-              configs[ee_db_name]['migrations_paths'] = migrations_paths.uniq
-              configs[ee_db_name]['schema_migrations_path'] = File.join('ee', 'db', ee_db_name, 'schema_migrations') if configs[ee_db_name]['schema_migrations_path'].blank?
+              parsed_config[ee_db_name]['migrations_paths'] = migrations_paths.uniq
+              parsed_config[ee_db_name]['schema_migrations_path'] = File.join('ee', 'db', ee_db_name, 'schema_migrations') if parsed_config[ee_db_name]['schema_migrations_path'].blank?
             end
           end
 
-          [env, configs]
+          [env, parsed_config]
+        end
+      end
+
+      private
+
+      def parse_extra_config(configs)
+        command = configs.delete('config_command')
+        return configs unless command.present?
+
+        config_from_command = extra_config_from_command(command)
+        return configs unless config_from_command.present?
+
+        configs.deep_merge(config_from_command)
+      end
+
+      def extra_config_from_command(command)
+        cmd = command.split(" ")
+        output, exit_status = Gitlab::Popen.popen(cmd)
+
+        if exit_status != 0
+          raise CommandExecutionError,
+            "database.yml: Execution of `#{command}` failed with exit code #{exit_status}. Output: #{output}"
         end
+
+        YAML.safe_load(output).deep_stringify_keys
+      rescue Psych::SyntaxError => e
+        error_message = <<~MSG
+          database.yml: Execution of `#{command}` generated invalid yaml.
+          Error: #{e.problem} #{e.context} at line #{e.line} column #{e.column}
+        MSG
+
+        raise CommandExecutionError, error_message
       end
     end
   end