Add latest changes from gitlab-org/gitlab@13-7-stable-eev13.7.0-rc42

8 files changed, 201 insertions, 56 deletions

diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake
index e2c92054d62..e03c78d5a40 100644
--- a/lib/tasks/gettext.rake
+++ b/lib/tasks/gettext.rake
@@ -1,39 +1,38 @@
+# frozen_string_literal: true
+
 require "gettext_i18n_rails/tasks"
 
 namespace :gettext do
-  # Customize list of translatable files
-  # See: https://github.com/grosser/gettext_i18n_rails#customizing-list-of-translatable-files
-  def files_to_translate
-    folders = %W(ee app lib config #{locale_path}).join(',')
-    exts = %w(rb erb haml slim rhtml js jsx vue handlebars hbs mustache).join(',')
-
-    Dir.glob(
-      "{#{folders}}/**/*.{#{exts}}"
-    )
-  end
-
-  # Disallow HTML from translatable strings
-  # See: https://docs.gitlab.com/ee/development/i18n/externalization.html#html
-  def html_todolist
-    return @html_todolist if defined?(@html_todolist)
-
-    @html_todolist = YAML.load_file(Rails.root.join('lib/gitlab/i18n/html_todo.yml'))
-  end
-
   task :compile do
     # See: https://gitlab.com/gitlab-org/gitlab-foss/issues/33014#note_31218998
-    FileUtils.touch(File.join(Rails.root, 'locale/gitlab.pot'))
+    FileUtils.touch(pot_file_path)
 
     Rake::Task['gettext:po_to_json'].invoke
   end
 
   desc 'Regenerate gitlab.pot file'
   task :regenerate do
-    pot_file = 'locale/gitlab.pot'
-    # Remove all translated files, this speeds up finding
-    FileUtils.rm Dir['locale/**/gitlab.*']
+    ensure_locale_folder_presence!
+
+    # Clean up folders that do not contain a gitlab.po file
+    Pathname.new(locale_path).children.each do |child|
+      next unless child.directory?
+
+      folder_path = child.to_path
+
+      if File.exist?("#{folder_path}/gitlab.po")
+        # remove all translated files to speed up finding
+        FileUtils.rm Dir["#{folder_path}/gitlab.*"]
+      else
+        # remove empty translation folders so we don't generate un-needed .po files
+        puts "Deleting #{folder_path} as it does not contain a 'gitlab.po' file."
+
+        FileUtils.rm_r folder_path
+      end
+    end
+
     # remove the `pot` file to ensure it's completely regenerated
-    FileUtils.rm_f pot_file
+    FileUtils.rm_f(pot_file_path)
 
     Rake::Task['gettext:find'].invoke
 
@@ -42,10 +41,12 @@ namespace :gettext do
       raise 'failed to cleanup generated locale/*/gitlab.po files'
     end
 
+    raise 'gitlab.pot file not generated' unless File.exist?(pot_file_path)
+
     # Remove timestamps from the pot file
-    pot_content = File.read pot_file
+    pot_content = File.read pot_file_path
     pot_content.gsub!(/^"POT?\-(?:Creation|Revision)\-Date\:.*\n/, '')
-    File.write pot_file, pot_content
+    File.write pot_file_path, pot_content
 
     puts <<~MSG
     All done. Please commit the changes to `locale/gitlab.pot`.
@@ -64,11 +65,10 @@ namespace :gettext do
     linters = files.map do |file|
       locale = File.basename(File.dirname(file))
 
-      Gitlab::I18n::PoLinter.new(po_path: file, html_todolist: html_todolist, locale: locale)
+      Gitlab::I18n::PoLinter.new(po_path: file, locale: locale)
     end
 
-    pot_file = Rails.root.join('locale/gitlab.pot')
-    linters.unshift(Gitlab::I18n::PoLinter.new(po_path: pot_file, html_todolist: html_todolist))
+    linters.unshift(Gitlab::I18n::PoLinter.new(po_path: pot_file_path))
 
     failed_linters = linters.select { |linter| linter.errors.any? }
 
@@ -84,12 +84,11 @@ namespace :gettext do
   end
 
   task :updated_check do
-    pot_file = 'locale/gitlab.pot'
     # Removing all pre-translated files speeds up `gettext:find` as the
     # files don't need to be merged.
     # Having `LC_MESSAGES/gitlab.mo files present also confuses the output.
     FileUtils.rm Dir['locale/**/gitlab.*']
-    FileUtils.rm_f pot_file
+    FileUtils.rm_f pot_file_path
 
     # `gettext:find` writes touches to temp files to `stderr` which would cause
     # `static-analysis` to report failures. We can ignore these.
@@ -97,18 +96,18 @@ namespace :gettext do
       Rake::Task['gettext:find'].invoke
     end
 
-    pot_diff = `git diff -- #{pot_file} | grep -E '^(\\+|-)msgid'`.strip
+    pot_diff = `git diff -- #{pot_file_path} | grep -E '^(\\+|-)msgid'`.strip
 
     # reset the locale folder for potential next tasks
     `git checkout -- locale`
 
     if pot_diff.present?
       raise <<~MSG
-        Changes in translated strings found, please update file `#{pot_file}` by running:
+        Changes in translated strings found, please update file `#{pot_file_path}` by running:
 
           bin/rake gettext:regenerate
 
-        Then commit and push the resulting changes to `#{pot_file}`.
+        Then commit and push the resulting changes to `#{pot_file_path}`.
 
         The diff was:
 
@@ -117,6 +116,27 @@ namespace :gettext do
     end
   end
 
+  private
+
+  # Customize list of translatable files
+  # See: https://github.com/grosser/gettext_i18n_rails#customizing-list-of-translatable-files
+  def files_to_translate
+    folders = %W(ee app lib config #{locale_path}).join(',')
+    exts = %w(rb erb haml slim rhtml js jsx vue handlebars hbs mustache).join(',')
+
+    Dir.glob(
+      "{#{folders}}/**/*.{#{exts}}"
+    )
+  end
+
+  # Disallow HTML from translatable strings
+  # See: https://docs.gitlab.com/ee/development/i18n/externalization.html#html
+  def html_todolist
+    return @html_todolist if defined?(@html_todolist)
+
+    @html_todolist = YAML.safe_load(File.read(Rails.root.join('lib/gitlab/i18n/html_todo.yml')))
+  end
+
   def report_errors_for_file(file, errors_for_file)
     puts "Errors in `#{file}`:"
 
@@ -140,4 +160,21 @@ namespace :gettext do
     $stderr.reopen(old_stderr)
     old_stderr.close
   end
+
+  def ensure_locale_folder_presence!
+    unless Dir.exist?(locale_path)
+      raise <<~MSG
+      Cannot find '#{locale_path}' folder. Please ensure you're running this task from the gitlab repo.
+
+      MSG
+    end
+  end
+
+  def locale_path
+    @locale_path ||= Rails.root.join('locale')
+  end
+
+  def pot_file_path
+    @pot_file_path ||= File.join(locale_path, 'gitlab.pot')
+  end
 end
diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake
index ab2d77eeaf0..54e74fd9c8b 100644
--- a/lib/tasks/gitlab/assets.rake
+++ b/lib/tasks/gitlab/assets.rake
@@ -81,7 +81,10 @@ namespace :gitlab do
 
       if head_assets_md5 != master_assets_md5 || !public_assets_webpack_dir_exists
         FileUtils.rm_r(Tasks::Gitlab::Assets::PUBLIC_ASSETS_WEBPACK_DIR) if public_assets_webpack_dir_exists
-        system('yarn webpack')
+
+        unless system('yarn webpack')
+          abort 'Error: Unable to compile webpack production bundle.'.color(:red)
+        end
       end
     end
 
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index a3f20f31f64..901e349ea31 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -192,16 +192,42 @@ namespace :gitlab do
         exit
       end
 
-      indexes = if args[:index_name]
-                  [Gitlab::Database::PostgresIndex.by_identifier(args[:index_name])]
-                else
-                  Gitlab::Database::Reindexing.candidate_indexes.random_few(2)
-                end
+      indexes = Gitlab::Database::Reindexing.candidate_indexes
+
+      if identifier = args[:index_name]
+        raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/
+
+        indexes = indexes.where(identifier: identifier)
+
+        raise "Index not found or not supported: #{args[:index_name]}" if indexes.empty?
+      end
+
+      ActiveRecord::Base.logger = Logger.new(STDOUT) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false)
 
       Gitlab::Database::Reindexing.perform(indexes)
     rescue => e
       Gitlab::AppLogger.error(e)
       raise
     end
+
+    desc 'Check if there have been user additions to the database'
+    task active: :environment do
+      if ActiveRecord::Base.connection.migration_context.needs_migration?
+        puts "Migrations pending. Database not active"
+        exit 1
+      end
+
+      # A list of projects that GitLab creates automatically on install/upgrade
+      # gc = Gitlab::CurrentSettings.current_application_settings
+      seed_projects = [Gitlab::CurrentSettings.current_application_settings.self_monitoring_project]
+
+      if (Project.count - seed_projects.count {|x| !x.nil? }).eql?(0)
+        puts "No user created projects. Database not active"
+        exit 1
+      end
+
+      puts "Found user created projects. Database active"
+      exit 0
+    end
   end
 end
diff --git a/lib/tasks/gitlab/ldap.rake b/lib/tasks/gitlab/ldap.rake
index 0459de27c96..fe7920c621f 100644
--- a/lib/tasks/gitlab/ldap.rake
+++ b/lib/tasks/gitlab/ldap.rake
@@ -36,5 +36,23 @@ namespace :gitlab do
         puts "Successfully updated #{plural_updated_count} out of #{plural_id_count} total"
       end
     end
+
+    namespace :secret do
+      desc 'GitLab | LDAP | Secret | Write LDAP secrets'
+      task write: [:environment] do
+        content = STDIN.tty? ? STDIN.gets : STDIN.read
+        Gitlab::EncryptedLdapCommand.write(content)
+      end
+
+      desc 'GitLab | LDAP | Secret | Edit LDAP secrets'
+      task edit: [:environment] do
+        Gitlab::EncryptedLdapCommand.edit
+      end
+
+      desc 'GitLab | LDAP | Secret | Show LDAP secrets'
+      task show: [:environment] do
+        Gitlab::EncryptedLdapCommand.show
+      end
+    end
   end
 end
diff --git a/lib/tasks/gitlab/packages/events.rake b/lib/tasks/gitlab/packages/events.rake
index 3484b9b6072..ca507fb5320 100644
--- a/lib/tasks/gitlab/packages/events.rake
+++ b/lib/tasks/gitlab/packages/events.rake
@@ -5,11 +5,29 @@ namespace :gitlab do
   namespace :packages do
     namespace :events do
       task generate: :environment do
+        Rake::Task["gitlab:packages:events:generate_guest"].invoke
+        Rake::Task["gitlab:packages:events:generate_unique"].invoke
+      rescue => e
+        logger.error("Error building events list: #{e}")
+      end
+
+      task generate_guest: :environment do
         logger = Logger.new(STDOUT)
         logger.info('Building list of package events...')
 
-        path = File.join(File.dirname(::Gitlab::UsageDataCounters::HLLRedisCounter::KNOWN_EVENTS_PATH), 'package_events.yml')
+        path = Gitlab::UsageDataCounters::GuestPackageEventCounter::KNOWN_EVENTS_PATH
+        File.open(path, "w") { |file| file << guest_events_list.to_yaml }
+
+        logger.info("Events file `#{path}` generated successfully")
+      rescue => e
+        logger.error("Error building events list: #{e}")
+      end
+
+      task generate_unique: :environment do
+        logger = Logger.new(STDOUT)
+        logger.info('Building list of package events...')
 
+        path = File.join(File.dirname(Gitlab::UsageDataCounters::HLLRedisCounter::KNOWN_EVENTS_PATH), 'package_events.yml')
         File.open(path, "w") { |file| file << generate_unique_events_list.to_yaml }
 
         logger.info("Events file `#{path}` generated successfully")
@@ -17,23 +35,34 @@ namespace :gitlab do
         logger.error("Error building events list: #{e}")
       end
 
+      private
+
       def event_pairs
-        ::Packages::Event.event_types.keys.product(::Packages::Event.originator_types.keys)
+        Packages::Event.event_types.keys.product(Packages::Event::EVENT_SCOPES.keys)
       end
 
       def generate_unique_events_list
-        ::Packages::Event::EVENT_SCOPES.keys.each_with_object([]) do |event_scope, events|
-          event_pairs.each do |event_type, originator|
-            if name = ::Packages::Event.allowed_event_name(event_scope, event_type, originator)
+        events = event_pairs.each_with_object([]) do |(event_type, event_scope), events|
+          Packages::Event.originator_types.keys.excluding('guest').each do |originator|
+            if name = Packages::Event.allowed_event_name(event_scope, event_type, originator)
               events << {
                 "name" => name,
                 "category" => "#{event_scope}_packages",
                 "aggregation" => "weekly",
-                "redis_slot" => "package"
+                "redis_slot" => "package",
+                "feature_flag" => "collect_package_events_redis"
               }
             end
           end
         end
+
+        events.sort_by { |event| event["name"] }
+      end
+
+      def guest_events_list
+        event_pairs.map do |event_type, event_scope|
+          Packages::Event.allowed_event_name(event_scope, event_type, "guest")
+        end.compact.sort
       end
     end
   end
diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake
index 6f3db91c2b0..d6f5661d5eb 100644
--- a/lib/tasks/gitlab/usage_data.rake
+++ b/lib/tasks/gitlab/usage_data.rake
@@ -9,5 +9,17 @@ namespace :gitlab do
     task dump_sql_in_json: :environment do
       puts Gitlab::Json.pretty_generate(Gitlab::UsageDataQueries.uncached_data)
     end
+
+    desc 'GitLab | UsageData | Generate usage ping in JSON'
+    task generate: :environment do
+      puts Gitlab::Json.pretty_generate(Gitlab::UsageData.uncached_data)
+    end
+
+    desc 'GitLab | UsageData | Generate usage ping and send it to Versions Application'
+    task generate_and_send: :environment do
+      result = SubmitUsagePingService.new.execute
+
+      puts Gitlab::Json.pretty_generate(result.attributes)
+    end
   end
 end
diff --git a/lib/tasks/gitlab/user_management.rake b/lib/tasks/gitlab/user_management.rake
new file mode 100644
index 00000000000..f47e549e795
--- /dev/null
+++ b/lib/tasks/gitlab/user_management.rake
@@ -0,0 +1,13 @@
+namespace :gitlab do
+  namespace :user_management do
+    desc "GitLab | User management | Update all users of a group with personal project limit to 0 and can_create_group to false"
+    task :disable_project_and_group_creation, [:group_id] => :environment do |t, args|
+      group = Group.find(args.group_id)
+
+      result = User.where(id: group.direct_and_indirect_users_with_inactive.select(:id)).update_all(projects_limit: 0, can_create_group: false)
+      ids_count = group.direct_and_indirect_users_with_inactive.count
+      puts "Done".color(:green) if result == ids_count
+      puts "Something went wrong".color(:red) if result != ids_count
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake
index 15084a118b7..2d72a01f66f 100644
--- a/lib/tasks/gitlab/workhorse.rake
+++ b/lib/tasks/gitlab/workhorse.rake
@@ -8,18 +8,25 @@ namespace :gitlab do
         abort %(Please specify the directory where you want to install gitlab-workhorse:\n  rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]")
       end
 
+      # It used to be the case that the binaries in the target directory match
+      # the source code. An administrator could run `make` to rebuild the
+      # binaries for instance. Or they could read the source code, or run `git
+      # log` to see what changed. Or they could patch workhorse for some
+      # reason and recompile it. None of those things make sense anymore once
+      # the transition in https://gitlab.com/groups/gitlab-org/-/epics/4826 is
+      # done: there would be an outdated copy of the workhorse source code for
+      # the administrator to poke at.
+      #
+      # To prevent this possible confusion and make clear what is going on, we
+      # have created a special branch `workhorse-move-notice` in the old
+      # gitlab-workhorse repository which contains no Go files anymore, just a
+      # README explaining what is going on. See:
+      # https://gitlab.com/gitlab-org/gitlab-workhorse/tree/workhorse-move-notice
+      #
       args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-workhorse.git')
+      checkout_or_clone_version(version: 'workhorse-move-notice', repo: args.repo, target_dir: args.dir, clone_opts: %w[--depth 1])
 
-      version = Gitlab::Workhorse.version
-
-      checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir, clone_opts: %w[--depth 1])
-
-      _, status = Gitlab::Popen.popen(%w[which gmake])
-      command = status == 0 ? 'gmake' : 'make'
-
-      Dir.chdir(args.dir) do
-        run_command!([command])
-      end
+      Gitlab::SetupHelper::Workhorse.compile_into(args.dir)
     end
   end
 end