Add latest changes from gitlab-org/gitlab@master

13 files changed, 245 insertions, 22 deletions

diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb
index e419a025508..f7a39db7249 100644
--- a/lib/api/award_emoji.rb
+++ b/lib/api/award_emoji.rb
@@ -80,7 +80,7 @@ module API
           delete "#{endpoint}/:award_id", feature_category: awardable_params[:feature_category] do
             award = awardable.award_emoji.find(params[:award_id])
 
-            unauthorized! unless award.user == current_user || current_user&.admin?
+            unauthorized! unless award.user == current_user || current_user&.can_admin_all_resources?
 
             destroy_conditionally!(award)
           end
diff --git a/lib/api/ci/runners.rb b/lib/api/ci/runners.rb
index 988c3f4f566..4a6c58b4987 100644
--- a/lib/api/ci/runners.rb
+++ b/lib/api/ci/runners.rb
@@ -58,19 +58,19 @@ module API
         end
 
         def authenticate_show_runner!(runner)
-          return if runner.instance_type? || current_user.admin?
+          return if runner.instance_type? || current_user.can_read_all_resources?
 
           forbidden!("No access granted") unless can?(current_user, :read_runner, runner)
         end
 
         def authenticate_update_runner!(runner)
-          return if current_user.admin?
+          return if current_user.can_admin_all_resources?
 
           forbidden!("No access granted") unless can?(current_user, :update_runner, runner)
         end
 
         def authenticate_delete_runner!(runner)
-          return if current_user.admin?
+          return if current_user.can_admin_all_resources?
 
           forbidden!("Runner associated with more than one project") if runner.runner_projects.count > 1
           forbidden!("No access granted") unless can?(current_user, :delete_runner, runner)
@@ -79,14 +79,14 @@ module API
         def authenticate_enable_runner!(runner)
           forbidden!("Runner is a group runner") if runner.group_type?
 
-          return if current_user.admin?
+          return if current_user.can_admin_all_resources?
 
           forbidden!("Runner is locked") if runner.locked?
           forbidden!("No access granted") unless can?(current_user, :assign_runner, runner)
         end
 
         def authenticate_list_runners_jobs!(runner)
-          return if current_user.admin?
+          return if current_user.can_read_all_resources?
 
           forbidden!("No access granted") unless can?(current_user, :read_builds, runner)
         end
diff --git a/lib/api/entities/ci/runner_details.rb b/lib/api/entities/ci/runner_details.rb
index 9b1decca274..8aa134dc669 100644
--- a/lib/api/entities/ci/runner_details.rb
+++ b/lib/api/entities/ci/runner_details.rb
@@ -14,7 +14,7 @@ module API
 
         # rubocop: disable CodeReuse/ActiveRecord
         expose :projects, with: Entities::BasicProjectDetails do |runner, options|
-          if options[:current_user].admin? # rubocop: disable Cop/UserAdmin
+          if options[:current_user].can_read_all_resources?
             runner.projects
           else
             options[:current_user].authorized_projects.where(id: runner.runner_projects.pluck(:project_id))
@@ -23,7 +23,7 @@ module API
         # rubocop: enable CodeReuse/ActiveRecord
         # rubocop: disable CodeReuse/ActiveRecord
         expose :groups, with: Entities::BasicGroupDetails do |runner, options|
-          if options[:current_user].admin? # rubocop: disable Cop/UserAdmin
+          if options[:current_user].can_read_all_resources?
             runner.groups
           else
             options[:current_user].authorized_groups.where(id: runner.runner_namespaces.pluck(:namespace_id))
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index f9842c01db2..23db10dbdbf 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -100,7 +100,7 @@ module API
         options = {
           with: serializer,
           current_user: current_user,
-          statistics: params[:statistics] && current_user&.admin?
+          statistics: params[:statistics] && current_user&.can_read_all_resources?
         }
 
         groups = groups.with_statistics if options[:statistics]
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 75e7612bd5b..86763b2cd67 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -301,7 +301,7 @@ module API
 
     def authenticated_as_admin!
       authenticate!
-      forbidden! unless current_user.admin?
+      forbidden! unless current_user.can_admin_all_resources?
     end
 
     def authorize!(action, subject = :global, reason = nil)
@@ -710,7 +710,7 @@ module API
 
       unauthorized! unless initial_current_user
 
-      unless initial_current_user.admin?
+      unless initial_current_user.can_admin_all_resources?
         forbidden!('Must be admin to use sudo')
       end
 
diff --git a/lib/api/users.rb b/lib/api/users.rb
index c26c515f1d8..d2d45c94291 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -133,7 +133,7 @@ module API
       get feature_category: :users, urgency: :low do
         authenticated_as_admin! if params[:extern_uid].present? && params[:provider].present?
 
-        unless current_user&.admin?
+        unless current_user&.can_read_all_resources?
           params.except!(:created_after, :created_before, :order_by, :sort, :two_factor, :without_projects)
         end
 
@@ -151,7 +151,7 @@ module API
         users = UsersFinder.new(current_user, params).execute
         users = reorder_users(users)
 
-        entity = current_user&.admin? ? Entities::UserWithAdmin : Entities::UserBasic
+        entity = current_user&.can_read_all_resources? ? Entities::UserWithAdmin : Entities::UserBasic
 
         if entity == Entities::UserWithAdmin
           users = users.preload(:identities, :u2f_registrations, :webauthn_registrations, :namespace, :followers, :followees, :user_preference)
@@ -177,7 +177,7 @@ module API
       get ":id", feature_category: :users, urgency: :low do
         forbidden!('Not authorized!') unless current_user
 
-        unless current_user.admin?
+        unless current_user.can_read_all_resources?
           check_rate_limit!(:users_get_by_id,
             scope: current_user,
             users_allowlist: Gitlab::CurrentSettings.current_application_settings.users_get_by_id_limit_allowlist
@@ -188,7 +188,7 @@ module API
 
         not_found!('User') unless user && can?(current_user, :read_user, user)
 
-        opts = { with: current_user.admin? ? Entities::UserDetailsWithAdmin : Entities::User, current_user: current_user }
+        opts = { with: current_user.can_read_all_resources? ? Entities::UserDetailsWithAdmin : Entities::User, current_user: current_user }
         user, opts = with_custom_attributes(user, opts)
 
         present user, opts
@@ -373,7 +373,8 @@ module API
         user = User.find_by_id(params[:id])
         not_found!('User') unless user
 
-        forbidden!('Two-factor authentication for admins cannot be disabled via the API. Use the Rails console') if user.admin?
+        # We're disabling Cop/UserAdmin because it checks if the given user (not the current user) is an admin.
+        forbidden!('Two-factor authentication for admins cannot be disabled via the API. Use the Rails console') if user.admin? # rubocop:disable Cop/UserAdmin
 
         result = TwoFactor::DestroyService.new(current_user, user: user).execute
 
@@ -1008,7 +1009,8 @@ module API
         end
         get feature_category: :users, urgency: :low do
           entity =
-            if current_user.admin?
+            # We're disabling Cop/UserAdmin because it checks if the given user is an admin.
+            if current_user.admin? # rubocop:disable Cop/UserAdmin
               Entities::UserWithAdmin
             else
               Entities::UserPublic
diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb
index e4a26838746..db71e823b1d 100644
--- a/lib/api/v3/github.rb
+++ b/lib/api/v3/github.rb
@@ -78,13 +78,13 @@ module API
         # rubocop: enable CodeReuse/ActiveRecord
 
         def authorized_merge_requests
-          MergeRequestsFinder.new(current_user, authorized_only: !current_user.admin?)
+          MergeRequestsFinder.new(current_user, authorized_only: !current_user.can_read_all_resources?)
             .execute.with_jira_integration_associations
         end
 
         def authorized_merge_requests_for_project(project)
           MergeRequestsFinder
-            .new(current_user, authorized_only: !current_user.admin?, project_id: project.id)
+            .new(current_user, authorized_only: !current_user.can_read_all_resources?, project_id: project.id)
             .execute.with_jira_integration_associations
         end
 
diff --git a/lib/gitlab/github_gists_import/importer/gist_importer.rb b/lib/gitlab/github_gists_import/importer/gist_importer.rb
new file mode 100644
index 00000000000..a5e87d3cf7d
--- /dev/null
+++ b/lib/gitlab/github_gists_import/importer/gist_importer.rb
@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GithubGistsImport
+    module Importer
+      class GistImporter
+        attr_reader :gist, :user
+
+        FileCountLimitError = Class.new(StandardError)
+
+        # gist - An instance of `Gitlab::GithubGistsImport::Representation::Gist`.
+        def initialize(gist, user_id)
+          @gist = gist
+          @user = User.find(user_id)
+        end
+
+        def execute
+          snippet = build_snippet
+          import_repository(snippet) if snippet.save!
+
+          return ServiceResponse.success unless max_snippet_files_count_exceeded?(snippet)
+
+          fail_and_track(snippet)
+        end
+
+        private
+
+        def build_snippet
+          attrs = {
+            title: gist.truncated_title,
+            visibility_level: gist.visibility_level,
+            content: gist.first_file[:file_content],
+            file_name: gist.first_file[:file_name],
+            author: user,
+            created_at: gist.created_at,
+            updated_at: gist.updated_at
+          }
+
+          PersonalSnippet.new(attrs)
+        end
+
+        def import_repository(snippet)
+          resolved_address = get_resolved_address
+
+          snippet.create_repository
+          snippet.repository.fetch_as_mirror(gist.git_pull_url, forced: true, resolved_address: resolved_address)
+        rescue StandardError
+          remove_snippet_and_repository(snippet)
+
+          raise
+        end
+
+        def get_resolved_address
+          validated_pull_url, host = Gitlab::UrlBlocker.validate!(gist.git_pull_url,
+                                      schemes: Project::VALID_IMPORT_PROTOCOLS,
+                                      ports: Project::VALID_IMPORT_PORTS,
+                                      allow_localhost: allow_local_requests?,
+                                      allow_local_network: allow_local_requests?)
+
+          host.present? ? validated_pull_url.host.to_s : ''
+        end
+
+        def max_snippet_files_count_exceeded?(snippet)
+          snippet.all_files.size > Snippet.max_file_limit
+        end
+
+        def remove_snippet_and_repository(snippet)
+          snippet.repository.remove if snippet.repository_exists?
+          snippet.destroy
+        end
+
+        def allow_local_requests?
+          Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services?
+        end
+
+        def fail_and_track(snippet)
+          remove_snippet_and_repository(snippet)
+
+          ServiceResponse.error(message: 'Snippet max file count exceeded').track_exception(as: FileCountLimitError)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_gists_import/representation/gist.rb b/lib/gitlab/github_gists_import/representation/gist.rb
new file mode 100644
index 00000000000..0d309a98f38
--- /dev/null
+++ b/lib/gitlab/github_gists_import/representation/gist.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module GithubGistsImport
+    module Representation
+      class Gist
+        include Gitlab::GithubImport::Representation::ToHash
+        include Gitlab::GithubImport::Representation::ExposeAttribute
+
+        attr_reader :attributes
+
+        expose_attribute :id, :description, :is_public, :created_at, :updated_at, :files, :git_pull_url
+
+        # Builds a gist from a GitHub API response.
+        #
+        # gist - An instance of `Hash` containing the gist
+        #         details.
+        def self.from_api_response(gist, additional_data = {})
+          hash = {
+            id: gist[:id],
+            description: gist[:description],
+            is_public: gist[:public],
+            files: gist[:files],
+            git_pull_url: gist[:git_pull_url],
+            created_at: gist[:created_at],
+            updated_at: gist[:updated_at]
+          }
+
+          new(hash)
+        end
+
+        # Builds a new gist using a Hash that was built from a JSON payload.
+        def self.from_json_hash(raw_hash)
+          new(Gitlab::GithubImport::Representation.symbolize_hash(raw_hash))
+        end
+
+        # attributes - A hash containing the raw gist details. The keys of this
+        #              Hash (and any nested hashes) must be symbols.
+        def initialize(attributes)
+          @attributes = attributes
+        end
+
+        # Gist description can be an empty string, so we returning nil to use first file
+        # name as a title in such case on snippet creation
+        # Gist description has a limit of 256, while the snippet's title can be up to 255
+        def truncated_title
+          title = description.presence || first_file[:file_name]
+
+          title.truncate(255)
+        end
+
+        def visibility_level
+          is_public ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE
+        end
+
+        def first_file
+          _key, value = files.first
+
+          {
+            file_name: value[:filename],
+            file_content: Gitlab::HTTP.try_get(value[:raw_url])&.body
+          }
+        end
+
+        def github_identifiers
+          { id: id }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/memory/watchdog/configurator.rb b/lib/gitlab/memory/watchdog/configurator.rb
index 880f9800d96..2848d6c36e9 100644
--- a/lib/gitlab/memory/watchdog/configurator.rb
+++ b/lib/gitlab/memory/watchdog/configurator.rb
@@ -30,7 +30,7 @@ module Gitlab
               config.write_heap_dumps = write_heap_dumps?
               config.sleep_time_seconds = sidekiq_sleep_time
               config.monitors(&configure_monitors_for_sidekiq)
-              config.event_reporter = EventReporter.new(logger: ::Sidekiq.logger)
+              config.event_reporter = SidekiqEventReporter.new
             end
           end
 
diff --git a/lib/gitlab/memory/watchdog/sidekiq_event_reporter.rb b/lib/gitlab/memory/watchdog/sidekiq_event_reporter.rb
new file mode 100644
index 00000000000..db94edd0992
--- /dev/null
+++ b/lib/gitlab/memory/watchdog/sidekiq_event_reporter.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Memory
+    class Watchdog
+      class SidekiqEventReporter
+        include ::Gitlab::Utils::StrongMemoize
+
+        delegate :threshold_violated, :started, :stopped, :logger, to: :event_reporter
+
+        def initialize(logger: ::Sidekiq.logger)
+          @event_reporter = EventReporter.new(logger: logger)
+          init_prometheus_metrics
+        end
+
+        def strikes_exceeded(monitor_name, labels = {})
+          running_jobs = fetch_running_jobs
+          labels[:running_jobs] = running_jobs
+          increment_worker_counters(running_jobs)
+
+          event_reporter.strikes_exceeded(monitor_name, labels)
+        end
+
+        private
+
+        attr_reader :event_reporter
+
+        def fetch_running_jobs
+          running_jobs = []
+          Gitlab::SidekiqDaemon::Monitor.instance.with_running_jobs do |jobs|
+            running_jobs = jobs.map do |jid, job|
+              {
+                jid: jid,
+                worker_class: job[:worker_class].name
+              }
+            end
+          end
+          running_jobs
+        end
+
+        def increment_worker_counters(running_jobs)
+          running_jobs.each do |job|
+            @sidekiq_watchdog_running_jobs_counter.increment({ worker_class: job[:worker_class] })
+          end
+        end
+
+        def init_prometheus_metrics
+          @sidekiq_watchdog_running_jobs_counter = ::Gitlab::Metrics.counter(
+            :sidekiq_watchdog_running_jobs_total,
+            'Current running jobs when limit was reached'
+          )
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_daemon/monitor.rb b/lib/gitlab/sidekiq_daemon/monitor.rb
index 655e95c82d3..2079683a2c1 100644
--- a/lib/gitlab/sidekiq_daemon/monitor.rb
+++ b/lib/gitlab/sidekiq_daemon/monitor.rb
@@ -65,6 +65,12 @@ module Gitlab
         end
       end
 
+      def with_running_jobs
+        @jobs_mutex.synchronize do
+          yield @jobs.dup
+        end
+      end
+
       private
 
       def run_thread
diff --git a/lib/version_check.rb b/lib/version_check.rb
index 35014f3ddf0..eddcddbeb49 100644
--- a/lib/version_check.rb
+++ b/lib/version_check.rb
@@ -69,13 +69,17 @@ class VersionCheck
 
     case response&.code
     when 200
-      response.body
+      Gitlab::Json.parse(response.body)
+    else
+      { error: 'version check failed', status: response&.code }
     end
   end
 
   def response
     with_reactive_cache do |data|
-      Gitlab::Json.parse(data) if data
+      raise InvalidateReactiveCache if data[:error]
+
+      data
     end
   end
 end