Add latest changes from gitlab-org/gitlab@master

19 files changed, 196 insertions, 28 deletions

diff --git a/app/assets/javascripts/notes/components/comment_form.vue b/app/assets/javascripts/notes/components/comment_form.vue
index 4e03bed8737..8ef071034e5 100644
--- a/app/assets/javascripts/notes/components/comment_form.vue
+++ b/app/assets/javascripts/notes/components/comment_form.vue
@@ -111,7 +111,7 @@ export default {
       return this.getNoteableData.current_user.can_create_note;
     },
     canSetConfidential() {
-      return this.getNoteableData.current_user.can_update;
+      return this.getNoteableData.current_user.can_update && (this.isIssue || this.isEpic);
     },
     issueActionButtonTitle() {
       const openOrClose = this.isOpen ? 'close' : 'reopen';
@@ -166,6 +166,9 @@ export default {
     isIssue() {
       return constants.NOTEABLE_TYPE_MAPPING[this.noteableType] === constants.ISSUE_NOTEABLE_TYPE;
     },
+    isEpic() {
+      return constants.NOTEABLE_TYPE_MAPPING[this.noteableType] === constants.EPIC_NOTEABLE_TYPE;
+    },
     trackingLabel() {
       return slugifyWithUnderscore(`${this.commentButtonTitle} button`);
     },
diff --git a/app/assets/javascripts/packages_and_registries/package_registry/components/details/package_files.vue b/app/assets/javascripts/packages_and_registries/package_registry/components/details/package_files.vue
index 3724e371e01..378fb102bdd 100644
--- a/app/assets/javascripts/packages_and_registries/package_registry/components/details/package_files.vue
+++ b/app/assets/javascripts/packages_and_registries/package_registry/components/details/package_files.vue
@@ -1,5 +1,5 @@
 <script>
-import { GlLink, GlTable, GlDropdownItem, GlDropdown, GlIcon, GlButton } from '@gitlab/ui';
+import { GlLink, GlTableLite, GlDropdownItem, GlDropdown, GlIcon, GlButton } from '@gitlab/ui';
 import { last } from 'lodash';
 import { numberToHumanSize } from '~/lib/utils/number_utils';
 import { __ } from '~/locale';
@@ -12,7 +12,7 @@ export default {
   name: 'PackageFiles',
   components: {
     GlLink,
-    GlTable,
+    GlTableLite,
     GlIcon,
     GlDropdown,
     GlDropdownItem,
@@ -94,7 +94,7 @@ export default {
 <template>
   <div>
     <h3 class="gl-font-lg gl-mt-5">{{ __('Files') }}</h3>
-    <gl-table
+    <gl-table-lite
       :fields="filesTableHeaderFields"
       :items="filesTableRows"
       :tbody-tr-attr="{ 'data-testid': 'file-row' }"
@@ -162,6 +162,6 @@ export default {
           <file-sha v-if="item.fileSha1" data-testid="sha-1" title="SHA-1" :sha="item.fileSha1" />
         </div>
       </template>
-    </gl-table>
+    </gl-table-lite>
   </div>
 </template>
diff --git a/app/controllers/import/fogbugz_controller.rb b/app/controllers/import/fogbugz_controller.rb
index c223d9d211e..9a457cf673c 100644
--- a/app/controllers/import/fogbugz_controller.rb
+++ b/app/controllers/import/fogbugz_controller.rb
@@ -55,16 +55,16 @@ class Import::FogbugzController < Import::BaseController
   # rubocop: enable CodeReuse/ActiveRecord
 
   def create
-    repo = client.repo(params[:repo_id])
-    fb_session = { uri: session[:fogbugz_uri], token: session[:fogbugz_token] }
+    credentials = { uri: session[:fogbugz_uri], token: session[:fogbugz_token] }
+
     umap = session[:fogbugz_user_map] || client.user_map
 
-    project = Gitlab::FogbugzImport::ProjectCreator.new(repo, fb_session, current_user.namespace, current_user, umap).execute
+    result = Import::FogbugzService.new(client, current_user, params.merge(umap: umap)).execute(credentials)
 
-    if project.persisted?
-      render json: ProjectSerializer.new.represent(project, serializer: :import)
+    if result[:status] == :success
+      render json: ProjectSerializer.new.represent(result[:project], serializer: :import)
     else
-      render json: { errors: project_save_error(project) }, status: :unprocessable_entity
+      render json: { errors: result[:message] }, status: result[:http_status]
     end
   end
 
diff --git a/app/controllers/projects/environments_controller.rb b/app/controllers/projects/environments_controller.rb
index 1a2c0d64d19..ac3c85f3b40 100644
--- a/app/controllers/projects/environments_controller.rb
+++ b/app/controllers/projects/environments_controller.rb
@@ -207,7 +207,11 @@ class Projects::EnvironmentsController < Projects::ApplicationController
   private
 
   def deployments
-    environment.deployments.ordered.page(params[:page])
+    environment
+      .deployments
+      .with_environment_page_associations
+      .ordered
+      .page(params[:page])
   end
 
   def verify_api_request!
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index eea8086d71d..fd264b71b92 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -144,6 +144,7 @@ module Ci
 
     scope :eager_load_job_artifacts, -> { includes(:job_artifacts) }
     scope :eager_load_tags, -> { includes(:tags) }
+    scope :eager_load_for_archiving_trace, -> { includes(:project, :pending_state) }
 
     scope :eager_load_everything, -> do
       includes(
@@ -673,7 +674,7 @@ module Ci
     end
 
     def has_live_trace?
-      trace.live_trace_exist?
+      trace.live?
     end
 
     def has_archived_trace?
@@ -826,12 +827,26 @@ module Ci
     end
 
     def erase_erasable_artifacts!
+      if project.refreshing_build_artifacts_size?
+        Gitlab::ProjectStatsRefreshConflictsLogger.warn_artifact_deletion_during_stats_refresh(
+          method: 'Ci::Build#erase_erasable_artifacts!',
+          project_id: project_id
+        )
+      end
+
       job_artifacts.erasable.destroy_all # rubocop: disable Cop/DestroyAll
     end
 
     def erase(opts = {})
       return false unless erasable?
 
+      if project.refreshing_build_artifacts_size?
+        Gitlab::ProjectStatsRefreshConflictsLogger.warn_artifact_deletion_during_stats_refresh(
+          method: 'Ci::Build#erase',
+          project_id: project_id
+        )
+      end
+
       job_artifacts.destroy_all # rubocop: disable Cop/DestroyAll
       erase_trace!
       update_erased!(opts[:erased_by])
diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb
index c831ef12501..74234a4e470 100644
--- a/app/models/ci/job_artifact.rb
+++ b/app/models/ci/job_artifact.rb
@@ -139,6 +139,9 @@ module Ci
     scope :for_sha, ->(sha, project_id) { joins(job: :pipeline).where(ci_pipelines: { sha: sha, project_id: project_id }) }
     scope :for_job_ids, ->(job_ids) { where(job_id: job_ids) }
     scope :for_job_name, ->(name) { joins(:job).where(ci_builds: { name: name }) }
+    scope :created_at_before, ->(time) { where(arel_table[:created_at].lteq(time)) }
+    scope :id_after, ->(id) { where(arel_table[:id].gt(id)) }
+    scope :ordered_by_created_at_and_id_asc, -> { order(:created_at, :id) }
 
     scope :with_job, -> { joins(:job).includes(:job) }
 
@@ -187,7 +190,7 @@ module Ci
     scope :downloadable, -> { where(file_type: DOWNLOADABLE_TYPES) }
     scope :unlocked, -> { joins(job: :pipeline).merge(::Ci::Pipeline.unlocked) }
     scope :order_expired_asc, -> { order(expire_at: :asc) }
-    scope :with_destroy_preloads, -> { includes(project: [:route, :statistics]) }
+    scope :with_destroy_preloads, -> { includes(project: [:route, :statistics, :build_artifacts_size_refresh]) }
 
     scope :for_project, ->(project) { where(project_id: project) }
     scope :created_in_time_range, ->(from: nil, to: nil) { where(created_at: from..to) }
diff --git a/app/models/deployment.rb b/app/models/deployment.rb
index 4204ad707b2..f84857d3511 100644
--- a/app/models/deployment.rb
+++ b/app/models/deployment.rb
@@ -52,6 +52,7 @@ class Deployment < ApplicationRecord
   scope :upcoming, -> { where(status: %i[blocked running]) }
   scope :older_than, -> (deployment) { where('deployments.id < ?', deployment.id) }
   scope :with_api_entity_associations, -> { preload({ deployable: { runner: [], tags: [], user: [], job_artifacts_archive: [] } }) }
+  scope :with_environment_page_associations, -> { preload(project: [], environment: [], deployable: [:user, :metadata, :project, pipeline: [:manual_actions]]) }
 
   scope :finished_after, ->(date) { where('finished_at >= ?', date) }
   scope :finished_before, ->(date) { where('finished_at < ?', date) }
diff --git a/app/models/project.rb b/app/models/project.rb
index f4e39524e47..e46f2a38cb0 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -418,6 +418,8 @@ class Project < ApplicationRecord
   has_one :ci_project_mirror, class_name: 'Ci::ProjectMirror'
   has_many :sync_events, class_name: 'Projects::SyncEvent'
 
+  has_one :build_artifacts_size_refresh, class_name: 'Projects::BuildArtifactsSizeRefresh'
+
   accepts_nested_attributes_for :variables, allow_destroy: true
   accepts_nested_attributes_for :project_feature, update_only: true
   accepts_nested_attributes_for :project_setting, update_only: true
@@ -2900,6 +2902,10 @@ class Project < ApplicationRecord
       last_activity_at < ::Gitlab::CurrentSettings.inactive_projects_send_warning_email_after_months.months.ago
   end
 
+  def refreshing_build_artifacts_size?
+    build_artifacts_size_refresh&.started?
+  end
+
   private
 
   # overridden in EE
diff --git a/app/models/projects/build_artifacts_size_refresh.rb b/app/models/projects/build_artifacts_size_refresh.rb
index 959f486a50a..f411dbe88ed 100644
--- a/app/models/projects/build_artifacts_size_refresh.rb
+++ b/app/models/projects/build_artifacts_size_refresh.rb
@@ -83,9 +83,14 @@ module Projects
 
     def next_batch(limit:)
       project.job_artifacts.select(:id, :size)
-        .where('created_at <= ? AND id > ?', refresh_started_at, last_job_artifact_id.to_i)
-        .order(:created_at)
+        .created_at_before(refresh_started_at)
+        .id_after(last_job_artifact_id.to_i)
+        .ordered_by_created_at_and_id_asc
         .limit(limit)
     end
+
+    def started?
+      !created?
+    end
   end
 end
diff --git a/app/models/repository.rb b/app/models/repository.rb
index dc0b5b54fb0..c6bfd69f4e6 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -13,6 +13,7 @@ class Repository
   REF_KEEP_AROUND = 'keep-around'
   REF_ENVIRONMENTS = 'environments'
   REF_PIPELINES = 'pipelines'
+  REF_TMP = 'tmp'
 
   ARCHIVE_CACHE_TIME = 60 # Cache archives referred to by a (mutable) ref for 1 minute
   ARCHIVE_CACHE_TIME_IMMUTABLE = 3600 # Cache archives referred to by an immutable reference for 1 hour
diff --git a/app/presenters/blob_presenter.rb b/app/presenters/blob_presenter.rb
index aeab914dc9e..2dcc6cd5df3 100644
--- a/app/presenters/blob_presenter.rb
+++ b/app/presenters/blob_presenter.rb
@@ -27,11 +27,11 @@ class BlobPresenter < Gitlab::View::Presenter::Delegated
   end
 
   def blob_data(to)
-    @_blob_data ||= Gitlab::Diff::CustomDiff.transformed_blob_data(blob) || limited_blob_data(to: to)
+    @_blob_data ||= limited_blob_data(to: to)
   end
 
   def blob_language
-    @_blob_language ||= Gitlab::Diff::CustomDiff.transformed_blob_language(blob) || gitattr_language || detect_language
+    @_blob_language ||= gitattr_language || detect_language
   end
 
   def raw_plain_data
diff --git a/app/services/ci/job_artifacts/destroy_batch_service.rb b/app/services/ci/job_artifacts/destroy_batch_service.rb
index 5121a8b0a8b..e8d21f14ee6 100644
--- a/app/services/ci/job_artifacts/destroy_batch_service.rb
+++ b/app/services/ci/job_artifacts/destroy_batch_service.rb
@@ -25,6 +25,8 @@ module Ci
 
       # rubocop: disable CodeReuse/ActiveRecord
       def execute(update_stats: true)
+        track_artifacts_undergoing_stats_refresh
+
         # Detect and fix artifacts that had `expire_at` wrongly backfilled by migration
         # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47723
         detect_and_fix_wrongly_expired_artifacts
@@ -154,6 +156,19 @@ module Ci
 
         Gitlab::AppLogger.info(message: "Fixed expire_at from artifacts.", fixed_artifacts_expire_at_count: artifacts.count)
       end
+
+      def track_artifacts_undergoing_stats_refresh
+        project_ids = @job_artifacts.find_all do |artifact|
+          artifact.project.refreshing_build_artifacts_size?
+        end.map(&:project_id).uniq
+
+        project_ids.each do |project_id|
+          Gitlab::ProjectStatsRefreshConflictsLogger.warn_artifact_deletion_during_stats_refresh(
+            method: 'Ci::JobArtifacts::DestroyBatchService#execute',
+            project_id: project_id
+          )
+        end
+      end
     end
   end
 end
diff --git a/app/services/import/base_service.rb b/app/services/import/base_service.rb
index 4a43b2f7425..ab3e9c7abba 100644
--- a/app/services/import/base_service.rb
+++ b/app/services/import/base_service.rb
@@ -8,6 +8,10 @@ module Import
       @params = params
     end
 
+    def authorized?
+      can?(current_user, :create_projects, target_namespace)
+    end
+
     private
 
     def find_or_create_namespace(namespace, owner)
diff --git a/app/services/import/bitbucket_server_service.rb b/app/services/import/bitbucket_server_service.rb
index d1c22f06464..20f6c987c92 100644
--- a/app/services/import/bitbucket_server_service.rb
+++ b/app/services/import/bitbucket_server_service.rb
@@ -72,10 +72,6 @@ module Import
       @url ||= params[:bitbucket_server_url]
     end
 
-    def authorized?
-      can?(current_user, :create_projects, target_namespace)
-    end
-
     def allow_local_requests?
       Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services?
     end
diff --git a/app/services/import/fogbugz_service.rb b/app/services/import/fogbugz_service.rb
new file mode 100644
index 00000000000..d1003823456
--- /dev/null
+++ b/app/services/import/fogbugz_service.rb
@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Import
+  class FogbugzService < Import::BaseService
+    attr_reader :client, :params, :current_user
+
+    def execute(credentials)
+      url = credentials[:uri]
+
+      if blocked_url?(url)
+        return log_and_return_error("Invalid URL: #{url}", _("Invalid URL: %{url}") % { url: url }, :bad_request)
+      end
+
+      unless authorized?
+        return log_and_return_error(
+          "You don't have permissions to create this project",
+          _("You don't have permissions to create this project"),
+          :unauthorized
+        )
+      end
+
+      unless repo
+        return log_and_return_error(
+          "Project #{repo_id} could not be found",
+          s_("Fogbugz|Project %{repo} could not be found") % { repo: repo_id },
+          :unprocessable_entity)
+      end
+
+      project = create_project(credentials)
+
+      if project.persisted?
+        success(project)
+      elsif project.errors[:import_source_disabled].present?
+        error(project.errors[:import_source_disabled], :forbidden)
+      else
+        error(project_save_error(project), :unprocessable_entity)
+      end
+    rescue StandardError => e
+      log_and_return_error(
+        "Fogbugz import failed due to an error: #{e}",
+        s_("Fogbugz|Fogbugz import failed due to an error: %{error}" % { error: e }),
+        :bad_request)
+    end
+
+    private
+
+    def create_project(credentials)
+      Gitlab::FogbugzImport::ProjectCreator.new(
+        repo,
+        project_name,
+        target_namespace,
+        current_user,
+        credentials,
+        umap
+      ).execute
+    end
+
+    def repo_id
+      @repo_id ||= params[:repo_id]
+    end
+
+    def repo
+      @repo ||= client.repo(repo_id)
+    end
+
+    def project_name
+      @project_name ||= params[:new_name].presence || repo.name
+    end
+
+    def namespace_path
+      @namespace_path ||= params[:target_namespace].presence || current_user.namespace_path
+    end
+
+    def target_namespace
+      @target_namespace ||= find_or_create_namespace(namespace_path, current_user.namespace_path)
+    end
+
+    def umap
+      @umap ||= params[:umap]
+    end
+
+    def allow_local_requests?
+      Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services?
+    end
+
+    def blocked_url?(url)
+      Gitlab::UrlBlocker.blocked_url?(
+        url,
+        allow_localhost: allow_local_requests?,
+        allow_local_network: allow_local_requests?,
+        schemes: %w(http https)
+      )
+    end
+
+    def log_and_return_error(message, translated_message, error_type)
+      log_error(message)
+      error(translated_message, error_type)
+    end
+
+    def log_error(message)
+      Gitlab::Import::Logger.error(
+        message: 'Import failed due to a Fogbugz error',
+        error: message
+      )
+    end
+  end
+end
diff --git a/app/services/import/github_service.rb b/app/services/import/github_service.rb
index 033f6bcb043..ff5d5d2c4c1 100644
--- a/app/services/import/github_service.rb
+++ b/app/services/import/github_service.rb
@@ -89,10 +89,6 @@ module Import
       end
     end
 
-    def authorized?
-      can?(current_user, :create_projects, target_namespace)
-    end
-
     def url
       @url ||= params[:github_hostname]
     end
diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb
index a73244c6971..f2b108fe9fc 100644
--- a/app/services/projects/destroy_service.rb
+++ b/app/services/projects/destroy_service.rb
@@ -191,6 +191,10 @@ module Projects
     # rubocop: enable CodeReuse/ActiveRecord
 
     def destroy_ci_records!
+      # Make sure to destroy this first just in case the project is undergoing stats refresh.
+      # This is to avoid logging the artifact deletion in Ci::JobArtifacts::DestroyBatchService.
+      project.build_artifacts_size_refresh&.destroy
+
       project.all_pipelines.find_each(batch_size: BATCH_SIZE) do |pipeline| # rubocop: disable CodeReuse/ActiveRecord
         # Destroy artifacts, then builds, then pipelines
         # All builds have already been dropped by Ci::AbortPipelinesService,
diff --git a/app/views/admin/application_settings/_visibility_and_access.html.haml b/app/views/admin/application_settings/_visibility_and_access.html.haml
index e3c044ff979..cc62ca354d1 100644
--- a/app/views/admin/application_settings/_visibility_and_access.html.haml
+++ b/app/views/admin/application_settings/_visibility_and_access.html.haml
@@ -63,4 +63,6 @@
       %label.label-bold= s_('AdminSettings|Feed token')
       = f.gitlab_ui_checkbox_component :disable_feed_token, s_('AdminSettings|Disable feed token')
 
+    = render_if_exists 'admin/application_settings/globally_allowed_ips', form: f
+
   = f.submit _('Save changes'), class: "gl-button btn btn-confirm"
diff --git a/app/workers/ci/archive_trace_worker.rb b/app/workers/ci/archive_trace_worker.rb
index 5a22a5c74ee..47d77c15b4a 100644
--- a/app/workers/ci/archive_trace_worker.rb
+++ b/app/workers/ci/archive_trace_worker.rb
@@ -4,13 +4,19 @@ module Ci
   class ArchiveTraceWorker # rubocop:disable Scalability/IdempotentWorker
     include ApplicationWorker
 
-    data_consistency :always
+    data_consistency :sticky, feature_flag: :sticky_ci_archive_trace_worker
 
     sidekiq_options retry: 3
     include PipelineBackgroundQueue
 
     def perform(job_id)
-      Ci::Build.without_archived_trace.find_by_id(job_id).try do |job|
+      archivable_jobs = Ci::Build.without_archived_trace
+
+      if Feature.enabled?(:sticky_ci_archive_trace_worker)
+        archivable_jobs = archivable_jobs.eager_load_for_archiving_trace
+      end
+
+      archivable_jobs.find_by_id(job_id).try do |job|
         Ci::ArchiveTraceService.new.execute(job, worker_name: self.class.name)
       end
     end