diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-11-17 14:33:21 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-11-17 14:33:21 +0300 |
commit | 7021455bd1ed7b125c55eb1b33c5a01f2bc55ee0 (patch) | |
tree | 5bdc2229f5198d516781f8d24eace62fc7e589e9 /app/services/ci | |
parent | 185b095e93520f96e9cfc31d9c3e69b498cdab7c (diff) |
Add latest changes from gitlab-org/gitlab@15-6-stable-eev15.6.0-rc42
Diffstat (limited to 'app/services/ci')
18 files changed, 130 insertions, 119 deletions
diff --git a/app/services/ci/after_requeue_job_service.rb b/app/services/ci/after_requeue_job_service.rb index 9d54207d75d..4374ccd52e0 100644 --- a/app/services/ci/after_requeue_job_service.rb +++ b/app/services/ci/after_requeue_job_service.rb @@ -23,8 +23,6 @@ module Ci # rubocop: disable CodeReuse/ActiveRecord def dependent_jobs - return legacy_dependent_jobs unless ::Feature.enabled?(:ci_requeue_with_dag_object_hierarchy, project) - ordered_by_dag( @processable.pipeline.processables .from_union(needs_dependent_jobs, stage_dependent_jobs) @@ -50,24 +48,6 @@ module Ci ).descendants end - def legacy_skipped_jobs - @legacy_skipped_jobs ||= @processable.pipeline.processables.skipped - end - - def legacy_dependent_jobs - ordered_by_dag( - legacy_stage_dependent_jobs.or(legacy_needs_dependent_jobs).ordered_by_stage.preload(:needs) - ) - end - - def legacy_stage_dependent_jobs - legacy_skipped_jobs.after_stage(@processable.stage_idx) - end - - def legacy_needs_dependent_jobs - legacy_skipped_jobs.scheduling_type_dag.with_needs([@processable.name]) - end - def ordered_by_dag(jobs) sorted_job_names = sort_jobs(jobs).each_with_index.to_h diff --git a/app/services/ci/archive_trace_service.rb b/app/services/ci/archive_trace_service.rb index 566346a4b09..3d548c824c8 100644 --- a/app/services/ci/archive_trace_service.rb +++ b/app/services/ci/archive_trace_service.rb @@ -68,7 +68,7 @@ module Ci Gitlab::ErrorTracking .track_and_raise_for_dev_exception(error, issue_url: 'https://gitlab.com/gitlab-org/gitlab-foss/issues/51502', - job_id: job.id ) + job_id: job.id) end end end diff --git a/app/services/ci/build_erase_service.rb b/app/services/ci/build_erase_service.rb index 8a468e094eb..71b4c5481b3 100644 --- a/app/services/ci/build_erase_service.rb +++ b/app/services/ci/build_erase_service.rb @@ -33,9 +33,7 @@ module Ci attr_reader :build, :current_user def destroy_artifacts - # fix_expire_at is false because in this case we want to explicitly delete the job artifacts - # this flag is a workaround that will be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/355833 - Ci::JobArtifacts::DestroyBatchService.new(build.job_artifacts, fix_expire_at: false).execute + Ci::JobArtifacts::DestroyBatchService.new(build.job_artifacts).execute end def erase_trace! diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index 0b49beffcb5..4106dfe0ecc 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -30,6 +30,7 @@ module Ci Gitlab::Ci::Pipeline::Chain::Limit::Deployments, Gitlab::Ci::Pipeline::Chain::Validate::External, Gitlab::Ci::Pipeline::Chain::Populate, + Gitlab::Ci::Pipeline::Chain::PopulateMetadata, Gitlab::Ci::Pipeline::Chain::StopDryRun, Gitlab::Ci::Pipeline::Chain::EnsureEnvironments, Gitlab::Ci::Pipeline::Chain::EnsureResourceGroups, @@ -118,17 +119,6 @@ module Ci end # rubocop: enable Metrics/ParameterLists - def execute!(*args, &block) - source = args[0] - params = Hash(args[1]) - - execute(source, **params, &block).tap do |response| - unless response.payload.persisted? - raise CreateError, pipeline.full_error_messages - end - end - end - private def commit diff --git a/app/services/ci/job_artifacts/destroy_associations_service.rb b/app/services/ci/job_artifacts/destroy_associations_service.rb index 08d7f7f6f02..794d24eadf2 100644 --- a/app/services/ci/job_artifacts/destroy_associations_service.rb +++ b/app/services/ci/job_artifacts/destroy_associations_service.rb @@ -12,7 +12,7 @@ module Ci def destroy_records @job_artifacts_relation.each_batch(of: BATCH_SIZE) do |relation| - service = Ci::JobArtifacts::DestroyBatchService.new(relation, pick_up_at: Time.current, fix_expire_at: false) + service = Ci::JobArtifacts::DestroyBatchService.new(relation, pick_up_at: Time.current) result = service.execute(update_stats: false) updates = result[:statistics_updates] diff --git a/app/services/ci/job_artifacts/destroy_batch_service.rb b/app/services/ci/job_artifacts/destroy_batch_service.rb index 54ec2c671c6..e0307d9bd53 100644 --- a/app/services/ci/job_artifacts/destroy_batch_service.rb +++ b/app/services/ci/job_artifacts/destroy_batch_service.rb @@ -17,10 +17,9 @@ module Ci # +pick_up_at+:: When to pick up for deletion of files # Returns: # +Hash+:: A hash with status and destroyed_artifacts_count keys - def initialize(job_artifacts, pick_up_at: nil, fix_expire_at: fix_expire_at?, skip_projects_on_refresh: false) + def initialize(job_artifacts, pick_up_at: nil, skip_projects_on_refresh: false) @job_artifacts = job_artifacts.with_destroy_preloads.to_a @pick_up_at = pick_up_at - @fix_expire_at = fix_expire_at @skip_projects_on_refresh = skip_projects_on_refresh end @@ -32,9 +31,7 @@ module Ci track_artifacts_undergoing_stats_refresh end - # Detect and fix artifacts that had `expire_at` wrongly backfilled by migration - # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47723 - detect_and_fix_wrongly_expired_artifacts + exclude_trace_artifacts return success(destroyed_artifacts_count: 0, statistics_updates: {}) if @job_artifacts.empty? @@ -113,55 +110,9 @@ module Ci end end - # This detects and fixes job artifacts that have `expire_at` wrongly backfilled by the migration - # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47723. - # These job artifacts will not be deleted and will have their `expire_at` removed. - # - # The migration would have backfilled `expire_at` - # to midnight on the 22nd of the month of the local timezone, - # storing it as UTC time in the database. - # - # If the timezone setting has changed since the migration, - # the `expire_at` stored in the database could have changed to a different local time other than midnight. - # For example: - # - changing timezone from UTC+02:00 to UTC+02:30 would change the `expire_at` in local time 00:00:00 to 00:30:00. - # - changing timezone from UTC+00:00 to UTC-01:00 would change the `expire_at` in local time 00:00:00 to 23:00:00 on the previous day (21st). - # - # Therefore job artifacts that have `expire_at` exactly on the 00, 30 or 45 minute mark - # on the dates 21, 22, 23 of the month will not be deleted. - # https://en.wikipedia.org/wiki/List_of_UTC_time_offsets - def detect_and_fix_wrongly_expired_artifacts - return unless @fix_expire_at - - wrongly_expired_artifacts, @job_artifacts = @job_artifacts.partition { |artifact| wrongly_expired?(artifact) } - - remove_expire_at(wrongly_expired_artifacts) if wrongly_expired_artifacts.any? - end - - def fix_expire_at? - Feature.enabled?(:ci_detect_wrongly_expired_artifacts) - end - - def wrongly_expired?(artifact) - return false unless artifact.expire_at.present? - - # Although traces should never have expiration dates that don't match time & date here. - # we can explicitly exclude them by type since they should never be destroyed. - artifact.trace? || (match_date?(artifact.expire_at) && match_time?(artifact.expire_at)) - end - - def match_date?(expire_at) - [21, 22, 23].include?(expire_at.day) - end - - def match_time?(expire_at) - %w[00:00.000 30:00.000 45:00.000].include?(expire_at.strftime('%M:%S.%L')) - end - - def remove_expire_at(artifacts) - Ci::JobArtifact.id_in(artifacts).update_all(expire_at: nil) - - Gitlab::AppLogger.info(message: "Fixed expire_at from artifacts.", fixed_artifacts_expire_at_count: artifacts.count) + # Traces should never be destroyed. + def exclude_trace_artifacts + _trace_artifacts, @job_artifacts = @job_artifacts.partition(&:trace?) end def track_artifacts_undergoing_stats_refresh diff --git a/app/services/ci/job_artifacts/track_artifact_report_service.rb b/app/services/ci/job_artifacts/track_artifact_report_service.rb index 1be1d98394f..0230a5e19ce 100644 --- a/app/services/ci/job_artifacts/track_artifact_report_service.rb +++ b/app/services/ci/job_artifacts/track_artifact_report_service.rb @@ -5,7 +5,7 @@ module Ci class TrackArtifactReportService include Gitlab::Utils::UsageData - REPORT_TRACKED = %i[test].freeze + REPORT_TRACKED = %i[test coverage].freeze def execute(pipeline) REPORT_TRACKED.each do |report| diff --git a/app/services/ci/list_config_variables_service.rb b/app/services/ci/list_config_variables_service.rb index 3890882b3d4..df4963d1b33 100644 --- a/app/services/ci/list_config_variables_service.rb +++ b/app/services/ci/list_config_variables_service.rb @@ -30,7 +30,7 @@ module Ci user: current_user, sha: sha).execute - result.valid? ? result.variables_with_data : {} + result.valid? ? result.root_variables_with_prefill_data : {} end # Required for ReactiveCaching, it is also used in `reactive_cache_worker_finder` diff --git a/app/services/ci/pipeline_artifacts/coverage_report_service.rb b/app/services/ci/pipeline_artifacts/coverage_report_service.rb index 9c6fdb7a405..de66a4cb045 100644 --- a/app/services/ci/pipeline_artifacts/coverage_report_service.rb +++ b/app/services/ci/pipeline_artifacts/coverage_report_service.rb @@ -39,7 +39,7 @@ module Ci def carrierwave_file strong_memoize(:carrier_wave_file) do CarrierWaveStringFile.new_file( - file_content: report.to_json, + file_content: Gitlab::Json.dump(report), filename: Ci::PipelineArtifact::DEFAULT_FILE_NAMES.fetch(:code_coverage), content_type: 'application/json' ) diff --git a/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb b/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb index a0746ef32b2..57b663dc293 100644 --- a/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb +++ b/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb @@ -77,9 +77,9 @@ module Ci end def build_quality_mr_diff_report(mr_diff_report) - mr_diff_report.each_with_object({}) do |diff_report, hash| + Gitlab::Json.dump(mr_diff_report.each_with_object({}) do |diff_report, hash| hash[diff_report.first] = Ci::CodequalityMrDiffReportSerializer.new.represent(diff_report.second) # rubocop: disable CodeReuse/Serializer - end.to_json + end) end end end diff --git a/app/services/ci/pipeline_schedules/take_ownership_service.rb b/app/services/ci/pipeline_schedules/take_ownership_service.rb new file mode 100644 index 00000000000..9b4001c74bd --- /dev/null +++ b/app/services/ci/pipeline_schedules/take_ownership_service.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Ci + module PipelineSchedules + class TakeOwnershipService + def initialize(schedule, user) + @schedule = schedule + @user = user + end + + def execute + return forbidden unless allowed? + + if schedule.update(owner: user) + ServiceResponse.success(payload: schedule) + else + ServiceResponse.error(message: schedule.errors.full_messages) + end + end + + private + + attr_reader :schedule, :user + + def allowed? + user.can?(:take_ownership_pipeline_schedule, schedule) + end + + def forbidden + ServiceResponse.error(message: _('Failed to change the owner'), reason: :access_denied) + end + end + end +end diff --git a/app/services/ci/pipeline_trigger_service.rb b/app/services/ci/pipeline_trigger_service.rb index 39ac9bf33e9..d7065680053 100644 --- a/app/services/ci/pipeline_trigger_service.rb +++ b/app/services/ci/pipeline_trigger_service.rb @@ -93,7 +93,7 @@ module Ci def payload_variable { key: PAYLOAD_VARIABLE_KEY, - value: params.except(*PAYLOAD_VARIABLE_HIDDEN_PARAMS).to_json, + value: Gitlab::Json.dump(params.except(*PAYLOAD_VARIABLE_HIDDEN_PARAMS)), variable_type: :file } end diff --git a/app/services/ci/play_build_service.rb b/app/services/ci/play_build_service.rb index fbf2aad1991..b7aec57f3e3 100644 --- a/app/services/ci/play_build_service.rb +++ b/app/services/ci/play_build_service.rb @@ -5,21 +5,27 @@ module Ci def execute(build, job_variables_attributes = nil) check_access!(build, job_variables_attributes) - # Try to enqueue the build, otherwise create a duplicate. - # - if build.enqueue - build.tap do |build| - build.update!(user: current_user, job_variables_attributes: job_variables_attributes || []) - - AfterRequeueJobService.new(project, current_user).execute(build) - end + if build.can_enqueue? + build.user = current_user + build.job_variables_attributes = job_variables_attributes || [] + build.enqueue! + + AfterRequeueJobService.new(project, current_user).execute(build) + + build else - Ci::RetryJobService.new(project, current_user).execute(build)[:job] + retry_build(build) end + rescue StateMachines::InvalidTransition + retry_build(build.reset) end private + def retry_build(build) + Ci::RetryJobService.new(project, current_user).execute(build)[:job] + end + def check_access!(build, job_variables_attributes) raise Gitlab::Access::AccessDeniedError unless can?(current_user, :play_job, build) diff --git a/app/services/ci/process_build_service.rb b/app/services/ci/process_build_service.rb index 22cd267806d..cb51d918fc2 100644 --- a/app/services/ci/process_build_service.rb +++ b/app/services/ci/process_build_service.rb @@ -15,6 +15,8 @@ module Ci private def process(build) + return enqueue(build) if Feature.enabled?(:ci_retry_job_fix, project) && build.enqueue_immediately? + if build.schedulable? build.schedule elsif build.action? @@ -25,7 +27,7 @@ module Ci end def enqueue(build) - return build.drop!(:failed_outdated_deployment_job) if build.prevent_rollback_deployment? + return build.drop!(:failed_outdated_deployment_job) if build.outdated_deployment? build.enqueue end diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb index 0bd4bf8cc86..f11577feb88 100644 --- a/app/services/ci/register_job_service.rb +++ b/app/services/ci/register_job_service.rb @@ -42,7 +42,7 @@ module Ci if !db_all_caught_up && !result.build metrics.increment_queue_operation(:queue_replication_lag) - ::Ci::RegisterJobService::Result.new(nil, false) # rubocop:disable Cop/AvoidReturnFromBlocks + ::Ci::RegisterJobService::Result.new(nil, nil, false) # rubocop:disable Cop/AvoidReturnFromBlocks else result end @@ -226,7 +226,7 @@ module Ci log_artifacts_context(build) log_build_dependencies_size(presented_build) - build_json = ::API::Entities::Ci::JobRequest::Response.new(presented_build).to_json + build_json = Gitlab::Json.dump(::API::Entities::Ci::JobRequest::Response.new(presented_build)) Result.new(build, build_json, true) end diff --git a/app/services/ci/retry_job_service.rb b/app/services/ci/retry_job_service.rb index 25bda8a6380..74ebaef48b1 100644 --- a/app/services/ci/retry_job_service.rb +++ b/app/services/ci/retry_job_service.rb @@ -19,7 +19,7 @@ module Ci end # rubocop: disable CodeReuse/ActiveRecord - def clone!(job, variables: []) + def clone!(job, variables: [], enqueue_if_actionable: false) # Cloning a job requires a strict type check to ensure # the attributes being used for the clone are taken straight # from the model and not overridden by other abstractions. @@ -28,6 +28,9 @@ module Ci check_access!(job) new_job = job.clone(current_user: current_user, new_job_variables_attributes: variables) + if Feature.enabled?(:ci_retry_job_fix, project) && enqueue_if_actionable && new_job.action? + new_job.set_enqueue_immediately! + end new_job.run_after_commit do ::Ci::CopyCrossDatabaseAssociationsService.new.execute(job, new_job) @@ -56,13 +59,20 @@ module Ci def check_assignable_runners!(job); end def retry_job(job, variables: []) - clone!(job, variables: variables).tap do |new_job| + clone!(job, variables: variables, enqueue_if_actionable: true).tap do |new_job| check_assignable_runners!(new_job) if new_job.is_a?(Ci::Build) next if new_job.failed? - Gitlab::OptimisticLocking.retry_lock(new_job, name: 'retry_build', &:enqueue) + Gitlab::OptimisticLocking.retry_lock(new_job, name: 'retry_build', &:enqueue) if Feature.disabled?( + :ci_retry_job_fix, project) + AfterRequeueJobService.new(project, current_user).execute(job) + + if Feature.enabled?(:ci_retry_job_fix, project) + Ci::PipelineCreation::StartPipelineService.new(job.pipeline).execute + new_job.reset + end end end diff --git a/app/services/ci/runners/bulk_delete_runners_service.rb b/app/services/ci/runners/bulk_delete_runners_service.rb index ce07aa541c2..b6b07746e61 100644 --- a/app/services/ci/runners/bulk_delete_runners_service.rb +++ b/app/services/ci/runners/bulk_delete_runners_service.rb @@ -7,29 +7,69 @@ module Ci RUNNER_LIMIT = 50 - # @param runners [Array<Ci::Runner, Integer>] the runners to unregister/destroy - def initialize(runners:) + # @param runners [Array<Ci::Runner>] the runners to unregister/destroy + # @param current_user [User] the user performing the operation + def initialize(runners:, current_user:) @runners = runners + @current_user = current_user end def execute if @runners # Delete a few runners immediately - return ServiceResponse.success(payload: delete_runners) + return delete_runners end - ServiceResponse.success(payload: { deleted_count: 0, deleted_ids: [] }) + ServiceResponse.success(payload: { deleted_count: 0, deleted_ids: [], errors: [] }) end private def delete_runners + runner_count = @runners.limit(RUNNER_LIMIT + 1).count + authorized_runners_ids, unauthorized_runners_ids = compute_authorized_runners # rubocop:disable CodeReuse/ActiveRecord - runners_to_be_deleted = Ci::Runner.where(id: @runners).limit(RUNNER_LIMIT) + runners_to_be_deleted = + Ci::Runner + .where(id: authorized_runners_ids) + .preload([:taggings, :runner_namespaces, :runner_projects]) # rubocop:enable CodeReuse/ActiveRecord - deleted_ids = runners_to_be_deleted.destroy_all.map(&:id) # rubocop: disable Cop/DestroyAll + deleted_ids = runners_to_be_deleted.destroy_all.map(&:id) # rubocop:disable Cop/DestroyAll - { deleted_count: deleted_ids.count, deleted_ids: deleted_ids } + ServiceResponse.success( + payload: { + deleted_count: deleted_ids.count, + deleted_ids: deleted_ids, + errors: error_messages(runner_count, authorized_runners_ids, unauthorized_runners_ids) + }) + end + + def compute_authorized_runners + # rubocop:disable CodeReuse/ActiveRecord + @current_user.ci_owned_runners.load # preload the owned runners to avoid an N+1 + authorized_runners, unauthorized_runners = + @runners.limit(RUNNER_LIMIT) + .partition { |runner| Ability.allowed?(@current_user, :delete_runner, runner) } + # rubocop:enable CodeReuse/ActiveRecord + + [authorized_runners.map(&:id), unauthorized_runners.map(&:id)] + end + + def error_messages(runner_count, authorized_runners_ids, unauthorized_runners_ids) + errors = [] + + if runner_count > RUNNER_LIMIT + errors << "Can only delete up to #{RUNNER_LIMIT} runners per call. Ignored the remaining runner(s)." + end + + if authorized_runners_ids.empty? + errors << "User does not have permission to delete any of the runners" + elsif unauthorized_runners_ids.any? + failed_ids = unauthorized_runners_ids.map { |runner_id| "##{runner_id}" }.join(', ') + errors << "User does not have permission to delete runner(s) #{failed_ids}" + end + + errors end end end diff --git a/app/services/ci/runners/set_runner_associated_projects_service.rb b/app/services/ci/runners/set_runner_associated_projects_service.rb index 7930776749d..5e33fdae2f4 100644 --- a/app/services/ci/runners/set_runner_associated_projects_service.rb +++ b/app/services/ci/runners/set_runner_associated_projects_service.rb @@ -17,7 +17,7 @@ module Ci return ServiceResponse.error(message: 'user not allowed to assign runner', http_status: :forbidden) end - return ServiceResponse.success if project_ids.blank? + return ServiceResponse.success if project_ids.nil? set_associated_projects end |