diff options
Diffstat (limited to 'app/services/ci')
11 files changed, 66 insertions, 36 deletions
diff --git a/app/services/ci/after_requeue_job_service.rb b/app/services/ci/after_requeue_job_service.rb index 634c547a623..9d54207d75d 100644 --- a/app/services/ci/after_requeue_job_service.rb +++ b/app/services/ci/after_requeue_job_service.rb @@ -26,7 +26,7 @@ module Ci return legacy_dependent_jobs unless ::Feature.enabled?(:ci_requeue_with_dag_object_hierarchy, project) ordered_by_dag( - ::Ci::Processable + @processable.pipeline.processables .from_union(needs_dependent_jobs, stage_dependent_jobs) .skipped .ordered_by_stage diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index af175b8da1c..0b49beffcb5 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -26,6 +26,7 @@ module Ci Gitlab::Ci::Pipeline::Chain::AssignPartition, Gitlab::Ci::Pipeline::Chain::Seed, Gitlab::Ci::Pipeline::Chain::Limit::Size, + Gitlab::Ci::Pipeline::Chain::Limit::ActiveJobs, Gitlab::Ci::Pipeline::Chain::Limit::Deployments, Gitlab::Ci::Pipeline::Chain::Validate::External, Gitlab::Ci::Pipeline::Chain::Populate, @@ -36,7 +37,6 @@ module Ci Gitlab::Ci::Pipeline::Chain::CreateDeployments, Gitlab::Ci::Pipeline::Chain::CreateCrossDatabaseAssociations, Gitlab::Ci::Pipeline::Chain::Limit::Activity, - Gitlab::Ci::Pipeline::Chain::Limit::JobActivity, Gitlab::Ci::Pipeline::Chain::CancelPendingPipelines, Gitlab::Ci::Pipeline::Chain::Metrics, Gitlab::Ci::Pipeline::Chain::TemplateUsage, @@ -140,7 +140,7 @@ module Ci end def create_namespace_onboarding_action - Namespaces::OnboardingPipelineCreatedWorker.perform_async(project.namespace_id) + Onboarding::PipelineCreatedWorker.perform_async(project.namespace_id) end def extra_options(content: nil, dry_run: false) diff --git a/app/services/ci/generate_kubeconfig_service.rb b/app/services/ci/generate_kubeconfig_service.rb index 894ab8e8505..347bc99dbf5 100644 --- a/app/services/ci/generate_kubeconfig_service.rb +++ b/app/services/ci/generate_kubeconfig_service.rb @@ -14,7 +14,8 @@ module Ci url: Gitlab::Kas.tunnel_url ) - agents.each do |agent| + agent_authorizations.each do |authorization| + agent = authorization.agent user = user_name(agent) template.add_user( @@ -24,6 +25,7 @@ module Ci template.add_context( name: context_name(agent), + namespace: context_namespace(authorization), cluster: cluster_name, user: user ) @@ -36,8 +38,8 @@ module Ci attr_reader :pipeline, :token, :template - def agents - pipeline.authorized_cluster_agents + def agent_authorizations + pipeline.cluster_agent_authorizations end def cluster_name @@ -52,6 +54,10 @@ module Ci [agent.project.full_path, agent.name].join(delimiter) end + def context_namespace(authorization) + authorization.config['default_namespace'] + end + def agent_token(agent) ['ci', agent.id, token].join(delimiter) end diff --git a/app/services/ci/job_artifacts/delete_service.rb b/app/services/ci/job_artifacts/delete_service.rb index 65cae03312e..c9d590eccc4 100644 --- a/app/services/ci/job_artifacts/delete_service.rb +++ b/app/services/ci/job_artifacts/delete_service.rb @@ -15,13 +15,23 @@ module Ci method: 'Ci::JobArtifacts::DeleteService#execute', project_id: build.project_id ) + return ServiceResponse.error( + message: 'Action temporarily disabled. The project this job belongs to is undergoing stats refresh.', + reason: :project_stats_refresh + ) end - # fix_expire_at is false because in this case we want to explicitly delete the job artifacts - # this flag is a workaround that will be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/355833 - Ci::JobArtifacts::DestroyBatchService.new(build.job_artifacts.erasable, fix_expire_at: false).execute + result = Ci::JobArtifacts::DestroyBatchService.new(build.job_artifacts.erasable).execute - ServiceResponse.success + if result.fetch(:status) == :success + ServiceResponse.success(payload: + { + destroyed_artifacts_count: result.fetch(:destroyed_artifacts_count), + statistics_updates: result.fetch(:statistics_updates) + }) + else + ServiceResponse.error(message: result.fetch(:message)) + end end private diff --git a/app/services/ci/parse_dotenv_artifact_service.rb b/app/services/ci/parse_dotenv_artifact_service.rb index fd13ed245cf..14e8dc41cf5 100644 --- a/app/services/ci/parse_dotenv_artifact_service.rb +++ b/app/services/ci/parse_dotenv_artifact_service.rb @@ -40,7 +40,7 @@ module Ci key, value = scan_line!(line) variables[key] = Ci::JobVariable.new(job_id: artifact.job_id, - source: :dotenv, key: key, value: value) + source: :dotenv, key: key, value: value, raw: false) end end diff --git a/app/services/ci/pipeline_artifacts/coverage_report_service.rb b/app/services/ci/pipeline_artifacts/coverage_report_service.rb index 99877603554..9c6fdb7a405 100644 --- a/app/services/ci/pipeline_artifacts/coverage_report_service.rb +++ b/app/services/ci/pipeline_artifacts/coverage_report_service.rb @@ -27,18 +27,13 @@ module Ci end def pipeline_artifact_params - attributes = { + { pipeline: pipeline, file_type: :code_coverage, file: carrierwave_file, - size: carrierwave_file['tempfile'].size + size: carrierwave_file['tempfile'].size, + locked: pipeline.locked } - - if ::Feature.enabled?(:ci_update_unlocked_pipeline_artifacts, pipeline.project) - attributes[:locked] = pipeline.locked - end - - attributes end def carrierwave_file diff --git a/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb b/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb index aeb68a75f88..a0746ef32b2 100644 --- a/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb +++ b/app/services/ci/pipeline_artifacts/create_code_quality_mr_diff_report_service.rb @@ -23,20 +23,15 @@ module Ci def artifact_attributes file = build_carrierwave_file! - attributes = { + { project_id: pipeline.project_id, file_type: :code_quality_mr_diff, file_format: Ci::PipelineArtifact::REPORT_TYPES.fetch(:code_quality_mr_diff), size: file["tempfile"].size, file: file, - expire_at: Ci::PipelineArtifact::EXPIRATION_DATE.from_now + expire_at: Ci::PipelineArtifact::EXPIRATION_DATE.from_now, + locked: pipeline.locked } - - if ::Feature.enabled?(:ci_update_unlocked_pipeline_artifacts, pipeline.project) - attributes[:locked] = pipeline.locked - end - - attributes end def merge_requests diff --git a/app/services/ci/pipeline_artifacts/destroy_all_expired_service.rb b/app/services/ci/pipeline_artifacts/destroy_all_expired_service.rb index 17c039885e5..8dddf3c3f6c 100644 --- a/app/services/ci/pipeline_artifacts/destroy_all_expired_service.rb +++ b/app/services/ci/pipeline_artifacts/destroy_all_expired_service.rb @@ -3,20 +3,26 @@ module Ci module PipelineArtifacts class DestroyAllExpiredService + include ::Gitlab::ExclusiveLeaseHelpers include ::Gitlab::LoopHelpers include ::Gitlab::Utils::StrongMemoize BATCH_SIZE = 100 - LOOP_TIMEOUT = 5.minutes LOOP_LIMIT = 1000 + LOOP_TIMEOUT = 5.minutes + LOCK_TIMEOUT = 10.minutes + EXCLUSIVE_LOCK_KEY = 'expired_pipeline_artifacts:destroy:lock' def initialize @removed_artifacts_count = 0 + @start_at = Time.current end def execute - loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do - destroy_artifacts_batch + in_lock(EXCLUSIVE_LOCK_KEY, ttl: LOCK_TIMEOUT, retries: 1) do + destroy_unlocked_pipeline_artifacts + + legacy_destroy_pipeline_artifacts end @removed_artifacts_count @@ -24,10 +30,30 @@ module Ci private + def destroy_unlocked_pipeline_artifacts + loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do + artifacts = Ci::PipelineArtifact.expired_before(@start_at).artifact_unlocked.limit(BATCH_SIZE) + + break if artifacts.empty? + + destroy_batch(artifacts) + end + end + + def legacy_destroy_pipeline_artifacts + loop_until(timeout: LOOP_TIMEOUT, limit: LOOP_LIMIT) do + destroy_artifacts_batch + end + end + def destroy_artifacts_batch artifacts = ::Ci::PipelineArtifact.unlocked.expired.limit(BATCH_SIZE).to_a return false if artifacts.empty? + destroy_batch(artifacts) + end + + def destroy_batch(artifacts) artifacts.each(&:destroy!) increment_stats(artifacts.size) diff --git a/app/services/ci/process_build_service.rb b/app/services/ci/process_build_service.rb index e6ec65fcc91..22cd267806d 100644 --- a/app/services/ci/process_build_service.rb +++ b/app/services/ci/process_build_service.rb @@ -25,6 +25,8 @@ module Ci end def enqueue(build) + return build.drop!(:failed_outdated_deployment_job) if build.prevent_rollback_deployment? + build.enqueue end diff --git a/app/services/ci/runners/register_runner_service.rb b/app/services/ci/runners/register_runner_service.rb index ae9b8bc8a16..abd32610cec 100644 --- a/app/services/ci/runners/register_runner_service.rb +++ b/app/services/ci/runners/register_runner_service.rb @@ -59,7 +59,7 @@ module Ci end def runner_registrar_valid?(type) - Feature.disabled?(:runner_registration_control) || Gitlab::CurrentSettings.valid_runner_registrars.include?(type) + Gitlab::CurrentSettings.valid_runner_registrars.include?(type) end def token_scope diff --git a/app/services/ci/unlock_artifacts_service.rb b/app/services/ci/unlock_artifacts_service.rb index 1fee31da4fc..574cdae6480 100644 --- a/app/services/ci/unlock_artifacts_service.rb +++ b/app/services/ci/unlock_artifacts_service.rb @@ -11,8 +11,6 @@ module Ci unlocked_pipeline_artifacts: 0 } - unlock_pipeline_artifacts_enabled = ::Feature.enabled?(:ci_update_unlocked_pipeline_artifacts, ci_ref.project) - if ::Feature.enabled?(:ci_update_unlocked_job_artifacts, ci_ref.project) loop do unlocked_pipelines = [] @@ -22,9 +20,7 @@ module Ci unlocked_pipelines = unlock_pipelines(ci_ref, before_pipeline) unlocked_job_artifacts = unlock_job_artifacts(unlocked_pipelines) - if unlock_pipeline_artifacts_enabled - results[:unlocked_pipeline_artifacts] += unlock_pipeline_artifacts(unlocked_pipelines) - end + results[:unlocked_pipeline_artifacts] += unlock_pipeline_artifacts(unlocked_pipelines) end break if unlocked_pipelines.empty? |