diff options
Diffstat (limited to 'app/models/ci/build.rb')
| -rw-r--r-- | app/models/ci/build.rb | 216 |
1 files changed, 151 insertions, 65 deletions
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 1e066b69c6e..41446946a5e 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -3,8 +3,10 @@ module Ci prepend ArtifactMigratable include TokenAuthenticatable include AfterCommitQueue + include ObjectStorage::BackgroundMove include Presentable include Importable + include Gitlab::Utils::StrongMemoize MissingDependenciesError = Class.new(StandardError) @@ -17,18 +19,26 @@ module Ci has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment' has_many :trace_sections, class_name: 'Ci::BuildTraceSection' + has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id - has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id - # The "environment" field for builds is a String, and is the unexpanded name + has_one :metadata, class_name: 'Ci::BuildMetadata' + delegate :timeout, to: :metadata, prefix: true, allow_nil: true + delegate :gitlab_deploy_token, to: :project + + ## + # The "environment" field for builds is a String, and is the unexpanded name! + # def persisted_environment - @persisted_environment ||= Environment.find_by( - name: expanded_environment_name, - project: project - ) + return unless has_environment? + + strong_memoize(:persisted_environment) do + Environment.find_by(name: expanded_environment_name, project: project) + end end serialize :options # rubocop:disable Cop/ActiveRecordSerialize @@ -45,11 +55,18 @@ module Ci where('(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)', '', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').archive) end + + scope :without_archived_trace, ->() do + where('NOT EXISTS (?)', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').trace) + end + + scope :with_artifacts_stored_locally, -> { with_artifacts_archive.where(artifacts_file_store: [nil, LegacyArtifactUploader::Store::LOCAL]) } scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) } scope :ref_protected, -> { where(protected: true) } + scope :with_live_trace, -> { where('EXISTS (?)', Ci::BuildTraceChunk.where('ci_builds.id = ci_build_trace_chunks.build_id').select(1)) } scope :matches_tag_ids, -> (tag_ids) do matcher = ::ActsAsTaggableOn::Tagging @@ -81,12 +98,13 @@ module Ci before_save :ensure_token before_destroy { unscoped_project } + before_create :ensure_metadata after_create unless: :importing? do |build| run_after_commit { BuildHooksWorker.perform_async(build.id) } end - after_commit :update_project_statistics_after_save, on: [:create, :update] - after_commit :update_project_statistics, on: :destroy + after_save :update_project_statistics_after_save, if: :artifacts_size_changed? + after_destroy :update_project_statistics_after_destroy, unless: :project_destroyed? class << self # This is needed for url_for to work, @@ -132,6 +150,7 @@ module Ci after_transition any => [:success] do |build| build.run_after_commit do BuildSuccessWorker.perform_async(id) + PagesWorker.perform_async(:deploy, id) if build.pages_generator? end end @@ -151,6 +170,14 @@ module Ci before_transition any => [:running] do |build| build.validates_dependencies! unless Feature.enabled?('ci_disable_validates_dependencies') end + + after_transition pending: :running do |build| + build.ensure_metadata.update_timeout_state + end + end + + def ensure_metadata + metadata || build_metadata(project: project) end def detailed_status(current_user) @@ -163,8 +190,13 @@ module Ci pipeline.manual_actions.where.not(name: name) end + def pages_generator? + Gitlab.config.pages.enabled && + self.name == 'pages' + end + def playable? - action? && (manual? || complete?) + action? && (manual? || retryable?) end def action? @@ -198,7 +230,11 @@ module Ci end def expanded_environment_name - ExpandVariables.expand(environment, simple_variables) if environment + return unless has_environment? + + strong_memoize(:expanded_environment_name) do + ExpandVariables.expand(environment, simple_variables) + end end def has_environment? @@ -229,10 +265,6 @@ module Ci latest_builds.where('stage_idx < ?', stage_idx) end - def timeout - project.build_timeout - end - def triggered_by?(current_user) user == current_user end @@ -248,31 +280,52 @@ module Ci Gitlab::Utils.slugify(ref.to_s) end - # Variables whose value does not depend on environment - def simple_variables - variables(environment: nil) - end - - # All variables, including those dependent on environment, which could - # contain unexpanded variables. - def variables(environment: persisted_environment) - collection = Gitlab::Ci::Variables::Collection.new.tap do |variables| + ## + # Variables in the environment name scope. + # + def scoped_variables(environment: expanded_environment_name) + Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.concat(predefined_variables) variables.concat(project.predefined_variables) variables.concat(pipeline.predefined_variables) variables.concat(runner.predefined_variables) if runner - variables.concat(project.deployment_variables(environment: environment)) if has_environment? + variables.concat(project.deployment_variables(environment: environment)) if environment variables.concat(yaml_variables) variables.concat(user_variables) - variables.concat(project.group.secret_variables_for(ref, project)) if project.group - variables.concat(secret_variables(environment: environment)) + variables.concat(secret_group_variables) + variables.concat(secret_project_variables(environment: environment)) variables.concat(trigger_request.user_variables) if trigger_request variables.concat(pipeline.variables) variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline.pipeline_schedule - variables.concat(persisted_environment_variables) if environment end + end - collection.to_runner_variables + ## + # Variables that do not depend on the environment name. + # + def simple_variables + strong_memoize(:simple_variables) do + scoped_variables(environment: nil).to_runner_variables + end + end + + ## + # All variables, including persisted environment variables. + # + def variables + Gitlab::Ci::Variables::Collection.new + .concat(persisted_variables) + .concat(scoped_variables) + .concat(persisted_environment_variables) + .to_runner_variables + end + + ## + # Regular Ruby hash of scoped variables, without duplicates that are + # possible to be present in an array of hashes returned from `variables`. + # + def scoped_variables_hash + scoped_variables.to_hash end def features @@ -361,17 +414,21 @@ module Ci build_data = Gitlab::DataBuilder::Build.build(self) project.execute_hooks(build_data.dup, :job_hooks) project.execute_services(build_data.dup, :job_hooks) - PagesService.new(build_data).execute - project.running_or_pending_build_count(force: true) + end + + def browsable_artifacts? + artifacts_metadata? end def artifacts_metadata_entry(path, **options) - metadata = Gitlab::Ci::Build::Artifacts::Metadata.new( - artifacts_metadata.path, - path, - **options) + artifacts_metadata.use_file do |metadata_path| + metadata = Gitlab::Ci::Build::Artifacts::Metadata.new( + metadata_path, + path, + **options) - metadata.to_entry + metadata.to_entry + end end def erase_artifacts! @@ -434,7 +491,7 @@ module Ci def user_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| - return variables if user.blank? + break variables if user.blank? variables.append(key: 'GITLAB_USER_ID', value: user.id.to_s) variables.append(key: 'GITLAB_USER_EMAIL', value: user.email) @@ -443,9 +500,14 @@ module Ci end end - def secret_variables(environment: persisted_environment) + def secret_group_variables + return [] unless project.group + + project.group.secret_variables_for(ref, project) + end + + def secret_project_variables(environment: persisted_environment) project.secret_variables_for(ref: ref, environment: environment) - .map(&:to_runner_variable) end def steps @@ -542,24 +604,37 @@ module Ci CI_REGISTRY_USER = 'gitlab-ci-token'.freeze + def persisted_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + break variables unless persisted? + + variables + .concat(pipeline.persisted_variables) + .append(key: 'CI_JOB_ID', value: id.to_s) + .append(key: 'CI_JOB_URL', value: Gitlab::Routing.url_helpers.project_job_url(project, self)) + .append(key: 'CI_JOB_TOKEN', value: token, public: false) + .append(key: 'CI_BUILD_ID', value: id.to_s) + .append(key: 'CI_BUILD_TOKEN', value: token, public: false) + .append(key: 'CI_REGISTRY_USER', value: CI_REGISTRY_USER) + .append(key: 'CI_REGISTRY_PASSWORD', value: token, public: false) + .append(key: 'CI_REPOSITORY_URL', value: repo_url, public: false) + .concat(deploy_token_variables) + end + end + def predefined_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'CI', value: 'true') variables.append(key: 'GITLAB_CI', value: 'true') - variables.append(key: 'GITLAB_FEATURES', value: project.namespace.features.join(',')) + variables.append(key: 'GITLAB_FEATURES', value: project.licensed_features.join(',')) variables.append(key: 'CI_SERVER_NAME', value: 'GitLab') variables.append(key: 'CI_SERVER_VERSION', value: Gitlab::VERSION) - variables.append(key: 'CI_SERVER_REVISION', value: Gitlab::REVISION) - variables.append(key: 'CI_JOB_ID', value: id.to_s) + variables.append(key: 'CI_SERVER_REVISION', value: Gitlab.revision) variables.append(key: 'CI_JOB_NAME', value: name) variables.append(key: 'CI_JOB_STAGE', value: stage) - variables.append(key: 'CI_JOB_TOKEN', value: token, public: false) variables.append(key: 'CI_COMMIT_SHA', value: sha) variables.append(key: 'CI_COMMIT_REF_NAME', value: ref) variables.append(key: 'CI_COMMIT_REF_SLUG', value: ref_slug) - variables.append(key: 'CI_REGISTRY_USER', value: CI_REGISTRY_USER) - variables.append(key: 'CI_REGISTRY_PASSWORD', value: token, public: false) - variables.append(key: 'CI_REPOSITORY_URL', value: repo_url, public: false) variables.append(key: "CI_COMMIT_TAG", value: ref) if tag? variables.append(key: "CI_PIPELINE_TRIGGERED", value: 'true') if trigger_request variables.append(key: "CI_JOB_MANUAL", value: 'true') if action? @@ -567,9 +642,23 @@ module Ci end end + def legacy_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'CI_BUILD_REF', value: sha) + variables.append(key: 'CI_BUILD_BEFORE_SHA', value: before_sha) + variables.append(key: 'CI_BUILD_REF_NAME', value: ref) + variables.append(key: 'CI_BUILD_REF_SLUG', value: ref_slug) + variables.append(key: 'CI_BUILD_NAME', value: name) + variables.append(key: 'CI_BUILD_STAGE', value: stage) + variables.append(key: "CI_BUILD_TAG", value: ref) if tag? + variables.append(key: "CI_BUILD_TRIGGERED", value: 'true') if trigger_request + variables.append(key: "CI_BUILD_MANUAL", value: 'true') if action? + end + end + def persisted_environment_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| - return variables unless persisted_environment + break variables unless persisted? && persisted_environment.present? variables.concat(persisted_environment.predefined_variables) @@ -580,19 +669,12 @@ module Ci end end - def legacy_variables + def deploy_token_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| - variables.append(key: 'CI_BUILD_ID', value: id.to_s) - variables.append(key: 'CI_BUILD_TOKEN', value: token, public: false) - variables.append(key: 'CI_BUILD_REF', value: sha) - variables.append(key: 'CI_BUILD_BEFORE_SHA', value: before_sha) - variables.append(key: 'CI_BUILD_REF_NAME', value: ref) - variables.append(key: 'CI_BUILD_REF_SLUG', value: ref_slug) - variables.append(key: 'CI_BUILD_NAME', value: name) - variables.append(key: 'CI_BUILD_STAGE', value: stage) - variables.append(key: "CI_BUILD_TAG", value: ref) if tag? - variables.append(key: "CI_BUILD_TRIGGERED", value: 'true') if trigger_request - variables.append(key: "CI_BUILD_MANUAL", value: 'true') if action? + break variables unless gitlab_deploy_token + + variables.append(key: 'CI_DEPLOY_USER', value: gitlab_deploy_token.username) + variables.append(key: 'CI_DEPLOY_PASSWORD', value: gitlab_deploy_token.token, public: false) end end @@ -606,16 +688,20 @@ module Ci pipeline.config_processor.build_attributes(name) end - def update_project_statistics - return unless project + def update_project_statistics_after_save + update_project_statistics(read_attribute(:artifacts_size).to_i - artifacts_size_was.to_i) + end - ProjectCacheWorker.perform_async(project_id, [], [:build_artifacts_size]) + def update_project_statistics_after_destroy + update_project_statistics(-artifacts_size) end - def update_project_statistics_after_save - if previous_changes.include?('artifacts_size') - update_project_statistics - end + def update_project_statistics(difference) + ProjectStatistics.increment_statistic(project_id, :build_artifacts_size, difference) + end + + def project_destroyed? + project.pending_delete? end end end |