diff options
30 files changed, 538 insertions, 17 deletions
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index b8068a22c02..21dbbae1747 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -23,6 +23,7 @@ module Ci belongs_to :runner belongs_to :trigger_request belongs_to :erased_by, class_name: 'User' + belongs_to :resource_group, class_name: 'Ci::ResourceGroup', inverse_of: :builds RUNNER_FEATURES = { upload_multiple_artifacts: -> (build) { build.publishes_artifacts_reports? }, @@ -34,6 +35,7 @@ module Ci }.freeze has_one :deployment, as: :deployable, class_name: 'Deployment' + has_one :resource, class_name: 'Ci::Resource', inverse_of: :build has_many :trace_sections, class_name: 'Ci::BuildTraceSection' has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id @@ -441,6 +443,15 @@ module Ci environment.present? end + def requires_resource? + Feature.enabled?(:ci_resource_group, project) && + self.resource_group_id.present? && resource.nil? + end + + def retains_resource? + self.resource_group_id.present? && resource.present? + end + def starts_environment? has_environment? && self.environment_action == 'start' end diff --git a/app/models/ci/resource.rb b/app/models/ci/resource.rb new file mode 100644 index 00000000000..ee5b6546165 --- /dev/null +++ b/app/models/ci/resource.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Ci + class Resource < ApplicationRecord + extend Gitlab::Ci::Model + + belongs_to :resource_group, class_name: 'Ci::ResourceGroup', inverse_of: :resources + belongs_to :build, class_name: 'Ci::Build', inverse_of: :resource + + scope :free, -> { where(build: nil) } + scope :retained_by, -> (build) { where(build: build) } + end +end diff --git a/app/models/ci/resource_group.rb b/app/models/ci/resource_group.rb new file mode 100644 index 00000000000..fb562783d3d --- /dev/null +++ b/app/models/ci/resource_group.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Ci + class ResourceGroup < ApplicationRecord + extend Gitlab::Ci::Model + + belongs_to :project, inverse_of: :resource_groups + + has_many :resources, class_name: 'Ci::Resource', inverse_of: :resource_group + has_many :builds, class_name: 'Ci::Build', inverse_of: :resource_group + + validates :key, + length: { maximum: 255 }, + format: { with: Gitlab::Regex.environment_name_regex, + message: Gitlab::Regex.environment_name_regex_message } + + before_create :ensure_resource + + def retain_resource_for(build) + resources.free.limit(1).update_all(build_id: build.id) > 0 + end + + def release_resource_from(build) + resources.retained_by(build).update_all(build_id: nil) > 0 + end + + private + + def ensure_resource + # Currently we only support one resource per group, which means + # maximum one build can be set to the resource group, thus builds + # belong to the same resource group are executed once at time. + self.resources.build if self.resources.empty? + end + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 84289e1d5ac..eacaaac8a6c 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -285,6 +285,7 @@ class Project < ApplicationRecord has_many :pipeline_schedules, class_name: 'Ci::PipelineSchedule' has_many :project_deploy_tokens has_many :deploy_tokens, through: :project_deploy_tokens + has_many :resource_groups, class_name: 'Ci::ResourceGroup', inverse_of: :project has_one :auto_devops, class_name: 'ProjectAutoDevops', inverse_of: :project, autosave: true has_many :custom_attributes, class_name: 'ProjectCustomAttribute' diff --git a/app/services/ci/retry_build_service.rb b/app/services/ci/retry_build_service.rb index 7a5e33c61ba..5abfbd26641 100644 --- a/app/services/ci/retry_build_service.rb +++ b/app/services/ci/retry_build_service.rb @@ -5,7 +5,7 @@ module Ci CLONE_ACCESSORS = %i[pipeline project ref tag options name allow_failure stage stage_id stage_idx trigger_request yaml_variables when environment coverage_regex - description tag_list protected needs].freeze + description tag_list protected needs resource_group].freeze def execute(build) reprocess!(build).tap do |new_build| diff --git a/changelogs/unreleased/ci-resource-group-model.yml b/changelogs/unreleased/ci-resource-group-model.yml new file mode 100644 index 00000000000..98bc0159626 --- /dev/null +++ b/changelogs/unreleased/ci-resource-group-model.yml @@ -0,0 +1,5 @@ +--- +title: Add Ci Resource Group models +merge_request: 20950 +author: +type: other diff --git a/db/migrate/20191128145231_add_ci_resource_groups.rb b/db/migrate/20191128145231_add_ci_resource_groups.rb new file mode 100644 index 00000000000..0b730e47dda --- /dev/null +++ b/db/migrate/20191128145231_add_ci_resource_groups.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class AddCiResourceGroups < ActiveRecord::Migration[5.2] + DOWNTIME = false + + def change + create_table :ci_resource_groups do |t| + t.timestamps_with_timezone + t.references :project, null: false, index: false, foreign_key: { on_delete: :cascade } + t.string :key, null: false, limit: 255 + t.index %i[project_id key], unique: true + end + + create_table :ci_resources do |t| + t.timestamps_with_timezone + t.references :resource_group, null: false, index: false, foreign_key: { to_table: :ci_resource_groups, on_delete: :cascade } + t.references :build, null: true, index: true, foreign_key: { to_table: :ci_builds, on_delete: :nullify } + t.index %i[resource_group_id build_id], unique: true + end + + add_column :ci_builds, :resource_group_id, :bigint + add_column :ci_builds, :waiting_for_resource_at, :datetime_with_timezone + end +end diff --git a/db/migrate/20191129144631_add_index_to_resource_group_id.rb b/db/migrate/20191129144631_add_index_to_resource_group_id.rb new file mode 100644 index 00000000000..0e5a84f094d --- /dev/null +++ b/db/migrate/20191129144631_add_index_to_resource_group_id.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddIndexToResourceGroupId < ActiveRecord::Migration[5.2] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'index_for_resource_group'.freeze + + disable_ddl_transaction! + + def up + add_concurrent_index :ci_builds, %i[resource_group_id id], where: 'resource_group_id IS NOT NULL', name: INDEX_NAME + add_concurrent_foreign_key :ci_builds, :ci_resource_groups, column: :resource_group_id, on_delete: :nullify + end + + def down + remove_foreign_key_if_exists :ci_builds, column: :resource_group_id + remove_concurrent_index_by_name :ci_builds, INDEX_NAME + end +end diff --git a/db/schema.rb b/db/schema.rb index a369db5e4df..5aa13fa8c72 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -683,6 +683,8 @@ ActiveRecord::Schema.define(version: 2019_12_08_071112) do t.datetime_with_timezone "scheduled_at" t.string "token_encrypted" t.integer "upstream_pipeline_id" + t.bigint "resource_group_id" + t.datetime_with_timezone "waiting_for_resource_at" t.index ["artifacts_expire_at"], name: "index_ci_builds_on_artifacts_expire_at", where: "(artifacts_file <> ''::text)" t.index ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id" t.index ["commit_id", "artifacts_expire_at", "id"], name: "index_ci_builds_on_commit_id_and_artifacts_expireatandidpartial", where: "(((type)::text = 'Ci::Build'::text) AND ((retried = false) OR (retried IS NULL)) AND ((name)::text = ANY (ARRAY[('sast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('sast:container'::character varying)::text, ('container_scanning'::character varying)::text, ('dast'::character varying)::text])))" @@ -696,6 +698,7 @@ ActiveRecord::Schema.define(version: 2019_12_08_071112) do t.index ["project_id"], name: "index_ci_builds_on_project_id_for_successfull_pages_deploy", where: "(((type)::text = 'GenericCommitStatus'::text) AND ((stage)::text = 'deploy'::text) AND ((name)::text = 'pages:deploy'::text) AND ((status)::text = 'success'::text))" t.index ["protected"], name: "index_ci_builds_on_protected" t.index ["queued_at"], name: "index_ci_builds_on_queued_at" + t.index ["resource_group_id", "id"], name: "index_for_resource_group", where: "(resource_group_id IS NOT NULL)" t.index ["runner_id"], name: "index_ci_builds_on_runner_id" t.index ["scheduled_at"], name: "partial_index_ci_builds_on_scheduled_at_with_scheduled_jobs", where: "((scheduled_at IS NOT NULL) AND ((type)::text = 'Ci::Build'::text) AND ((status)::text = 'scheduled'::text))" t.index ["stage_id", "stage_idx"], name: "tmp_build_stage_position_index", where: "(stage_idx IS NOT NULL)" @@ -870,6 +873,23 @@ ActiveRecord::Schema.define(version: 2019_12_08_071112) do t.index ["user_id"], name: "index_ci_pipelines_on_user_id" end + create_table "ci_resource_groups", force: :cascade do |t| + t.datetime_with_timezone "created_at", null: false + t.datetime_with_timezone "updated_at", null: false + t.bigint "project_id", null: false + t.string "key", limit: 255, null: false + t.index ["project_id", "key"], name: "index_ci_resource_groups_on_project_id_and_key", unique: true + end + + create_table "ci_resources", force: :cascade do |t| + t.datetime_with_timezone "created_at", null: false + t.datetime_with_timezone "updated_at", null: false + t.bigint "resource_group_id", null: false + t.bigint "build_id" + t.index ["build_id"], name: "index_ci_resources_on_build_id" + t.index ["resource_group_id", "build_id"], name: "index_ci_resources_on_resource_group_id_and_build_id", unique: true + end + create_table "ci_runner_namespaces", id: :serial, force: :cascade do |t| t.integer "runner_id" t.integer "namespace_id" @@ -4364,6 +4384,7 @@ ActiveRecord::Schema.define(version: 2019_12_08_071112) do add_foreign_key "ci_builds", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_a2141b1522", on_delete: :nullify add_foreign_key "ci_builds", "ci_pipelines", column: "commit_id", name: "fk_d3130c9a7f", on_delete: :cascade add_foreign_key "ci_builds", "ci_pipelines", column: "upstream_pipeline_id", name: "fk_87f4cefcda", on_delete: :cascade + add_foreign_key "ci_builds", "ci_resource_groups", column: "resource_group_id", name: "fk_6661f4f0e8", on_delete: :nullify add_foreign_key "ci_builds", "ci_stages", column: "stage_id", name: "fk_3a9eaa254d", on_delete: :cascade add_foreign_key "ci_builds", "projects", name: "fk_befce0568a", on_delete: :cascade add_foreign_key "ci_builds_metadata", "ci_builds", column: "build_id", on_delete: :cascade @@ -4384,6 +4405,9 @@ ActiveRecord::Schema.define(version: 2019_12_08_071112) do add_foreign_key "ci_pipelines", "external_pull_requests", name: "fk_190998ef09", on_delete: :nullify add_foreign_key "ci_pipelines", "merge_requests", name: "fk_a23be95014", on_delete: :cascade add_foreign_key "ci_pipelines", "projects", name: "fk_86635dbd80", on_delete: :cascade + add_foreign_key "ci_resource_groups", "projects", on_delete: :cascade + add_foreign_key "ci_resources", "ci_builds", column: "build_id", on_delete: :nullify + add_foreign_key "ci_resources", "ci_resource_groups", column: "resource_group_id", on_delete: :cascade add_foreign_key "ci_runner_namespaces", "ci_runners", column: "runner_id", on_delete: :cascade add_foreign_key "ci_runner_namespaces", "namespaces", on_delete: :cascade add_foreign_key "ci_runner_projects", "projects", name: "fk_4478a6f1e4", on_delete: :cascade diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index c352a5a4fd6..cdd8cecbf8e 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -1024,6 +1024,33 @@ Additional job configuration may be added to rules in the future. If something useful isn't available, please [open an issue](https://gitlab.com/gitlab-org/gitlab/issues). +### `workflow:rules` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/29654) in GitLab 12.5 + +The top-level `workflow:` key applies to the entirety of a pipeline, and will +determine whether or not a pipeline is created. It currently accepts a single +`rules:` key that operates similarly to [`rules:` defined within jobs](#rules), +enabling dynamic configuration of the pipeline. + +The configuration options currently available for `workflow:rules` are: + +- [`if`](#rulesif): Define a rule. +- [`when`](#when): May be set to `always` or `never` only. If not provided, the default value is `always`. + +The list of `if` rules is evaluated until a single one is matched. If none +match, the last `when` will be used: + +```yaml +workflow: + rules: + - if: $CI_COMMIT_REF_NAME =~ /-wip$/ + when: never + - if: $CI_COMMIT_TAG + when: never + - when: always +``` + ### `tags` `tags` is used to select specific Runners from the list of all Runners that are diff --git a/doc/development/rake_tasks.md b/doc/development/rake_tasks.md index ae2dbc5c340..ff978ee2899 100644 --- a/doc/development/rake_tasks.md +++ b/doc/development/rake_tasks.md @@ -93,9 +93,9 @@ There are a few environment flags you can pass to change how projects are seeded In order to run the test you can use the following commands: - `bin/rake spec` to run the rspec suite -- `bin/rake spec:unit` to run the only the unit tests -- `bin/rake spec:integration` to run the only the integration tests -- `bin/rake spec:system` to run the only the system tests +- `bin/rake spec:unit` to run only the unit tests +- `bin/rake spec:integration` to run only the integration tests +- `bin/rake spec:system` to run only the system tests - `bin/rake karma` to run the Karma test suite Note: `bin/rake spec` takes significant time to pass. @@ -188,11 +188,10 @@ task, then check the dimensions of the new spritesheet and update the ## Updating project templates Starting a project from a template needs this project to be exported. On a -up to date master branch with run: +up to date master branch run: ``` -gdk run -# In a new terminal window +gdk start bundle exec rake gitlab:update_project_templates git checkout -b update-project-templates git add vendor/project_templates diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb index 4f503377a77..625db1fce32 100644 --- a/lib/feature/gitaly.rb +++ b/lib/feature/gitaly.rb @@ -8,7 +8,6 @@ class Feature SERVER_FEATURE_FLAGS = %w[ inforef_uploadpack_cache - get_all_lfs_pointers_go get_tag_messages_go filter_shas_with_signatures_go ].freeze diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index eea59ecb937..860ceae2ab8 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -16,7 +16,8 @@ module Gitlab ALLOWED_KEYS = %i[tags script only except rules type image services allow_failure type stage when start_in artifacts cache dependencies before_script needs after_script variables - environment coverage retry parallel extends interruptible timeout].freeze + environment coverage retry parallel extends interruptible timeout + resource_group].freeze REQUIRED_BY_NEEDS = %i[stage].freeze @@ -48,6 +49,7 @@ module Gitlab validates :dependencies, array_of_strings: true validates :extends, array_of_strings_or_string: true validates :rules, array_of_hashes: true + validates :resource_group, type: String end validates :start_in, duration: { limit: '1 week' }, if: :delayed? @@ -156,7 +158,7 @@ module Gitlab attributes :script, :tags, :allow_failure, :when, :dependencies, :needs, :retry, :parallel, :extends, :start_in, :rules, - :interruptible, :timeout + :interruptible, :timeout, :resource_group def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -236,7 +238,8 @@ module Gitlab artifacts: artifacts_value, after_script: after_script_value, ignore: ignored?, - needs: needs_defined? ? needs_value : nil } + needs: needs_defined? ? needs_value : nil, + resource_group: resource_group } end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 590c7f4d1dd..98b4b4593e0 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -18,6 +18,7 @@ module Gitlab @seed_attributes = attributes @previous_stages = previous_stages @needs_attributes = dig(:needs_attributes) + @resource_group_key = attributes.delete(:resource_group_key) @using_rules = attributes.key?(:rules) @using_only = attributes.key?(:only) @@ -78,6 +79,7 @@ module Gitlab else ::Ci::Build.new(attributes).tap do |job| job.deployment = Seed::Deployment.new(job).to_resource + job.resource_group = Seed::Build::ResourceGroup.new(job, @resource_group_key).to_resource end end end diff --git a/lib/gitlab/ci/pipeline/seed/build/resource_group.rb b/lib/gitlab/ci/pipeline/seed/build/resource_group.rb new file mode 100644 index 00000000000..100eb1d4084 --- /dev/null +++ b/lib/gitlab/ci/pipeline/seed/build/resource_group.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Seed + class Build + class ResourceGroup < Seed::Base + include Gitlab::Utils::StrongMemoize + + attr_reader :build, :resource_group_key + + def initialize(build, resource_group_key) + @build = build + @resource_group_key = resource_group_key + end + + def to_resource + return unless Feature.enabled?(:ci_resource_group, build.project) + return unless resource_group_key.present? + + resource_group = build.project.resource_groups + .safe_find_or_create_by(key: expanded_resource_group_key) + + resource_group if resource_group.persisted? + end + + private + + def expanded_resource_group_key + strong_memoize(:expanded_resource_group_key) do + ExpandVariables.expand(resource_group_key, -> { build.simple_variables }) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 833c545fc5b..6f1ab79064a 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -44,6 +44,7 @@ module Gitlab interruptible: job[:interruptible], rules: job[:rules], cache: job[:cache], + resource_group_key: job[:resource_group], options: { image: job[:image], services: job[:services], diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 2fd4f18b756..f822e931bdb 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -203,6 +203,8 @@ excluded_attributes: - :artifacts_metadata_store - :artifacts_size - :commands + - :resource_group_id + - :waiting_for_resource_at push_event_payload: - :event_id project_badges: diff --git a/spec/factories/ci/builds.rb b/spec/factories/ci/builds.rb index ecb1f1996d9..a38935c89ba 100644 --- a/spec/factories/ci/builds.rb +++ b/spec/factories/ci/builds.rb @@ -207,6 +207,14 @@ FactoryBot.define do trigger_request factory: :ci_trigger_request end + trait :resource_group do + waiting_for_resource_at { 5.minutes.ago } + + after(:build) do |build, evaluator| + build.resource_group = create(:ci_resource_group, project: build.project) + end + end + after(:build) do |build, evaluator| build.project ||= build.pipeline.project end diff --git a/spec/factories/ci/resource.rb b/spec/factories/ci/resource.rb new file mode 100644 index 00000000000..d47b3ba4635 --- /dev/null +++ b/spec/factories/ci/resource.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :ci_resource, class: Ci::Resource do + resource_group factory: :ci_resource_group + + trait(:retained) do + build factory: :ci_build + end + end +end diff --git a/spec/factories/ci/resource_group.rb b/spec/factories/ci/resource_group.rb new file mode 100644 index 00000000000..bdfc0740a45 --- /dev/null +++ b/spec/factories/ci/resource_group.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :ci_resource_group, class: Ci::ResourceGroup do + project + sequence(:key) { |n| "IOS_#{n}" } + end +end diff --git a/spec/lib/gitlab/ci/pipeline/seed/build/resource_group_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build/resource_group_spec.rb new file mode 100644 index 00000000000..bf6985156d3 --- /dev/null +++ b/spec/lib/gitlab/ci/pipeline/seed/build/resource_group_spec.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Ci::Pipeline::Seed::Build::ResourceGroup do + let_it_be(:project) { create(:project) } + let(:job) { build(:ci_build, project: project) } + let(:seed) { described_class.new(job, resource_group_key) } + + describe '#to_resource' do + subject { seed.to_resource } + + context 'when resource group key is specified' do + let(:resource_group_key) { 'iOS' } + + it 'returns a resource group object' do + is_expected.to be_a(Ci::ResourceGroup) + expect(subject.key).to eq('iOS') + end + + context 'when environment has an invalid URL' do + let(:resource_group_key) { ':::' } + + it 'returns nothing' do + is_expected.to be_nil + end + end + + context 'when there is a resource group already' do + let!(:resource_group) { create(:ci_resource_group, project: project, key: 'iOS') } + + it 'does not create a new resource group' do + expect { subject }.not_to change { Ci::ResourceGroup.count } + end + end + end + + context 'when resource group key is nil' do + let(:resource_group_key) { nil } + + it 'returns nothing' do + is_expected.to be_nil + end + end + end +end diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb index 2ae513aea1b..5526ec9e16f 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb @@ -231,6 +231,15 @@ describe Gitlab::Ci::Pipeline::Seed::Build do end end end + + context 'when job belongs to a resource group' do + let(:attributes) { { name: 'rspec', ref: 'master', resource_group_key: 'iOS' } } + + it 'returns a job with resource group' do + expect(subject.resource_group).not_to be_nil + expect(subject.resource_group.key).to eq('iOS') + end + end end context 'when job is a bridge' do diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index ff8849f84d5..fc57341b5bc 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -239,6 +239,21 @@ module Gitlab end end end + + describe 'resource group' do + context 'when resource group is defined' do + let(:config) do + YAML.dump(rspec: { + script: 'test', + resource_group: 'iOS' + }) + end + + it 'has the attributes' do + expect(subject[:resource_group_key]).to eq 'iOS' + end + end + end end describe '#stages_attributes' do diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 8d436fb28e0..16fe5f23d14 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -444,6 +444,7 @@ project: - service_desk_setting - import_failures - container_expiration_policy +- resource_groups award_emoji: - awardable - user diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index acc54338a10..bcab621ab6a 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -1275,6 +1275,68 @@ describe Ci::Build do end end + describe '#requires_resource?' do + subject { build.requires_resource? } + + context 'when build needs a resource from a resource group' do + let(:resource_group) { create(:ci_resource_group, project: project) } + let(:build) { create(:ci_build, resource_group: resource_group, project: project) } + + context 'when build has not retained a resource' do + it { is_expected.to eq(true) } + end + + context 'when build has retained a resource' do + before do + resource_group.retain_resource_for(build) + end + + it { is_expected.to eq(false) } + + context 'when ci_resource_group feature flag is disabled' do + before do + stub_feature_flags(ci_resource_group: false) + end + + it { is_expected.to eq(false) } + end + end + end + + context 'when build does not need a resource from a resource group' do + let(:build) { create(:ci_build, project: project) } + + it { is_expected.to eq(false) } + end + end + + describe '#retains_resource?' do + subject { build.retains_resource? } + + context 'when build needs a resource from a resource group' do + let(:resource_group) { create(:ci_resource_group, project: project) } + let(:build) { create(:ci_build, resource_group: resource_group, project: project) } + + context 'when build has retained a resource' do + before do + resource_group.retain_resource_for(build) + end + + it { is_expected.to eq(true) } + end + + context 'when build has not retained a resource' do + it { is_expected.to eq(false) } + end + end + + context 'when build does not need a resource from a resource group' do + let(:build) { create(:ci_build, project: project) } + + it { is_expected.to eq(false) } + end + end + describe '#stops_environment?' do subject { build.stops_environment? } diff --git a/spec/models/ci/resource_group_spec.rb b/spec/models/ci/resource_group_spec.rb new file mode 100644 index 00000000000..213a57c2d78 --- /dev/null +++ b/spec/models/ci/resource_group_spec.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Ci::ResourceGroup do + describe 'validation' do + it 'valids when key includes allowed character' do + resource_group = build(:ci_resource_group, key: 'test') + + expect(resource_group).to be_valid + end + + it 'invalids when key includes invalid character' do + resource_group = build(:ci_resource_group, key: ':::') + + expect(resource_group).not_to be_valid + end + end + + describe '#ensure_resource' do + it 'creates one resource when resource group is created' do + resource_group = create(:ci_resource_group) + + expect(resource_group.resources.count).to eq(1) + expect(resource_group.resources.all?(&:persisted?)).to eq(true) + end + end + + describe '#retain_resource_for' do + subject { resource_group.retain_resource_for(build) } + + let(:build) { create(:ci_build) } + let(:resource_group) { create(:ci_resource_group) } + + it 'retains resource for the build' do + expect(resource_group.resources.first.build).to be_nil + + is_expected.to eq(true) + + expect(resource_group.resources.first.build).to eq(build) + end + + context 'when there are no free resources' do + before do + resource_group.retain_resource_for(create(:ci_build)) + end + + it 'fails to retain resource' do + is_expected.to eq(false) + end + end + + context 'when the build has already retained a resource' do + let!(:another_resource) { create(:ci_resource, resource_group: resource_group, build: build) } + + it 'fails to retain resource' do + expect { subject }.to raise_error(ActiveRecord::RecordNotUnique) + end + end + end + + describe '#release_resource_from' do + subject { resource_group.release_resource_from(build) } + + let(:build) { create(:ci_build) } + let(:resource_group) { create(:ci_resource_group) } + + context 'when the build has already retained a resource' do + before do + resource_group.retain_resource_for(build) + end + + it 'releases resource from the build' do + expect(resource_group.resources.first.build).to eq(build) + + is_expected.to eq(true) + + expect(resource_group.resources.first.build).to be_nil + end + end + + context 'when the build has already released a resource' do + it 'fails to release resource' do + is_expected.to eq(false) + end + end + end +end diff --git a/spec/models/ci/resource_spec.rb b/spec/models/ci/resource_spec.rb new file mode 100644 index 00000000000..27e512e2c45 --- /dev/null +++ b/spec/models/ci/resource_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Ci::Resource do + describe '.free' do + subject { described_class.free } + + let(:resource_group) { create(:ci_resource_group) } + let!(:free_resource) { resource_group.resources.take } + let!(:retained_resource) { create(:ci_resource, :retained, resource_group: resource_group) } + + it 'returns free resources' do + is_expected.to eq([free_resource]) + end + end + + describe '.retained_by' do + subject { described_class.retained_by(build) } + + let(:build) { create(:ci_build) } + let!(:resource) { create(:ci_resource, build: build) } + + it 'returns retained resources' do + is_expected.to eq([resource]) + end + end +end diff --git a/spec/requests/api/internal/base_spec.rb b/spec/requests/api/internal/base_spec.rb index 9b3762877e0..ecbb81294a0 100644 --- a/spec/requests/api/internal/base_spec.rb +++ b/spec/requests/api/internal/base_spec.rb @@ -326,7 +326,7 @@ describe API::Internal::Base do expect(json_response["gitaly"]["repository"]["relative_path"]).to eq(project.repository.gitaly_repository.relative_path) expect(json_response["gitaly"]["address"]).to eq(Gitlab::GitalyClient.address(project.repository_storage)) expect(json_response["gitaly"]["token"]).to eq(Gitlab::GitalyClient.token(project.repository_storage)) - expect(json_response["gitaly"]["features"]).to eq('gitaly-feature-get-all-lfs-pointers-go' => 'true', 'gitaly-feature-inforef-uploadpack-cache' => 'true', 'gitaly-feature-get-tag-messages-go' => 'true', 'gitaly-feature-filter-shas-with-signatures-go' => 'true') + expect(json_response["gitaly"]["features"]).to eq('gitaly-feature-inforef-uploadpack-cache' => 'true', 'gitaly-feature-get-tag-messages-go' => 'true', 'gitaly-feature-filter-shas-with-signatures-go' => 'true') expect(user.reload.last_activity_on).to eql(Date.today) end end @@ -346,7 +346,7 @@ describe API::Internal::Base do expect(json_response["gitaly"]["repository"]["relative_path"]).to eq(project.repository.gitaly_repository.relative_path) expect(json_response["gitaly"]["address"]).to eq(Gitlab::GitalyClient.address(project.repository_storage)) expect(json_response["gitaly"]["token"]).to eq(Gitlab::GitalyClient.token(project.repository_storage)) - expect(json_response["gitaly"]["features"]).to eq('gitaly-feature-get-all-lfs-pointers-go' => 'true', 'gitaly-feature-inforef-uploadpack-cache' => 'true', 'gitaly-feature-get-tag-messages-go' => 'true', 'gitaly-feature-filter-shas-with-signatures-go' => 'true') + expect(json_response["gitaly"]["features"]).to eq('gitaly-feature-inforef-uploadpack-cache' => 'true', 'gitaly-feature-get-tag-messages-go' => 'true', 'gitaly-feature-filter-shas-with-signatures-go' => 'true') expect(user.reload.last_activity_on).to be_nil end end @@ -588,7 +588,7 @@ describe API::Internal::Base do expect(json_response["gitaly"]["repository"]["relative_path"]).to eq(project.repository.gitaly_repository.relative_path) expect(json_response["gitaly"]["address"]).to eq(Gitlab::GitalyClient.address(project.repository_storage)) expect(json_response["gitaly"]["token"]).to eq(Gitlab::GitalyClient.token(project.repository_storage)) - expect(json_response["gitaly"]["features"]).to eq('gitaly-feature-get-all-lfs-pointers-go' => 'true', 'gitaly-feature-inforef-uploadpack-cache' => 'true', 'gitaly-feature-get-tag-messages-go' => 'true', 'gitaly-feature-filter-shas-with-signatures-go' => 'true') + expect(json_response["gitaly"]["features"]).to eq('gitaly-feature-inforef-uploadpack-cache' => 'true', 'gitaly-feature-get-tag-messages-go' => 'true', 'gitaly-feature-filter-shas-with-signatures-go' => 'true') end end diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb index 3d2d8691c68..4c83f9229d9 100644 --- a/spec/services/ci/create_pipeline_service_spec.rb +++ b/spec/services/ci/create_pipeline_service_spec.rb @@ -914,6 +914,44 @@ describe Ci::CreatePipelineService do end end + context 'with resource group' do + context 'when resource group is defined' do + before do + config = YAML.dump( + test: { stage: 'test', script: 'ls', resource_group: resource_group_key } + ) + + stub_ci_pipeline_yaml_file(config) + end + + let(:resource_group_key) { 'iOS' } + + it 'persists the association correctly' do + result = execute_service + deploy_job = result.builds.find_by_name!(:test) + resource_group = project.resource_groups.find_by_key!(resource_group_key) + + expect(result).to be_persisted + expect(deploy_job.resource_group.key).to eq(resource_group_key) + expect(project.resource_groups.count).to eq(1) + expect(resource_group.builds.count).to eq(1) + expect(resource_group.resources.count).to eq(1) + expect(resource_group.resources.first.build).to eq(nil) + end + + context 'when resourc group key includes predefined variables' do + let(:resource_group_key) { '$CI_COMMIT_REF_NAME-$CI_JOB_NAME' } + + it 'interpolates the variables into the key correctly' do + result = execute_service + + expect(result).to be_persisted + expect(project.resource_groups.exists?(key: 'master-test')).to eq(true) + end + end + end + end + context 'with timeout' do context 'when builds with custom timeouts are configured' do before do diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb index b1368f7776b..76fe6f53a11 100644 --- a/spec/services/ci/retry_build_service_spec.rb +++ b/spec/services/ci/retry_build_service_spec.rb @@ -31,7 +31,7 @@ describe Ci::RetryBuildService do job_artifacts_container_scanning job_artifacts_dast job_artifacts_license_management job_artifacts_performance job_artifacts_codequality job_artifacts_metrics scheduled_at - job_variables].freeze + job_variables waiting_for_resource_at].freeze IGNORE_ACCESSORS = %i[type lock_version target_url base_tags trace_sections @@ -40,14 +40,14 @@ describe Ci::RetryBuildService do user_id auto_canceled_by_id retried failure_reason sourced_pipelines artifacts_file_store artifacts_metadata_store metadata runner_session trace_chunks upstream_pipeline_id - artifacts_file artifacts_metadata artifacts_size commands].freeze + artifacts_file artifacts_metadata artifacts_size commands resource resource_group_id].freeze shared_examples 'build duplication' do let(:another_pipeline) { create(:ci_empty_pipeline, project: project) } let(:build) do create(:ci_build, :failed, :expired, :erased, :queued, :coverage, :tags, - :allowed_to_fail, :on_tag, :triggered, :teardown_environment, + :allowed_to_fail, :on_tag, :triggered, :teardown_environment, :resource_group, description: 'my-job', stage: 'test', stage_id: stage.id, pipeline: pipeline, auto_canceled_by: another_pipeline, scheduled_at: 10.seconds.since) |