From f211e47eb20959fdebc5ae0aabf66267ea521ee7 Mon Sep 17 00:00:00 2001 From: haseeb Date: Fri, 9 Feb 2018 23:53:55 +0530 Subject: closed by field added --- app/models/issue.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/models') diff --git a/app/models/issue.rb b/app/models/issue.rb index c81f7e52bb1..8e51448770d 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -23,6 +23,7 @@ class Issue < ActiveRecord::Base belongs_to :project belongs_to :moved_to, class_name: 'Issue' + belongs_to :closed_by, class_name: 'User' has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent -- cgit v1.2.3 From 9b0e8a09b1b35cc8fc893003afc59c78f5dc846f Mon Sep 17 00:00:00 2001 From: haseeb Date: Tue, 27 Feb 2018 21:48:01 +0530 Subject: added forign key and specs --- app/models/user.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index 982080763d2..ef888051841 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -135,6 +135,7 @@ class User < ActiveRecord::Base has_many :issue_assignees has_many :assigned_issues, class_name: "Issue", through: :issue_assignees, source: :issue has_many :assigned_merge_requests, dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" # rubocop:disable Cop/ActiveRecordDependent + has_many :closed_issues, class_name: 'Issue', dependent: :nullify, foreign_key: :closed_by_id has_many :custom_attributes, class_name: 'UserCustomAttribute' has_many :callouts, class_name: 'UserCallout' -- cgit v1.2.3 From d4df931a6b23b5a065532c421fa5ba793eba117e Mon Sep 17 00:00:00 2001 From: haseeb Date: Wed, 28 Feb 2018 19:41:53 +0530 Subject: removed hasmany association from users --- app/models/user.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index ef888051841..982080763d2 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -135,7 +135,6 @@ class User < ActiveRecord::Base has_many :issue_assignees has_many :assigned_issues, class_name: "Issue", through: :issue_assignees, source: :issue has_many :assigned_merge_requests, dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" # rubocop:disable Cop/ActiveRecordDependent - has_many :closed_issues, class_name: 'Issue', dependent: :nullify, foreign_key: :closed_by_id has_many :custom_attributes, class_name: 'UserCustomAttribute' has_many :callouts, class_name: 'UserCallout' -- cgit v1.2.3 From 52c3b8f31264230814d2ffa79d0987c1491676b3 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 8 Jun 2017 05:29:35 +0000 Subject: Merge branch 'zj-object-store-artifacts' into 'master' Object store for artifacts Closes gitlab-ce#29203 See merge request !1762 --- app/models/ci/build.rb | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 432f3f242eb..39d0647b182 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -295,17 +295,27 @@ module Ci !artifacts_expired? && artifacts_file.exists? end + def browsable_artifacts? + artifacts_metadata? + end + + def downloadable_single_artifacts_file? + artifacts_metadata? && artifacts_file.file_storage? + end + def artifacts_metadata? artifacts? && artifacts_metadata.exists? end def artifacts_metadata_entry(path, **options) - metadata = Gitlab::Ci::Build::Artifacts::Metadata.new( - artifacts_metadata.path, - path, - **options) + artifacts_metadata.use_file do |metadata_path| + metadata = Gitlab::Ci::Build::Artifacts::Metadata.new( + metadata_path, + path, + **options) - metadata.to_entry + metadata.to_entry + end end def erase_artifacts! -- cgit v1.2.3 From bc76062774f01208403685965f4d780da4e03ebb Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 7 Sep 2017 21:27:04 +0000 Subject: Merge branch 'jej/lfs-object-storage' into 'master' Can migrate LFS objects to S3 style object storage Closes #2841 See merge request !2760 --- app/models/ci/build.rb | 1 + app/models/lfs_object.rb | 2 ++ 2 files changed, 3 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index c41355f5aff..1c8364b00da 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -33,6 +33,7 @@ module Ci scope :with_artifacts, ->() { where.not(artifacts_file: [nil, '']) } scope :with_artifacts_not_expired, ->() { with_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) } + scope :with_artifacts_stored_locally, ->() { with_artifacts.where(artifacts_file_store: [nil, ArtifactUploader::LOCAL_STORE]) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) } scope :ref_protected, -> { where(protected: true) } diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index b7cf96abe83..0056b0f80f4 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -4,6 +4,8 @@ class LfsObject < ActiveRecord::Base validates :oid, presence: true, uniqueness: true + scope :with_files_stored_locally, ->() { where(file_store: [nil, LfsObjectUploader::LOCAL_STORE]) } + mount_uploader :file, LfsObjectUploader def storage_project(project) -- cgit v1.2.3 From 6ca02a41500790b3e9061dd8836540955b9aaf7c Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 5 Dec 2017 14:31:33 +0000 Subject: Merge branch 'zj-multiple-artifacts-ee' into 'master' Multiple artifacts ee See merge request gitlab-org/gitlab-ee!3276 --- app/models/ci/build.rb | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index ddd075e1fcb..7cf8bdd968b 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -45,7 +45,7 @@ module Ci end scope :with_artifacts_not_expired, ->() { with_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) } - scope :with_artifacts_stored_locally, ->() { with_artifacts.where(artifacts_file_store: [nil, ArtifactUploader::LOCAL_STORE]) } + scope :with_artifacts_stored_locally, ->() { with_artifacts.where(artifacts_file_store: [nil, LegacyArtifactUploader::LOCAL_STORE]) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) } scope :ref_protected, -> { where(protected: true) } @@ -361,22 +361,10 @@ module Ci project.running_or_pending_build_count(force: true) end - def artifacts? - !artifacts_expired? && artifacts_file.exists? - end - def browsable_artifacts? artifacts_metadata? end - def downloadable_single_artifacts_file? - artifacts_metadata? && artifacts_file.file_storage? - end - - def artifacts_metadata? - artifacts? && artifacts_metadata.exists? - end - def artifacts_metadata_entry(path, **options) artifacts_metadata.use_file do |metadata_path| metadata = Gitlab::Ci::Build::Artifacts::Metadata.new( -- cgit v1.2.3 From 87f11d2cf539d9539b439b54355f0dadaf4ebf76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Fri, 8 Dec 2017 09:09:06 +0000 Subject: Merge branch 'zj-auto-upload-job-artifacts' into 'master' Transfer job archives after creation See merge request gitlab-org/gitlab-ee!3646 --- app/models/ci/job_artifact.rb | 7 +++++++ app/models/lfs_object.rb | 9 +++++++++ 2 files changed, 16 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 84fc6863567..1aea897aaca 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -1,5 +1,6 @@ module Ci class JobArtifact < ActiveRecord::Base + include AfterCommitQueue extend Gitlab::Ci::Model belongs_to :project @@ -9,6 +10,12 @@ module Ci mount_uploader :file, JobArtifactUploader + after_save if: :file_changed?, on: [:create, :update] do + run_after_commit do + file.schedule_migration_to_object_storage + end + end + enum file_type: { archive: 1, metadata: 2 diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index 38b8c41024a..6ad792aab30 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -1,4 +1,7 @@ class LfsObject < ActiveRecord::Base + prepend EE::LfsObject + include AfterCommitQueue + has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :lfs_objects_projects @@ -8,6 +11,12 @@ class LfsObject < ActiveRecord::Base mount_uploader :file, LfsObjectUploader + after_save if: :file_changed?, on: [:create, :update] do + run_after_commit do + file.schedule_migration_to_object_storage + end + end + def project_allowed_access?(project) projects.exists?(project.lfs_storage_project.id) end -- cgit v1.2.3 From a7dae52e9d27adde427ef8aa066c0761071a3cd9 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Fri, 2 Feb 2018 13:59:43 +0000 Subject: Merge branch '4163-move-uploads-to-object-storage' into 'master' Move uploads to object storage Closes #4163 See merge request gitlab-org/gitlab-ee!3867 --- app/models/appearance.rb | 1 + app/models/ci/build.rb | 2 +- app/models/concerns/avatarable.rb | 24 +++++++++++++++++++ app/models/group.rb | 20 ++-------------- app/models/lfs_object.rb | 2 +- app/models/note.rb | 1 + app/models/project.rb | 14 +---------- app/models/upload.rb | 50 ++++++++++++++++++++++++--------------- app/models/user.rb | 16 ++----------- 9 files changed, 64 insertions(+), 66 deletions(-) (limited to 'app/models') diff --git a/app/models/appearance.rb b/app/models/appearance.rb index 76cfe28742a..dcd14c08f3c 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -11,6 +11,7 @@ class Appearance < ActiveRecord::Base mount_uploader :logo, AttachmentUploader mount_uploader :header_logo, AttachmentUploader + has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent CACHE_KEY = 'current_appearance'.freeze diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index b65daa376d2..4eeccd4d934 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -45,7 +45,7 @@ module Ci end scope :with_artifacts_not_expired, ->() { with_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) } - scope :with_artifacts_stored_locally, ->() { with_artifacts.where(artifacts_file_store: [nil, LegacyArtifactUploader::LOCAL_STORE]) } + scope :with_artifacts_stored_locally, ->() { with_artifacts.where(artifacts_file_store: [nil, LegacyArtifactUploader::Store::LOCAL]) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) } scope :ref_protected, -> { where(protected: true) } diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb index 10659030910..d35e37935fb 100644 --- a/app/models/concerns/avatarable.rb +++ b/app/models/concerns/avatarable.rb @@ -1,6 +1,30 @@ module Avatarable extend ActiveSupport::Concern + included do + prepend ShadowMethods + + validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? } + validates :avatar, file_size: { maximum: 200.kilobytes.to_i } + + mount_uploader :avatar, AvatarUploader + end + + module ShadowMethods + def avatar_url(**args) + # We use avatar_path instead of overriding avatar_url because of carrierwave. + # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864 + + avatar_path(only_path: args.fetch(:only_path, true)) || super + end + end + + def avatar_type + unless self.avatar.image? + self.errors.add :avatar, "only images allowed" + end + end + def avatar_path(only_path: true) return unless self[:avatar].present? diff --git a/app/models/group.rb b/app/models/group.rb index fddace03387..5d1e2f62982 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -29,18 +29,14 @@ class Group < Namespace has_many :variables, class_name: 'Ci::GroupVariable' has_many :custom_attributes, class_name: 'GroupCustomAttribute' - validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? } + has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + validate :visibility_level_allowed_by_projects validate :visibility_level_allowed_by_sub_groups validate :visibility_level_allowed_by_parent - validates :avatar, file_size: { maximum: 200.kilobytes.to_i } - validates :two_factor_grace_period, presence: true, numericality: { greater_than_or_equal_to: 0 } - mount_uploader :avatar, AvatarUploader - has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - after_create :post_create_hook after_destroy :post_destroy_hook after_save :update_two_factor_requirement @@ -116,12 +112,6 @@ class Group < Namespace visibility_level_allowed_by_sub_groups?(level) end - def avatar_url(**args) - # We use avatar_path instead of overriding avatar_url because of carrierwave. - # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864 - avatar_path(args) - end - def lfs_enabled? return false unless Gitlab.config.lfs.enabled return Gitlab.config.lfs.enabled if self[:lfs_enabled].nil? @@ -193,12 +183,6 @@ class Group < Namespace owners.include?(user) && owners.size == 1 end - def avatar_type - unless self.avatar.image? - self.errors.add :avatar, "only images allowed" - end - end - def post_create_hook Gitlab::AppLogger.info("Group \"#{name}\" was created") diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index 6ad792aab30..65c157d61ca 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -7,7 +7,7 @@ class LfsObject < ActiveRecord::Base validates :oid, presence: true, uniqueness: true - scope :with_files_stored_locally, ->() { where(file_store: [nil, LfsObjectUploader::LOCAL_STORE]) } + scope :with_files_stored_locally, -> { where(file_store: [nil, LfsObjectUploader::Store::LOCAL]) } mount_uploader :file, LfsObjectUploader diff --git a/app/models/note.rb b/app/models/note.rb index 184fbd5f5ae..a84db8982e5 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -88,6 +88,7 @@ class Note < ActiveRecord::Base end end + # @deprecated attachments are handler by the MarkdownUploader mount_uploader :attachment, AttachmentUploader # Scopes diff --git a/app/models/project.rb b/app/models/project.rb index fbe65e700a4..b3c2b599129 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -255,9 +255,6 @@ class Project < ActiveRecord::Base validates :star_count, numericality: { greater_than_or_equal_to: 0 } validate :check_limit, on: :create validate :check_repository_path_availability, on: :update, if: ->(project) { project.renamed? } - validate :avatar_type, - if: ->(project) { project.avatar.present? && project.avatar_changed? } - validates :avatar, file_size: { maximum: 200.kilobytes.to_i } validate :visibility_level_allowed_by_group validate :visibility_level_allowed_as_fork validate :check_wiki_path_conflict @@ -265,7 +262,6 @@ class Project < ActiveRecord::Base presence: true, inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } } - mount_uploader :avatar, AvatarUploader has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent # Scopes @@ -917,20 +913,12 @@ class Project < ActiveRecord::Base issues_tracker.to_param == 'jira' end - def avatar_type - unless self.avatar.image? - self.errors.add :avatar, 'only images allowed' - end - end - def avatar_in_git repository.avatar end def avatar_url(**args) - # We use avatar_path instead of overriding avatar_url because of carrierwave. - # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864 - avatar_path(args) || (Gitlab::Routing.url_helpers.project_avatar_url(self) if avatar_in_git) + Gitlab::Routing.url_helpers.project_avatar_url(self) if avatar_in_git end # For compatibility with old code diff --git a/app/models/upload.rb b/app/models/upload.rb index f194d7bdb80..e227baea994 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -9,44 +9,52 @@ class Upload < ActiveRecord::Base validates :model, presence: true validates :uploader, presence: true - before_save :calculate_checksum, if: :foreground_checksum? - after_commit :schedule_checksum, unless: :foreground_checksum? + before_save :calculate_checksum!, if: :foreground_checksummable? + after_commit :schedule_checksum, if: :checksummable? - def self.remove_path(path) - where(path: path).destroy_all - end - - def self.record(uploader) - remove_path(uploader.relative_path) - - create( - size: uploader.file.size, - path: uploader.relative_path, - model: uploader.model, - uploader: uploader.class.to_s - ) + def self.hexdigest(path) + Digest::SHA256.file(path).hexdigest end def absolute_path + raise ObjectStorage::RemoteStoreError, "Remote object has no absolute path." unless local? return path unless relative_path? uploader_class.absolute_path(self) end - def calculate_checksum - return unless exist? + def calculate_checksum! + self.checksum = nil + return unless checksummable? self.checksum = Digest::SHA256.file(absolute_path).hexdigest end + def build_uploader + uploader_class.new(model).tap do |uploader| + uploader.upload = self + uploader.retrieve_from_store!(identifier) + end + end + def exist? File.exist?(absolute_path) end private - def foreground_checksum? - size <= CHECKSUM_THRESHOLD + def checksummable? + checksum.nil? && local? && exist? + end + + def local? + return true if store.nil? + + store == ObjectStorage::Store::LOCAL + end + + def foreground_checksummable? + checksummable? && size <= CHECKSUM_THRESHOLD end def schedule_checksum @@ -57,6 +65,10 @@ class Upload < ActiveRecord::Base !path.start_with?('/') end + def identifier + File.basename(path) + end + def uploader_class Object.const_get(uploader) end diff --git a/app/models/user.rb b/app/models/user.rb index 4484ee9ff4c..eb6d12b5ec5 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -134,6 +134,7 @@ class User < ActiveRecord::Base has_many :assigned_merge_requests, dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" # rubocop:disable Cop/ActiveRecordDependent has_many :custom_attributes, class_name: 'UserCustomAttribute' + has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent # # Validations @@ -156,12 +157,10 @@ class User < ActiveRecord::Base validate :namespace_uniq, if: :username_changed? validate :namespace_move_dir_allowed, if: :username_changed? - validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? } validate :unique_email, if: :email_changed? validate :owns_notification_email, if: :notification_email_changed? validate :owns_public_email, if: :public_email_changed? validate :signup_domain_valid?, on: :create, if: ->(user) { !user.created_by_id } - validates :avatar, file_size: { maximum: 200.kilobytes.to_i } before_validation :sanitize_attrs before_validation :set_notification_email, if: :email_changed? @@ -223,9 +222,6 @@ class User < ActiveRecord::Base end end - mount_uploader :avatar, AvatarUploader - has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - # Scopes scope :admins, -> { where(admin: true) } scope :blocked, -> { with_states(:blocked, :ldap_blocked) } @@ -521,12 +517,6 @@ class User < ActiveRecord::Base end end - def avatar_type - unless avatar.image? - errors.add :avatar, "only images allowed" - end - end - def unique_email if !emails.exists?(email: email) && Email.exists?(email: email) errors.add(:email, 'has already been taken') @@ -854,9 +844,7 @@ class User < ActiveRecord::Base end def avatar_url(size: nil, scale: 2, **args) - # We use avatar_path instead of overriding avatar_url because of carrierwave. - # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864 - avatar_path(args) || GravatarService.new.execute(email, size, scale, username: username) + GravatarService.new.execute(email, size, scale, username: username) end def primary_email_verified? -- cgit v1.2.3 From 999118f0ec6edabc9e13c089381ad664970a080a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Tue, 6 Feb 2018 14:18:32 +0000 Subject: Merge branch 'feature/sm/artifacts-trace-ee' into 'master' EE: Trace as artifacts (FileStorage and ObjectStorage) Closes #4171 See merge request gitlab-org/gitlab-ee!4258 --- app/models/ci/build.rb | 1 + app/models/ci/job_artifact.rb | 5 ++++- app/models/concerns/artifact_migratable.rb | 3 +-- 3 files changed, 6 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 4eeccd4d934..47eb4460320 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -20,6 +20,7 @@ module Ci has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id + has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id # The "environment" field for builds is a String, and is the unexpanded name def persisted_environment diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 1aea897aaca..2dfd8d4ef58 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -16,9 +16,12 @@ module Ci end end + delegate :open, :exists?, to: :file + enum file_type: { archive: 1, - metadata: 2 + metadata: 2, + trace: 3 } def self.artifacts_size_for(project) diff --git a/app/models/concerns/artifact_migratable.rb b/app/models/concerns/artifact_migratable.rb index 0460439e9e6..ff52ca64459 100644 --- a/app/models/concerns/artifact_migratable.rb +++ b/app/models/concerns/artifact_migratable.rb @@ -39,7 +39,6 @@ module ArtifactMigratable end def artifacts_size - read_attribute(:artifacts_size).to_i + - job_artifacts_archive&.size.to_i + job_artifacts_metadata&.size.to_i + read_attribute(:artifacts_size).to_i + job_artifacts.sum(:size).to_i end end -- cgit v1.2.3 From 8af23def1d6450420d06b8de54d23311a978de20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Wed, 28 Feb 2018 21:09:34 +0100 Subject: Revert "Merge branch '3867-port-to-ce' into 'master'" This reverts commit 54a575f1bbba44573ab92dc58a4242f1ee734c5d, reversing changes made to c63af942e5baf7849a94fa99da8494bcba28e3f8. --- app/models/appearance.rb | 1 - app/models/concerns/avatarable.rb | 24 ------------------- app/models/group.rb | 14 +++++++++-- app/models/note.rb | 1 - app/models/project.rb | 15 +++++++++++- app/models/upload.rb | 49 ++++++++++++++++----------------------- app/models/user.rb | 16 +++++++++++-- 7 files changed, 60 insertions(+), 60 deletions(-) (limited to 'app/models') diff --git a/app/models/appearance.rb b/app/models/appearance.rb index dcd14c08f3c..76cfe28742a 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -11,7 +11,6 @@ class Appearance < ActiveRecord::Base mount_uploader :logo, AttachmentUploader mount_uploader :header_logo, AttachmentUploader - has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent CACHE_KEY = 'current_appearance'.freeze diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb index d35e37935fb..10659030910 100644 --- a/app/models/concerns/avatarable.rb +++ b/app/models/concerns/avatarable.rb @@ -1,30 +1,6 @@ module Avatarable extend ActiveSupport::Concern - included do - prepend ShadowMethods - - validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? } - validates :avatar, file_size: { maximum: 200.kilobytes.to_i } - - mount_uploader :avatar, AvatarUploader - end - - module ShadowMethods - def avatar_url(**args) - # We use avatar_path instead of overriding avatar_url because of carrierwave. - # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864 - - avatar_path(only_path: args.fetch(:only_path, true)) || super - end - end - - def avatar_type - unless self.avatar.image? - self.errors.add :avatar, "only images allowed" - end - end - def avatar_path(only_path: true) return unless self[:avatar].present? diff --git a/app/models/group.rb b/app/models/group.rb index 62b1322ebe6..fddace03387 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -29,14 +29,18 @@ class Group < Namespace has_many :variables, class_name: 'Ci::GroupVariable' has_many :custom_attributes, class_name: 'GroupCustomAttribute' - has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - + validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? } validate :visibility_level_allowed_by_projects validate :visibility_level_allowed_by_sub_groups validate :visibility_level_allowed_by_parent + validates :avatar, file_size: { maximum: 200.kilobytes.to_i } + validates :two_factor_grace_period, presence: true, numericality: { greater_than_or_equal_to: 0 } + mount_uploader :avatar, AvatarUploader + has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + after_create :post_create_hook after_destroy :post_destroy_hook after_save :update_two_factor_requirement @@ -112,6 +116,12 @@ class Group < Namespace visibility_level_allowed_by_sub_groups?(level) end + def avatar_url(**args) + # We use avatar_path instead of overriding avatar_url because of carrierwave. + # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864 + avatar_path(args) + end + def lfs_enabled? return false unless Gitlab.config.lfs.enabled return Gitlab.config.lfs.enabled if self[:lfs_enabled].nil? diff --git a/app/models/note.rb b/app/models/note.rb index a84db8982e5..184fbd5f5ae 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -88,7 +88,6 @@ class Note < ActiveRecord::Base end end - # @deprecated attachments are handler by the MarkdownUploader mount_uploader :attachment, AttachmentUploader # Scopes diff --git a/app/models/project.rb b/app/models/project.rb index 90f5df6265d..4def590a7a9 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -256,6 +256,9 @@ class Project < ActiveRecord::Base validates :star_count, numericality: { greater_than_or_equal_to: 0 } validate :check_limit, on: :create validate :check_repository_path_availability, on: :update, if: ->(project) { project.renamed? } + validate :avatar_type, + if: ->(project) { project.avatar.present? && project.avatar_changed? } + validates :avatar, file_size: { maximum: 200.kilobytes.to_i } validate :visibility_level_allowed_by_group validate :visibility_level_allowed_as_fork validate :check_wiki_path_conflict @@ -263,6 +266,7 @@ class Project < ActiveRecord::Base presence: true, inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } } + mount_uploader :avatar, AvatarUploader has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent # Scopes @@ -285,6 +289,7 @@ class Project < ActiveRecord::Base scope :non_archived, -> { where(archived: false) } scope :for_milestones, ->(ids) { joins(:milestones).where('milestones.id' => ids).distinct } scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) } + scope :with_project_feature, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id') } scope :with_statistics, -> { includes(:statistics) } scope :with_shared_runners, -> { where(shared_runners_enabled: true) } @@ -918,12 +923,20 @@ class Project < ActiveRecord::Base issues_tracker.to_param == 'jira' end + def avatar_type + unless self.avatar.image? + self.errors.add :avatar, 'only images allowed' + end + end + def avatar_in_git repository.avatar end def avatar_url(**args) - Gitlab::Routing.url_helpers.project_avatar_url(self) if avatar_in_git + # We use avatar_path instead of overriding avatar_url because of carrierwave. + # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864 + avatar_path(args) || (Gitlab::Routing.url_helpers.project_avatar_url(self) if avatar_in_git) end # For compatibility with old code diff --git a/app/models/upload.rb b/app/models/upload.rb index fb55fd8007b..f194d7bdb80 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -9,11 +9,22 @@ class Upload < ActiveRecord::Base validates :model, presence: true validates :uploader, presence: true - before_save :calculate_checksum!, if: :foreground_checksummable? - after_commit :schedule_checksum, if: :checksummable? + before_save :calculate_checksum, if: :foreground_checksum? + after_commit :schedule_checksum, unless: :foreground_checksum? - def self.hexdigest(path) - Digest::SHA256.file(path).hexdigest + def self.remove_path(path) + where(path: path).destroy_all + end + + def self.record(uploader) + remove_path(uploader.relative_path) + + create( + size: uploader.file.size, + path: uploader.relative_path, + model: uploader.model, + uploader: uploader.class.to_s + ) end def absolute_path @@ -22,18 +33,10 @@ class Upload < ActiveRecord::Base uploader_class.absolute_path(self) end - def calculate_checksum! - self.checksum = nil - return unless checksummable? + def calculate_checksum + return unless exist? - self.checksum = self.class.hexdigest(absolute_path) - end - - def build_uploader - uploader_class.new(model).tap do |uploader| - uploader.upload = self - uploader.retrieve_from_store!(identifier) - end + self.checksum = Digest::SHA256.file(absolute_path).hexdigest end def exist? @@ -42,16 +45,8 @@ class Upload < ActiveRecord::Base private - def checksummable? - checksum.nil? && local? && exist? - end - - def local? - true - end - - def foreground_checksummable? - checksummable? && size <= CHECKSUM_THRESHOLD + def foreground_checksum? + size <= CHECKSUM_THRESHOLD end def schedule_checksum @@ -62,10 +57,6 @@ class Upload < ActiveRecord::Base !path.start_with?('/') end - def identifier - File.basename(path) - end - def uploader_class Object.const_get(uploader) end diff --git a/app/models/user.rb b/app/models/user.rb index 89e787c3274..fb5d56a68b0 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -137,7 +137,6 @@ class User < ActiveRecord::Base has_many :assigned_merge_requests, dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" # rubocop:disable Cop/ActiveRecordDependent has_many :custom_attributes, class_name: 'UserCustomAttribute' - has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent # # Validations @@ -160,10 +159,12 @@ class User < ActiveRecord::Base validate :namespace_uniq, if: :username_changed? validate :namespace_move_dir_allowed, if: :username_changed? + validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? } validate :unique_email, if: :email_changed? validate :owns_notification_email, if: :notification_email_changed? validate :owns_public_email, if: :public_email_changed? validate :signup_domain_valid?, on: :create, if: ->(user) { !user.created_by_id } + validates :avatar, file_size: { maximum: 200.kilobytes.to_i } before_validation :sanitize_attrs before_validation :set_notification_email, if: :email_changed? @@ -224,6 +225,9 @@ class User < ActiveRecord::Base end end + mount_uploader :avatar, AvatarUploader + has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + # Scopes scope :admins, -> { where(admin: true) } scope :blocked, -> { with_states(:blocked, :ldap_blocked) } @@ -523,6 +527,12 @@ class User < ActiveRecord::Base end end + def avatar_type + unless avatar.image? + errors.add :avatar, "only images allowed" + end + end + def unique_email if !emails.exists?(email: email) && Email.exists?(email: email) errors.add(:email, 'has already been taken') @@ -850,7 +860,9 @@ class User < ActiveRecord::Base end def avatar_url(size: nil, scale: 2, **args) - GravatarService.new.execute(email, size, scale, username: username) + # We use avatar_path instead of overriding avatar_url because of carrierwave. + # See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11001/diffs#note_28659864 + avatar_path(args) || GravatarService.new.execute(email, size, scale, username: username) end def primary_email_verified? -- cgit v1.2.3 From 0f1d348d683fdef6c36c3b244c85e59f582ff886 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mica=C3=ABl=20Bergeron?= Date: Wed, 21 Feb 2018 11:43:21 -0500 Subject: port the object storage to CE --- app/models/appearance.rb | 2 ++ app/models/ci/job_artifact.rb | 15 ++++++++------- app/models/concerns/avatarable.rb | 1 + app/models/lfs_object.rb | 8 +++++++- app/models/upload.rb | 18 ++++++++++-------- 5 files changed, 28 insertions(+), 16 deletions(-) (limited to 'app/models') diff --git a/app/models/appearance.rb b/app/models/appearance.rb index dcd14c08f3c..2a6406d63c7 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -1,5 +1,7 @@ class Appearance < ActiveRecord::Base include CacheMarkdownField + include AfterCommitQueue + include ObjectStorage::BackgroundMove cache_markdown_field :description cache_markdown_field :new_project_guidelines diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 2dfd8d4ef58..df57b4f65e3 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -1,6 +1,7 @@ module Ci class JobArtifact < ActiveRecord::Base include AfterCommitQueue + include ObjectStorage::BackgroundMove extend Gitlab::Ci::Model belongs_to :project @@ -8,15 +9,11 @@ module Ci before_save :set_size, if: :file_changed? - mount_uploader :file, JobArtifactUploader + scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) } - after_save if: :file_changed?, on: [:create, :update] do - run_after_commit do - file.schedule_migration_to_object_storage - end - end + mount_uploader :file, JobArtifactUploader - delegate :open, :exists?, to: :file + delegate :exists?, :open, to: :file enum file_type: { archive: 1, @@ -28,6 +25,10 @@ module Ci self.where(project: project).sum(:size) end + def local_store? + [nil, ::JobArtifactUploader::Store::LOCAL].include?(self.file_store) + end + def set_size self.size = file.size end diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb index d35e37935fb..4d40a2c483e 100644 --- a/app/models/concerns/avatarable.rb +++ b/app/models/concerns/avatarable.rb @@ -3,6 +3,7 @@ module Avatarable included do prepend ShadowMethods + include ObjectStorage::BackgroundMove validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? } validates :avatar, file_size: { maximum: 200.kilobytes.to_i } diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index 65c157d61ca..04c75d827e0 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -1,10 +1,12 @@ class LfsObject < ActiveRecord::Base - prepend EE::LfsObject include AfterCommitQueue + include ObjectStorage::BackgroundMove has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :lfs_objects_projects + scope :with_files_stored_locally, -> { where(file_store: [nil, LfsObjectUploader::Store::LOCAL]) } + validates :oid, presence: true, uniqueness: true scope :with_files_stored_locally, -> { where(file_store: [nil, LfsObjectUploader::Store::LOCAL]) } @@ -21,6 +23,10 @@ class LfsObject < ActiveRecord::Base projects.exists?(project.lfs_storage_project.id) end + def local_store? + [nil, LfsObjectUploader::Store::LOCAL].include?(self.file_store) + end + def self.destroy_unreferenced joins("LEFT JOIN lfs_objects_projects ON lfs_objects_projects.lfs_object_id = #{table_name}.id") .where(lfs_objects_projects: { id: nil }) diff --git a/app/models/upload.rb b/app/models/upload.rb index 3aca452616c..cf71a7b76fc 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -9,6 +9,8 @@ class Upload < ActiveRecord::Base validates :model, presence: true validates :uploader, presence: true + scope :with_files_stored_locally, -> { where(store: [nil, ObjectStorage::Store::LOCAL]) } + before_save :calculate_checksum!, if: :foreground_checksummable? after_commit :schedule_checksum, if: :checksummable? @@ -34,8 +36,8 @@ class Upload < ActiveRecord::Base self.checksum = Digest::SHA256.file(absolute_path).hexdigest end - def build_uploader - uploader_class.new(model, mount_point, **uploader_context).tap do |uploader| + def build_uploader(mounted_as = nil) + uploader_class.new(model, mounted_as || mount_point).tap do |uploader| uploader.upload = self uploader.retrieve_from_store!(identifier) end @@ -52,6 +54,12 @@ class Upload < ActiveRecord::Base }.compact end + def local? + return true if store.nil? + + store == ObjectStorage::Store::LOCAL + end + private def delete_file! @@ -62,12 +70,6 @@ class Upload < ActiveRecord::Base checksum.nil? && local? && exist? end - def local? - return true if store.nil? - - store == ObjectStorage::Store::LOCAL - end - def foreground_checksummable? checksummable? && size <= CHECKSUM_THRESHOLD end -- cgit v1.2.3 From 40a3b467f25df62fab4b92a3700846f06ef7534c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mica=C3=ABl=20Bergeron?= Date: Mon, 26 Feb 2018 14:14:00 -0500 Subject: add the Ci::Build `*_store` column --- app/models/ci/build.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 53a0d787988..715ab55a100 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -3,6 +3,7 @@ module Ci prepend ArtifactMigratable include TokenAuthenticatable include AfterCommitQueue + include ObjectStorage::BackgroundMove include Presentable include Importable @@ -41,10 +42,13 @@ module Ci scope :unstarted, ->() { where(runner_id: nil) } scope :ignore_failures, ->() { where(allow_failure: false) } + scope :with_artifacts, ->() do where('(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)', '', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id')) end + scope :with_artifacts_stored_locally, -> { with_artifacts.where(artifacts_file_store: [nil, LegacyArtifactUploader::Store::LOCAL]) } + scope :with_artifacts_not_expired, ->() { with_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) } scope :with_artifacts_stored_locally, ->() { with_artifacts.where(artifacts_file_store: [nil, LegacyArtifactUploader::Store::LOCAL]) } -- cgit v1.2.3 From 0d458b96e8349a50877ebd55932bf806e93caa21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mica=C3=ABl=20Bergeron?= Date: Thu, 1 Mar 2018 16:30:31 -0500 Subject: remove geo specific code --- app/models/lfs_object.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index 04c75d827e0..54840fb084d 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -15,7 +15,7 @@ class LfsObject < ActiveRecord::Base after_save if: :file_changed?, on: [:create, :update] do run_after_commit do - file.schedule_migration_to_object_storage + file.schedule_background_upload end end -- cgit v1.2.3 From b03b31659b0101dc4a5838bec7dca193706661cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mica=C3=ABl=20Bergeron?= Date: Fri, 2 Mar 2018 16:19:17 -0500 Subject: fixing some broken merges --- app/models/lfs_object.rb | 8 -------- 1 file changed, 8 deletions(-) (limited to 'app/models') diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index 54840fb084d..6c78c086662 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -9,16 +9,8 @@ class LfsObject < ActiveRecord::Base validates :oid, presence: true, uniqueness: true - scope :with_files_stored_locally, -> { where(file_store: [nil, LfsObjectUploader::Store::LOCAL]) } - mount_uploader :file, LfsObjectUploader - after_save if: :file_changed?, on: [:create, :update] do - run_after_commit do - file.schedule_background_upload - end - end - def project_allowed_access?(project) projects.exists?(project.lfs_storage_project.id) end -- cgit v1.2.3 From f0859cc937f7c4251062b12957253c4832bd1263 Mon Sep 17 00:00:00 2001 From: haseeb Date: Mon, 5 Mar 2018 19:57:53 +0530 Subject: changed the way of nullifying colums --- app/models/issue.rb | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'app/models') diff --git a/app/models/issue.rb b/app/models/issue.rb index 8e51448770d..f1075d07372 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -77,6 +77,11 @@ class Issue < ActiveRecord::Base before_transition any => :closed do |issue| issue.closed_at = Time.zone.now end + + before_transition closed: :opened do |issue| + issue.closed_at = nil + issue.closed_by = nil + end end class << self -- cgit v1.2.3 From edbcde8877f497ea675fde811065679286a1aa56 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Mon, 5 Mar 2018 11:16:17 +0000 Subject: [cherry-pick] '4862-verify-file-checksums' See merge request gitlab-org/gitlab-ee!4753 --- app/models/lfs_object.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'app/models') diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index 6c78c086662..64e88d5a6a2 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -24,4 +24,8 @@ class LfsObject < ActiveRecord::Base .where(lfs_objects_projects: { id: nil }) .destroy_all end + + def self.calculate_oid(path) + Digest::SHA256.file(path).hexdigest + end end -- cgit v1.2.3 From f7b8ae3fe2adc95f18f779d737e60eac45a14223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mica=C3=ABl=20Bergeron?= Date: Fri, 9 Mar 2018 10:31:31 -0500 Subject: apply feedback --- app/models/ci/build.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 715ab55a100..0056b329f40 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -51,7 +51,6 @@ module Ci scope :with_artifacts_not_expired, ->() { with_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) } - scope :with_artifacts_stored_locally, ->() { with_artifacts.where(artifacts_file_store: [nil, LegacyArtifactUploader::Store::LOCAL]) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) } scope :ref_protected, -> { where(protected: true) } -- cgit v1.2.3 From 53915c5c54c06182717b457375ae771ee01558fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Sat, 17 Mar 2018 12:17:40 +0100 Subject: Alias secret_key and secret_value to key and value --- app/models/ci/group_variable.rb | 3 +++ app/models/ci/variable.rb | 3 +++ 2 files changed, 6 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/group_variable.rb b/app/models/ci/group_variable.rb index 1dd0e050ba9..65399557289 100644 --- a/app/models/ci/group_variable.rb +++ b/app/models/ci/group_variable.rb @@ -6,6 +6,9 @@ module Ci belongs_to :group + alias_attribute :secret_key, :key + alias_attribute :secret_value, :value + validates :key, uniqueness: { scope: :group_id, message: "(%{value}) has already been taken" diff --git a/app/models/ci/variable.rb b/app/models/ci/variable.rb index 7c71291de84..bcad55f115f 100644 --- a/app/models/ci/variable.rb +++ b/app/models/ci/variable.rb @@ -6,6 +6,9 @@ module Ci belongs_to :project + alias_attribute :secret_key, :key + alias_attribute :secret_value, :value + validates :key, uniqueness: { scope: [:project_id, :environment_scope], message: "(%{value}) has already been taken" -- cgit v1.2.3 From 68c6e410bd02207b621f9339edf3fc53d0bde7e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Sat, 17 Mar 2018 16:18:36 +0100 Subject: Use secret_key and secret_value in Pipeline Schedule variables --- app/models/ci/pipeline_schedule_variable.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/pipeline_schedule_variable.rb b/app/models/ci/pipeline_schedule_variable.rb index af989fb14b4..2e30612a88e 100644 --- a/app/models/ci/pipeline_schedule_variable.rb +++ b/app/models/ci/pipeline_schedule_variable.rb @@ -5,6 +5,9 @@ module Ci belongs_to :pipeline_schedule + alias_attribute :secret_key, :key + alias_attribute :secret_value, :value + validates :key, uniqueness: { scope: :pipeline_schedule_id } end end -- cgit v1.2.3 From ab0cd2a8bf54209e5e1b9594752d5158d3cf5faa Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 21 Mar 2018 09:33:14 +0100 Subject: Introduce pipeline build seeds --- app/models/ci/pipeline.rb | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index f2edcdd61fd..2e8d7f9dbfe 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -6,6 +6,7 @@ module Ci include AfterCommitQueue include Presentable include Gitlab::OptimisticLocking + include Gitlab::Utils::StrongMemoize belongs_to :project, inverse_of: :pipelines belongs_to :user @@ -472,6 +473,22 @@ module Ci end end + # TODO specs + # + def protected_ref? + strong_memoize(:protected_ref) do + project.protected_for?(ref) + end + end + + # TODO specs + # + def legacy_trigger + strong_memoize(:legacy_trigger) do + trigger_requests.first + end + end + def predefined_variables Gitlab::Ci::Variables::Collection.new .append(key: 'CI_PIPELINE_ID', value: id.to_s) -- cgit v1.2.3 From ad61a1815400e8c9fce1758618379874d93868e6 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 21 Mar 2018 09:53:26 +0100 Subject: Support building all pipeline resource before saving --- app/models/ci/pipeline.rb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 2e8d7f9dbfe..8889195ec24 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -476,17 +476,13 @@ module Ci # TODO specs # def protected_ref? - strong_memoize(:protected_ref) do - project.protected_for?(ref) - end + strong_memoize(:protected_ref) { project.protected_for?(ref) } end # TODO specs # def legacy_trigger - strong_memoize(:legacy_trigger) do - trigger_requests.first - end + strong_memoize(:legacy_trigger) { trigger_requests.first } end def predefined_variables -- cgit v1.2.3 From b82de0f0087a91165a7306fa9fe11a0cbc48fcd5 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 21 Mar 2018 12:12:45 +0100 Subject: Reduce stage seeds coupling between pipeline and YAML This is a first step to decouple pipeline from YAML processing. It reduces the coupling by removing some methods that introduce coupling and by moving logic into separate chain element that is being used to populate pipelines with stages and builds. --- app/models/ci/pipeline.rb | 8 -------- 1 file changed, 8 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 8889195ec24..6b38c2eff98 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -365,18 +365,10 @@ module Ci @stage_seeds ||= config_processor.stage_seeds(self) end - def seeds_size - @seeds_size ||= stage_seeds.sum(&:size) - end - def has_kubernetes_active? project.deployment_platform&.active? end - def has_stage_seeds? - stage_seeds.any? - end - def has_warnings? builds.latest.failed_but_allowed.any? end -- cgit v1.2.3 From 9b5a912f9377a77e52e9acdf7471acd27c709264 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 21 Mar 2018 13:08:30 +0100 Subject: Make it easier to test pipeline stage seeds --- app/models/ci/pipeline.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 6b38c2eff98..bac5b3860e4 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -381,6 +381,9 @@ module Ci end end + ## + # TODO, setting yaml_errors should be moved to the pipeline creation chain. + # def config_processor return unless ci_yaml_file return @config_processor if defined?(@config_processor) -- cgit v1.2.3 From 008a6a6ce6fa943adcfecf3a606b845cfa282680 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Wed, 14 Mar 2018 14:42:49 +0100 Subject: Route path lookups through legacy_disk_path --- app/models/project.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 5f9d9785d64..f60660d00a4 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -498,7 +498,7 @@ class Project < ActiveRecord::Base end def repository_storage_path - Gitlab.config.repositories.storages[repository_storage].try(:[], 'path') + Gitlab.config.repositories.storages[repository_storage]&.legacy_disk_path end def team -- cgit v1.2.3 From 39c9928cfbf5b72042755c58bd5c484551c5f1ad Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Tue, 20 Mar 2018 15:51:54 +0000 Subject: Fix N+1 in `MergeRequest#merge_request_diff_for` Previously, this would issue a query for each unique `diff_refs_or_sha` passed. This was because we didn't want to load other MR diffs into memory, as they had some very large columns. Now they are actually very small, and it's more efficient to just load them all at once and do the finding in Ruby. --- app/models/merge_request.rb | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) (limited to 'app/models') diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 7e6d89ec9c7..91d8be5559b 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -536,18 +536,25 @@ class MergeRequest < ActiveRecord::Base merge_request_diff(true) end + def viewable_diffs + @viewable_diffs ||= merge_request_diffs.viewable.to_a + end + def merge_request_diff_for(diff_refs_or_sha) - @merge_request_diffs_by_diff_refs_or_sha ||= Hash.new do |h, diff_refs_or_sha| - diffs = merge_request_diffs.viewable - h[diff_refs_or_sha] = - if diff_refs_or_sha.is_a?(Gitlab::Diff::DiffRefs) - diffs.find_by_diff_refs(diff_refs_or_sha) - else - diffs.find_by(head_commit_sha: diff_refs_or_sha) - end - end + matcher = + if diff_refs_or_sha.is_a?(Gitlab::Diff::DiffRefs) + { + 'start_commit_sha' => diff_refs_or_sha.start_sha, + 'head_commit_sha' => diff_refs_or_sha.head_sha, + 'base_commit_sha' => diff_refs_or_sha.base_sha + } + else + { 'head_commit_sha' => diff_refs_or_sha } + end - @merge_request_diffs_by_diff_refs_or_sha[diff_refs_or_sha] + viewable_diffs.find do |diff| + diff.attributes.slice(*matcher.keys) == matcher + end end def version_params_for(diff_refs) -- cgit v1.2.3 From 05103f080cf0e40b8fe5e1774b8dd1f8084105e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Thu, 22 Mar 2018 12:08:16 +0100 Subject: Make Variable key not secret --- app/models/ci/group_variable.rb | 1 - app/models/ci/pipeline_schedule_variable.rb | 1 - app/models/ci/variable.rb | 1 - 3 files changed, 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/group_variable.rb b/app/models/ci/group_variable.rb index 65399557289..62d768cc6cf 100644 --- a/app/models/ci/group_variable.rb +++ b/app/models/ci/group_variable.rb @@ -6,7 +6,6 @@ module Ci belongs_to :group - alias_attribute :secret_key, :key alias_attribute :secret_value, :value validates :key, uniqueness: { diff --git a/app/models/ci/pipeline_schedule_variable.rb b/app/models/ci/pipeline_schedule_variable.rb index 2e30612a88e..03df4e3e638 100644 --- a/app/models/ci/pipeline_schedule_variable.rb +++ b/app/models/ci/pipeline_schedule_variable.rb @@ -5,7 +5,6 @@ module Ci belongs_to :pipeline_schedule - alias_attribute :secret_key, :key alias_attribute :secret_value, :value validates :key, uniqueness: { scope: :pipeline_schedule_id } diff --git a/app/models/ci/variable.rb b/app/models/ci/variable.rb index bcad55f115f..452cb910bca 100644 --- a/app/models/ci/variable.rb +++ b/app/models/ci/variable.rb @@ -6,7 +6,6 @@ module Ci belongs_to :project - alias_attribute :secret_key, :key alias_attribute :secret_value, :value validates :key, uniqueness: { -- cgit v1.2.3 From 44f37504fb229ab78606a5fd11f75316ebc2667b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mica=C3=ABl=20Bergeron?= Date: Fri, 9 Mar 2018 10:09:00 -0500 Subject: Backport ee-40781-os-to-ce --- app/models/project.rb | 3 ++- app/models/upload.rb | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 0cef4d335dd..452c6784bd4 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -188,6 +188,8 @@ class Project < ActiveRecord::Base has_many :todos has_many :notification_settings, as: :source, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent + has_many :internal_ids + has_one :import_data, class_name: 'ProjectImportData', inverse_of: :project, autosave: true has_one :project_feature, inverse_of: :project has_one :statistics, class_name: 'ProjectStatistics' @@ -290,7 +292,6 @@ class Project < ActiveRecord::Base scope :non_archived, -> { where(archived: false) } scope :for_milestones, ->(ids) { joins(:milestones).where('milestones.id' => ids).distinct } scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) } - scope :with_project_feature, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id') } scope :with_statistics, -> { includes(:statistics) } scope :with_shared_runners, -> { where(shared_runners_enabled: true) } diff --git a/app/models/upload.rb b/app/models/upload.rb index cf71a7b76fc..e94a81c8770 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -1,4 +1,6 @@ class Upload < ActiveRecord::Base + prepend EE::Upload + # Upper limit for foreground checksum processing CHECKSUM_THRESHOLD = 100.megabytes -- cgit v1.2.3 From 000f9d01f7a8c6eb20c16927f95a2e3de42bc283 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 22 Mar 2018 13:57:39 +0100 Subject: Decouple YAML processor from pipeline objects --- app/models/ci/pipeline.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index bac5b3860e4..acc2dbb59fd 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -362,7 +362,13 @@ module Ci def stage_seeds return [] unless config_processor - @stage_seeds ||= config_processor.stage_seeds(self) + strong_memoize(:stage_seeds) do + seeds = config_processor.stages.map do |attributes| + Gitlab::Ci::Pipeline::Seed::Stage.new(self, attributes) + end + + seeds.select(&:included?) + end end def has_kubernetes_active? -- cgit v1.2.3 From b40d5d0fbb4d61ab8d5ee393eb91a395fb70b236 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 22 Mar 2018 15:22:50 +0100 Subject: Fix static analysis and tests related to YAML processing --- app/models/ci/pipeline.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index acc2dbb59fd..9642c94a231 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -363,7 +363,7 @@ module Ci return [] unless config_processor strong_memoize(:stage_seeds) do - seeds = config_processor.stages.map do |attributes| + seeds = config_processor.stages_attributes.map do |attributes| Gitlab::Ci::Pipeline::Seed::Stage.new(self, attributes) end -- cgit v1.2.3 From f45358db311f76d9a3aa8c68dd1f503228691ae1 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 22 Mar 2018 15:32:49 +0100 Subject: Bring seeds size method back to CI/CD pipeline class --- app/models/ci/pipeline.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 9642c94a231..643a8324c7d 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -371,6 +371,10 @@ module Ci end end + def seeds_size + stage_seeds.sum(&:size) + end + def has_kubernetes_active? project.deployment_platform&.active? end -- cgit v1.2.3 From 3b69e41018e48af6aa11512f3809e7d1fa43e7e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mica=C3=ABl=20Bergeron?= Date: Thu, 22 Mar 2018 13:00:58 -0400 Subject: remove EE-only code --- app/models/upload.rb | 2 -- 1 file changed, 2 deletions(-) (limited to 'app/models') diff --git a/app/models/upload.rb b/app/models/upload.rb index e94a81c8770..cf71a7b76fc 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -1,6 +1,4 @@ class Upload < ActiveRecord::Base - prepend EE::Upload - # Upper limit for foreground checksum processing CHECKSUM_THRESHOLD = 100.megabytes -- cgit v1.2.3 From 839589fde30cff9ecf963dab775bfd9b9b2fe17b Mon Sep 17 00:00:00 2001 From: Fabian Schneider Date: Sun, 11 Mar 2018 20:56:07 +0100 Subject: Change avatar error message to include allowed file formats --- app/models/concerns/avatarable.rb | 2 +- app/models/group.rb | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb index d35e37935fb..318df11727e 100644 --- a/app/models/concerns/avatarable.rb +++ b/app/models/concerns/avatarable.rb @@ -21,7 +21,7 @@ module Avatarable def avatar_type unless self.avatar.image? - self.errors.add :avatar, "only images allowed" + errors.add :avatar, "file format is not supported. Please try one of the following supported formats: #{AvatarUploader::IMAGE_EXT.join(', ')}" end end diff --git a/app/models/group.rb b/app/models/group.rb index 8d183006c65..8e391412b52 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -189,12 +189,6 @@ class Group < Namespace owners.include?(user) && owners.size == 1 end - def avatar_type - unless self.avatar.image? - self.errors.add :avatar, "only images allowed" - end - end - def post_create_hook Gitlab::AppLogger.info("Group \"#{name}\" was created") -- cgit v1.2.3 From 9d45951fcaeda4f01a2e4be2480d980a3e7cd37e Mon Sep 17 00:00:00 2001 From: Rob Watson Date: Wed, 3 Jan 2018 08:07:03 +0000 Subject: Add HTTPS-only pages Closes #28857 --- app/models/pages_domain.rb | 10 ++++++++-- app/models/project.rb | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index 588bd50ed77..2e478a24778 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -6,8 +6,10 @@ class PagesDomain < ActiveRecord::Base validates :domain, hostname: { allow_numeric_hostname: true } validates :domain, uniqueness: { case_sensitive: false } - validates :certificate, certificate: true, allow_nil: true, allow_blank: true - validates :key, certificate_key: true, allow_nil: true, allow_blank: true + validates :certificate, presence: { message: 'must be present if HTTPS-only is enabled' }, if: ->(domain) { domain.project&.pages_https_only? } + validates :certificate, certificate: true, if: ->(domain) { domain.certificate.present? } + validates :key, presence: { message: 'must be present if HTTPS-only is enabled' }, if: ->(domain) { domain.project&.pages_https_only? } + validates :key, certificate_key: true, if: ->(domain) { domain.key.present? } validates :verification_code, presence: true, allow_blank: false validate :validate_pages_domain @@ -46,6 +48,10 @@ class PagesDomain < ActiveRecord::Base !Gitlab::CurrentSettings.pages_domain_verification_enabled? || enabled_until.present? end + def https? + certificate.present? + end + def to_param domain end diff --git a/app/models/project.rb b/app/models/project.rb index 250680e2a2c..48a81ddb82e 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -267,6 +267,7 @@ class Project < ActiveRecord::Base validate :visibility_level_allowed_by_group validate :visibility_level_allowed_as_fork validate :check_wiki_path_conflict + validate :validate_pages_https_only, if: -> { changes.has_key?(:pages_https_only) } validates :repository_storage, presence: true, inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } } @@ -737,6 +738,26 @@ class Project < ActiveRecord::Base end end + def pages_https_only + return false unless Gitlab.config.pages.external_https + + super + end + + def pages_https_only? + return false unless Gitlab.config.pages.external_https + + super + end + + def validate_pages_https_only + return unless pages_https_only? + + unless pages_domains.all?(&:https?) + errors.add(:pages_https_only, "cannot be enabled unless all domains have TLS certificates") + end + end + def to_param if persisted? && errors.include?(:path) path_was -- cgit v1.2.3 From ca59bfdc6657fc20a27e93f41fc9e38c41498389 Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Wed, 21 Mar 2018 08:59:40 -0600 Subject: Rearrange DeploymentPlatform methods - Change find_cluster_platform_kubernetes to ignore environment argument - Fix broken specs CE Port of: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/5047/ --- app/models/clusters/cluster.rb | 2 ++ app/models/concerns/deployment_platform.rb | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'app/models') diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 49eb069016a..bfdfc5ae6fe 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -10,6 +10,7 @@ module Clusters Applications::Prometheus.application_name => Applications::Prometheus, Applications::Runner.application_name => Applications::Runner }.freeze + DEFAULT_ENVIRONMENT = '*'.freeze belongs_to :user @@ -50,6 +51,7 @@ module Clusters scope :enabled, -> { where(enabled: true) } scope :disabled, -> { where(enabled: false) } + scope :default_environment, -> { where(environment_scope: DEFAULT_ENVIRONMENT) } def status_name if provider diff --git a/app/models/concerns/deployment_platform.rb b/app/models/concerns/deployment_platform.rb index faa94204e33..6ddfc7b0d7b 100644 --- a/app/models/concerns/deployment_platform.rb +++ b/app/models/concerns/deployment_platform.rb @@ -1,16 +1,18 @@ module DeploymentPlatform - # EE would override this and utilize the extra argument + # EE would override this and utilize environment argument def deployment_platform(environment: nil) @deployment_platform ||= - find_cluster_platform_kubernetes || + find_cluster_platform_kubernetes(environment: environment) || find_kubernetes_service_integration || build_cluster_and_deployment_platform end private - def find_cluster_platform_kubernetes - clusters.find_by(enabled: true)&.platform_kubernetes + # EE would override this and utilize environment argument + def find_cluster_platform_kubernetes(environment: nil) + clusters.enabled.default_environment + .last&.platform_kubernetes end def find_kubernetes_service_integration -- cgit v1.2.3 From 96d6193cabacc92f43bd441f48d63b226d9dad21 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 23 Mar 2018 12:55:27 +0100 Subject: Make it possible to access static builds variables This makes it possible to access static build variables even when an object is not persisted yet. This will allow us to evaluate build variables using only/except policies before deciding whether we actually want to persist the build in the database, or not. --- app/models/ci/build.rb | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index c1da2081465..fa21b560736 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -547,16 +547,16 @@ module Ci variables.append(key: 'CI_SERVER_NAME', value: 'GitLab') variables.append(key: 'CI_SERVER_VERSION', value: Gitlab::VERSION) variables.append(key: 'CI_SERVER_REVISION', value: Gitlab::REVISION) - variables.append(key: 'CI_JOB_ID', value: id.to_s) + variables.append(key: 'CI_JOB_ID', value: id.to_s) if persisted? variables.append(key: 'CI_JOB_NAME', value: name) variables.append(key: 'CI_JOB_STAGE', value: stage) - variables.append(key: 'CI_JOB_TOKEN', value: token, public: false) + variables.append(key: 'CI_JOB_TOKEN', value: token, public: false) if persisted? variables.append(key: 'CI_COMMIT_SHA', value: sha) variables.append(key: 'CI_COMMIT_REF_NAME', value: ref) variables.append(key: 'CI_COMMIT_REF_SLUG', value: ref_slug) - variables.append(key: 'CI_REGISTRY_USER', value: CI_REGISTRY_USER) - variables.append(key: 'CI_REGISTRY_PASSWORD', value: token, public: false) - variables.append(key: 'CI_REPOSITORY_URL', value: repo_url, public: false) + variables.append(key: 'CI_REGISTRY_USER', value: CI_REGISTRY_USER) if persisted? + variables.append(key: 'CI_REGISTRY_PASSWORD', value: token, public: false) if persisted? + variables.append(key: 'CI_REPOSITORY_URL', value: repo_url, public: false) if persisted? variables.append(key: "CI_COMMIT_TAG", value: ref) if tag? variables.append(key: "CI_PIPELINE_TRIGGERED", value: 'true') if trigger_request variables.append(key: "CI_JOB_MANUAL", value: 'true') if action? @@ -579,8 +579,11 @@ module Ci def legacy_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| - variables.append(key: 'CI_BUILD_ID', value: id.to_s) - variables.append(key: 'CI_BUILD_TOKEN', value: token, public: false) + if persisted? + variables.append(key: 'CI_BUILD_ID', value: id.to_s) + variables.append(key: 'CI_BUILD_TOKEN', value: token, public: false) + end + variables.append(key: 'CI_BUILD_REF', value: sha) variables.append(key: 'CI_BUILD_BEFORE_SHA', value: before_sha) variables.append(key: 'CI_BUILD_REF_NAME', value: ref) -- cgit v1.2.3 From 1c2530ded3b52db0459095d5365460f12d104adb Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 23 Mar 2018 15:21:31 +0100 Subject: Expose evaluable variables method from build object --- app/models/ci/build.rb | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index fa21b560736..cc277fb5266 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -244,13 +244,24 @@ module Ci Gitlab::Utils.slugify(ref.to_s) end + ## # Variables whose value does not depend on environment + # def simple_variables variables(environment: nil) end - # All variables, including those dependent on environment, which could - # contain unexpanded variables. + ## + # Variables that are available for evaluation using variables policy. + # + def evaluable_variables + Gitlab::Ci::Variables::Collection.new + .concat(simple_variables) + end + + ## All variables, including those dependent on environment, which could + # contain unexpanded variables. + # def variables(environment: persisted_environment) collection = Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.concat(predefined_variables) -- cgit v1.2.3 From 783868e9faa1e3dff27765f8244dc59898a2f7b2 Mon Sep 17 00:00:00 2001 From: Andreas Brandl Date: Fri, 23 Mar 2018 13:58:46 +0100 Subject: Remove N+1 query for Noteable association. For each event with `Event#target_type` of "Note", we had a query to load the associated instance `noteable`. For example, if `Note` was related to an issue, we'd load each `Issue` with its own query (N+1 problem). Closes #43150. --- app/models/event.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/event.rb b/app/models/event.rb index 17a198d52c7..3805f6cf857 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -52,12 +52,12 @@ class Event < ActiveRecord::Base belongs_to :target, -> { # If the association for "target" defines an "author" association we want to # eager-load this so Banzai & friends don't end up performing N+1 queries to - # get the authors of notes, issues, etc. - if reflections['events'].active_record.reflect_on_association(:author) - includes(:author) - else - self + # get the authors of notes, issues, etc. (likewise for "noteable"). + incs = %i(author noteable).select do |a| + reflections['events'].active_record.reflect_on_association(a) end + + incs.reduce(self) { |obj, a| obj.includes(a) } }, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations has_one :push_event_payload -- cgit v1.2.3 From 95f2826e1aa23809412f32263c399c67f005df16 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 23 Mar 2018 15:43:17 +0100 Subject: Remove outdates TODOs from pipeline class --- app/models/ci/pipeline.rb | 4 ---- 1 file changed, 4 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 643a8324c7d..156ad194d2e 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -478,14 +478,10 @@ module Ci end end - # TODO specs - # def protected_ref? strong_memoize(:protected_ref) { project.protected_for?(ref) } end - # TODO specs - # def legacy_trigger strong_memoize(:legacy_trigger) { trigger_requests.first } end -- cgit v1.2.3 From 99b01e23598e6b0b2bcae891939ea28c67f7b2e9 Mon Sep 17 00:00:00 2001 From: YarNayar Date: Tue, 25 Jul 2017 14:56:09 +0300 Subject: Send notification emails when push to a merge request Closes #23460 --- app/models/commit.rb | 2 +- app/models/notification_recipient.rb | 15 ++++++++++++--- app/models/notification_setting.rb | 7 ++++++- 3 files changed, 19 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/commit.rb b/app/models/commit.rb index cceae5efb72..b64462fb768 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -175,7 +175,7 @@ class Commit if safe_message.blank? no_commit_message else - safe_message.split("\n", 2).first + safe_message.split(/[\r\n]/, 2).first end end diff --git a/app/models/notification_recipient.rb b/app/models/notification_recipient.rb index e95655e19f8..b3ffad00a07 100644 --- a/app/models/notification_recipient.rb +++ b/app/models/notification_recipient.rb @@ -48,7 +48,7 @@ class NotificationRecipient when :custom custom_enabled? || %i[participating mention].include?(@type) when :watch, :participating - !excluded_watcher_action? + !action_excluded? when :mention @type == :mention else @@ -96,13 +96,22 @@ class NotificationRecipient end end + def action_excluded? + excluded_watcher_action? || excluded_participating_action? + end + def excluded_watcher_action? - return false unless @custom_action - return false if notification_level == :custom + return false unless @custom_action && notification_level == :watch NotificationSetting::EXCLUDED_WATCHER_EVENTS.include?(@custom_action) end + def excluded_participating_action? + return false unless @custom_action && notification_level == :participating + + NotificationSetting::EXCLUDED_PARTICIPATING_EVENTS.include?(@custom_action) + end + private def read_ability diff --git a/app/models/notification_setting.rb b/app/models/notification_setting.rb index 245f8dddcf9..f6d9b0215fc 100644 --- a/app/models/notification_setting.rb +++ b/app/models/notification_setting.rb @@ -33,6 +33,7 @@ class NotificationSetting < ActiveRecord::Base :close_issue, :reassign_issue, :new_merge_request, + :push_to_merge_request, :reopen_merge_request, :close_merge_request, :reassign_merge_request, @@ -41,10 +42,14 @@ class NotificationSetting < ActiveRecord::Base :success_pipeline ].freeze - EXCLUDED_WATCHER_EVENTS = [ + EXCLUDED_PARTICIPATING_EVENTS = [ :success_pipeline ].freeze + EXCLUDED_WATCHER_EVENTS = [ + :push_to_merge_request + ].push(*EXCLUDED_PARTICIPATING_EVENTS).freeze + def self.find_or_create_for(source) setting = find_or_initialize_by(source: source) -- cgit v1.2.3 From 5f22dbdb3af5d7d1472f23a5ed81299f05fa45da Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Thu, 22 Mar 2018 15:25:46 +0000 Subject: Improves User#owned_projects query performance --- app/models/user.rb | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index b8c55205ab8..fa54581d220 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -623,9 +623,7 @@ class User < ActiveRecord::Base end def owned_projects - @owned_projects ||= - Project.where('namespace_id IN (?) OR namespace_id = ?', - owned_groups.select(:id), namespace.id).joins(:namespace) + @owned_projects ||= Project.from("(#{owned_projects_union.to_sql}) AS projects") end # Returns projects which user can admin issues on (for example to move an issue to that project). @@ -1196,6 +1194,15 @@ class User < ActiveRecord::Base private + def owned_projects_union + Gitlab::SQL::Union.new([ + Project.where(namespace: namespace), + Project.joins(:project_authorizations) + .where("projects.namespace_id <> ?", namespace.id) + .where(project_authorizations: { user_id: id, access_level: Gitlab::Access::OWNER }) + ], remove_duplicates: false) + end + def ci_projects_union scope = { access_level: [Gitlab::Access::MASTER, Gitlab::Access::OWNER] } groups = groups_projects.where(members: scope) -- cgit v1.2.3 From 1a79beff6ea15c8ce37bc461103d6a5f46ff2955 Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Mon, 26 Mar 2018 16:14:23 -0600 Subject: Re-arrange deployment_platform to avoid cop complainments --- app/models/concerns/deployment_platform.rb | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/deployment_platform.rb b/app/models/concerns/deployment_platform.rb index 6ddfc7b0d7b..52851b3d0b2 100644 --- a/app/models/concerns/deployment_platform.rb +++ b/app/models/concerns/deployment_platform.rb @@ -1,14 +1,20 @@ module DeploymentPlatform # EE would override this and utilize environment argument + # rubocop:disable Gitlab/ModuleWithInstanceVariables def deployment_platform(environment: nil) - @deployment_platform ||= - find_cluster_platform_kubernetes(environment: environment) || - find_kubernetes_service_integration || - build_cluster_and_deployment_platform + @deployment_platform ||= {} + + @deployment_platform[environment] ||= find_deployment_platform(environment) end private + def find_deployment_platform(environment) + find_cluster_platform_kubernetes(environment: environment) || + find_kubernetes_service_integration || + build_cluster_and_deployment_platform + end + # EE would override this and utilize environment argument def find_cluster_platform_kubernetes(environment: nil) clusters.enabled.default_environment -- cgit v1.2.3 From 04c5e637f827d869f8bbfb21ca41eb552c3324d6 Mon Sep 17 00:00:00 2001 From: Alessio Caiazza Date: Fri, 23 Mar 2018 11:07:22 +0100 Subject: Port LFS direct_upload from EE --- app/models/lfs_object.rb | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'app/models') diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index 64e88d5a6a2..b7de46fa202 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -11,6 +11,12 @@ class LfsObject < ActiveRecord::Base mount_uploader :file, LfsObjectUploader + before_save :update_file_store + + def update_file_store + self.file_store = file.object_store + end + def project_allowed_access?(project) projects.exists?(project.lfs_storage_project.id) end -- cgit v1.2.3 From da6bfd9b801c4d1c91593c6b4a755b097b644ab0 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 27 Mar 2018 13:05:29 +0200 Subject: Decouple build variables from persisted environment --- app/models/ci/build.rb | 70 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 28 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 627064d3dec..5f149ad3057 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -23,7 +23,9 @@ module Ci has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id - # The "environment" field for builds is a String, and is the unexpanded name + ## + # The "environment" field for builds is a String, and is the unexpanded name! + # def persisted_environment @persisted_environment ||= Environment.find_by( name: expanded_environment_name, @@ -198,7 +200,9 @@ module Ci end def expanded_environment_name - ExpandVariables.expand(environment, simple_variables) if environment + if has_environment? + ExpandVariables.expand(environment, simple_variables) + end end def has_environment? @@ -249,41 +253,46 @@ module Ci end ## - # Variables whose value does not depend on environment - # - def simple_variables - variables(environment: nil) - end - - ## - # Variables that are available for evaluation using variables policy. - # - def evaluable_variables - Gitlab::Ci::Variables::Collection.new - .concat(simple_variables) - end - - ## All variables, including those dependent on environment, which could - # contain unexpanded variables. + # Variables in the environment name scope. # - def variables(environment: persisted_environment) - collection = Gitlab::Ci::Variables::Collection.new.tap do |variables| + def scoped_variables(environment: expanded_environment_name) + Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.concat(predefined_variables) variables.concat(project.predefined_variables) variables.concat(pipeline.predefined_variables) variables.concat(runner.predefined_variables) if runner - variables.concat(project.deployment_variables(environment: environment)) if has_environment? + variables.concat(project.deployment_variables(environment: environment)) if environment variables.concat(yaml_variables) variables.concat(user_variables) - variables.concat(project.group.secret_variables_for(ref, project)) if project.group - variables.concat(secret_variables(environment: environment)) + variables.concat(secret_group_variables) + variables.concat(secret_project_variables(environment: environment)) variables.concat(trigger_request.user_variables) if trigger_request variables.concat(pipeline.variables) variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline.pipeline_schedule - variables.concat(persisted_environment_variables) if environment end + end - collection.to_runner_variables + ## + # Variables that do not depend on the environment name. + # + def simple_variables + scoped_variables(environment: nil).to_runner_variables + end + + ## + # All variables, including persisted environment variables. + # + def variables + scoped_variables + .concat(persisted_environment_variables) + .to_runner_variables + end + + ## + # TODO, add specs + # + def variables_hash + scoped_variables.to_hash end def features @@ -454,9 +463,14 @@ module Ci end end - def secret_variables(environment: persisted_environment) + def secret_group_variables + return [] unless project.group + + project.group.secret_variables_for(ref, project) + end + + def secret_project_variables(environment: persisted_environment) project.secret_variables_for(ref: ref, environment: environment) - .map(&:to_runner_variable) end def steps @@ -580,7 +594,7 @@ module Ci def persisted_environment_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| - return variables unless persisted_environment + return variables unless persisted? && persisted_environment.present? variables.concat(persisted_environment.predefined_variables) -- cgit v1.2.3 From a60ccef9fec1d798dd0178d10b6a6a109612c17d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 27 Mar 2018 13:46:22 +0200 Subject: Add specs for method that exposes build variables hash --- app/models/ci/build.rb | 3 --- 1 file changed, 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 5f149ad3057..68aafdd7304 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -288,9 +288,6 @@ module Ci .to_runner_variables end - ## - # TODO, add specs - # def variables_hash scoped_variables.to_hash end -- cgit v1.2.3 From 2308ab7a9f0e3efcc1c3a21626aaffc6207cf1aa Mon Sep 17 00:00:00 2001 From: Elias Werberich Date: Tue, 27 Mar 2018 12:16:12 +0000 Subject: 'Assigned Issues' and 'Assigned Merge Requests' as dashboard user choices --- app/models/user.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index fa54581d220..187878f4fb5 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -187,7 +187,7 @@ class User < ActiveRecord::Base # User's Dashboard preference # Note: When adding an option, it MUST go on the end of the array. - enum dashboard: [:projects, :stars, :project_activity, :starred_project_activity, :groups, :todos] + enum dashboard: [:projects, :stars, :project_activity, :starred_project_activity, :groups, :todos, :issues, :merge_requests] # User's Project preference # Note: When adding an option, it MUST go on the end of the array. -- cgit v1.2.3 From c43e18fc495f66c4a96c9a72bdda72392fb0ea32 Mon Sep 17 00:00:00 2001 From: "Jacob Vosmaer (GitLab)" Date: Wed, 28 Mar 2018 09:21:32 +0000 Subject: Remove some easy cases of 'path_to_repo' use --- app/models/project_services/gemnasium_service.rb | 2 +- app/models/repository.rb | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/project_services/gemnasium_service.rb b/app/models/project_services/gemnasium_service.rb index 017a9b2df6e..26cbfd784ad 100644 --- a/app/models/project_services/gemnasium_service.rb +++ b/app/models/project_services/gemnasium_service.rb @@ -36,7 +36,7 @@ class GemnasiumService < Service after: data[:after], token: token, api_key: api_key, - repo: project.repository.path_to_repo + repo: project.repository.path_to_repo # Gitaly: fixed by https://gitlab.com/gitlab-org/security-products/gemnasium-migration/issues/9 ) end end diff --git a/app/models/repository.rb b/app/models/repository.rb index 42f1ac43e29..2ba1c6cb8c9 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -93,10 +93,6 @@ class Repository "#<#{self.class.name}:#{@disk_path}>" end - def create_hooks - Gitlab::Git::Repository.create_hooks(path_to_repo, Gitlab.config.gitlab_shell.hooks_path) - end - def commit(ref = 'HEAD') return nil unless exists? return ref if ref.is_a?(::Commit) -- cgit v1.2.3 From 834f473821b816515504abb7c6bc91ab2dee4450 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 19 Feb 2018 18:23:12 +0100 Subject: Override project-defined timeout with runner-defined one --- app/models/ci/build.rb | 7 +++++++ app/models/ci/runner.rb | 4 ++++ 2 files changed, 11 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 08bb5915d10..e15c4bc6ceb 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -232,9 +232,16 @@ module Ci end def timeout + return runner.job_upper_timeout if should_use_runner_timeout + project.build_timeout end + def should_use_runner_timeout + runner && runner.defines_job_upper_timeout? && runner.job_upper_timeout < project.build_timeout + end + private :should_use_runner_timeout + def triggered_by?(current_user) user == current_user end diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 7173f88f1c7..b28e892dcb6 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -167,6 +167,10 @@ module Ci end end + def defines_job_upper_timeout? + job_upper_timeout && job_upper_timeout > 0 + end + private def cleanup_runner_queue -- cgit v1.2.3 From b6d26f979c29ce594a8645b81ec0fd1028e79b64 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 19 Feb 2018 18:41:50 +0100 Subject: Add UI support for per-runner job timeout --- app/models/ci/runner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index b28e892dcb6..be15fb0f729 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -8,7 +8,7 @@ module Ci ONLINE_CONTACT_TIMEOUT = 1.hour UPDATE_DB_RUNNER_INFO_EVERY = 40.minutes AVAILABLE_SCOPES = %w[specific shared active paused online].freeze - FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level].freeze + FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level job_upper_timeout].freeze has_many :builds has_many :runner_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent -- cgit v1.2.3 From a4ea9a93db98461479dcb8e1d7b8425a77018f1e Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 21 Feb 2018 01:09:59 +0100 Subject: Add ChroniDurationAttribute concern --- app/models/ci/runner.rb | 3 +++ app/models/concerns/chronic_duration_attribute.rb | 25 +++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 app/models/concerns/chronic_duration_attribute.rb (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index be15fb0f729..cf91ed3c9dc 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -3,6 +3,7 @@ module Ci extend Gitlab::Ci::Model include Gitlab::SQL::Pattern include RedisCacheable + include ChronicDurationAttribute RUNNER_QUEUE_EXPIRY_TIME = 60.minutes ONLINE_CONTACT_TIMEOUT = 1.hour @@ -51,6 +52,8 @@ module Ci cached_attr_reader :version, :revision, :platform, :architecture, :contacted_at, :ip_address + chronic_duration_attribute :job_upper_timeout_user_readable, :job_upper_timeout + # Searches for runners matching the given query. # # This method uses ILIKE on PostgreSQL and LIKE on MySQL. diff --git a/app/models/concerns/chronic_duration_attribute.rb b/app/models/concerns/chronic_duration_attribute.rb new file mode 100644 index 00000000000..2bf33174640 --- /dev/null +++ b/app/models/concerns/chronic_duration_attribute.rb @@ -0,0 +1,25 @@ +module ChronicDurationAttribute + extend ActiveSupport::Concern + + class_methods do + def chronic_duration_attribute(virtual_attribute, source_attribute) + chronic_duration_attribute_reader(virtual_attribute, source_attribute) + chronic_duration_attribute_writer(virtual_attribute, source_attribute) + end + + def chronic_duration_attribute_reader(virtual_attribute, source_attribute) + define_method(virtual_attribute) do + value = self.send(source_attribute) # rubocop:disable GitlabSecurity/PublicSend + ChronicDuration.output(value, format: :short) unless value.nil? + end + end + + def chronic_duration_attribute_writer(virtual_attribute, source_attribute) + define_method("#{virtual_attribute}=") do |value| + new_value = ChronicDuration.parse(value).to_i + self.send("#{source_attribute}=", new_value) # rubocop:disable GitlabSecurity/PublicSend + new_value + end + end + end +end -- cgit v1.2.3 From d633bc8134fe472137fb668c1eb78de45dc9bb57 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 21 Feb 2018 01:21:59 +0100 Subject: Rename job_upper_timeout to maximum_job_timeout --- app/models/ci/build.rb | 4 ++-- app/models/ci/runner.rb | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index e15c4bc6ceb..61a0ef08dde 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -232,13 +232,13 @@ module Ci end def timeout - return runner.job_upper_timeout if should_use_runner_timeout + return runner.maximum_job_timeout if should_use_runner_timeout project.build_timeout end def should_use_runner_timeout - runner && runner.defines_job_upper_timeout? && runner.job_upper_timeout < project.build_timeout + !runner.nil? && runner.defines_maximum_job_timeout? && runner.maximum_job_timeout < project.build_timeout end private :should_use_runner_timeout diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index cf91ed3c9dc..3d83e00ccfe 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -9,7 +9,7 @@ module Ci ONLINE_CONTACT_TIMEOUT = 1.hour UPDATE_DB_RUNNER_INFO_EVERY = 40.minutes AVAILABLE_SCOPES = %w[specific shared active paused online].freeze - FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level job_upper_timeout].freeze + FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level maximum_job_timeout_user_readable].freeze has_many :builds has_many :runner_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -52,7 +52,7 @@ module Ci cached_attr_reader :version, :revision, :platform, :architecture, :contacted_at, :ip_address - chronic_duration_attribute :job_upper_timeout_user_readable, :job_upper_timeout + chronic_duration_attribute :maximum_job_timeout_user_readable, :maximum_job_timeout # Searches for runners matching the given query. # @@ -170,8 +170,8 @@ module Ci end end - def defines_job_upper_timeout? - job_upper_timeout && job_upper_timeout > 0 + def defines_maximum_job_timeout? + !maximum_job_timeout.nil? && maximum_job_timeout > 0 end private -- cgit v1.2.3 From 78a4189ece4f8d125eefbfdf6619d3452820bb8e Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 21 Feb 2018 04:03:12 +0100 Subject: Show timeout information on job's page --- app/models/ci/build.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 61a0ef08dde..3072817f443 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -6,6 +6,7 @@ module Ci include ObjectStorage::BackgroundMove include Presentable include Importable + include ChronicDurationAttribute MissingDependenciesError = Class.new(StandardError) @@ -90,6 +91,8 @@ module Ci after_commit :update_project_statistics_after_save, on: [:create, :update] after_commit :update_project_statistics, on: :destroy + chronic_duration_attribute_reader :used_timeout_user_readable, :used_timeout + class << self # This is needed for url_for to work, # as the controller is JobsController @@ -120,6 +123,10 @@ module Ci end after_transition pending: :running do |build| + build.used_timeout = build.timeout + build.timeout_source = build.should_use_runner_timeout? ? 'Runner' : 'Project' + build.save! + build.run_after_commit do BuildHooksWorker.perform_async(id) end @@ -232,15 +239,14 @@ module Ci end def timeout - return runner.maximum_job_timeout if should_use_runner_timeout + return runner.maximum_job_timeout if should_use_runner_timeout? project.build_timeout end - def should_use_runner_timeout + def should_use_runner_timeout? !runner.nil? && runner.defines_maximum_job_timeout? && runner.maximum_job_timeout < project.build_timeout end - private :should_use_runner_timeout def triggered_by?(current_user) user == current_user -- cgit v1.2.3 From 4ce376c0649faf4dea01857c312379771bdf37b7 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 26 Feb 2018 16:26:39 +0100 Subject: Rename chronic_duration_attribute* to chronic_duration_attr* --- app/models/ci/build.rb | 2 +- app/models/ci/runner.rb | 2 +- app/models/concerns/chronic_duration_attribute.rb | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 3072817f443..5b9e06ab203 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -91,7 +91,7 @@ module Ci after_commit :update_project_statistics_after_save, on: [:create, :update] after_commit :update_project_statistics, on: :destroy - chronic_duration_attribute_reader :used_timeout_user_readable, :used_timeout + chronic_duration_attr_reader :used_timeout_user_readable, :used_timeout class << self # This is needed for url_for to work, diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 3d83e00ccfe..f95afd4c40c 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -52,7 +52,7 @@ module Ci cached_attr_reader :version, :revision, :platform, :architecture, :contacted_at, :ip_address - chronic_duration_attribute :maximum_job_timeout_user_readable, :maximum_job_timeout + chronic_duration_attr :maximum_job_timeout_user_readable, :maximum_job_timeout # Searches for runners matching the given query. # diff --git a/app/models/concerns/chronic_duration_attribute.rb b/app/models/concerns/chronic_duration_attribute.rb index 2bf33174640..ae3aeda1709 100644 --- a/app/models/concerns/chronic_duration_attribute.rb +++ b/app/models/concerns/chronic_duration_attribute.rb @@ -2,19 +2,19 @@ module ChronicDurationAttribute extend ActiveSupport::Concern class_methods do - def chronic_duration_attribute(virtual_attribute, source_attribute) - chronic_duration_attribute_reader(virtual_attribute, source_attribute) - chronic_duration_attribute_writer(virtual_attribute, source_attribute) + def chronic_duration_attr(virtual_attribute, source_attribute) + chronic_duration_attr_reader(virtual_attribute, source_attribute) + chronic_duration_attr_writer(virtual_attribute, source_attribute) end - def chronic_duration_attribute_reader(virtual_attribute, source_attribute) + def chronic_duration_attr_reader(virtual_attribute, source_attribute) define_method(virtual_attribute) do value = self.send(source_attribute) # rubocop:disable GitlabSecurity/PublicSend ChronicDuration.output(value, format: :short) unless value.nil? end end - def chronic_duration_attribute_writer(virtual_attribute, source_attribute) + def chronic_duration_attr_writer(virtual_attribute, source_attribute) define_method("#{virtual_attribute}=") do |value| new_value = ChronicDuration.parse(value).to_i self.send("#{source_attribute}=", new_value) # rubocop:disable GitlabSecurity/PublicSend -- cgit v1.2.3 From 36753b78c065a54d7501f37f69fb49506f26688c Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 26 Feb 2018 16:35:08 +0100 Subject: Replace user_readable with human_readable --- app/models/ci/build.rb | 2 +- app/models/ci/runner.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 5b9e06ab203..f47cbe0a206 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -91,7 +91,7 @@ module Ci after_commit :update_project_statistics_after_save, on: [:create, :update] after_commit :update_project_statistics, on: :destroy - chronic_duration_attr_reader :used_timeout_user_readable, :used_timeout + chronic_duration_attr_reader :used_timeout_human_readable, :used_timeout class << self # This is needed for url_for to work, diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index f95afd4c40c..baf57423682 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -9,7 +9,7 @@ module Ci ONLINE_CONTACT_TIMEOUT = 1.hour UPDATE_DB_RUNNER_INFO_EVERY = 40.minutes AVAILABLE_SCOPES = %w[specific shared active paused online].freeze - FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level maximum_job_timeout_user_readable].freeze + FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level maximum_job_timeout_human_readable].freeze has_many :builds has_many :runner_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -52,7 +52,7 @@ module Ci cached_attr_reader :version, :revision, :platform, :architecture, :contacted_at, :ip_address - chronic_duration_attr :maximum_job_timeout_user_readable, :maximum_job_timeout + chronic_duration_attr :maximum_job_timeout_human_readable, :maximum_job_timeout # Searches for runners matching the given query. # -- cgit v1.2.3 From 3e7f3bc7980018910f6c6e7a19f706ad18ab2793 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 26 Feb 2018 18:51:56 +0100 Subject: Refactor timeout selection mechanism --- app/models/ci/build.rb | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index f47cbe0a206..f9a83965199 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -124,7 +124,7 @@ module Ci after_transition pending: :running do |build| build.used_timeout = build.timeout - build.timeout_source = build.should_use_runner_timeout? ? 'Runner' : 'Project' + build.timeout_source = build.timeout < build.project.build_timeout ? 'Runner' : 'Project' build.save! build.run_after_commit do @@ -239,13 +239,7 @@ module Ci end def timeout - return runner.maximum_job_timeout if should_use_runner_timeout? - - project.build_timeout - end - - def should_use_runner_timeout? - !runner.nil? && runner.defines_maximum_job_timeout? && runner.maximum_job_timeout < project.build_timeout + [project.build_timeout, runner&.maximum_job_timeout].compact.min end def triggered_by?(current_user) -- cgit v1.2.3 From cb502b626659001af1e0ca82288a2b427e346d18 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 26 Feb 2018 18:56:39 +0100 Subject: Remove unused method --- app/models/ci/runner.rb | 4 ---- 1 file changed, 4 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index baf57423682..413607e0eff 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -170,10 +170,6 @@ module Ci end end - def defines_maximum_job_timeout? - !maximum_job_timeout.nil? && maximum_job_timeout > 0 - end - private def cleanup_runner_queue -- cgit v1.2.3 From 7ef24a47714802edd8ad5470a6147b60172a0a28 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 28 Feb 2018 20:56:14 +0100 Subject: Properly handle empty value for chronic duration attribute --- app/models/concerns/chronic_duration_attribute.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/concerns/chronic_duration_attribute.rb b/app/models/concerns/chronic_duration_attribute.rb index ae3aeda1709..87c60a2b7d5 100644 --- a/app/models/concerns/chronic_duration_attribute.rb +++ b/app/models/concerns/chronic_duration_attribute.rb @@ -10,14 +10,20 @@ module ChronicDurationAttribute def chronic_duration_attr_reader(virtual_attribute, source_attribute) define_method(virtual_attribute) do value = self.send(source_attribute) # rubocop:disable GitlabSecurity/PublicSend - ChronicDuration.output(value, format: :short) unless value.nil? + + return '' if value.nil? + + ChronicDuration.output(value, format: :short) end end def chronic_duration_attr_writer(virtual_attribute, source_attribute) define_method("#{virtual_attribute}=") do |value| new_value = ChronicDuration.parse(value).to_i + new_value = nil if new_value <= 0 + self.send("#{source_attribute}=", new_value) # rubocop:disable GitlabSecurity/PublicSend + new_value end end -- cgit v1.2.3 From 8ffa48098b77e1f35f9c01f6977d9ccf1d166a24 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 28 Feb 2018 21:07:57 +0100 Subject: Downcase timeout_source value --- app/models/ci/build.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index f9a83965199..2f01a098b0e 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -124,7 +124,7 @@ module Ci after_transition pending: :running do |build| build.used_timeout = build.timeout - build.timeout_source = build.timeout < build.project.build_timeout ? 'Runner' : 'Project' + build.timeout_source = build.timeout < build.project.build_timeout ? 'runner' : 'project' build.save! build.run_after_commit do -- cgit v1.2.3 From 1b0b8b9c02642ac19b9f5019cdd38fcec280c2a7 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 28 Feb 2018 21:36:01 +0100 Subject: Change timeout_source to enum --- app/models/ci/build.rb | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 2f01a098b0e..cbd2cc6c58f 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -93,6 +93,12 @@ module Ci chronic_duration_attr_reader :used_timeout_human_readable, :used_timeout + enum timeout_source: { + unknown_timeout_source: nil, + project_timeout_source: 1, + runner_timeout_source: 2 + } + class << self # This is needed for url_for to work, # as the controller is JobsController @@ -123,10 +129,6 @@ module Ci end after_transition pending: :running do |build| - build.used_timeout = build.timeout - build.timeout_source = build.timeout < build.project.build_timeout ? 'runner' : 'project' - build.save! - build.run_after_commit do BuildHooksWorker.perform_async(id) end @@ -160,6 +162,11 @@ module Ci before_transition any => [:running] do |build| build.validates_dependencies! unless Feature.enabled?('ci_disable_validates_dependencies') end + + before_transition pending: :running do |build| + build.used_timeout = build.timeout + build.timeout_source = build.timeout < build.project.build_timeout ? :runner_timeout_source : :project_timeout_source + end end def detailed_status(current_user) -- cgit v1.2.3 From 1dde609ca6b130aa0a3d39e929edee7e770e62fc Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 1 Mar 2018 03:12:32 +0100 Subject: Move job timeout information to new ci_builds_metadata table --- app/models/ci/build.rb | 18 +++++++----------- app/models/ci/build_metadata.rb | 25 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 11 deletions(-) create mode 100644 app/models/ci/build_metadata.rb (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index cbd2cc6c58f..4b4a988d600 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -6,7 +6,6 @@ module Ci include ObjectStorage::BackgroundMove include Presentable include Importable - include ChronicDurationAttribute MissingDependenciesError = Class.new(StandardError) @@ -25,6 +24,8 @@ module Ci has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id + has_one :metadata, class_name: 'Ci::BuildMetadata' + # The "environment" field for builds is a String, and is the unexpanded name def persisted_environment @persisted_environment ||= Environment.find_by( @@ -84,6 +85,10 @@ module Ci before_save :ensure_token before_destroy { unscoped_project } + before_create do |build| + build.metadata = Ci::BuildMetadata.new + end + after_create unless: :importing? do |build| run_after_commit { BuildHooksWorker.perform_async(build.id) } end @@ -91,14 +96,6 @@ module Ci after_commit :update_project_statistics_after_save, on: [:create, :update] after_commit :update_project_statistics, on: :destroy - chronic_duration_attr_reader :used_timeout_human_readable, :used_timeout - - enum timeout_source: { - unknown_timeout_source: nil, - project_timeout_source: 1, - runner_timeout_source: 2 - } - class << self # This is needed for url_for to work, # as the controller is JobsController @@ -164,8 +161,7 @@ module Ci end before_transition pending: :running do |build| - build.used_timeout = build.timeout - build.timeout_source = build.timeout < build.project.build_timeout ? :runner_timeout_source : :project_timeout_source + build.metadata.save_timeout_state! end end diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb new file mode 100644 index 00000000000..7a1315dfcf9 --- /dev/null +++ b/app/models/ci/build_metadata.rb @@ -0,0 +1,25 @@ +module Ci + class BuildMetadata < ActiveRecord::Base + extend Gitlab::Ci::Model + include Presentable + include ChronicDurationAttribute + + self.table_name = 'ci_builds_metadata' + + belongs_to :build, class_name: 'Ci::Build' + + chronic_duration_attr_reader :used_timeout_human_readable, :used_timeout + + enum timeout_source: { + unknown_timeout_source: nil, + project_timeout_source: 1, + runner_timeout_source: 2 + } + + def save_timeout_state! + self.used_timeout = build.timeout + self.timeout_source = build.timeout < build.project.build_timeout ? :runner_timeout_source : :project_timeout_source + save! + end + end +end -- cgit v1.2.3 From 9ef86b1b12e80bcc3a6277e2b6a36b793c828489 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 1 Mar 2018 20:04:15 +0100 Subject: Fix transition failure when metadata not exists --- app/models/ci/build.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 4b4a988d600..cc29c6150d3 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -161,7 +161,7 @@ module Ci end before_transition pending: :running do |build| - build.metadata.save_timeout_state! + build.metadata.save_timeout_state! unless build.metadata.nil? end end -- cgit v1.2.3 From bb64b20a1f7dd831e4c200343b531f9d048c02a8 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 5 Mar 2018 17:35:06 +0100 Subject: Refactorize Ci::Build and Ci::BuildMetadata models --- app/models/ci/build.rb | 14 +++++++------- app/models/ci/build_metadata.rb | 16 +++++++++++----- 2 files changed, 18 insertions(+), 12 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index cc29c6150d3..7a12d7a3deb 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -24,7 +24,7 @@ module Ci has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id - has_one :metadata, class_name: 'Ci::BuildMetadata' + has_one :build_metadata, class_name: 'Ci::BuildMetadata' # The "environment" field for builds is a String, and is the unexpanded name def persisted_environment @@ -85,10 +85,6 @@ module Ci before_save :ensure_token before_destroy { unscoped_project } - before_create do |build| - build.metadata = Ci::BuildMetadata.new - end - after_create unless: :importing? do |build| run_after_commit { BuildHooksWorker.perform_async(build.id) } end @@ -161,10 +157,14 @@ module Ci end before_transition pending: :running do |build| - build.metadata.save_timeout_state! unless build.metadata.nil? + build.metadata.save_timeout_state! end end + def metadata + self.build_metadata ||= Ci::BuildMetadata.new + end + def detailed_status(current_user) Gitlab::Ci::Status::Build::Factory .new(self, current_user) @@ -242,7 +242,7 @@ module Ci end def timeout - [project.build_timeout, runner&.maximum_job_timeout].compact.min + metadata.used_timeout end def triggered_by?(current_user) diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index 7a1315dfcf9..9043bed86bf 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -1,4 +1,6 @@ module Ci + # The purpose of this class is to store Build related data that can be disposed. + # Data that should be persisted forever, should be stored with Ci::Build model. class BuildMetadata < ActiveRecord::Base extend Gitlab::Ci::Model include Presentable @@ -11,14 +13,18 @@ module Ci chronic_duration_attr_reader :used_timeout_human_readable, :used_timeout enum timeout_source: { - unknown_timeout_source: nil, - project_timeout_source: 1, - runner_timeout_source: 2 + unknown_timeout_source: 1, + project_timeout_source: 2, + runner_timeout_source: 3 } def save_timeout_state! - self.used_timeout = build.timeout - self.timeout_source = build.timeout < build.project.build_timeout ? :runner_timeout_source : :project_timeout_source + project_timeout = build.project&.build_timeout + timeout = [project_timeout, build.runner&.maximum_job_timeout].compact.min + + self.used_timeout = timeout + self.timeout_source = timeout < project_timeout ? :runner_timeout_source : :project_timeout_source + save! end end -- cgit v1.2.3 From 1e138767a652d86458d38665b98c9c2e5d4c3cb8 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 5 Mar 2018 20:22:00 +0100 Subject: Fix tests failures --- app/models/concerns/chronic_duration_attribute.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/chronic_duration_attribute.rb b/app/models/concerns/chronic_duration_attribute.rb index 87c60a2b7d5..15095d0b758 100644 --- a/app/models/concerns/chronic_duration_attribute.rb +++ b/app/models/concerns/chronic_duration_attribute.rb @@ -19,8 +19,8 @@ module ChronicDurationAttribute def chronic_duration_attr_writer(virtual_attribute, source_attribute) define_method("#{virtual_attribute}=") do |value| - new_value = ChronicDuration.parse(value).to_i - new_value = nil if new_value <= 0 + new_value = ChronicDuration.parse(value).to_i unless value.nil? + new_value = nil if !new_value.nil? && new_value <= 0 self.send("#{source_attribute}=", new_value) # rubocop:disable GitlabSecurity/PublicSend -- cgit v1.2.3 From f5e602ee0f8d95617adf6fb9b5a1a132a471fb12 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 6 Mar 2018 16:14:23 +0100 Subject: Rename maximum_job_timeout to maximum_timeout --- app/models/ci/build_metadata.rb | 2 +- app/models/ci/runner.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index 9043bed86bf..89fee37b1b6 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -20,7 +20,7 @@ module Ci def save_timeout_state! project_timeout = build.project&.build_timeout - timeout = [project_timeout, build.runner&.maximum_job_timeout].compact.min + timeout = [project_timeout, build.runner&.maximum_timeout].compact.min self.used_timeout = timeout self.timeout_source = timeout < project_timeout ? :runner_timeout_source : :project_timeout_source diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 413607e0eff..62cc636db22 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -9,7 +9,7 @@ module Ci ONLINE_CONTACT_TIMEOUT = 1.hour UPDATE_DB_RUNNER_INFO_EVERY = 40.minutes AVAILABLE_SCOPES = %w[specific shared active paused online].freeze - FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level maximum_job_timeout_human_readable].freeze + FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level maximum_timeout_human_readable].freeze has_many :builds has_many :runner_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -52,7 +52,7 @@ module Ci cached_attr_reader :version, :revision, :platform, :architecture, :contacted_at, :ip_address - chronic_duration_attr :maximum_job_timeout_human_readable, :maximum_job_timeout + chronic_duration_attr :maximum_timeout_human_readable, :maximum_timeout # Searches for runners matching the given query. # -- cgit v1.2.3 From d58d3098f159a17fbcf1ae27165c249722990988 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 6 Mar 2018 16:25:13 +0100 Subject: Rename used_timeout to timeout --- app/models/ci/build.rb | 2 +- app/models/ci/build_metadata.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 7a12d7a3deb..405c89d0103 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -242,7 +242,7 @@ module Ci end def timeout - metadata.used_timeout + metadata.timeout end def triggered_by?(current_user) diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index 89fee37b1b6..335209e8ec2 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -10,7 +10,7 @@ module Ci belongs_to :build, class_name: 'Ci::Build' - chronic_duration_attr_reader :used_timeout_human_readable, :used_timeout + chronic_duration_attr_reader :timeout_human_readable, :timeout enum timeout_source: { unknown_timeout_source: 1, @@ -22,7 +22,7 @@ module Ci project_timeout = build.project&.build_timeout timeout = [project_timeout, build.runner&.maximum_timeout].compact.min - self.used_timeout = timeout + self.timeout = timeout self.timeout_source = timeout < project_timeout ? :runner_timeout_source : :project_timeout_source save! -- cgit v1.2.3 From 12401758a40868b1683c2c208d756105fd43de7b Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 6 Mar 2018 16:43:44 +0100 Subject: BuildMetadata styling improvements --- app/models/ci/build_metadata.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index 335209e8ec2..424b6da1014 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -19,13 +19,13 @@ module Ci } def save_timeout_state! - project_timeout = build.project&.build_timeout - timeout = [project_timeout, build.runner&.maximum_timeout].compact.min + return unless build.runner.present? - self.timeout = timeout - self.timeout_source = timeout < project_timeout ? :runner_timeout_source : :project_timeout_source + project_timeout = build.project&.build_timeout + timeout = [project_timeout, build.runner.maximum_timeout].compact.min + timeout_source = timeout < project_timeout ? :runner_timeout_source : :project_timeout_source - save! + update_attributes(timeout: timeout, timeout_source: timeout_source) end end end -- cgit v1.2.3 From afcc57abfdcb11001803655f938187cbdc96b67c Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 20 Mar 2018 23:21:17 +0100 Subject: Rename metadata relation and methods --- app/models/ci/build.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 405c89d0103..0b3c6ac4fee 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -24,7 +24,7 @@ module Ci has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id - has_one :build_metadata, class_name: 'Ci::BuildMetadata' + has_one :metadata, class_name: 'Ci::BuildMetadata' # The "environment" field for builds is a String, and is the unexpanded name def persisted_environment @@ -157,12 +157,12 @@ module Ci end before_transition pending: :running do |build| - build.metadata.save_timeout_state! + build.ensure_metadata.save_timeout_state! end end - def metadata - self.build_metadata ||= Ci::BuildMetadata.new + def ensure_metadata + metadata || build_metadata end def detailed_status(current_user) @@ -242,7 +242,7 @@ module Ci end def timeout - metadata.timeout + ensure_metadata.timeout end def triggered_by?(current_user) -- cgit v1.2.3 From a1cd3390e550c0008e83146e8627fdee3f58e422 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 21 Mar 2018 01:20:15 +0100 Subject: Add project_id column to Ci::BuildMetadata --- app/models/ci/build_metadata.rb | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index 424b6da1014..c221f43384b 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -9,9 +9,12 @@ module Ci self.table_name = 'ci_builds_metadata' belongs_to :build, class_name: 'Ci::Build' + belongs_to :project chronic_duration_attr_reader :timeout_human_readable, :timeout + after_initialize :set_project_id + enum timeout_source: { unknown_timeout_source: 1, project_timeout_source: 2, @@ -27,5 +30,13 @@ module Ci update_attributes(timeout: timeout, timeout_source: timeout_source) end + + private + + def set_project_id + return unless self.project_id.nil? + + self.project_id = build&.project&.id + end end end -- cgit v1.2.3 From c747d9bc0b35d2982f45e2e8a0bdaa305f504f38 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 21 Mar 2018 17:22:52 +0100 Subject: Add validation for chronic_duration_attr_writer --- app/models/concerns/chronic_duration_attribute.rb | 27 +++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/chronic_duration_attribute.rb b/app/models/concerns/chronic_duration_attribute.rb index 15095d0b758..ad8934f58d8 100644 --- a/app/models/concerns/chronic_duration_attribute.rb +++ b/app/models/concerns/chronic_duration_attribute.rb @@ -18,13 +18,32 @@ module ChronicDurationAttribute end def chronic_duration_attr_writer(virtual_attribute, source_attribute) + virtual_attribute_validator = "#{virtual_attribute}_validator".to_sym + validation_error = "#{virtual_attribute}_error".to_sym + + validate virtual_attribute_validator + attr_accessor validation_error + define_method("#{virtual_attribute}=") do |value| - new_value = ChronicDuration.parse(value).to_i unless value.nil? - new_value = nil if !new_value.nil? && new_value <= 0 + begin + self.send("#{validation_error}=", '') # rubocop:disable GitlabSecurity/PublicSend - self.send("#{source_attribute}=", new_value) # rubocop:disable GitlabSecurity/PublicSend + new_value = + if value.blank? + nil + else + ChronicDuration.parse(value).to_i + end + + self.send("#{source_attribute}=", new_value) # rubocop:disable GitlabSecurity/PublicSend + rescue ChronicDuration::DurationParseError => ex + self.send("#{validation_error}=", ex.message) # rubocop:disable GitlabSecurity/PublicSend + end + end - new_value + define_method(virtual_attribute_validator) do + error = self.send(validation_error) # rubocop:disable GitlabSecurity/PublicSend + self.send('errors').add(source_attribute, error) unless error.blank? # rubocop:disable GitlabSecurity/PublicSend end end end -- cgit v1.2.3 From 7d7b0688b846e346a0799340875d459d26c1718d Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 21 Mar 2018 17:34:55 +0100 Subject: Add validation for max_timeout in Ci::Runner --- app/models/ci/runner.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 62cc636db22..5a4c56ec0dc 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -54,6 +54,10 @@ module Ci chronic_duration_attr :maximum_timeout_human_readable, :maximum_timeout + validates :maximum_timeout, allow_nil: true, + numericality: { greater_than_or_equal_to: 600, + message: 'needs to be at least 10 minutes' } + # Searches for runners matching the given query. # # This method uses ILIKE on PostgreSQL and LIKE on MySQL. -- cgit v1.2.3 From 973e4030b13adbcc4eb7fad347b928a5164a04ff Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 22 Mar 2018 17:52:28 +0100 Subject: Refactor build_metadata --- app/models/ci/build.rb | 2 +- app/models/ci/build_metadata.rb | 16 ++++------------ 2 files changed, 5 insertions(+), 13 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 0b3c6ac4fee..355d1c0523f 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -162,7 +162,7 @@ module Ci end def ensure_metadata - metadata || build_metadata + metadata || build_metadata(project: project) end def detailed_status(current_user) diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index c221f43384b..6d8a895d509 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -11,9 +11,9 @@ module Ci belongs_to :build, class_name: 'Ci::Build' belongs_to :project - chronic_duration_attr_reader :timeout_human_readable, :timeout + validates :project, presence: true - after_initialize :set_project_id + chronic_duration_attr_reader :timeout_human_readable, :timeout enum timeout_source: { unknown_timeout_source: 1, @@ -24,19 +24,11 @@ module Ci def save_timeout_state! return unless build.runner.present? - project_timeout = build.project&.build_timeout + project_timeout = project&.build_timeout timeout = [project_timeout, build.runner.maximum_timeout].compact.min timeout_source = timeout < project_timeout ? :runner_timeout_source : :project_timeout_source - update_attributes(timeout: timeout, timeout_source: timeout_source) - end - - private - - def set_project_id - return unless self.project_id.nil? - - self.project_id = build&.project&.id + update!(timeout: timeout, timeout_source: timeout_source) end end end -- cgit v1.2.3 From c2bc153314f14566abfdcbf92020cff41fc05a7e Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Fri, 23 Mar 2018 00:12:15 +0100 Subject: Refactorize ChronicDurationAttribute concern --- app/models/concerns/chronic_duration_attribute.rb | 51 +++++++++-------------- 1 file changed, 20 insertions(+), 31 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/chronic_duration_attribute.rb b/app/models/concerns/chronic_duration_attribute.rb index ad8934f58d8..fa1eafb1d7a 100644 --- a/app/models/concerns/chronic_duration_attribute.rb +++ b/app/models/concerns/chronic_duration_attribute.rb @@ -2,49 +2,38 @@ module ChronicDurationAttribute extend ActiveSupport::Concern class_methods do - def chronic_duration_attr(virtual_attribute, source_attribute) - chronic_duration_attr_reader(virtual_attribute, source_attribute) - chronic_duration_attr_writer(virtual_attribute, source_attribute) - end - def chronic_duration_attr_reader(virtual_attribute, source_attribute) define_method(virtual_attribute) do - value = self.send(source_attribute) # rubocop:disable GitlabSecurity/PublicSend - - return '' if value.nil? - - ChronicDuration.output(value, format: :short) + chronic_duration_attributes[virtual_attribute] || output_chronic_duration_attribute(source_attribute) end end def chronic_duration_attr_writer(virtual_attribute, source_attribute) - virtual_attribute_validator = "#{virtual_attribute}_validator".to_sym - validation_error = "#{virtual_attribute}_error".to_sym - - validate virtual_attribute_validator - attr_accessor validation_error + chronic_duration_attr_reader(virtual_attribute, source_attribute) define_method("#{virtual_attribute}=") do |value| + chronic_duration_attributes[virtual_attribute] = value.presence || '' + begin - self.send("#{validation_error}=", '') # rubocop:disable GitlabSecurity/PublicSend - - new_value = - if value.blank? - nil - else - ChronicDuration.parse(value).to_i - end - - self.send("#{source_attribute}=", new_value) # rubocop:disable GitlabSecurity/PublicSend - rescue ChronicDuration::DurationParseError => ex - self.send("#{validation_error}=", ex.message) # rubocop:disable GitlabSecurity/PublicSend + new_value = ChronicDuration.parse(value).to_i if value.present? + assign_attributes(source_attribute => new_value) + rescue ChronicDuration::DurationParseError + # ignore error as it will be caught by validation end end - define_method(virtual_attribute_validator) do - error = self.send(validation_error) # rubocop:disable GitlabSecurity/PublicSend - self.send('errors').add(source_attribute, error) unless error.blank? # rubocop:disable GitlabSecurity/PublicSend - end + validates virtual_attribute, allow_nil: true, duration: true end + + alias_method :chronic_duration_attr, :chronic_duration_attr_writer + end + + def chronic_duration_attributes + @chronic_duration_attributes ||= {} + end + + def output_chronic_duration_attribute(source_attribute) + value = attributes[source_attribute.to_s] + ChronicDuration.output(value, format: :short) if value end end -- cgit v1.2.3 From 7008ed1e33bff510126a0eb0e4f7bf1a7adb02bd Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 26 Mar 2018 17:47:46 +0200 Subject: Change and rename behavior of save_timeout_state! --- app/models/ci/build.rb | 2 +- app/models/ci/build_metadata.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 355d1c0523f..073d73f0426 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -157,7 +157,7 @@ module Ci end before_transition pending: :running do |build| - build.ensure_metadata.save_timeout_state! + build.ensure_metadata.update_timeout_state end end diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index 6d8a895d509..de5b4170201 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -21,14 +21,14 @@ module Ci runner_timeout_source: 3 } - def save_timeout_state! + def update_timeout_state return unless build.runner.present? project_timeout = project&.build_timeout timeout = [project_timeout, build.runner.maximum_timeout].compact.min timeout_source = timeout < project_timeout ? :runner_timeout_source : :project_timeout_source - update!(timeout: timeout, timeout_source: timeout_source) + update(timeout: timeout, timeout_source: timeout_source) end end end -- cgit v1.2.3 From 403decbbad26bea620125aed34b440a9b6611172 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 26 Mar 2018 18:58:35 +0200 Subject: Add explicit primary key for ci_builds_metadata table --- app/models/ci/build_metadata.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/models') diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index de5b4170201..96762f8845c 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -11,6 +11,7 @@ module Ci belongs_to :build, class_name: 'Ci::Build' belongs_to :project + validates :build, presence: true validates :project, presence: true chronic_duration_attr_reader :timeout_human_readable, :timeout -- cgit v1.2.3 From 6ecde0076afa83e30608ea9caba924bbab66a123 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 26 Mar 2018 19:26:52 +0200 Subject: Remove Ci::Build#timeout --- app/models/ci/build.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 073d73f0426..ed02af05e3d 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -26,6 +26,8 @@ module Ci has_one :metadata, class_name: 'Ci::BuildMetadata' + delegate :timeout, to: :metadata, prefix: true, allow_nil: true + # The "environment" field for builds is a String, and is the unexpanded name def persisted_environment @persisted_environment ||= Environment.find_by( @@ -241,10 +243,6 @@ module Ci latest_builds.where('stage_idx < ?', stage_idx) end - def timeout - ensure_metadata.timeout - end - def triggered_by?(current_user) user == current_user end -- cgit v1.2.3 From ce69419a60a04b78616e7fe7d26ed7ed936e7c62 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Mon, 5 Mar 2018 13:02:36 +0100 Subject: Remove permanent redirects Removes permanent redirects, this means that redirects will only be possible as long as the old route isn't taken by a new project/group. --- app/models/redirect_route.rb | 28 ---------------------------- app/models/route.rb | 26 +++++--------------------- 2 files changed, 5 insertions(+), 49 deletions(-) (limited to 'app/models') diff --git a/app/models/redirect_route.rb b/app/models/redirect_route.rb index 20532527346..31de204d824 100644 --- a/app/models/redirect_route.rb +++ b/app/models/redirect_route.rb @@ -17,32 +17,4 @@ class RedirectRoute < ActiveRecord::Base where(wheres, path, "#{sanitize_sql_like(path)}/%") end - - scope :permanent, -> do - if column_permanent_exists? - where(permanent: true) - else - none - end - end - - scope :temporary, -> do - if column_permanent_exists? - where(permanent: [false, nil]) - else - all - end - end - - default_value_for :permanent, false - - def permanent=(value) - if self.class.column_permanent_exists? - super - end - end - - def self.column_permanent_exists? - ActiveRecord::Base.connection.column_exists?(:redirect_routes, :permanent) - end end diff --git a/app/models/route.rb b/app/models/route.rb index 07d96c21cf1..2d609920051 100644 --- a/app/models/route.rb +++ b/app/models/route.rb @@ -10,8 +10,6 @@ class Route < ActiveRecord::Base presence: true, uniqueness: { case_sensitive: false } - validate :ensure_permanent_paths, if: :path_changed? - before_validation :delete_conflicting_orphaned_routes after_create :delete_conflicting_redirects after_update :delete_conflicting_redirects, if: :path_changed? @@ -45,7 +43,7 @@ class Route < ActiveRecord::Base # We are not calling route.delete_conflicting_redirects here, in hopes # of avoiding deadlocks. The parent (self, in this method) already # called it, which deletes conflicts for all descendants. - route.create_redirect(old_path, permanent: permanent_redirect?) if attributes[:path] + route.create_redirect(old_path) if attributes[:path] end end end @@ -55,31 +53,17 @@ class Route < ActiveRecord::Base end def conflicting_redirects - RedirectRoute.temporary.matching_path_and_descendants(path) + RedirectRoute.matching_path_and_descendants(path) end - def create_redirect(path, permanent: false) - RedirectRoute.create(source: source, path: path, permanent: permanent) + def create_redirect(path) + RedirectRoute.create(source: source, path: path) end private def create_redirect_for_old_path - create_redirect(path_was, permanent: permanent_redirect?) if path_changed? - end - - def permanent_redirect? - source_type != "Project" - end - - def ensure_permanent_paths - return if path.nil? - - errors.add(:path, "has been taken before") if conflicting_redirect_exists? - end - - def conflicting_redirect_exists? - RedirectRoute.permanent.matching_path_and_descendants(path).exists? + create_redirect(path_was) if path_changed? end def delete_conflicting_orphaned_routes -- cgit v1.2.3 From 05b4b2d9679ccd09cf9723207603d0d3f670df30 Mon Sep 17 00:00:00 2001 From: Takuya Noguchi Date: Tue, 27 Mar 2018 00:57:13 +0900 Subject: Reuse root_ref_hash for performance on Branches --- app/models/repository.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/repository.rb b/app/models/repository.rb index 42f1ac43e29..904bfe06ce6 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -253,13 +253,13 @@ class Repository end def diverging_commit_counts(branch) - root_ref_hash = raw_repository.commit(root_ref).id + @root_ref_hash ||= raw_repository.commit(root_ref).id cache.fetch(:"diverging_commit_counts_#{branch.name}") do # Rugged seems to throw a `ReferenceError` when given branch_names rather # than SHA-1 hashes number_commits_behind, number_commits_ahead = raw_repository.count_commits_between( - root_ref_hash, + @root_ref_hash, branch.dereferenced_target.sha, left_right: true, max_count: MAX_DIVERGING_COUNT) -- cgit v1.2.3 From 7fd9e39d885c7d7520ba0a7e45e8c58847f1c11d Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Tue, 27 Mar 2018 16:33:29 +0100 Subject: When a Service templates are invalid newly created projects will have them inactive --- app/models/service.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/models') diff --git a/app/models/service.rb b/app/models/service.rb index 1dcb79157a2..7424cef0fc0 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -273,6 +273,7 @@ class Service < ActiveRecord::Base def self.build_from_template(project_id, template) service = template.dup + service.active = false unless service.valid? service.template = false service.project_id = project_id service -- cgit v1.2.3 From 7bca902a23d0cccaa3bce422fce7fdc71e722db4 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 28 Mar 2018 18:54:15 +0200 Subject: Fall back on ghost user when deploy key user is not set --- app/models/deploy_key.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'app/models') diff --git a/app/models/deploy_key.rb b/app/models/deploy_key.rb index c2e0a5fa126..89a74b7dcb1 100644 --- a/app/models/deploy_key.rb +++ b/app/models/deploy_key.rb @@ -27,6 +27,10 @@ class DeployKey < Key self.private? end + def user + super || User.ghost + end + def has_access_to?(project) deploy_keys_project_for(project).present? end -- cgit v1.2.3 From 931151972189d0d203e2bacedb78638f256a5055 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 28 Mar 2018 18:54:31 +0200 Subject: =?UTF-8?q?Don=E2=80=99t=20delete=20deploy=20key=20when=20user=20w?= =?UTF-8?q?ho=20created=20it=20is=20deleted?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/user.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index b8c55205ab8..8d5d0bbbfa0 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -82,11 +82,8 @@ class User < ActiveRecord::Base has_one :namespace, -> { where(type: nil) }, dependent: :destroy, foreign_key: :owner_id, inverse_of: :owner, autosave: true # rubocop:disable Cop/ActiveRecordDependent # Profile - has_many :keys, -> do - type = Key.arel_table[:type] - where(type.not_eq('DeployKey').or(type.eq(nil))) - end, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :deploy_keys, -> { where(type: 'DeployKey') }, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :keys, -> { where(type: ['Key', nil]) }, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :deploy_keys, -> { where(type: 'DeployKey') }, dependent: :nullify # rubocop:disable Cop/ActiveRecordDependent has_many :gpg_keys has_many :emails, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent -- cgit v1.2.3 From 1a84f96a0602c9e6a9dfd2e2de3cfbe9385470ff Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Mar 2018 14:42:23 +0200 Subject: Extract build variables that depend on persistence --- app/models/ci/build.rb | 55 ++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 24 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 7e724de9e77..f8755e1a4ff 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -292,7 +292,9 @@ module Ci # All variables, including persisted environment variables. # def variables - scoped_variables + Gitlab::Ci::Variables::Collection.new + .concat(persisted_variables) + .concat(scoped_variables) .concat(persisted_environment_variables) .to_runner_variables end @@ -579,6 +581,21 @@ module Ci CI_REGISTRY_USER = 'gitlab-ci-token'.freeze + def persisted_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + return variables unless persisted? + + variables + .append(key: 'CI_JOB_ID', value: id.to_s) + .append(key: 'CI_JOB_TOKEN', value: token, public: false) + .append(key: 'CI_BUILD_ID', value: id.to_s) + .append(key: 'CI_BUILD_TOKEN', value: token, public: false) + .append(key: 'CI_REGISTRY_USER', value: CI_REGISTRY_USER) + .append(key: 'CI_REGISTRY_PASSWORD', value: token, public: false) + .append(key: 'CI_REPOSITORY_URL', value: repo_url, public: false) + end + end + def predefined_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'CI', value: 'true') @@ -587,16 +604,11 @@ module Ci variables.append(key: 'CI_SERVER_NAME', value: 'GitLab') variables.append(key: 'CI_SERVER_VERSION', value: Gitlab::VERSION) variables.append(key: 'CI_SERVER_REVISION', value: Gitlab::REVISION) - variables.append(key: 'CI_JOB_ID', value: id.to_s) if persisted? variables.append(key: 'CI_JOB_NAME', value: name) variables.append(key: 'CI_JOB_STAGE', value: stage) - variables.append(key: 'CI_JOB_TOKEN', value: token, public: false) if persisted? variables.append(key: 'CI_COMMIT_SHA', value: sha) variables.append(key: 'CI_COMMIT_REF_NAME', value: ref) variables.append(key: 'CI_COMMIT_REF_SLUG', value: ref_slug) - variables.append(key: 'CI_REGISTRY_USER', value: CI_REGISTRY_USER) if persisted? - variables.append(key: 'CI_REGISTRY_PASSWORD', value: token, public: false) if persisted? - variables.append(key: 'CI_REPOSITORY_URL', value: repo_url, public: false) if persisted? variables.append(key: "CI_COMMIT_TAG", value: ref) if tag? variables.append(key: "CI_PIPELINE_TRIGGERED", value: 'true') if trigger_request variables.append(key: "CI_JOB_MANUAL", value: 'true') if action? @@ -604,26 +616,8 @@ module Ci end end - def persisted_environment_variables - Gitlab::Ci::Variables::Collection.new.tap do |variables| - return variables unless persisted? && persisted_environment.present? - - variables.concat(persisted_environment.predefined_variables) - - # Here we're passing unexpanded environment_url for runner to expand, - # and we need to make sure that CI_ENVIRONMENT_NAME and - # CI_ENVIRONMENT_SLUG so on are available for the URL be expanded. - variables.append(key: 'CI_ENVIRONMENT_URL', value: environment_url) if environment_url - end - end - def legacy_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| - if persisted? - variables.append(key: 'CI_BUILD_ID', value: id.to_s) - variables.append(key: 'CI_BUILD_TOKEN', value: token, public: false) - end - variables.append(key: 'CI_BUILD_REF', value: sha) variables.append(key: 'CI_BUILD_BEFORE_SHA', value: before_sha) variables.append(key: 'CI_BUILD_REF_NAME', value: ref) @@ -636,6 +630,19 @@ module Ci end end + def persisted_environment_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + return variables unless persisted? && persisted_environment.present? + + variables.concat(persisted_environment.predefined_variables) + + # Here we're passing unexpanded environment_url for runner to expand, + # and we need to make sure that CI_ENVIRONMENT_NAME and + # CI_ENVIRONMENT_SLUG so on are available for the URL be expanded. + variables.append(key: 'CI_ENVIRONMENT_URL', value: environment_url) if environment_url + end + end + def environment_url options&.dig(:environment, :url) || persisted_environment&.external_url end -- cgit v1.2.3 From e09f99b2a3a969baa97608b9358a8551e87655d4 Mon Sep 17 00:00:00 2001 From: Alessio Caiazza Date: Thu, 29 Mar 2018 11:24:08 +0200 Subject: Add port number to artifacts links to gitlab-pages, if needed --- app/models/ci/artifact_blob.rb | 7 +++---- app/models/project.rb | 19 +++++++++---------- 2 files changed, 12 insertions(+), 14 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/artifact_blob.rb b/app/models/ci/artifact_blob.rb index ec56cc53aea..760f01f225b 100644 --- a/app/models/ci/artifact_blob.rb +++ b/app/models/ci/artifact_blob.rb @@ -36,16 +36,15 @@ module Ci def external_url(project, job) return unless external_link?(job) - full_path_parts = project.full_path_components - top_level_group = full_path_parts.shift + url_project_path = project.full_path.partition('/').last artifact_path = [ - '-', *full_path_parts, '-', + '-', url_project_path, '-', 'jobs', job.id, 'artifacts', path ].join('/') - "#{pages_config.protocol}://#{top_level_group}.#{pages_config.host}/#{artifact_path}" + "#{project.pages_group_url}/#{artifact_path}" end def external_link?(job) diff --git a/app/models/project.rb b/app/models/project.rb index 6a420663644..da03080f440 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1346,20 +1346,19 @@ class Project < ActiveRecord::Base Dir.exist?(public_pages_path) end - def pages_url - subdomain, _, url_path = full_path.partition('/') - - # The hostname always needs to be in downcased - # All web servers convert hostname to lowercase - host = "#{subdomain}.#{Settings.pages.host}".downcase - + def pages_group_url # The host in URL always needs to be downcased - url = Gitlab.config.pages.url.sub(%r{^https?://}) do |prefix| - "#{prefix}#{subdomain}." + Gitlab.config.pages.url.sub(%r{^https?://}) do |prefix| + "#{prefix}#{pages_subdomain}." end.downcase + end + + def pages_url + url = pages_group_url + url_path = full_path.partition('/').last # If the project path is the same as host, we serve it as group page - return url if host == url_path + return url if url == "#{Settings.pages.protocol}://#{url_path}" "#{url}/#{url_path}" end -- cgit v1.2.3 From 1cc96d7a622cd754674ae8184268acf41a0be7d7 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 29 Mar 2018 14:46:27 +0200 Subject: Memoize environment-specific methods in build class The purpose of this memoization is to make getting persisted environment name, and related scoped variables, a little more performant task, because it can be invoked multiple times. --- app/models/ci/build.rb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index f8755e1a4ff..b42555e46e4 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -6,6 +6,7 @@ module Ci include ObjectStorage::BackgroundMove include Presentable include Importable + include Gitlab::Utils::StrongMemoize MissingDependenciesError = Class.new(StandardError) @@ -31,10 +32,11 @@ module Ci # The "environment" field for builds is a String, and is the unexpanded name! # def persisted_environment - @persisted_environment ||= Environment.find_by( - name: expanded_environment_name, - project: project - ) + return unless has_environment? + + strong_memoize(:persisted_environment) do + Environment.find_by(name: expanded_environment_name, project: project) + end end serialize :options # rubocop:disable Cop/ActiveRecordSerialize @@ -213,7 +215,9 @@ module Ci end def expanded_environment_name - if has_environment? + return unless has_environment? + + strong_memoize(:expanded_environment_name) do ExpandVariables.expand(environment, simple_variables) end end -- cgit v1.2.3 From a5f9e49f1317345d1b72566ecff188dff5f74913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Thu, 29 Mar 2018 21:17:18 +0200 Subject: Add user_provided and gcp_provided cluster scopes --- app/models/clusters/cluster.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'app/models') diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 49eb069016a..dcfc3e02f0e 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -50,6 +50,8 @@ module Clusters scope :enabled, -> { where(enabled: true) } scope :disabled, -> { where(enabled: false) } + scope :user_provided, -> { where(provider_type: ::Clusters::Cluster.provider_types[:user]) } + scope :gcp_provided, -> { where(provider_type: ::Clusters::Cluster.provider_types[:gcp]) } def status_name if provider -- cgit v1.2.3 From 982e5436e1ef9b309f8ee3a205f1e168273386b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Thu, 29 Mar 2018 21:24:10 +0200 Subject: Add installed scope to cluster applications --- app/models/clusters/applications/helm.rb | 2 ++ app/models/clusters/applications/ingress.rb | 2 ++ app/models/clusters/applications/prometheus.rb | 2 ++ app/models/clusters/applications/runner.rb | 2 ++ 4 files changed, 8 insertions(+) (limited to 'app/models') diff --git a/app/models/clusters/applications/helm.rb b/app/models/clusters/applications/helm.rb index 58de3448577..f7dfae56df6 100644 --- a/app/models/clusters/applications/helm.rb +++ b/app/models/clusters/applications/helm.rb @@ -8,6 +8,8 @@ module Clusters default_value_for :version, Gitlab::Kubernetes::Helm::HELM_VERSION + scope :installed, -> { where(status: ::Clusters::Applications::Helm.state_machines[:status].states[:installed].value) } + def set_initial_status return unless not_installable? diff --git a/app/models/clusters/applications/ingress.rb b/app/models/clusters/applications/ingress.rb index 27fc3b85465..8cd88e8aeaf 100644 --- a/app/models/clusters/applications/ingress.rb +++ b/app/models/clusters/applications/ingress.rb @@ -11,6 +11,8 @@ module Clusters default_value_for :ingress_type, :nginx default_value_for :version, :nginx + scope :installed, -> { where(status: ::Clusters::Applications::Ingress.state_machines[:status].states[:installed].value) } + enum ingress_type: { nginx: 1 } diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb index 7b25d8c4089..2ae3e006047 100644 --- a/app/models/clusters/applications/prometheus.rb +++ b/app/models/clusters/applications/prometheus.rb @@ -13,6 +13,8 @@ module Clusters default_value_for :version, VERSION + scope :installed, -> { where(status: ::Clusters::Applications::Prometheus.state_machines[:status].states[:installed].value) } + state_machine :status do after_transition any => [:installed] do |application| application.cluster.projects.each do |project| diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index 16efe90fa27..6dc51d1baa4 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -14,6 +14,8 @@ module Clusters default_value_for :version, VERSION + scope :installed, -> { where(status: ::Clusters::Applications::Runner.state_machines[:status].states[:installed].value) } + def chart "#{name}/gitlab-runner" end -- cgit v1.2.3 From f6fa84298282c78229e4028655cb87011372e2b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Thu, 29 Mar 2018 22:08:13 +0200 Subject: Extract installed cluster application scope to concern --- app/models/clusters/applications/helm.rb | 2 -- app/models/clusters/applications/ingress.rb | 2 -- app/models/clusters/applications/prometheus.rb | 2 -- app/models/clusters/applications/runner.rb | 2 -- app/models/clusters/concerns/application_status.rb | 2 ++ 5 files changed, 2 insertions(+), 8 deletions(-) (limited to 'app/models') diff --git a/app/models/clusters/applications/helm.rb b/app/models/clusters/applications/helm.rb index f7dfae56df6..58de3448577 100644 --- a/app/models/clusters/applications/helm.rb +++ b/app/models/clusters/applications/helm.rb @@ -8,8 +8,6 @@ module Clusters default_value_for :version, Gitlab::Kubernetes::Helm::HELM_VERSION - scope :installed, -> { where(status: ::Clusters::Applications::Helm.state_machines[:status].states[:installed].value) } - def set_initial_status return unless not_installable? diff --git a/app/models/clusters/applications/ingress.rb b/app/models/clusters/applications/ingress.rb index 8cd88e8aeaf..27fc3b85465 100644 --- a/app/models/clusters/applications/ingress.rb +++ b/app/models/clusters/applications/ingress.rb @@ -11,8 +11,6 @@ module Clusters default_value_for :ingress_type, :nginx default_value_for :version, :nginx - scope :installed, -> { where(status: ::Clusters::Applications::Ingress.state_machines[:status].states[:installed].value) } - enum ingress_type: { nginx: 1 } diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb index 2ae3e006047..7b25d8c4089 100644 --- a/app/models/clusters/applications/prometheus.rb +++ b/app/models/clusters/applications/prometheus.rb @@ -13,8 +13,6 @@ module Clusters default_value_for :version, VERSION - scope :installed, -> { where(status: ::Clusters::Applications::Prometheus.state_machines[:status].states[:installed].value) } - state_machine :status do after_transition any => [:installed] do |application| application.cluster.projects.each do |project| diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index 6dc51d1baa4..16efe90fa27 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -14,8 +14,6 @@ module Clusters default_value_for :version, VERSION - scope :installed, -> { where(status: ::Clusters::Applications::Runner.state_machines[:status].states[:installed].value) } - def chart "#{name}/gitlab-runner" end diff --git a/app/models/clusters/concerns/application_status.rb b/app/models/clusters/concerns/application_status.rb index 7b7c8eac773..8f3eb75bfa9 100644 --- a/app/models/clusters/concerns/application_status.rb +++ b/app/models/clusters/concerns/application_status.rb @@ -4,6 +4,8 @@ module Clusters extend ActiveSupport::Concern included do + scope :installed, -> { where(status: self.state_machines[:status].states[:installed].value) } + state_machine :status, initial: :not_installable do state :not_installable, value: -2 state :errored, value: -1 -- cgit v1.2.3 From 0a203f408786ae2e28616a0e1bf488595f7f115b Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 30 Mar 2018 10:32:28 +0200 Subject: Improve naming and optimize methods of the build class --- app/models/ci/build.rb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index b42555e46e4..18e96389199 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -289,7 +289,9 @@ module Ci # Variables that do not depend on the environment name. # def simple_variables - scoped_variables(environment: nil).to_runner_variables + strong_memoize(:simple_variables) do + scoped_variables(environment: nil).to_runner_variables + end end ## @@ -303,7 +305,11 @@ module Ci .to_runner_variables end - def variables_hash + ## + # Regular Ruby hash of scoped variables, without duplicates that are + # possible to be present in an array of hashes returned from `variables`. + # + def scoped_variables_hash scoped_variables.to_hash end -- cgit v1.2.3 From 5ab75649f3ea00b64cb63e7e5283100c6b70cfb5 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Fri, 30 Mar 2018 13:25:46 +0100 Subject: Only send issue due emails to participants and custom subscribers --- app/models/notification_setting.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/notification_setting.rb b/app/models/notification_setting.rb index f6d9b0215fc..9195408551f 100644 --- a/app/models/notification_setting.rb +++ b/app/models/notification_setting.rb @@ -47,7 +47,8 @@ class NotificationSetting < ActiveRecord::Base ].freeze EXCLUDED_WATCHER_EVENTS = [ - :push_to_merge_request + :push_to_merge_request, + :issue_due ].push(*EXCLUDED_PARTICIPATING_EVENTS).freeze def self.find_or_create_for(source) -- cgit v1.2.3 From 2db218f8bf186c509c927ce3e9d0502fee4f8349 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Fri, 30 Mar 2018 14:05:20 +0100 Subject: Send emails for issues due tomorrow Also, refactor the mail sending slightly: instead of one worker sending all emails, create a worker per project with issues due, which will send all emails for that project. --- app/models/issue.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/models') diff --git a/app/models/issue.rb b/app/models/issue.rb index 7bfc45c1f43..f65cd8bf896 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -48,6 +48,7 @@ class Issue < ActiveRecord::Base scope :without_due_date, -> { where(due_date: nil) } scope :due_before, ->(date) { where('issues.due_date < ?', date) } scope :due_between, ->(from_date, to_date) { where('issues.due_date >= ?', from_date).where('issues.due_date <= ?', to_date) } + scope :due_tomorrow, -> { where(due_date: Date.tomorrow) } scope :order_due_date_asc, -> { reorder('issues.due_date IS NULL, issues.due_date ASC') } scope :order_due_date_desc, -> { reorder('issues.due_date IS NULL, issues.due_date DESC') } -- cgit v1.2.3 From 22b05a1ff74d4f64490f93995259602b3d07c3cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Javier=20L=C3=B3pez?= Date: Fri, 30 Mar 2018 15:45:59 +0000 Subject: Extend API for exporting a project with direct upload URL --- app/models/project.rb | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index da03080f440..b343786d2c9 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1544,8 +1544,8 @@ class Project < ActiveRecord::Base @errors = original_errors end - def add_export_job(current_user:, params: {}) - job_id = ProjectExportWorker.perform_async(current_user.id, self.id, params) + def add_export_job(current_user:, after_export_strategy: nil, params: {}) + job_id = ProjectExportWorker.perform_async(current_user.id, self.id, after_export_strategy, params) if job_id Rails.logger.info "Export job started for project ID #{self.id} with job ID #{job_id}" @@ -1571,6 +1571,8 @@ class Project < ActiveRecord::Base def export_status if export_in_progress? :started + elsif after_export_in_progress? + :after_export_action elsif export_project_path :finished else @@ -1582,12 +1584,22 @@ class Project < ActiveRecord::Base import_export_shared.active_export_count > 0 end + def after_export_in_progress? + import_export_shared.after_export_in_progress? + end + def remove_exports return nil unless export_path.present? FileUtils.rm_rf(export_path) end + def remove_exported_project_file + return unless export_project_path.present? + + FileUtils.rm_f(export_project_path) + end + def full_path_slug Gitlab::Utils.slugify(full_path.to_s) end -- cgit v1.2.3 From b26913a30c608f81c44abece263da8d01faf0126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Fri, 30 Mar 2018 17:34:10 +0200 Subject: Extract cluster installed query to scope --- app/models/clusters/cluster.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'app/models') diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 05e8d266bcf..77947d515c1 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -53,6 +53,8 @@ module Clusters scope :disabled, -> { where(enabled: false) } scope :user_provided, -> { where(provider_type: ::Clusters::Cluster.provider_types[:user]) } scope :gcp_provided, -> { where(provider_type: ::Clusters::Cluster.provider_types[:gcp]) } + scope :gcp_installed, -> { gcp_provided.includes(:provider_gcp).where(cluster_providers_gcp: { status: ::Clusters::Providers::Gcp.state_machines[:status].states[:created].value }) } + scope :default_environment, -> { where(environment_scope: DEFAULT_ENVIRONMENT) } def status_name -- cgit v1.2.3 From ddfc661f794ca02654853a9e223b9a5f3fb983ab Mon Sep 17 00:00:00 2001 From: Ahmad Sherif Date: Mon, 26 Mar 2018 20:21:49 +0200 Subject: Use shard name in Git::GitlabProjects instead of shard path Closes gitaly#1110 --- app/models/project.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 6a420663644..7d45156a825 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -566,9 +566,7 @@ class Project < ActiveRecord::Base def add_import_job job_id = if forked? - RepositoryForkWorker.perform_async(id, - forked_from_project.repository_storage_path, - forked_from_project.disk_path) + RepositoryForkWorker.perform_async(id) elsif gitlab_project_import? # Do not retry on Import/Export until https://gitlab.com/gitlab-org/gitlab-ce/issues/26189 is solved. RepositoryImportWorker.set(retry: false).perform_async(self.id) -- cgit v1.2.3 From 65664c2eaeed853396c97a9b46e404c05209c42e Mon Sep 17 00:00:00 2001 From: Jan Provaznik Date: Tue, 3 Apr 2018 16:03:00 +0000 Subject: Refactor discussions/notes code --- app/models/note.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/note.rb b/app/models/note.rb index 787a80f0196..0f5fb529a87 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -379,12 +379,15 @@ class Note < ActiveRecord::Base def expire_etag_cache return unless noteable&.discussions_rendered_on_frontend? - key = Gitlab::Routing.url_helpers.project_noteable_notes_path( + Gitlab::EtagCaching::Store.new.touch(etag_key) + end + + def etag_key + Gitlab::Routing.url_helpers.project_noteable_notes_path( project, target_type: noteable_type.underscore, target_id: noteable_id ) - Gitlab::EtagCaching::Store.new.touch(key) end def touch(*args) -- cgit v1.2.3 From 3d3b46f344748420a638f8a66745d974fa2c2748 Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Wed, 4 Apr 2018 09:19:47 +0000 Subject: [Rails5] Rename `sort` methods to `sort_by_attribute` --- app/models/concerns/issuable.rb | 2 +- app/models/concerns/milestoneish.rb | 4 ++-- app/models/group.rb | 2 +- app/models/issue.rb | 2 +- app/models/member.rb | 2 +- app/models/milestone.rb | 2 +- app/models/project.rb | 2 +- app/models/todo.rb | 2 +- app/models/user.rb | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 5a566f3ac02..b45395343cc 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -137,7 +137,7 @@ module Issuable fuzzy_search(query, [:title, :description]) end - def sort(method, excluded_labels: []) + def sort_by_attribute(method, excluded_labels: []) sorted = case method.to_s when 'downvotes_desc' then order_downvotes_desc diff --git a/app/models/concerns/milestoneish.rb b/app/models/concerns/milestoneish.rb index caf8afa97f9..5130ecec472 100644 --- a/app/models/concerns/milestoneish.rb +++ b/app/models/concerns/milestoneish.rb @@ -45,11 +45,11 @@ module Milestoneish end def sorted_issues(user) - issues_visible_to_user(user).preload_associations.sort('label_priority') + issues_visible_to_user(user).preload_associations.sort_by_attribute('label_priority') end def sorted_merge_requests - merge_requests.sort('label_priority') + merge_requests.sort_by_attribute('label_priority') end def upcoming? diff --git a/app/models/group.rb b/app/models/group.rb index d99af79b5fe..3cfe21ac93b 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -53,7 +53,7 @@ class Group < Namespace Gitlab::Database.postgresql? end - def sort(method) + def sort_by_attribute(method) if method == 'storage_size_desc' # storage_size is a virtual column so we need to # pass a string to avoid AR adding the table name diff --git a/app/models/issue.rb b/app/models/issue.rb index 6a94d60c828..13abc6c1a0d 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -116,7 +116,7 @@ class Issue < ActiveRecord::Base 'project_id' end - def self.sort(method, excluded_labels: []) + def self.sort_by_attribute(method, excluded_labels: []) case method.to_s when 'due_date' then order_due_date_asc when 'due_date_asc' then order_due_date_asc diff --git a/app/models/member.rb b/app/models/member.rb index e1a32148538..eac4a22a03f 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -96,7 +96,7 @@ class Member < ActiveRecord::Base joins(:user).merge(User.search(query)) end - def sort(method) + def sort_by_attribute(method) case method.to_s when 'access_level_asc' then reorder(access_level: :asc) when 'access_level_desc' then reorder(access_level: :desc) diff --git a/app/models/milestone.rb b/app/models/milestone.rb index e7d397f40f5..dafae58d121 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -138,7 +138,7 @@ class Milestone < ActiveRecord::Base User.joins(assigned_issues: :milestone).where("milestones.id = ?", id).uniq end - def self.sort(method) + def self.sort_by_attribute(method) case method.to_s when 'due_date_asc' reorder(Gitlab::Database.nulls_last_order('due_date', 'ASC')) diff --git a/app/models/project.rb b/app/models/project.rb index b343786d2c9..3bd0d3445fe 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -436,7 +436,7 @@ class Project < ActiveRecord::Base Gitlab::VisibilityLevel.options end - def sort(method) + def sort_by_attribute(method) case method.to_s when 'storage_size_desc' # storage_size is a joined column so we need to diff --git a/app/models/todo.rb b/app/models/todo.rb index 8afacd188e0..a2ab405fdbe 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -50,7 +50,7 @@ class Todo < ActiveRecord::Base # Priority sorting isn't displayed in the dropdown, because we don't show # milestones, but still show something if the user has a URL with that # selected. - def sort(method) + def sort_by_attribute(method) sorted = case method.to_s when 'priority', 'label_priority' then order_by_labels_priority diff --git a/app/models/user.rb b/app/models/user.rb index f934b654225..ba51595e6a3 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -256,7 +256,7 @@ class User < ActiveRecord::Base end end - def sort(method) + def sort_by_attribute(method) order_method = method || 'id_desc' case order_method.to_s -- cgit v1.2.3 From 16ea3315f2bb2a7764f2c101625422e6a374fa6f Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Wed, 4 Apr 2018 14:10:22 +0200 Subject: Revert "Allow CI/CD Jobs being grouped on version strings" This reverts commit 4f2cdb51df0f2729055ec4dc6960ae347163da16. --- app/models/commit_status.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 9fb5b7efec6..3469d5d795c 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -141,7 +141,7 @@ class CommitStatus < ActiveRecord::Base end def group_name - name.to_s.gsub(%r{\d+[\.\s:/\\]+\d+\s*}, '').strip + name.to_s.gsub(%r{\d+[\s:/\\]+\d+\s*}, '').strip end def failed_but_allowed? -- cgit v1.2.3 From 964933c9638c7d039a2843375d76c76c0b2cf25e Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 3 Apr 2018 16:58:27 +0200 Subject: Create metadata object on build object creation --- app/models/ci/build.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 18e96389199..4aa65bf4273 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -90,6 +90,7 @@ module Ci before_save :ensure_token before_destroy { unscoped_project } + before_create :ensure_metadata after_create unless: :importing? do |build| run_after_commit { BuildHooksWorker.perform_async(build.id) } end -- cgit v1.2.3 From a069aa494a71450f3a6627b723bd5312bbf20133 Mon Sep 17 00:00:00 2001 From: Omar Mekky Date: Wed, 4 Apr 2018 15:04:03 +0000 Subject: Add banzai filter to detect commit message trailers and properly link the users --- app/models/commit.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/commit.rb b/app/models/commit.rb index b64462fb768..3f7f36e83c0 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -32,7 +32,8 @@ class Commit COMMIT_SHA_PATTERN = /\h{#{MIN_SHA_LENGTH},40}/.freeze def banzai_render_context(field) - context = { pipeline: :single_line, project: self.project } + pipeline = field == :description ? :commit_description : :single_line + context = { pipeline: pipeline, project: self.project } context[:author] = self.author if self.author context -- cgit v1.2.3 From 99caa5bb04dea578f767e89296301b00d7401c5d Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 27 Feb 2018 01:25:52 +0100 Subject: Use ChronicDurationAttribute to handle CI/CD timeout setting --- app/models/project.rb | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 714a15ade9c..8a325f49ecb 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -21,6 +21,7 @@ class Project < ActiveRecord::Base include Gitlab::SQL::Pattern include DeploymentPlatform include ::Gitlab::Utils::StrongMemoize + include ChronicDurationAttribute extend Gitlab::ConfigHelper @@ -325,6 +326,12 @@ class Project < ActiveRecord::Base enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 } + chronic_duration_attr :build_timeout_human_readable, :build_timeout + + validates :build_timeout, allow_nil: true, + numericality: { greater_than_or_equal_to: 600, + message: 'needs to be at least 10 minutes' } + # Returns a collection of projects that is either public or visible to the # logged in user. def self.public_or_visible_to_user(user = nil) @@ -1299,14 +1306,6 @@ class Project < ActiveRecord::Base self.runners_token && ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.runners_token) end - def build_timeout_in_minutes - build_timeout / 60 - end - - def build_timeout_in_minutes=(value) - self.build_timeout = value.to_i * 60 - end - def open_issues_count Projects::OpenIssuesCountService.new(self).count end -- cgit v1.2.3 From e40c0085ef300aca38076af3ea2f227761084038 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Thu, 29 Mar 2018 13:57:21 +0200 Subject: Store override params as import data on projects This means import data doesn't necessarily have to have an import_url anymore. The `ProjectImportData` could just contain the override data in it's serialized data attribute. The import data is automatically cleaned up after it is finished by the state machine. --- app/models/project.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 714a15ade9c..71d2f30698e 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -630,7 +630,7 @@ class Project < ActiveRecord::Base end def create_or_update_import_data(data: nil, credentials: nil) - return unless import_url.present? && valid_import_url? + return if data.nil? && credentials.nil? project_import_data = import_data || build_import_data if data -- cgit v1.2.3 From a52e3edd1a1869c2656193c95f8f2e8fd3bc8fa2 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Wed, 4 Apr 2018 21:31:56 +0200 Subject: Specify default value for Project#build_timeout --- app/models/concerns/chronic_duration_attribute.rb | 6 +++--- app/models/project.rb | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/chronic_duration_attribute.rb b/app/models/concerns/chronic_duration_attribute.rb index fa1eafb1d7a..593a9b3d71d 100644 --- a/app/models/concerns/chronic_duration_attribute.rb +++ b/app/models/concerns/chronic_duration_attribute.rb @@ -8,14 +8,14 @@ module ChronicDurationAttribute end end - def chronic_duration_attr_writer(virtual_attribute, source_attribute) + def chronic_duration_attr_writer(virtual_attribute, source_attribute, parameters = {}) chronic_duration_attr_reader(virtual_attribute, source_attribute) define_method("#{virtual_attribute}=") do |value| - chronic_duration_attributes[virtual_attribute] = value.presence || '' + chronic_duration_attributes[virtual_attribute] = value.presence || parameters[:default].presence.to_s begin - new_value = ChronicDuration.parse(value).to_i if value.present? + new_value = value.present? ? ChronicDuration.parse(value).to_i : parameters[:default].presence assign_attributes(source_attribute => new_value) rescue ChronicDuration::DurationParseError # ignore error as it will be caught by validation diff --git a/app/models/project.rb b/app/models/project.rb index 8a325f49ecb..cfd00e8a73c 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -326,7 +326,7 @@ class Project < ActiveRecord::Base enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 } - chronic_duration_attr :build_timeout_human_readable, :build_timeout + chronic_duration_attr :build_timeout_human_readable, :build_timeout, default: 3600 validates :build_timeout, allow_nil: true, numericality: { greater_than_or_equal_to: 600, -- cgit v1.2.3 From accc2cab20cb1546605f05aa68545d2d55c1f522 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 26 Mar 2018 20:45:18 +0900 Subject: Add ci_job_trace_chunks table --- app/models/ci/job_trace_chunk.rb | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 app/models/ci/job_trace_chunk.rb (limited to 'app/models') diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb new file mode 100644 index 00000000000..8998ed920a5 --- /dev/null +++ b/app/models/ci/job_trace_chunk.rb @@ -0,0 +1,7 @@ +module Ci + class JobTraceChunk < ActiveRecord::Base + extend Gitlab::Ci::Model + + belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id + end +end -- cgit v1.2.3 From 2fac77b0819fc951bb9e896d2615f8a550093707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Wed, 4 Apr 2018 12:19:17 +0200 Subject: Simpler chunking :) --- app/models/ci/build.rb | 2 + app/models/ci/job_trace_chunk.rb | 107 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 4aa65bf4273..b471fb80536 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -25,6 +25,8 @@ module Ci has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id + has_many :chunks, class_name: 'Ci::JobTraceChunk', foreign_key: :job_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_one :metadata, class_name: 'Ci::BuildMetadata' delegate :timeout, to: :metadata, prefix: true, allow_nil: true diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb index 8998ed920a5..85b67997d1e 100644 --- a/app/models/ci/job_trace_chunk.rb +++ b/app/models/ci/job_trace_chunk.rb @@ -3,5 +3,112 @@ module Ci extend Gitlab::Ci::Model belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id + + after_destroy :redis_delete_data, if: :redis? + + default_value_for :data_store, :redis + + CHUNK_SIZE = 8 + CHUNK_REDIS_TTL = 1.month + + enum data_store: { + redis: 1, + db: 2, + } + + def data + case + when redis? + redis_data + when db? + raw_data + else + raise 'Unsupported data store' + end + end + + def set_data(value) + raise 'too much data' if value.length > CHUNK_SIZE + + case + when redis? + redis_set_data(value) + when db? + self.raw_data = value + else + raise 'Unsupported data store' + end + + save if changed? + schedule_to_db if fullfilled? + end + + def truncate(offset = 0) + self.append("", offset) + end + + def append(new_data, offset) + current_data = self.data || "" + raise 'Outside of if data' if offset > current_data.bytesize + + self.set_data(current_data.byteslice(0, offset) + new_data) + end + + def size + data&.bytesize.to_i + end + + def start_offset + chunk_index * CHUNK_SIZE + end + + def end_offset + start_offset + size + end + + def range + (start_offset...end_offset) + end + + def use_database! + return if db? + + self.update!(raw_data: data, data_store: :db) + redis_delete_data + end + + private + + def schedule_to_db + return if db? + + self.use_database! + end + + def fullfilled? + size == CHUNK_SIZE + end + + def redis_data + Gitlab::Redis::SharedState.with do |redis| + redis.get(redis_key) + end + end + + def redis_set_data(data) + Gitlab::Redis::SharedState.with do |redis| + redis.set(redis_key, data, ex: CHUNK_REDIS_TTL) + end + end + + def redis_delete_data + Gitlab::Redis::SharedState.with do |redis| + redis.del(redis_key) + end + end + + def redis_key + "gitlab:ci:trace:#{job_id}:chunks:#{chunk_index}" + end end end -- cgit v1.2.3 From 26fec9d460163373043e56c13ae4518c50e98367 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 5 Apr 2018 03:16:51 +0900 Subject: Fix #read to increament tell correctly --- app/models/ci/job_trace_chunk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb index 85b67997d1e..329f2fd01ae 100644 --- a/app/models/ci/job_trace_chunk.rb +++ b/app/models/ci/job_trace_chunk.rb @@ -8,7 +8,7 @@ module Ci default_value_for :data_store, :redis - CHUNK_SIZE = 8 + CHUNK_SIZE = 32.kilobytes CHUNK_REDIS_TTL = 1.month enum data_store: { -- cgit v1.2.3 From 52967b107b7b2f1472b4c005f70f21346079cd95 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 3 Apr 2018 11:00:33 +0000 Subject: Merge branch 'jej/mattermost-notification-confidentiality-10-6' into 'security-10-6' [10.6] Prevent notes on confidential issues from being sent to chat See merge request gitlab/gitlabhq!2366 # Conflicts: # app/helpers/services_helper.rb --- app/models/hooks/project_hook.rb | 1 + app/models/note.rb | 4 ++++ app/models/project_services/chat_notification_service.rb | 14 ++++++++++++-- app/models/project_services/hipchat_service.rb | 2 +- app/models/service.rb | 8 ++++++-- 5 files changed, 24 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/hooks/project_hook.rb b/app/models/hooks/project_hook.rb index b6dd39b860b..ec072882cc9 100644 --- a/app/models/hooks/project_hook.rb +++ b/app/models/hooks/project_hook.rb @@ -7,6 +7,7 @@ class ProjectHook < WebHook :issue_hooks, :confidential_issue_hooks, :note_hooks, + :confidential_note_hooks, :merge_request_hooks, :job_hooks, :pipeline_hooks, diff --git a/app/models/note.rb b/app/models/note.rb index 0f5fb529a87..109405d3f17 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -268,6 +268,10 @@ class Note < ActiveRecord::Base self.special_role = Note::SpecialRole::FIRST_TIME_CONTRIBUTOR end + def confidential? + noteable.try(:confidential?) + end + def editable? !system? end diff --git a/app/models/project_services/chat_notification_service.rb b/app/models/project_services/chat_notification_service.rb index dab0ea1a681..7591ab4f478 100644 --- a/app/models/project_services/chat_notification_service.rb +++ b/app/models/project_services/chat_notification_service.rb @@ -21,8 +21,16 @@ class ChatNotificationService < Service end end + def confidential_issue_channel + properties['confidential_issue_channel'].presence || properties['issue_channel'] + end + + def confidential_note_channel + properties['confidential_note_channel'].presence || properties['note_channel'] + end + def self.supported_events - %w[push issue confidential_issue merge_request note tag_push + %w[push issue confidential_issue merge_request note confidential_note tag_push pipeline wiki_page] end @@ -55,7 +63,9 @@ class ChatNotificationService < Service return false unless message - channel_name = get_channel_field(object_kind).presence || channel + event_type = data[:event_type] || object_kind + + channel_name = get_channel_field(event_type).presence || channel opts = {} opts[:channel] = channel_name if channel_name diff --git a/app/models/project_services/hipchat_service.rb b/app/models/project_services/hipchat_service.rb index f31c3f02af2..dce878e485f 100644 --- a/app/models/project_services/hipchat_service.rb +++ b/app/models/project_services/hipchat_service.rb @@ -46,7 +46,7 @@ class HipchatService < Service end def self.supported_events - %w(push issue confidential_issue merge_request note tag_push pipeline) + %w(push issue confidential_issue merge_request note confidential_note tag_push pipeline) end def execute(data) diff --git a/app/models/service.rb b/app/models/service.rb index 7424cef0fc0..e9b6f005aec 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -14,6 +14,7 @@ class Service < ActiveRecord::Base default_value_for :merge_requests_events, true default_value_for :tag_push_events, true default_value_for :note_events, true + default_value_for :confidential_note_events, true default_value_for :job_events, true default_value_for :pipeline_events, true default_value_for :wiki_page_events, true @@ -42,6 +43,7 @@ class Service < ActiveRecord::Base scope :confidential_issue_hooks, -> { where(confidential_issues_events: true, active: true) } scope :merge_request_hooks, -> { where(merge_requests_events: true, active: true) } scope :note_hooks, -> { where(note_events: true, active: true) } + scope :confidential_note_hooks, -> { where(confidential_note_events: true, active: true) } scope :job_hooks, -> { where(job_events: true, active: true) } scope :pipeline_hooks, -> { where(pipeline_events: true, active: true) } scope :wiki_page_hooks, -> { where(wiki_page_events: true, active: true) } @@ -168,8 +170,10 @@ class Service < ActiveRecord::Base def self.prop_accessor(*args) args.each do |arg| class_eval %{ - def #{arg} - properties['#{arg}'] + unless method_defined?(arg) + def #{arg} + properties['#{arg}'] + end end def #{arg}=(value) -- cgit v1.2.3 From 54aec65e3ed3897d0f7a62bb0df1d55e7caa5023 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 5 Apr 2018 16:19:32 +0900 Subject: Add a gurad logic for appending oversized data --- app/models/ci/job_trace_chunk.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb index 329f2fd01ae..1878e75153d 100644 --- a/app/models/ci/job_trace_chunk.rb +++ b/app/models/ci/job_trace_chunk.rb @@ -28,7 +28,7 @@ module Ci end def set_data(value) - raise 'too much data' if value.length > CHUNK_SIZE + raise 'too much data' if value.bytesize > CHUNK_SIZE case when redis? @@ -49,7 +49,8 @@ module Ci def append(new_data, offset) current_data = self.data || "" - raise 'Outside of if data' if offset > current_data.bytesize + raise 'Offset is out of bound' if offset > current_data.bytesize + raise 'Outside of chunk size' if CHUNK_SIZE < offset + new_data.bytesize self.set_data(current_data.byteslice(0, offset) + new_data) end -- cgit v1.2.3 From 446086f18c83dbfe12675a6f186e82cc04bac0f3 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 5 Apr 2018 16:57:28 +0900 Subject: Add sidekiq worker for writing operation --- app/models/ci/job_trace_chunk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb index 1878e75153d..aa92bd7e3c1 100644 --- a/app/models/ci/job_trace_chunk.rb +++ b/app/models/ci/job_trace_chunk.rb @@ -83,7 +83,7 @@ module Ci def schedule_to_db return if db? - self.use_database! + StashTraceChunkWorker.perform_async(id) end def fullfilled? -- cgit v1.2.3 From 48b17e991a960c006da278d4c1f534b1db81d070 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Wed, 4 Apr 2018 16:40:58 +0200 Subject: Add helper for accessing lfs_objects for project This makes accessing LFS Objects for a project easier project.lfs_storage_project.lfs_objects` becomes project.all_lfs_objects This will make the refactor in https://gitlab.com/gitlab-org/gitlab-ce/issues/39769 easier to deal with. --- app/models/project.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 714a15ade9c..32289106f28 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1066,6 +1066,16 @@ class Project < ActiveRecord::Base end end + # This will return all `lfs_objects` that are accessible to the project. + # So this might be `self.lfs_objects` if the project is not part of a fork + # network, or it is the base of the fork network. + # + # TODO: refactor this to get the correct lfs objects when implementing + # https://gitlab.com/gitlab-org/gitlab-ce/issues/39769 + def all_lfs_objects + lfs_storage_project.lfs_objects + end + def personal? !group end -- cgit v1.2.3 From 7de250fb81fb24f8e905cfb330072206c4a8a40a Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Thu, 5 Apr 2018 11:14:26 +0200 Subject: Ensure internal users (ghost, support bot) get assigned a namespace --- app/models/user.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index ba51595e6a3..7b6857a0d34 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -164,12 +164,15 @@ class User < ActiveRecord::Base before_validation :sanitize_attrs before_validation :set_notification_email, if: :email_changed? + before_save :set_notification_email, if: :email_changed? # in case validation is skipped before_validation :set_public_email, if: :public_email_changed? + before_save :set_public_email, if: :public_email_changed? # in case validation is skipped before_save :ensure_incoming_email_token before_save :ensure_user_rights_and_limits, if: ->(user) { user.new_record? || user.external_changed? } before_save :skip_reconfirmation!, if: ->(user) { user.email_changed? && user.read_only_attribute?(:email) } before_save :check_for_verified_email, if: ->(user) { user.email_changed? && !user.new_record? } before_validation :ensure_namespace_correct + before_save :ensure_namespace_correct # in case validation is skipped after_validation :set_username_errors after_update :username_changed_hook, if: :username_changed? after_destroy :post_destroy_hook @@ -408,7 +411,6 @@ class User < ActiveRecord::Base unique_internal(where(ghost: true), 'ghost', email) do |u| u.bio = 'This is a "Ghost User", created to hold all issues authored by users that have since been deleted. This user cannot be removed.' u.name = 'Ghost User' - u.notification_email = email end end end -- cgit v1.2.3 From 9e146233d489894fac32c0d2e81bb3224b6f4ff3 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 5 Apr 2018 18:59:07 +0900 Subject: Add job_trace_chunk_spec with small fixes --- app/models/ci/job_trace_chunk.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb index aa92bd7e3c1..baa48a602f1 100644 --- a/app/models/ci/job_trace_chunk.rb +++ b/app/models/ci/job_trace_chunk.rb @@ -39,7 +39,7 @@ module Ci raise 'Unsupported data store' end - save if changed? + save! if changed? schedule_to_db if fullfilled? end @@ -49,7 +49,7 @@ module Ci def append(new_data, offset) current_data = self.data || "" - raise 'Offset is out of bound' if offset > current_data.bytesize + raise 'Offset is out of bound' if offset > current_data.bytesize || offset < 0 raise 'Outside of chunk size' if CHUNK_SIZE < offset + new_data.bytesize self.set_data(current_data.byteslice(0, offset) + new_data) @@ -73,6 +73,7 @@ module Ci def use_database! return if db? + return unless size > 0 self.update!(raw_data: data, data_store: :db) redis_delete_data -- cgit v1.2.3 From c1946364707947f56048cc00caa6d67fe635f3c7 Mon Sep 17 00:00:00 2001 From: Jan Provaznik Date: Fri, 30 Mar 2018 19:27:03 +0200 Subject: Better group support notes-related code Updates notes-related services and rendering so this code can be easily used for group-scoped resources (specifically Epics). Related to gitlab-ee!5205 --- app/models/note.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'app/models') diff --git a/app/models/note.rb b/app/models/note.rb index 0f5fb529a87..19bd41a34be 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -313,6 +313,10 @@ class Note < ActiveRecord::Base !system? && !for_snippet? end + def can_create_notification? + true + end + def discussion_class(noteable = nil) # When commit notes are rendered on an MR's Discussion page, they are # displayed in one discussion instead of individually. -- cgit v1.2.3 From 242b4afa8fd1ffd0686038ba955f8ad41dc95383 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 5 Apr 2018 20:39:35 +0900 Subject: Chnaged Chunk size to 128kb. Add spec. --- app/models/ci/job_trace_chunk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb index baa48a602f1..7d9ccd5e5ad 100644 --- a/app/models/ci/job_trace_chunk.rb +++ b/app/models/ci/job_trace_chunk.rb @@ -8,7 +8,7 @@ module Ci default_value_for :data_store, :redis - CHUNK_SIZE = 32.kilobytes + CHUNK_SIZE = 128.kilobytes CHUNK_REDIS_TTL = 1.month enum data_store: { -- cgit v1.2.3 From f8f62ea5b6e81162cfb8781730040e54993cd847 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 5 Apr 2018 21:15:18 +0900 Subject: Rename to SwapTraceChunkWorker from StashTraceChunkWorker --- app/models/ci/job_trace_chunk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb index 7d9ccd5e5ad..c2f05dd1f03 100644 --- a/app/models/ci/job_trace_chunk.rb +++ b/app/models/ci/job_trace_chunk.rb @@ -84,7 +84,7 @@ module Ci def schedule_to_db return if db? - StashTraceChunkWorker.perform_async(id) + SwapTraceChunkWorker.perform_async(id) end def fullfilled? -- cgit v1.2.3 From 678620cce67cc283b19b75137f747f9415aaf942 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Tue, 3 Apr 2018 18:47:33 +0200 Subject: Add `direct_upload` setting for artifacts --- app/models/ci/job_artifact.rb | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index df57b4f65e3..fbb95fe16df 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -7,6 +7,7 @@ module Ci belongs_to :project belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id + before_save :update_file_store before_save :set_size, if: :file_changed? scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) } @@ -21,6 +22,10 @@ module Ci trace: 3 } + def update_file_store + self.file_store = file.object_store + end + def self.artifacts_size_for(project) self.where(project: project).sum(:size) end -- cgit v1.2.3 From 902cec12b5b531434ccf7ecf6df22ddb4249c253 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Thu, 5 Apr 2018 12:13:10 +0200 Subject: Don't export `Project#description_html` Since we can regenerate `description_html` from the `description`, we should not export it. This avoids some complexity when overriding the description during an import/export where we would need to invalidate this cached field. Now we refresh the markdown cache after the import --- app/models/project.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 71d2f30698e..1d583831156 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1478,6 +1478,7 @@ class Project < ActiveRecord::Base remove_import_jid update_project_counter_caches after_create_default_branch + refresh_markdown_cache! end def update_project_counter_caches -- cgit v1.2.3 From d42909955554b6f1eae15f744250d8ddf30dbbf9 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 6 Apr 2018 00:19:41 +0900 Subject: Add tests for Trace::Stream --- app/models/ci/job_trace_chunk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb index c2f05dd1f03..a68b2a16efb 100644 --- a/app/models/ci/job_trace_chunk.rb +++ b/app/models/ci/job_trace_chunk.rb @@ -24,7 +24,7 @@ module Ci raw_data else raise 'Unsupported data store' - end + end&.force_encoding(Encoding::BINARY) end def set_data(value) -- cgit v1.2.3 From 1a71dd049b627a396297e601aeb767f2b600e666 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 6 Apr 2018 00:57:05 +0900 Subject: Fix rubocop --- app/models/ci/job_trace_chunk.rb | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb index a68b2a16efb..38374907e32 100644 --- a/app/models/ci/job_trace_chunk.rb +++ b/app/models/ci/job_trace_chunk.rb @@ -13,14 +13,13 @@ module Ci enum data_store: { redis: 1, - db: 2, + db: 2 } def data - case - when redis? + if redis? redis_data - when db? + elsif db? raw_data else raise 'Unsupported data store' @@ -30,10 +29,9 @@ module Ci def set_data(value) raise 'too much data' if value.bytesize > CHUNK_SIZE - case - when redis? + if redis? redis_set_data(value) - when db? + elsif db? self.raw_data = value else raise 'Unsupported data store' -- cgit v1.2.3 From 22423d284704c3ec2f7715736a75155d14ff413d Mon Sep 17 00:00:00 2001 From: Felipe Artur Date: Wed, 4 Apr 2018 18:43:33 -0300 Subject: Show issues of subgroups in group-level issue board --- app/models/issue.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/issue.rb b/app/models/issue.rb index 13abc6c1a0d..13d9e42bcc8 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -272,11 +272,17 @@ class Issue < ActiveRecord::Base def as_json(options = {}) super(options).tap do |json| - if options.key?(:sidebar_endpoints) && project + if options.key?(:issue_endpoints) && project url_helper = Gitlab::Routing.url_helpers - json.merge!(issue_sidebar_endpoint: url_helper.project_issue_path(project, self, format: :json, serializer: 'sidebar'), - toggle_subscription_endpoint: url_helper.toggle_subscription_project_issue_path(project, self)) + issue_reference = options[:include_full_project_path] ? to_reference(full: true) : to_reference + + json.merge!( + reference_path: issue_reference, + real_path: url_helper.project_issue_path(project, self), + issue_sidebar_endpoint: url_helper.project_issue_path(project, self, format: :json, serializer: 'sidebar'), + toggle_subscription_endpoint: url_helper.toggle_subscription_project_issue_path(project, self) + ) end if options.key?(:labels) -- cgit v1.2.3 From 62574be35d5a92c6c825912e1ab9969c5acdb7d9 Mon Sep 17 00:00:00 2001 From: Felipe Artur Date: Thu, 5 Apr 2018 12:28:49 -0300 Subject: Fix specs --- app/models/milestone.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/milestone.rb b/app/models/milestone.rb index dafae58d121..a66a0015827 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -34,8 +34,8 @@ class Milestone < ActiveRecord::Base scope :for_projects_and_groups, -> (project_ids, group_ids) do conditions = [] - conditions << arel_table[:project_id].in(project_ids) if project_ids.compact.any? - conditions << arel_table[:group_id].in(group_ids) if group_ids.compact.any? + conditions << arel_table[:project_id].in(project_ids) if project_ids&.compact&.any? + conditions << arel_table[:group_id].in(group_ids) if group_ids&.compact&.any? where(conditions.reduce(:or)) end -- cgit v1.2.3 From b099c6ff9fcbd81beb75fd12bff8d1d5b9c16083 Mon Sep 17 00:00:00 2001 From: m b Date: Fri, 23 Mar 2018 00:09:26 -0500 Subject: Deleting a MR you are assigned to should decrements counter The merge request counter in the UI was not decreasing when a merge request was deleting. This was just due to the cache not being refreshed on a delete action. fixes: https://gitlab.com/gitlab-org/gitlab-ce/issues/44458 --- app/models/user.rb | 5 ----- 1 file changed, 5 deletions(-) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index 7b6857a0d34..ce56b39b1c8 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1046,11 +1046,6 @@ class User < ActiveRecord::Base end end - def update_cache_counts - assigned_open_merge_requests_count(force: true) - assigned_open_issues_count(force: true) - end - def update_todos_count_cache todos_done_count(force: true) todos_pending_count(force: true) -- cgit v1.2.3 From d209a05486503f1dd6412c777c1fd29fe02f0a40 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 5 Apr 2018 12:52:20 -0700 Subject: Use the GitLab version as part of the appearances cache key This prevents us from having to manually flush the cache key in migrations every time a new column is added. Closes https://gitlab.com/gitlab-org/gitlab-ee/issues/5571 --- app/models/appearance.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/appearance.rb b/app/models/appearance.rb index 2a6406d63c7..fb66dd0b766 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -16,7 +16,7 @@ class Appearance < ActiveRecord::Base has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - CACHE_KEY = 'current_appearance'.freeze + CACHE_KEY = "current_appearance:#{Gitlab::VERSION}".freeze after_commit :flush_redis_cache -- cgit v1.2.3 From d54cf868f81ca957c8322661b11e6755d9ea5a85 Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Thu, 5 Apr 2018 21:04:42 +0000 Subject: Resolve "Show `failure_reason` and upgrade tooltips of jobs" --- app/models/concerns/presentable.rb | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'app/models') diff --git a/app/models/concerns/presentable.rb b/app/models/concerns/presentable.rb index 7b33b837004..bc4fbd19a02 100644 --- a/app/models/concerns/presentable.rb +++ b/app/models/concerns/presentable.rb @@ -1,4 +1,12 @@ module Presentable + extend ActiveSupport::Concern + + class_methods do + def present(attributes) + all.map { |klass_object| klass_object.present(attributes) } + end + end + def present(**attributes) Gitlab::View::Presenter::Factory .new(self, attributes) -- cgit v1.2.3 From 8fc9ff7f131aa850ce4b1a8cb86ab4fa9b46410d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 6 Apr 2018 12:10:30 +0200 Subject: Fix environment deployment platform filter method --- app/models/environment.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/environment.rb b/app/models/environment.rb index 9517723d9d9..fddb269af4b 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -224,7 +224,7 @@ class Environment < ActiveRecord::Base end def deployment_platform - project.deployment_platform(environment: self) + project.deployment_platform(environment: self.name) end private -- cgit v1.2.3 From 29b0a90c208f29606a05d1391a72b9ff7ff843b1 Mon Sep 17 00:00:00 2001 From: Andreas Brandl Date: Wed, 4 Apr 2018 17:14:19 +0200 Subject: Cache personal projects count. Closes #37462. --- app/models/user.rb | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index ce56b39b1c8..a14aefc61d2 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -700,10 +700,6 @@ class User < ActiveRecord::Base projects_limit - personal_projects_count end - def personal_projects_count - @personal_projects_count ||= personal_projects.count - end - def recent_push(project = nil) service = Users::LastPushEventService.new(self) @@ -1046,6 +1042,18 @@ class User < ActiveRecord::Base end end + def personal_projects_count(force: false) + Rails.cache.fetch(['users', id, 'personal_projects_count'], force: force, expires_in: 24.hours, raw: true) do + personal_projects.count + end.to_i + end + + def update_cache_counts + assigned_open_merge_requests_count(force: true) + assigned_open_issues_count(force: true) + personal_projects_count(force: true) + end + def update_todos_count_cache todos_done_count(force: true) todos_pending_count(force: true) @@ -1056,6 +1064,7 @@ class User < ActiveRecord::Base invalidate_merge_request_cache_counts invalidate_todos_done_count invalidate_todos_pending_count + invalidate_personal_projects_count end def invalidate_issue_cache_counts @@ -1074,6 +1083,10 @@ class User < ActiveRecord::Base Rails.cache.delete(['users', id, 'todos_pending_count']) end + def invalidate_personal_projects_count + Rails.cache.delete(['users', id, 'personal_projects_count']) + end + # This is copied from Devise::Models::Lockable#valid_for_authentication?, as our auth # flow means we don't call that automatically (and can't conveniently do so). # -- cgit v1.2.3 From fa46b19ddb82851523fabaea2fca4660c181db89 Mon Sep 17 00:00:00 2001 From: Andreas Brandl Date: Thu, 5 Apr 2018 15:48:18 +0200 Subject: Remove unused method. --- app/models/user.rb | 6 ------ 1 file changed, 6 deletions(-) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index a14aefc61d2..2b95be3f888 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1048,12 +1048,6 @@ class User < ActiveRecord::Base end.to_i end - def update_cache_counts - assigned_open_merge_requests_count(force: true) - assigned_open_issues_count(force: true) - personal_projects_count(force: true) - end - def update_todos_count_cache todos_done_count(force: true) todos_pending_count(force: true) -- cgit v1.2.3 From cffee49f7ffca39cb0e522dacc9b777e45d22680 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 6 Apr 2018 19:30:23 +0900 Subject: Change Redis TTL to 1day. Fixing nitpicks --- app/models/ci/job_trace_chunk.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb index 38374907e32..bec4405dbd6 100644 --- a/app/models/ci/job_trace_chunk.rb +++ b/app/models/ci/job_trace_chunk.rb @@ -9,7 +9,7 @@ module Ci default_value_for :data_store, :redis CHUNK_SIZE = 128.kilobytes - CHUNK_REDIS_TTL = 1.month + CHUNK_REDIS_TTL = 1.day enum data_store: { redis: 1, @@ -27,7 +27,7 @@ module Ci end def set_data(value) - raise 'too much data' if value.bytesize > CHUNK_SIZE + raise ArgumentError, 'too much data' if value.bytesize > CHUNK_SIZE if redis? redis_set_data(value) @@ -46,9 +46,9 @@ module Ci end def append(new_data, offset) - current_data = self.data || "" - raise 'Offset is out of bound' if offset > current_data.bytesize || offset < 0 - raise 'Outside of chunk size' if CHUNK_SIZE < offset + new_data.bytesize + current_data = self.data.to_s + raise ArgumentError, 'Offset is out of bound' if offset > current_data.bytesize || offset < 0 + raise ArgumentError, 'Outside of chunk size' if CHUNK_SIZE < offset + new_data.bytesize self.set_data(current_data.byteslice(0, offset) + new_data) end -- cgit v1.2.3 From 20695052db6860620365af9b86ab301759d1405d Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Fri, 6 Apr 2018 10:57:19 +0000 Subject: [Rails5] Update `type_cast_*_database` methods --- app/models/merge_request_diff_commit.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/merge_request_diff_commit.rb b/app/models/merge_request_diff_commit.rb index b75387e236e..1c2e57bb01f 100644 --- a/app/models/merge_request_diff_commit.rb +++ b/app/models/merge_request_diff_commit.rb @@ -17,7 +17,7 @@ class MergeRequestDiffCommit < ActiveRecord::Base commit_hash.merge( merge_request_diff_id: merge_request_diff_id, relative_order: index, - sha: sha_attribute.type_cast_for_database(sha), + sha: sha_attribute.serialize(sha), # rubocop:disable Cop/ActiveRecordSerialize authored_date: Gitlab::Database.sanitize_timestamp(commit_hash[:authored_date]), committed_date: Gitlab::Database.sanitize_timestamp(commit_hash[:committed_date]) ) -- cgit v1.2.3 From 180267d6869ce94922b929893729441772da95a9 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 6 Apr 2018 22:43:12 +0900 Subject: Change Redis TTL to 1 week --- app/models/ci/job_trace_chunk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb index bec4405dbd6..f45077f5f4f 100644 --- a/app/models/ci/job_trace_chunk.rb +++ b/app/models/ci/job_trace_chunk.rb @@ -9,7 +9,7 @@ module Ci default_value_for :data_store, :redis CHUNK_SIZE = 128.kilobytes - CHUNK_REDIS_TTL = 1.day + CHUNK_REDIS_TTL = 1.week enum data_store: { redis: 1, -- cgit v1.2.3 From 76485cbf8ba555c929fd2f54ca2051a382760f20 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 7 Apr 2018 00:08:35 +0900 Subject: Add ExclusiveLock in Ci::JobTraceChunk --- app/models/ci/job_trace_chunk.rb | 72 +++++++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 20 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb index f45077f5f4f..383c6596a51 100644 --- a/app/models/ci/job_trace_chunk.rb +++ b/app/models/ci/job_trace_chunk.rb @@ -8,8 +8,13 @@ module Ci default_value_for :data_store, :redis + WriteError = Class.new(StandardError) + CHUNK_SIZE = 128.kilobytes CHUNK_REDIS_TTL = 1.week + LOCK_RETRY = 100 + LOCK_SLEEP = 1 + LOCK_TTL = 5.minutes enum data_store: { redis: 1, @@ -27,18 +32,20 @@ module Ci end def set_data(value) - raise ArgumentError, 'too much data' if value.bytesize > CHUNK_SIZE - - if redis? - redis_set_data(value) - elsif db? - self.raw_data = value - else - raise 'Unsupported data store' + in_lock do + raise ArgumentError, 'too much data' if value.bytesize > CHUNK_SIZE + + if redis? + redis_set_data(value) + elsif db? + self.raw_data = value + else + raise 'Unsupported data store' + end + + save! if changed? + schedule_to_db if fullfilled? end - - save! if changed? - schedule_to_db if fullfilled? end def truncate(offset = 0) @@ -70,11 +77,13 @@ module Ci end def use_database! - return if db? - return unless size > 0 + in_lock do + return if db? + return unless size > 0 - self.update!(raw_data: data, data_store: :db) - redis_delete_data + self.update!(raw_data: data, data_store: :db) + redis_delete_data + end end private @@ -91,24 +100,47 @@ module Ci def redis_data Gitlab::Redis::SharedState.with do |redis| - redis.get(redis_key) + redis.get(redis_data_key) end end def redis_set_data(data) Gitlab::Redis::SharedState.with do |redis| - redis.set(redis_key, data, ex: CHUNK_REDIS_TTL) + redis.set(redis_data_key, data, ex: CHUNK_REDIS_TTL) end end def redis_delete_data Gitlab::Redis::SharedState.with do |redis| - redis.del(redis_key) + redis.del(redis_data_key) end end - def redis_key - "gitlab:ci:trace:#{job_id}:chunks:#{chunk_index}" + def redis_data_key + "gitlab:ci:trace:#{job_id}:chunks:#{chunk_index}:data" + end + + def redis_lock_key + "gitlab:ci:trace:#{job_id}:chunks:#{chunk_index}:lock" + end + + def in_lock + lease = Gitlab::ExclusiveLease.new(redis_lock_key, timeout: LOCK_TTL) + retry_count = 0 + + until uuid = lease.try_obtain + # Keep trying until we obtain the lease. To prevent hammering Redis too + # much we'll wait for a bit between retries. + sleep(LOCK_SLEEP) + break if LOCK_RETRY < (retry_count += 1) + end + + raise WriteError, 'Failed to obtain write lock' unless uuid + + self.reload if self.persisted? + return yield + ensure + Gitlab::ExclusiveLease.cancel(redis_lock_key, uuid) end end end -- cgit v1.2.3 From f20912df033d07c46b0989012244d96d0a12b66d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Javier=20L=C3=B3pez?= Date: Fri, 6 Apr 2018 15:23:49 +0000 Subject: Extend API for importing a project export with overwrite support --- app/models/group.rb | 4 ++++ app/models/namespace.rb | 4 ++++ app/models/project.rb | 4 +++- 3 files changed, 11 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/group.rb b/app/models/group.rb index 3cfe21ac93b..8ff781059cc 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -286,6 +286,10 @@ class Group < Namespace false end + def refresh_project_authorizations + refresh_members_authorized_projects(blocking: false) + end + private def update_two_factor_requirement diff --git a/app/models/namespace.rb b/app/models/namespace.rb index e350b675639..2b63aa33222 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -252,6 +252,10 @@ class Namespace < ActiveRecord::Base [] end + def refresh_project_authorizations + owner.refresh_authorized_projects + end + private def path_or_parent_changed? diff --git a/app/models/project.rb b/app/models/project.rb index 1b29cbf28d2..96907f3b23d 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1472,7 +1472,9 @@ class Project < ActiveRecord::Base end def rename_repo_notify! - send_move_instructions(full_path_was) + # When we import a project overwriting the original project, there + # is a move operation. In that case we don't want to send the instructions. + send_move_instructions(full_path_was) unless started? expires_full_path_cache self.old_path_with_namespace = full_path_was -- cgit v1.2.3 From 4dd1f906fd60bebd433dc9c4b5879c1e575fb735 Mon Sep 17 00:00:00 2001 From: Oswaldo Ferreira Date: Fri, 6 Apr 2018 12:13:18 -0300 Subject: Add support for patch link extension for commit links on GFM --- app/models/commit.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/commit.rb b/app/models/commit.rb index 3f7f36e83c0..de860df4b9c 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -30,6 +30,8 @@ class Commit MIN_SHA_LENGTH = Gitlab::Git::Commit::MIN_SHA_LENGTH COMMIT_SHA_PATTERN = /\h{#{MIN_SHA_LENGTH},40}/.freeze + # Used by GFM to match and present link extensions on node texts and hrefs. + LINK_EXTENSION_PATTERN = /(patch)/.freeze def banzai_render_context(field) pipeline = field == :description ? :commit_description : :single_line @@ -143,7 +145,8 @@ class Commit end def self.link_reference_pattern - @link_reference_pattern ||= super("commit", /(?#{COMMIT_SHA_PATTERN})/) + @link_reference_pattern ||= + super("commit", /(?#{COMMIT_SHA_PATTERN})?(\.(?#{LINK_EXTENSION_PATTERN}))?/) end def to_reference(from = nil, full: false) -- cgit v1.2.3 From db18993f652425b72c4b854e18a002e0ec44b196 Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Mon, 19 Mar 2018 10:11:12 -0600 Subject: Create barebones for Deploytoken Includes: - Model, factories, create service and controller actions - As usual, includes specs for everything - Builds UI (copy from PAT) - Add revoke action Closes #31591 --- app/models/deploy_token.rb | 25 +++++++++++++++++++++++++ app/models/project.rb | 1 + 2 files changed, 26 insertions(+) create mode 100644 app/models/deploy_token.rb (limited to 'app/models') diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb new file mode 100644 index 00000000000..185bd806b18 --- /dev/null +++ b/app/models/deploy_token.rb @@ -0,0 +1,25 @@ +class DeployToken < ActiveRecord::Base + include Expirable + include TokenAuthenticatable + add_authentication_token_field :token + + AVAILABLE_SCOPES = %w(read_repo read_registry).freeze + + serialize :scopes, Array # rubocop:disable Cop/ActiveRecordSerialize + + validates :scopes, presence: true + + belongs_to :project + + before_save :ensure_token + + scope :active, -> { where("revoked = false AND (expires_at >= NOW() OR expires_at IS NULL)") } + + def revoke! + update!(revoked: true) + end + + def self.redis_shared_state_key(user_id) + "gitlab:personal_access_token:#{user_id}" + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 96907f3b23d..3cfb163abf4 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -222,6 +222,7 @@ class Project < ActiveRecord::Base has_many :environments has_many :deployments has_many :pipeline_schedules, class_name: 'Ci::PipelineSchedule' + has_many :deploy_tokens has_many :active_runners, -> { active }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' -- cgit v1.2.3 From 370fc05da7f95bf6621867a71d51493cf3899e25 Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Thu, 29 Mar 2018 16:56:35 -0600 Subject: Implement 'read_repo' for DeployTokens This will allow to download a repo using the token from the DeployToken --- app/models/deploy_token.rb | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'app/models') diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index 185bd806b18..089b4343f41 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -22,4 +22,12 @@ class DeployToken < ActiveRecord::Base def self.redis_shared_state_key(user_id) "gitlab:personal_access_token:#{user_id}" end + + def active? + !revoked + end + + def username + User.ghost.username + end end -- cgit v1.2.3 From 345ac03b7afb1dc9b941c53bc45cc3dfcf22e61c Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Thu, 29 Mar 2018 20:11:36 -0600 Subject: Address UX review - Keep 'Deploy Section' open upon save, otherwise the token might get lost - When an error appears, display the error inside the form and also keep the Deploy Section open - Changue copy of revoke modal --- app/models/deploy_token.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index 089b4343f41..475ad06906a 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -19,8 +19,8 @@ class DeployToken < ActiveRecord::Base update!(revoked: true) end - def self.redis_shared_state_key(user_id) - "gitlab:personal_access_token:#{user_id}" + def redis_shared_state_key(user_id) + "gitlab:deploy_token:#{project_id}:#{user_id}" end def active? -- cgit v1.2.3 From 171b2625b128e5954ce0a150a4fc923a22164e4e Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Wed, 4 Apr 2018 18:43:41 -0500 Subject: Addreses backend review suggestions - Remove extra method for authorize_admin_project - Ensure project presence - Rename 'read_repo' to 'read_repository' to be more verbose --- app/models/deploy_token.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index 475ad06906a..b4df44d295a 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -3,11 +3,12 @@ class DeployToken < ActiveRecord::Base include TokenAuthenticatable add_authentication_token_field :token - AVAILABLE_SCOPES = %w(read_repo read_registry).freeze + AVAILABLE_SCOPES = %w(read_repository read_registry).freeze serialize :scopes, Array # rubocop:disable Cop/ActiveRecordSerialize validates :scopes, presence: true + validates :project, presence: true belongs_to :project -- cgit v1.2.3 From 72220a99d1cdbcf8a914f9e765c43e63eaee2548 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Thu, 5 Apr 2018 15:49:18 +0200 Subject: Support Deploy Tokens properly without hacking abilities --- app/models/deploy_token.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index b4df44d295a..c70d1457afb 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -29,6 +29,10 @@ class DeployToken < ActiveRecord::Base end def username - User.ghost.username + "gitlab+deploy-token-#{id}" + end + + def has_access_to?(project) + self.project == project end end -- cgit v1.2.3 From 8315861c9a50675b4f4f4ca536f0da90f27994f3 Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Thu, 5 Apr 2018 12:22:34 -0500 Subject: Include ProjectDeployTokens Also: - Changes scopes from serializer to use boolean columns - Fixes broken specs --- app/models/deploy_token.rb | 41 ++++++++++++++++++++++++++------------ app/models/project.rb | 3 ++- app/models/project_deploy_token.rb | 14 +++++++++++++ 3 files changed, 44 insertions(+), 14 deletions(-) create mode 100644 app/models/project_deploy_token.rb (limited to 'app/models') diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index c70d1457afb..6639cb17287 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -3,36 +3,51 @@ class DeployToken < ActiveRecord::Base include TokenAuthenticatable add_authentication_token_field :token - AVAILABLE_SCOPES = %w(read_repository read_registry).freeze + AVAILABLE_SCOPES = %i(read_repository read_registry).freeze - serialize :scopes, Array # rubocop:disable Cop/ActiveRecordSerialize - - validates :scopes, presence: true - validates :project, presence: true - - belongs_to :project + has_many :project_deploy_tokens, inverse_of: :deploy_token + has_many :projects, through: :project_deploy_tokens + validate :ensure_at_least_one_scope before_save :ensure_token + accepts_nested_attributes_for :project_deploy_tokens + scope :active, -> { where("revoked = false AND (expires_at >= NOW() OR expires_at IS NULL)") } + scope :read_repository, -> { where(read_repository: true) } + scope :read_registry, -> { where(read_registry: true) } - def revoke! - update!(revoked: true) + def self.redis_shared_state_key(user_id) + "gitlab:deploy_token:user_#{user_id}" end - def redis_shared_state_key(user_id) - "gitlab:deploy_token:#{project_id}:#{user_id}" + def revoke! + update!(revoked: true) end def active? !revoked end + def scopes + AVAILABLE_SCOPES.select { |token_scope| send("#{token_scope}") } # rubocop:disable GitlabSecurity/PublicSend + end + def username "gitlab+deploy-token-#{id}" end - def has_access_to?(project) - self.project == project + def has_access_to?(requested_project) + self.projects.first == requested_project + end + + def project + projects.first + end + + private + + def ensure_at_least_one_scope + errors.add(:base, "Scopes can't be blank") unless read_repository || read_registry end end diff --git a/app/models/project.rb b/app/models/project.rb index 3cfb163abf4..3f805dd1fc9 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -222,7 +222,8 @@ class Project < ActiveRecord::Base has_many :environments has_many :deployments has_many :pipeline_schedules, class_name: 'Ci::PipelineSchedule' - has_many :deploy_tokens + has_many :project_deploy_tokens + has_many :deploy_tokens, through: :project_deploy_tokens has_many :active_runners, -> { active }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' diff --git a/app/models/project_deploy_token.rb b/app/models/project_deploy_token.rb new file mode 100644 index 00000000000..2831b01e378 --- /dev/null +++ b/app/models/project_deploy_token.rb @@ -0,0 +1,14 @@ +class ProjectDeployToken < ActiveRecord::Base + belongs_to :project + belongs_to :deploy_token, inverse_of: :project_deploy_tokens + + validates :deploy_token, presence: true + validates :project, presence: true + validates :deploy_token_id, uniqueness: { scope: [:project_id] } + + accepts_nested_attributes_for :deploy_token + + def redis_shared_state_key(user_id) + "gitlab:deploy_token:#{project_id}:#{user_id}" + end +end -- cgit v1.2.3 From c4f56a88029c1fe73bf6efb062b5f77a65282fed Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Thu, 5 Apr 2018 22:02:13 -0500 Subject: Increase test suite around deploy tokens behavior Also, fixes broken specs --- app/models/deploy_token.rb | 6 +----- app/models/project_deploy_token.rb | 6 ------ 2 files changed, 1 insertion(+), 11 deletions(-) (limited to 'app/models') diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index 6639cb17287..11add42b2c5 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -17,10 +17,6 @@ class DeployToken < ActiveRecord::Base scope :read_repository, -> { where(read_repository: true) } scope :read_registry, -> { where(read_registry: true) } - def self.redis_shared_state_key(user_id) - "gitlab:deploy_token:user_#{user_id}" - end - def revoke! update!(revoked: true) end @@ -38,7 +34,7 @@ class DeployToken < ActiveRecord::Base end def has_access_to?(requested_project) - self.projects.first == requested_project + project == requested_project end def project diff --git a/app/models/project_deploy_token.rb b/app/models/project_deploy_token.rb index 2831b01e378..ab4482f0c0b 100644 --- a/app/models/project_deploy_token.rb +++ b/app/models/project_deploy_token.rb @@ -5,10 +5,4 @@ class ProjectDeployToken < ActiveRecord::Base validates :deploy_token, presence: true validates :project, presence: true validates :deploy_token_id, uniqueness: { scope: [:project_id] } - - accepts_nested_attributes_for :deploy_token - - def redis_shared_state_key(user_id) - "gitlab:deploy_token:#{project_id}:#{user_id}" - end end -- cgit v1.2.3 From 29913816309c6f6387b20c8702bcc8e90ef3a984 Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Fri, 6 Apr 2018 09:30:21 -0500 Subject: Addresses database comments - Adds a default on expires_at datetime - Modifies deploy tokens views to handle default expires at value - Use datetime_with_timezone where possible - Remove unused scopes --- app/models/deploy_token.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'app/models') diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index 11add42b2c5..688540dd67f 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -4,6 +4,7 @@ class DeployToken < ActiveRecord::Base add_authentication_token_field :token AVAILABLE_SCOPES = %i(read_repository read_registry).freeze + FUTURE_DATE = Date.new(3000 - 01 - 01) has_many :project_deploy_tokens, inverse_of: :deploy_token has_many :projects, through: :project_deploy_tokens @@ -13,9 +14,7 @@ class DeployToken < ActiveRecord::Base accepts_nested_attributes_for :project_deploy_tokens - scope :active, -> { where("revoked = false AND (expires_at >= NOW() OR expires_at IS NULL)") } - scope :read_repository, -> { where(read_repository: true) } - scope :read_registry, -> { where(read_registry: true) } + scope :active, -> { where("revoked = false AND expires_at >= NOW()") } def revoke! update!(revoked: true) @@ -26,7 +25,7 @@ class DeployToken < ActiveRecord::Base end def scopes - AVAILABLE_SCOPES.select { |token_scope| send("#{token_scope}") } # rubocop:disable GitlabSecurity/PublicSend + AVAILABLE_SCOPES.select { |token_scope| read_attribute(token_scope) } end def username @@ -37,6 +36,9 @@ class DeployToken < ActiveRecord::Base project == requested_project end + # This is temporal. Currently we limit DeployToken + # to a single project, later we're going to extend + # that to be for multiple projects and namespaces. def project projects.first end -- cgit v1.2.3 From 18a1569319af918fe3aff7564e344143d04d6aca Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Fri, 6 Apr 2018 11:23:45 -0500 Subject: Handles default expires_at date directly into DeployToken model --- app/models/deploy_token.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index 688540dd67f..bfdc5457157 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -4,7 +4,9 @@ class DeployToken < ActiveRecord::Base add_authentication_token_field :token AVAILABLE_SCOPES = %i(read_repository read_registry).freeze - FUTURE_DATE = Date.new(3000 - 01 - 01) + FOREVER = DateTime.new(3000, 1, 1) + + default_value_for :expires_at, FOREVER has_many :project_deploy_tokens, inverse_of: :deploy_token has_many :projects, through: :project_deploy_tokens -- cgit v1.2.3 From 5bc58bac2678aed9c8b2318f9f4d4825baa2b110 Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Fri, 6 Apr 2018 14:48:17 -0500 Subject: Handle limit for datetime attributes on MySQL The TIMESTAMP data type is used for values that contain both date and time parts. TIMESTAMP has a range of '1970-01-01 00:00:01' UTC to '2038-01-19 03:14:07' UTC. A Forever lib class was included to handle future dates for PostgreSQL and MySQL, also changes were made to DeployToken to enforce Forever.date Also removes extra conditional from JwtController --- app/models/deploy_token.rb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index bfdc5457157..fe726b156d4 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -4,9 +4,8 @@ class DeployToken < ActiveRecord::Base add_authentication_token_field :token AVAILABLE_SCOPES = %i(read_repository read_registry).freeze - FOREVER = DateTime.new(3000, 1, 1) - default_value_for :expires_at, FOREVER + default_value_for(:expires_at) { Forever.date } has_many :project_deploy_tokens, inverse_of: :deploy_token has_many :projects, through: :project_deploy_tokens @@ -45,6 +44,15 @@ class DeployToken < ActiveRecord::Base projects.first end + def expires_at + expires_at = read_attribute(:expires_at) + expires_at != Forever.date ? expires_at : nil + end + + def expires_at=(value) + write_attribute(:expires_at, value.presence || Forever.date) + end + private def ensure_at_least_one_scope -- cgit v1.2.3 From 432e57fccfcc854f8aedc578111c1d4a6b0f4a2d Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Sun, 8 Apr 2018 09:48:38 +1100 Subject: [Rails5] Add FALSE_VALUES constant to Service#boolean_accessor In Rails 5.0 the `ActiveRecord::ConnectionAdapters::Column::TRUE_VALUES` constant has been removed [1] and the remaining `FALSE_VALUES` constant has been moved to `ActiveModel::Type::Boolean` [2] [1]: https://github.com/rails/rails/commit/a502703c3d2151d4d3b421b29fefdac5ad05df61 [2]: https://github.com/rails/rails/commit/9cc8c6f3730df3d94c81a55be9ee1b7b4ffd29f6 --- app/models/service.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/service.rb b/app/models/service.rb index e9b6f005aec..f7e3f7590ad 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -206,7 +206,11 @@ class Service < ActiveRecord::Base args.each do |arg| class_eval %{ def #{arg}? - ActiveRecord::ConnectionAdapters::Column::TRUE_VALUES.include?(#{arg}) + if Gitlab.rails5? + !ActiveModel::Type::Boolean::FALSE_VALUES.include?(#{arg}) + else + ActiveRecord::ConnectionAdapters::Column::TRUE_VALUES.include?(#{arg}) + end end } end -- cgit v1.2.3 From 5e3733da3773f187c5a8c20c26f9d5c0432dea57 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Thu, 5 Apr 2018 16:39:16 +1000 Subject: Get GITLAB_FEATURES from Project#licensed_features instead of Namespace#features (#5320) --- app/models/ci/build.rb | 2 +- app/models/namespace.rb | 4 ---- app/models/project.rb | 4 ++++ 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 4aa65bf4273..3ee46453334 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -611,7 +611,7 @@ module Ci Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'CI', value: 'true') variables.append(key: 'GITLAB_CI', value: 'true') - variables.append(key: 'GITLAB_FEATURES', value: project.namespace.features.join(',')) + variables.append(key: 'GITLAB_FEATURES', value: project.licensed_features.join(',')) variables.append(key: 'CI_SERVER_NAME', value: 'GitLab') variables.append(key: 'CI_SERVER_VERSION', value: Gitlab::VERSION) variables.append(key: 'CI_SERVER_REVISION', value: Gitlab::REVISION) diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 2b63aa33222..c29a53e5ce7 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -248,10 +248,6 @@ class Namespace < ActiveRecord::Base all_projects.with_storage_feature(:repository).find_each(&:remove_exports) end - def features - [] - end - def refresh_project_authorizations owner.refresh_authorized_projects end diff --git a/app/models/project.rb b/app/models/project.rb index 3f805dd1fc9..ff4ea25987c 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1875,6 +1875,10 @@ class Project < ActiveRecord::Base memoized_results[cache_key] end + def licensed_features + [] + end + private def storage -- cgit v1.2.3 From 0e78c2e9c925d180a443d132658691adf18f26a1 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Tue, 27 Mar 2018 16:31:43 +1100 Subject: Allow group owner to enable runners from subgroups (#41981) --- app/models/user.rb | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index 2b95be3f888..01ca1446376 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -955,6 +955,10 @@ class User < ActiveRecord::Base Gitlab::GroupHierarchy.new(owned_and_master_groups).base_and_descendants end + def manageable_group_projects + Project.where(namespace: manageable_groups) + end + def namespaces namespace_ids = groups.pluck(:id) namespace_ids.push(namespace.id) @@ -1205,12 +1209,15 @@ class User < ActiveRecord::Base end def ci_projects_union - scope = { access_level: [Gitlab::Access::MASTER, Gitlab::Access::OWNER] } - groups = groups_projects.where(members: scope) - other = projects.where(members: scope) + manageable_other_projects = projects.where(members: { + access_level: [Gitlab::Access::MASTER, Gitlab::Access::OWNER] + }) - Gitlab::SQL::Union.new([personal_projects.select(:id), groups.select(:id), - other.select(:id)]) + Gitlab::SQL::Union.new([ + manageable_group_projects.select(:id), + personal_projects.select(:id), + manageable_other_projects.select(:id) + ]) end # Added according to https://github.com/plataformatec/devise/blob/7df57d5081f9884849ca15e4fde179ef164a575f/README.md#activejob-integration -- cgit v1.2.3 From 742d23a5d0f61c9939d42605762b6fc073ae4f22 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Thu, 5 Apr 2018 13:12:21 +1000 Subject: Use project_authorizations instead of members to calculate manageable CI projects to speed up query (#41981) --- app/models/user.rb | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index 01ca1446376..25441d2b68f 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -997,7 +997,7 @@ class User < ActiveRecord::Base def ci_authorized_runners @ci_authorized_runners ||= begin runner_ids = Ci::RunnerProject - .where("ci_runner_projects.project_id IN (#{ci_projects_union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection + .where(project: authorized_projects(Gitlab::Access::MASTER)) .select(:runner_id) Ci::Runner.specific.where(id: runner_ids) end @@ -1208,18 +1208,6 @@ class User < ActiveRecord::Base ], remove_duplicates: false) end - def ci_projects_union - manageable_other_projects = projects.where(members: { - access_level: [Gitlab::Access::MASTER, Gitlab::Access::OWNER] - }) - - Gitlab::SQL::Union.new([ - manageable_group_projects.select(:id), - personal_projects.select(:id), - manageable_other_projects.select(:id) - ]) - end - # Added according to https://github.com/plataformatec/devise/blob/7df57d5081f9884849ca15e4fde179ef164a575f/README.md#activejob-integration def send_devise_notification(notification, *args) return true unless can?(:receive_notifications) -- cgit v1.2.3 From f45b8888fc1a2694b8e2da64512137ab54a09a66 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Sat, 7 Apr 2018 07:31:52 +1000 Subject: Remove unused User#manageable_group_projects (#41981) --- app/models/user.rb | 4 ---- 1 file changed, 4 deletions(-) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index 25441d2b68f..42bb27d4753 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -955,10 +955,6 @@ class User < ActiveRecord::Base Gitlab::GroupHierarchy.new(owned_and_master_groups).base_and_descendants end - def manageable_group_projects - Project.where(namespace: manageable_groups) - end - def namespaces namespace_ids = groups.pluck(:id) namespace_ids.push(namespace.id) -- cgit v1.2.3 From 20fdbbe86a6cffbf467f08d50a0d8ef0f5c87f50 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 5 Apr 2018 13:19:24 +0200 Subject: Use Goldiloader for handling N+1 queries Goldiloader (https://github.com/salsify/goldiloader) can eager load associations automatically. This removes the need for adding "includes" calls in a variety of different places. This also comes with the added benefit of not having to eager load data if it's not used. --- app/models/clusters/cluster.rb | 2 +- app/models/concerns/issuable.rb | 2 +- app/models/concerns/resolvable_discussion.rb | 2 +- app/models/issue.rb | 2 +- app/models/label.rb | 4 ++-- app/models/project.rb | 6 +++--- app/models/todo.rb | 2 +- app/models/user.rb | 6 +++--- 8 files changed, 13 insertions(+), 13 deletions(-) (limited to 'app/models') diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 77947d515c1..e4a06f3f976 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -15,7 +15,7 @@ module Clusters belongs_to :user has_many :cluster_projects, class_name: 'Clusters::Project' - has_many :projects, through: :cluster_projects, class_name: '::Project' + has_many :projects, -> { auto_include(false) }, through: :cluster_projects, class_name: '::Project' # we force autosave to happen when we save `Cluster` model has_one :provider_gcp, class_name: 'Clusters::Providers::Gcp', autosave: true diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index b45395343cc..d9416352f9c 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -48,7 +48,7 @@ module Issuable end has_many :label_links, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :labels, through: :label_links + has_many :labels, -> { auto_include(false) }, through: :label_links has_many :todos, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_one :metrics diff --git a/app/models/concerns/resolvable_discussion.rb b/app/models/concerns/resolvable_discussion.rb index 7c236369793..399abb67c9d 100644 --- a/app/models/concerns/resolvable_discussion.rb +++ b/app/models/concerns/resolvable_discussion.rb @@ -102,7 +102,7 @@ module ResolvableDiscussion yield(notes_relation) # Set the notes array to the updated notes - @notes = notes_relation.fresh.to_a # rubocop:disable Gitlab/ModuleWithInstanceVariables + @notes = notes_relation.fresh.auto_include(false).to_a # rubocop:disable Gitlab/ModuleWithInstanceVariables self.class.memoized_values.each do |name| clear_memoization(name) diff --git a/app/models/issue.rb b/app/models/issue.rb index 13d9e42bcc8..c1ffe6512ea 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -34,7 +34,7 @@ class Issue < ActiveRecord::Base dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_many :issue_assignees - has_many :assignees, class_name: "User", through: :issue_assignees + has_many :assignees, -> { auto_include(false) }, class_name: "User", through: :issue_assignees validates :project, presence: true diff --git a/app/models/label.rb b/app/models/label.rb index de7f1d56c64..f3496884cff 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -18,8 +18,8 @@ class Label < ActiveRecord::Base has_many :lists, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :priorities, class_name: 'LabelPriority' has_many :label_links, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :issues, through: :label_links, source: :target, source_type: 'Issue' - has_many :merge_requests, through: :label_links, source: :target, source_type: 'MergeRequest' + has_many :issues, -> { auto_include(false) }, through: :label_links, source: :target, source_type: 'Issue' + has_many :merge_requests, -> { auto_include(false) }, through: :label_links, source: :target, source_type: 'MergeRequest' before_validation :strip_whitespace_from_title_and_color diff --git a/app/models/project.rb b/app/models/project.rb index 3f805dd1fc9..3f94b89c36b 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -179,7 +179,7 @@ class Project < ActiveRecord::Base has_many :members_and_requesters, as: :source, class_name: 'ProjectMember' has_many :deploy_keys_projects - has_many :deploy_keys, through: :deploy_keys_projects + has_many :deploy_keys, -> { auto_include(false) }, through: :deploy_keys_projects has_many :users_star_projects has_many :starrers, through: :users_star_projects, source: :user has_many :releases @@ -199,7 +199,7 @@ class Project < ActiveRecord::Base has_one :statistics, class_name: 'ProjectStatistics' has_one :cluster_project, class_name: 'Clusters::Project' - has_many :clusters, through: :cluster_project, class_name: 'Clusters::Cluster' + has_many :clusters, -> { auto_include(false) }, through: :cluster_project, class_name: 'Clusters::Cluster' # Container repositories need to remove data from the container registry, # which is not managed by the DB. Hence we're still using dependent: :destroy @@ -225,7 +225,7 @@ class Project < ActiveRecord::Base has_many :project_deploy_tokens has_many :deploy_tokens, through: :project_deploy_tokens - has_many :active_runners, -> { active }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' + has_many :active_runners, -> { active.auto_include(false) }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' has_one :auto_devops, class_name: 'ProjectAutoDevops' has_many :custom_attributes, class_name: 'ProjectCustomAttribute' diff --git a/app/models/todo.rb b/app/models/todo.rb index a2ab405fdbe..aad2c1dac4e 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -22,7 +22,7 @@ class Todo < ActiveRecord::Base belongs_to :author, class_name: "User" belongs_to :note belongs_to :project - belongs_to :target, polymorphic: true, touch: true # rubocop:disable Cop/PolymorphicAssociations + belongs_to :target, -> { auto_include(false) }, polymorphic: true, touch: true # rubocop:disable Cop/PolymorphicAssociations belongs_to :user delegate :name, :email, to: :author, prefix: true, allow_nil: true diff --git a/app/models/user.rb b/app/models/user.rb index 2b95be3f888..c7e1dfaf595 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -96,9 +96,9 @@ class User < ActiveRecord::Base # Groups has_many :members has_many :group_members, -> { where(requested_at: nil) }, source: 'GroupMember' - has_many :groups, through: :group_members - has_many :owned_groups, -> { where members: { access_level: Gitlab::Access::OWNER } }, through: :group_members, source: :group - has_many :masters_groups, -> { where members: { access_level: Gitlab::Access::MASTER } }, through: :group_members, source: :group + has_many :groups, -> { auto_include(false) }, through: :group_members + has_many :owned_groups, -> { where(members: { access_level: Gitlab::Access::OWNER }).auto_include(false) }, through: :group_members, source: :group + has_many :masters_groups, -> { where(members: { access_level: Gitlab::Access::MASTER }).auto_include(false) }, through: :group_members, source: :group # Projects has_many :groups_projects, through: :groups, source: :projects -- cgit v1.2.3 From 4ef3e3491e2ecc34e7f4de1221d5ad7b8b4a1e24 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Mon, 9 Apr 2018 12:19:18 +0100 Subject: Add cop for has_many :through without disabled autoloading Goldiloader is great, but has several issues with has_many :through relations: * https://github.com/salsify/goldiloader/issues/12 * https://github.com/salsify/goldiloader/issues/14 * https://github.com/salsify/goldiloader/issues/18 Rather than try to figure out which applies in each case, we should just do the drudge work of manually disabling autoloading for all relations of this type. We can always use regular preloading for specific cases, but this way we avoid generating invalid queries through Goldiloader's magic. --- app/models/ci/runner.rb | 2 +- app/models/deploy_key.rb | 2 +- app/models/deploy_token.rb | 2 +- app/models/fork_network.rb | 2 +- app/models/group.rb | 6 +++--- app/models/lfs_object.rb | 2 +- app/models/milestone.rb | 2 +- app/models/project.rb | 22 +++++++++++----------- app/models/user.rb | 16 ++++++++-------- 9 files changed, 28 insertions(+), 28 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 5a4c56ec0dc..ee0d8df8eb7 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -13,7 +13,7 @@ module Ci has_many :builds has_many :runner_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :projects, through: :runner_projects + has_many :projects, -> { auto_include(false) }, through: :runner_projects has_one :last_build, ->() { order('id DESC') }, class_name: 'Ci::Build' diff --git a/app/models/deploy_key.rb b/app/models/deploy_key.rb index 89a74b7dcb1..858b7ef533e 100644 --- a/app/models/deploy_key.rb +++ b/app/models/deploy_key.rb @@ -2,7 +2,7 @@ class DeployKey < Key include IgnorableColumn has_many :deploy_keys_projects, inverse_of: :deploy_key, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :projects, through: :deploy_keys_projects + has_many :projects, -> { auto_include(false) }, through: :deploy_keys_projects scope :in_projects, ->(projects) { joins(:deploy_keys_projects).where('deploy_keys_projects.project_id in (?)', projects) } scope :are_public, -> { where(public: true) } diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index fe726b156d4..b47b2ff4c3f 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -8,7 +8,7 @@ class DeployToken < ActiveRecord::Base default_value_for(:expires_at) { Forever.date } has_many :project_deploy_tokens, inverse_of: :deploy_token - has_many :projects, through: :project_deploy_tokens + has_many :projects, -> { auto_include(false) }, through: :project_deploy_tokens validate :ensure_at_least_one_scope before_save :ensure_token diff --git a/app/models/fork_network.rb b/app/models/fork_network.rb index 7f1728e8c77..aad3509b895 100644 --- a/app/models/fork_network.rb +++ b/app/models/fork_network.rb @@ -1,7 +1,7 @@ class ForkNetwork < ActiveRecord::Base belongs_to :root_project, class_name: 'Project' has_many :fork_network_members - has_many :projects, through: :fork_network_members + has_many :projects, -> { auto_include(false) }, through: :fork_network_members after_create :add_root_as_member, if: :root_project diff --git a/app/models/group.rb b/app/models/group.rb index 8ff781059cc..202988d743d 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -12,9 +12,9 @@ class Group < Namespace has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent alias_method :members, :group_members - has_many :users, through: :group_members + has_many :users, -> { auto_include(false) }, through: :group_members has_many :owners, - -> { where(members: { access_level: Gitlab::Access::OWNER }) }, + -> { where(members: { access_level: Gitlab::Access::OWNER }).auto_include(false) }, through: :group_members, source: :user @@ -23,7 +23,7 @@ class Group < Namespace has_many :milestones has_many :project_group_links, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :shared_projects, through: :project_group_links, source: :project + has_many :shared_projects, -> { auto_include(false) }, through: :project_group_links, source: :project has_many :notification_settings, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent has_many :labels, class_name: 'GroupLabel' has_many :variables, class_name: 'Ci::GroupVariable' diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index b7de46fa202..ed95613ee59 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -3,7 +3,7 @@ class LfsObject < ActiveRecord::Base include ObjectStorage::BackgroundMove has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :projects, through: :lfs_objects_projects + has_many :projects, -> { auto_include(false) }, through: :lfs_objects_projects scope :with_files_stored_locally, -> { where(file_store: [nil, LfsObjectUploader::Store::LOCAL]) } diff --git a/app/models/milestone.rb b/app/models/milestone.rb index a66a0015827..8e33bab81c1 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -22,7 +22,7 @@ class Milestone < ActiveRecord::Base belongs_to :group has_many :issues - has_many :labels, -> { distinct.reorder('labels.title') }, through: :issues + has_many :labels, -> { distinct.reorder('labels.title').auto_include(false) }, through: :issues has_many :merge_requests has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent diff --git a/app/models/project.rb b/app/models/project.rb index 3f94b89c36b..ffd78d3ab70 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -138,11 +138,11 @@ class Project < ActiveRecord::Base has_one :packagist_service # TODO: replace these relations with the fork network versions - has_one :forked_project_link, foreign_key: "forked_to_project_id" - has_one :forked_from_project, through: :forked_project_link + has_one :forked_project_link, foreign_key: "forked_to_project_id" + has_one :forked_from_project, -> { auto_include(false) }, through: :forked_project_link has_many :forked_project_links, foreign_key: "forked_from_project_id" - has_many :forks, through: :forked_project_links, source: :forked_to_project + has_many :forks, -> { auto_include(false) }, through: :forked_project_links, source: :forked_to_project # TODO: replace these relations with the fork network versions has_one :root_of_fork_network, @@ -150,7 +150,7 @@ class Project < ActiveRecord::Base inverse_of: :root_project, class_name: 'ForkNetwork' has_one :fork_network_member - has_one :fork_network, through: :fork_network_member + has_one :fork_network, -> { auto_include(false) }, through: :fork_network_member # Merge Requests for target project should be removed with it has_many :merge_requests, foreign_key: 'target_project_id' @@ -167,12 +167,12 @@ class Project < ActiveRecord::Base has_many :protected_tags has_many :project_authorizations - has_many :authorized_users, through: :project_authorizations, source: :user, class_name: 'User' + has_many :authorized_users, -> { auto_include(false) }, through: :project_authorizations, source: :user, class_name: 'User' has_many :project_members, -> { where(requested_at: nil) }, as: :source, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent alias_method :members, :project_members - has_many :users, through: :project_members + has_many :users, -> { auto_include(false) }, through: :project_members has_many :requesters, -> { where.not(requested_at: nil) }, as: :source, class_name: 'ProjectMember', dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent @@ -181,13 +181,13 @@ class Project < ActiveRecord::Base has_many :deploy_keys_projects has_many :deploy_keys, -> { auto_include(false) }, through: :deploy_keys_projects has_many :users_star_projects - has_many :starrers, through: :users_star_projects, source: :user + has_many :starrers, -> { auto_include(false) }, through: :users_star_projects, source: :user has_many :releases has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :lfs_objects, through: :lfs_objects_projects + has_many :lfs_objects, -> { auto_include(false) }, through: :lfs_objects_projects has_many :lfs_file_locks has_many :project_group_links - has_many :invited_groups, through: :project_group_links, source: :group + has_many :invited_groups, -> { auto_include(false) }, through: :project_group_links, source: :group has_many :pages_domains has_many :todos has_many :notification_settings, as: :source, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent @@ -216,14 +216,14 @@ class Project < ActiveRecord::Base has_many :builds, class_name: 'Ci::Build', inverse_of: :project, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :build_trace_section_names, class_name: 'Ci::BuildTraceSectionName' has_many :runner_projects, class_name: 'Ci::RunnerProject' - has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' + has_many :runners, -> { auto_include(false) }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' has_many :variables, class_name: 'Ci::Variable' has_many :triggers, class_name: 'Ci::Trigger' has_many :environments has_many :deployments has_many :pipeline_schedules, class_name: 'Ci::PipelineSchedule' has_many :project_deploy_tokens - has_many :deploy_tokens, through: :project_deploy_tokens + has_many :deploy_tokens, -> { auto_include(false) }, through: :project_deploy_tokens has_many :active_runners, -> { active.auto_include(false) }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' diff --git a/app/models/user.rb b/app/models/user.rb index c7e1dfaf595..fc87ada7447 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -101,18 +101,18 @@ class User < ActiveRecord::Base has_many :masters_groups, -> { where(members: { access_level: Gitlab::Access::MASTER }).auto_include(false) }, through: :group_members, source: :group # Projects - has_many :groups_projects, through: :groups, source: :projects - has_many :personal_projects, through: :namespace, source: :projects + has_many :groups_projects, -> { auto_include(false) }, through: :groups, source: :projects + has_many :personal_projects, -> { auto_include(false) }, through: :namespace, source: :projects has_many :project_members, -> { where(requested_at: nil) } - has_many :projects, through: :project_members - has_many :created_projects, foreign_key: :creator_id, class_name: 'Project' + has_many :projects, -> { auto_include(false) }, through: :project_members + has_many :created_projects, foreign_key: :creator_id, class_name: 'Project' has_many :users_star_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :starred_projects, through: :users_star_projects, source: :project + has_many :starred_projects, -> { auto_include(false) }, through: :users_star_projects, source: :project has_many :project_authorizations - has_many :authorized_projects, through: :project_authorizations, source: :project + has_many :authorized_projects, -> { auto_include(false) }, through: :project_authorizations, source: :project has_many :user_interacted_projects - has_many :project_interactions, through: :user_interacted_projects, source: :project, class_name: 'Project' + has_many :project_interactions, -> { auto_include(false) }, through: :user_interacted_projects, source: :project, class_name: 'Project' has_many :snippets, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent has_many :notes, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent @@ -132,7 +132,7 @@ class User < ActiveRecord::Base has_many :triggers, dependent: :destroy, class_name: 'Ci::Trigger', foreign_key: :owner_id # rubocop:disable Cop/ActiveRecordDependent has_many :issue_assignees - has_many :assigned_issues, class_name: "Issue", through: :issue_assignees, source: :issue + has_many :assigned_issues, -> { auto_include(false) }, class_name: "Issue", through: :issue_assignees, source: :issue has_many :assigned_merge_requests, dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" # rubocop:disable Cop/ActiveRecordDependent has_many :custom_attributes, class_name: 'UserCustomAttribute' -- cgit v1.2.3 From 3e35f65394fad201a9277667772f3ad9c6940d07 Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Tue, 10 Apr 2018 07:31:30 +0000 Subject: Verify that deploy token has valid access when pulling container registry image --- app/models/deploy_token.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index b47b2ff4c3f..8dae821a10e 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -34,7 +34,7 @@ class DeployToken < ActiveRecord::Base end def has_access_to?(requested_project) - project == requested_project + active? && project == requested_project end # This is temporal. Currently we limit DeployToken -- cgit v1.2.3 From 2cc0c692c082fddabcbb5ddc51e46d7996f5c3d7 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 10 Apr 2018 14:19:41 +0200 Subject: Ensure that enqueuing a commit status always sets 'queued_at' value --- app/models/commit_status.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 3469d5d795c..b6276c2fb50 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -87,7 +87,7 @@ class CommitStatus < ActiveRecord::Base transition [:created, :pending, :running, :manual] => :canceled end - before_transition created: [:pending, :running] do |commit_status| + before_transition [:created, :skipped, :manual] => :pending do |commit_status| commit_status.queued_at = Time.now end -- cgit v1.2.3 From 267a909600e02f0728fec1765adf817acc03d813 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Mon, 2 Apr 2018 19:05:47 +0200 Subject: Remove edit_note and update_note abilities in favor of admin_note --- app/models/ability.rb | 4 ---- 1 file changed, 4 deletions(-) (limited to 'app/models') diff --git a/app/models/ability.rb b/app/models/ability.rb index 6dae49f38dc..618d4af4272 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -46,10 +46,6 @@ class Ability end end - def can_edit_note?(user, note) - allowed?(user, :edit_note, note) - end - def allowed?(user, action, subject = :global, opts = {}) if subject.is_a?(Hash) opts, subject = subject, :global -- cgit v1.2.3 From 04c7d0d55500e6f118bd17153f3af11e83fce826 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Fri, 6 Apr 2018 20:19:37 +0200 Subject: Prevent awarding emoji when a project is archived This prevents performing the requests, and disables all emoji reaction buttons --- app/models/concerns/awardable.rb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/awardable.rb b/app/models/concerns/awardable.rb index d8394415362..fce37e7f78e 100644 --- a/app/models/concerns/awardable.rb +++ b/app/models/concerns/awardable.rb @@ -79,11 +79,7 @@ module Awardable end def user_can_award?(current_user, name) - if user_authored?(current_user) - !awardable_votes?(normalize_name(name)) - else - true - end + awardable_by_user?(current_user, name) && Ability.allowed?(current_user, :award_emoji, self) end def user_authored?(current_user) @@ -119,4 +115,12 @@ module Awardable def normalize_name(name) Gitlab::Emoji.normalize_emoji_name(name) end + + def awardable_by_user?(current_user, name) + if user_authored?(current_user) + !awardable_votes?(normalize_name(name)) + else + true + end + end end -- cgit v1.2.3 From c57cddf5b9bce8eeb700b59b964cad6e304d94c8 Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Wed, 11 Apr 2018 22:16:59 +1100 Subject: [Rails5] Update Event#subclass_from_attributes method In Rails 5.0 the `ActiveRecord::Inheritance::subclass_from_attributes` method was updated. Now it calls the `find_sti_class` method [1] which is overriden in the `Event` model and returns needed class (`Event` vs `PushEvent`). [1]: https://github.com/rails/rails/blob/5-0-stable/activerecord/lib/active_record/inheritance.rb#L209 This commit fixes the errors like ``` 143) User#contributed_projects doesn't include IDs for unrelated projects Failure/Error: action = attrs.with_indifferent_access[inheritance_column].to_i NoMethodError: undefined method `with_indifferent_access' for nil:NilClass # ./app/models/event.rb:118:in `subclass_from_attributes' ``` which are raised on the `RAILS5=1 rspec ...` command. --- app/models/event.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'app/models') diff --git a/app/models/event.rb b/app/models/event.rb index 3805f6cf857..741a84194e2 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -110,7 +110,10 @@ class Event < ActiveRecord::Base end end + # Remove this method when removing Gitlab.rails5? code. def subclass_from_attributes(attrs) + return super if Gitlab.rails5? + # Without this Rails will keep calling this method on the returned class, # resulting in an infinite loop. return unless self == Event -- cgit v1.2.3 From 1ce0595b18e3cb4caa551f9f0f03b264adb1e70e Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 11 Apr 2018 10:49:56 -0700 Subject: Backport: Propagate broadcast messages to secondaries --- app/models/broadcast_message.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/broadcast_message.rb b/app/models/broadcast_message.rb index 0b561203914..4aa236555cb 100644 --- a/app/models/broadcast_message.rb +++ b/app/models/broadcast_message.rb @@ -19,7 +19,7 @@ class BroadcastMessage < ActiveRecord::Base after_commit :flush_redis_cache def self.current - messages = Rails.cache.fetch(CACHE_KEY) { current_and_future_messages.to_a } + messages = Rails.cache.fetch(CACHE_KEY, expires_in: cache_expires_in) { current_and_future_messages.to_a } return messages if messages.empty? @@ -36,6 +36,10 @@ class BroadcastMessage < ActiveRecord::Base where('ends_at > :now', now: Time.zone.now).order_id_asc end + def self.cache_expires_in + nil + end + def active? started? && !ended? end -- cgit v1.2.3 From cbf2906496b5123fc7bf1be3094693790d047e11 Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Thu, 12 Apr 2018 13:36:31 +1100 Subject: [Rails5] Fix `undefined method 'arel_table' for Ci::Group:Class` error Some specs fail in rails5 branch with errors like ``` 1) Group#secret_variables_for when the ref is not protected contains only the secret variables Failure/Error: variables = Ci::GroupVariable.where(group: list_of_ids) NoMethodError: undefined method `arel_table' for Ci::Group:Class ``` This commit fixes it. --- app/models/ci/group_variable.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/group_variable.rb b/app/models/ci/group_variable.rb index 62d768cc6cf..44cb583e1bd 100644 --- a/app/models/ci/group_variable.rb +++ b/app/models/ci/group_variable.rb @@ -4,7 +4,7 @@ module Ci include HasVariable include Presentable - belongs_to :group + belongs_to :group, class_name: "::Group" alias_attribute :secret_value, :value -- cgit v1.2.3 From 74e5ec198cbafe5d83690fae970ff73e5ef4cfcb Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sat, 14 Apr 2018 15:36:36 -0700 Subject: Fix failing ./spec/lib/backup/repository_spec.rb by clearing the memoized value --- app/models/repository.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/models') diff --git a/app/models/repository.rb b/app/models/repository.rb index fd1afafe4df..5bdaa7f0720 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -331,6 +331,7 @@ class Repository return unless empty? expire_method_caches(%i(has_visible_content?)) + raw_repository.expire_has_local_branches_cache end def lookup_cache -- cgit v1.2.3 From 14acbf245582a1821896b94b567c5ca8ba064d4a Mon Sep 17 00:00:00 2001 From: Andreas Brandl Date: Mon, 16 Apr 2018 11:19:56 +0200 Subject: Double-check next value for internal ids. This is useful for a transition period to migrate away from `NoninternalAtomicId`. In a situation where both the old and new code to generate a iid value is run at the same time (for example, during a deploy different nodes may serve both versions), this will lead to problems regarding the correct `last_value`. That is, what we track in `InternalId` may get out of sync with the maximum iid present for issues. With this change, we double-check that and correct the `last_value` with the maximum iid found in issues if necessary. This is subject to be removed with the 10.8 release and tracked over here: https://gitlab.com/gitlab-org/gitlab-ce/issues/45389 Closes #45269. --- app/models/internal_id.rb | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) (limited to 'app/models') diff --git a/app/models/internal_id.rb b/app/models/internal_id.rb index cbec735c2dd..96a43006642 100644 --- a/app/models/internal_id.rb +++ b/app/models/internal_id.rb @@ -23,9 +23,12 @@ class InternalId < ActiveRecord::Base # # The operation locks the record and gathers a `ROW SHARE` lock (in PostgreSQL). # As such, the increment is atomic and safe to be called concurrently. - def increment_and_save! + # + # If a `maximum_iid` is passed in, this overrides the incremented value if it's + # greater than that. This can be used to correct the increment value if necessary. + def increment_and_save!(maximum_iid) lock! - self.last_value = (last_value || 0) + 1 + self.last_value = [(last_value || 0) + 1, (maximum_iid || 0) + 1].max save! last_value end @@ -89,7 +92,16 @@ class InternalId < ActiveRecord::Base # and increment its last value # # Note this will acquire a ROW SHARE lock on the InternalId record - (lookup || create_record).increment_and_save! + + # Note we always calculate the maximum iid present here and + # pass it in to correct the InternalId entry if it's last_value is off. + # + # This can happen in a transition phase where both `AtomicInternalId` and + # `NonatomicInternalId` code runs (e.g. during a deploy). + # + # This is subject to be cleaned up with the 10.8 release: + # https://gitlab.com/gitlab-org/gitlab-ce/issues/45389. + (lookup || create_record).increment_and_save!(maximum_iid) end end @@ -115,11 +127,15 @@ class InternalId < ActiveRecord::Base InternalId.create!( **scope, usage: usage_value, - last_value: init.call(subject) || 0 + last_value: maximum_iid ) end rescue ActiveRecord::RecordNotUnique lookup end + + def maximum_iid + @maximum_iid ||= init.call(subject) || 0 + end end end -- cgit v1.2.3 From 392c411bdc16386ef42c86afaf8c4d8e4cddb955 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 12 Apr 2018 14:07:44 +0200 Subject: Introduce new ProjectCiCdSetting This model and the corresponding table will be used for storing settings specific to CI/CD, starting with the "group_runners_enabled" boolean. The model is called ProjectCiCdSetting and not ProjectCiCdSettings. The project exporter doesn't like plural model names as it uses "classify" which turns those into singular (so "ProjectCiCdSettings" becomes "ProjectCiCdSetting", producing an error if said class is undefined). The initial work in this commit was done by Dylan Griffith, with most of the migration work being done by Yorick Peterse. --- app/models/project.rb | 6 ++++++ app/models/project_ci_cd_setting.rb | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 app/models/project_ci_cd_setting.rb (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index ffd78d3ab70..5412b1d49c6 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -68,6 +68,11 @@ class Project < ActiveRecord::Base after_save :update_project_statistics, if: :namespace_id_changed? after_create :create_project_feature, unless: :project_feature + + after_create :create_ci_cd_settings, + unless: :ci_cd_settings, + if: proc { ProjectCiCdSetting.available? } + after_create :set_last_activity_at after_create :set_last_repository_updated_at after_update :update_forks_visibility_level @@ -231,6 +236,7 @@ class Project < ActiveRecord::Base has_many :custom_attributes, class_name: 'ProjectCustomAttribute' has_many :project_badges, class_name: 'ProjectBadge' + has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting' accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature, update_only: true diff --git a/app/models/project_ci_cd_setting.rb b/app/models/project_ci_cd_setting.rb new file mode 100644 index 00000000000..9f10a93148c --- /dev/null +++ b/app/models/project_ci_cd_setting.rb @@ -0,0 +1,16 @@ +class ProjectCiCdSetting < ActiveRecord::Base + belongs_to :project + + # The version of the schema that first introduced this model/table. + MINIMUM_SCHEMA_VERSION = 20180403035759 + + def self.available? + @available ||= + ActiveRecord::Migrator.current_version >= MINIMUM_SCHEMA_VERSION + end + + def self.reset_column_information + @available = nil + super + end +end -- cgit v1.2.3 From 33de33198560844a32e4ed60f9f6476b7c6aa6aa Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 17 Apr 2018 15:51:53 +0200 Subject: Move Ci::BuildMetadata#update_timeout_state to after_transition callback --- app/models/ci/build.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 4aa65bf4273..3e3ef674dff 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -162,7 +162,7 @@ module Ci build.validates_dependencies! unless Feature.enabled?('ci_disable_validates_dependencies') end - before_transition pending: :running do |build| + after_transition pending: :running do |build| build.ensure_metadata.update_timeout_state end end -- cgit v1.2.3 From 36a5a2c331a9344ce089d1dbc38cc928b01b2ebb Mon Sep 17 00:00:00 2001 From: Brett Walker Date: Tue, 17 Apr 2018 17:42:19 +0200 Subject: for cached markdown fields, select the correct engine and also make sure that if a field is saved, then the existing cache version is maintained or only upgraded to the version with the same markdown engine. --- app/models/concerns/cache_markdown_field.rb | 30 +++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index 4ae5dd8c677..db8cf322ef7 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -11,7 +11,9 @@ module CacheMarkdownField extend ActiveSupport::Concern # Increment this number every time the renderer changes its output - CACHE_VERSION = 3 + CACHE_REDCARPET_VERSION = 3 + CACHE_COMMONMARK_VERSION_START = 10 + CACHE_COMMONMARK_VERSION = 11 # changes to these attributes cause the cache to be invalidates INVALIDATED_BY = %w[author project].freeze @@ -49,12 +51,14 @@ module CacheMarkdownField # Always include a project key, or Banzai complains project = self.project if self.respond_to?(:project) - group = self.group if self.respond_to?(:group) + group = self.group if self.respond_to?(:group) context = cached_markdown_fields[field].merge(project: project, group: group) # Banzai is less strict about authors, so don't always have an author key context[:author] = self.author if self.respond_to?(:author) + context[:markdown_engine] = markdown_engine + context end @@ -69,7 +73,7 @@ module CacheMarkdownField Banzai::Renderer.cacheless_render_field(self, markdown_field, options) ] end.to_h - updates['cached_markdown_version'] = CacheMarkdownField::CACHE_VERSION + updates['cached_markdown_version'] = latest_cached_markdown_version updates.each {|html_field, data| write_attribute(html_field, data) } end @@ -90,7 +94,7 @@ module CacheMarkdownField markdown_changed = attribute_changed?(markdown_field) || false html_changed = attribute_changed?(html_field) || false - CacheMarkdownField::CACHE_VERSION == cached_markdown_version && + latest_cached_markdown_version == cached_markdown_version && (html_changed || markdown_changed == html_changed) end @@ -109,6 +113,24 @@ module CacheMarkdownField __send__(cached_markdown_fields.html_field(markdown_field)) # rubocop:disable GitlabSecurity/PublicSend end + def latest_cached_markdown_version + return CacheMarkdownField::CACHE_REDCARPET_VERSION unless cached_markdown_version + + if cached_markdown_version < CacheMarkdownField::CACHE_COMMONMARK_VERSION_START + CacheMarkdownField::CACHE_REDCARPET_VERSION + else + CacheMarkdownField::CACHE_COMMONMARK_VERSION + end + end + + def markdown_engine + if latest_cached_markdown_version < CacheMarkdownField::CACHE_COMMONMARK_VERSION_START + :redcarpet + else + :common_mark + end + end + included do cattr_reader :cached_markdown_fields do FieldData.new -- cgit v1.2.3 From 731118d349c53a712c7afa67adc2b457895af048 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 18 Apr 2018 14:12:43 +0900 Subject: Put out schedule_to_db from inclock to avoid deadlock --- app/models/ci/job_trace_chunk.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb index 383c6596a51..5d8727e2b11 100644 --- a/app/models/ci/job_trace_chunk.rb +++ b/app/models/ci/job_trace_chunk.rb @@ -32,9 +32,9 @@ module Ci end def set_data(value) - in_lock do - raise ArgumentError, 'too much data' if value.bytesize > CHUNK_SIZE + raise ArgumentError, 'too much data' if value.bytesize > CHUNK_SIZE + in_lock do if redis? redis_set_data(value) elsif db? @@ -44,8 +44,9 @@ module Ci end save! if changed? - schedule_to_db if fullfilled? end + + schedule_to_db if fullfilled? end def truncate(offset = 0) -- cgit v1.2.3 From 1e817e0018af2c3fbb622ec74f02ae255e7be95f Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 18 Apr 2018 15:19:53 +0900 Subject: Align force_encoding strategy into Trace::Stream --- app/models/ci/job_trace_chunk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb index 5d8727e2b11..371417e7ff5 100644 --- a/app/models/ci/job_trace_chunk.rb +++ b/app/models/ci/job_trace_chunk.rb @@ -28,7 +28,7 @@ module Ci raw_data else raise 'Unsupported data store' - end&.force_encoding(Encoding::BINARY) + end&.force_encoding(Encoding::BINARY) # Redis/Database return UTF-8 string as default end def set_data(value) -- cgit v1.2.3 From c6b1043e9d1b7fe9912c330b6e7d4342f2a9694e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=99=88=20=20jacopo=20beschi=20=F0=9F=99=89?= Date: Wed, 18 Apr 2018 09:19:40 +0000 Subject: Resolve "Make a Rubocop that forbids returning from a block" --- app/models/ci/build.rb | 6 +++--- app/models/notification_recipient.rb | 12 ++++++------ app/models/project.rb | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 3e3ef674dff..1c8d9ca4aa5 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -479,7 +479,7 @@ module Ci def user_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| - return variables if user.blank? + break variables if user.blank? variables.append(key: 'GITLAB_USER_ID', value: user.id.to_s) variables.append(key: 'GITLAB_USER_EMAIL', value: user.email) @@ -594,7 +594,7 @@ module Ci def persisted_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| - return variables unless persisted? + break variables unless persisted? variables .append(key: 'CI_JOB_ID', value: id.to_s) @@ -643,7 +643,7 @@ module Ci def persisted_environment_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| - return variables unless persisted? && persisted_environment.present? + break variables unless persisted? && persisted_environment.present? variables.concat(persisted_environment.predefined_variables) diff --git a/app/models/notification_recipient.rb b/app/models/notification_recipient.rb index b3ffad00a07..2c3580bbdc6 100644 --- a/app/models/notification_recipient.rb +++ b/app/models/notification_recipient.rb @@ -83,14 +83,14 @@ class NotificationRecipient def has_access? DeclarativePolicy.subject_scope do - return false unless user.can?(:receive_notifications) - return true if @skip_read_ability + break false unless user.can?(:receive_notifications) + break true if @skip_read_ability - return false if @target && !user.can?(:read_cross_project) - return false if @project && !user.can?(:read_project, @project) + break false if @target && !user.can?(:read_cross_project) + break false if @project && !user.can?(:read_project, @project) - return true unless read_ability - return true unless DeclarativePolicy.has_policy?(@target) + break true unless read_ability + break true unless DeclarativePolicy.has_policy?(@target) user.can?(read_ability, @target) end diff --git a/app/models/project.rb b/app/models/project.rb index ffd78d3ab70..38139a9b137 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1637,7 +1637,7 @@ class Project < ActiveRecord::Base def container_registry_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| - return variables unless Gitlab.config.registry.enabled + break variables unless Gitlab.config.registry.enabled variables.append(key: 'CI_REGISTRY', value: Gitlab.config.registry.host_port) -- cgit v1.2.3 From 6f292eaa69c771cec8a81d2edaad19a26ab3eae6 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 18 Apr 2018 15:41:42 +0200 Subject: Revert the addition of goldiloader This reverts the addition of the "goldiloader" Gem and all use of it. While this Gem is very promising it's causing a variety of problems on GitLab.com due to it eager-loading too much data in places where we don't expect/can handle this. At least for the time being this means we have to go back to manually fixing N+1 query problems, but at least those should not cause a negative impact on availability. --- app/models/ci/runner.rb | 2 +- app/models/clusters/cluster.rb | 2 +- app/models/concerns/issuable.rb | 2 +- app/models/concerns/resolvable_discussion.rb | 2 +- app/models/deploy_key.rb | 2 +- app/models/deploy_token.rb | 2 +- app/models/fork_network.rb | 2 +- app/models/group.rb | 6 +++--- app/models/issue.rb | 2 +- app/models/label.rb | 4 ++-- app/models/lfs_object.rb | 2 +- app/models/milestone.rb | 2 +- app/models/project.rb | 28 ++++++++++++++-------------- app/models/todo.rb | 2 +- app/models/user.rb | 22 +++++++++++----------- 15 files changed, 41 insertions(+), 41 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index ee0d8df8eb7..5a4c56ec0dc 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -13,7 +13,7 @@ module Ci has_many :builds has_many :runner_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :projects, -> { auto_include(false) }, through: :runner_projects + has_many :projects, through: :runner_projects has_one :last_build, ->() { order('id DESC') }, class_name: 'Ci::Build' diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index e4a06f3f976..77947d515c1 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -15,7 +15,7 @@ module Clusters belongs_to :user has_many :cluster_projects, class_name: 'Clusters::Project' - has_many :projects, -> { auto_include(false) }, through: :cluster_projects, class_name: '::Project' + has_many :projects, through: :cluster_projects, class_name: '::Project' # we force autosave to happen when we save `Cluster` model has_one :provider_gcp, class_name: 'Clusters::Providers::Gcp', autosave: true diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index d9416352f9c..b45395343cc 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -48,7 +48,7 @@ module Issuable end has_many :label_links, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :labels, -> { auto_include(false) }, through: :label_links + has_many :labels, through: :label_links has_many :todos, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_one :metrics diff --git a/app/models/concerns/resolvable_discussion.rb b/app/models/concerns/resolvable_discussion.rb index 399abb67c9d..7c236369793 100644 --- a/app/models/concerns/resolvable_discussion.rb +++ b/app/models/concerns/resolvable_discussion.rb @@ -102,7 +102,7 @@ module ResolvableDiscussion yield(notes_relation) # Set the notes array to the updated notes - @notes = notes_relation.fresh.auto_include(false).to_a # rubocop:disable Gitlab/ModuleWithInstanceVariables + @notes = notes_relation.fresh.to_a # rubocop:disable Gitlab/ModuleWithInstanceVariables self.class.memoized_values.each do |name| clear_memoization(name) diff --git a/app/models/deploy_key.rb b/app/models/deploy_key.rb index 858b7ef533e..89a74b7dcb1 100644 --- a/app/models/deploy_key.rb +++ b/app/models/deploy_key.rb @@ -2,7 +2,7 @@ class DeployKey < Key include IgnorableColumn has_many :deploy_keys_projects, inverse_of: :deploy_key, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :projects, -> { auto_include(false) }, through: :deploy_keys_projects + has_many :projects, through: :deploy_keys_projects scope :in_projects, ->(projects) { joins(:deploy_keys_projects).where('deploy_keys_projects.project_id in (?)', projects) } scope :are_public, -> { where(public: true) } diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index 8dae821a10e..979e9232fda 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -8,7 +8,7 @@ class DeployToken < ActiveRecord::Base default_value_for(:expires_at) { Forever.date } has_many :project_deploy_tokens, inverse_of: :deploy_token - has_many :projects, -> { auto_include(false) }, through: :project_deploy_tokens + has_many :projects, through: :project_deploy_tokens validate :ensure_at_least_one_scope before_save :ensure_token diff --git a/app/models/fork_network.rb b/app/models/fork_network.rb index aad3509b895..7f1728e8c77 100644 --- a/app/models/fork_network.rb +++ b/app/models/fork_network.rb @@ -1,7 +1,7 @@ class ForkNetwork < ActiveRecord::Base belongs_to :root_project, class_name: 'Project' has_many :fork_network_members - has_many :projects, -> { auto_include(false) }, through: :fork_network_members + has_many :projects, through: :fork_network_members after_create :add_root_as_member, if: :root_project diff --git a/app/models/group.rb b/app/models/group.rb index 202988d743d..8ff781059cc 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -12,9 +12,9 @@ class Group < Namespace has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent alias_method :members, :group_members - has_many :users, -> { auto_include(false) }, through: :group_members + has_many :users, through: :group_members has_many :owners, - -> { where(members: { access_level: Gitlab::Access::OWNER }).auto_include(false) }, + -> { where(members: { access_level: Gitlab::Access::OWNER }) }, through: :group_members, source: :user @@ -23,7 +23,7 @@ class Group < Namespace has_many :milestones has_many :project_group_links, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :shared_projects, -> { auto_include(false) }, through: :project_group_links, source: :project + has_many :shared_projects, through: :project_group_links, source: :project has_many :notification_settings, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent has_many :labels, class_name: 'GroupLabel' has_many :variables, class_name: 'Ci::GroupVariable' diff --git a/app/models/issue.rb b/app/models/issue.rb index 702bfc77803..7611e83647c 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -34,7 +34,7 @@ class Issue < ActiveRecord::Base dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_many :issue_assignees - has_many :assignees, -> { auto_include(false) }, class_name: "User", through: :issue_assignees + has_many :assignees, class_name: "User", through: :issue_assignees validates :project, presence: true diff --git a/app/models/label.rb b/app/models/label.rb index f3496884cff..de7f1d56c64 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -18,8 +18,8 @@ class Label < ActiveRecord::Base has_many :lists, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :priorities, class_name: 'LabelPriority' has_many :label_links, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :issues, -> { auto_include(false) }, through: :label_links, source: :target, source_type: 'Issue' - has_many :merge_requests, -> { auto_include(false) }, through: :label_links, source: :target, source_type: 'MergeRequest' + has_many :issues, through: :label_links, source: :target, source_type: 'Issue' + has_many :merge_requests, through: :label_links, source: :target, source_type: 'MergeRequest' before_validation :strip_whitespace_from_title_and_color diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index ed95613ee59..b7de46fa202 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -3,7 +3,7 @@ class LfsObject < ActiveRecord::Base include ObjectStorage::BackgroundMove has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :projects, -> { auto_include(false) }, through: :lfs_objects_projects + has_many :projects, through: :lfs_objects_projects scope :with_files_stored_locally, -> { where(file_store: [nil, LfsObjectUploader::Store::LOCAL]) } diff --git a/app/models/milestone.rb b/app/models/milestone.rb index 8e33bab81c1..a66a0015827 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -22,7 +22,7 @@ class Milestone < ActiveRecord::Base belongs_to :group has_many :issues - has_many :labels, -> { distinct.reorder('labels.title').auto_include(false) }, through: :issues + has_many :labels, -> { distinct.reorder('labels.title') }, through: :issues has_many :merge_requests has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent diff --git a/app/models/project.rb b/app/models/project.rb index 38139a9b137..5c37cd85243 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -138,11 +138,11 @@ class Project < ActiveRecord::Base has_one :packagist_service # TODO: replace these relations with the fork network versions - has_one :forked_project_link, foreign_key: "forked_to_project_id" - has_one :forked_from_project, -> { auto_include(false) }, through: :forked_project_link + has_one :forked_project_link, foreign_key: "forked_to_project_id" + has_one :forked_from_project, through: :forked_project_link has_many :forked_project_links, foreign_key: "forked_from_project_id" - has_many :forks, -> { auto_include(false) }, through: :forked_project_links, source: :forked_to_project + has_many :forks, through: :forked_project_links, source: :forked_to_project # TODO: replace these relations with the fork network versions has_one :root_of_fork_network, @@ -150,7 +150,7 @@ class Project < ActiveRecord::Base inverse_of: :root_project, class_name: 'ForkNetwork' has_one :fork_network_member - has_one :fork_network, -> { auto_include(false) }, through: :fork_network_member + has_one :fork_network, through: :fork_network_member # Merge Requests for target project should be removed with it has_many :merge_requests, foreign_key: 'target_project_id' @@ -167,27 +167,27 @@ class Project < ActiveRecord::Base has_many :protected_tags has_many :project_authorizations - has_many :authorized_users, -> { auto_include(false) }, through: :project_authorizations, source: :user, class_name: 'User' + has_many :authorized_users, through: :project_authorizations, source: :user, class_name: 'User' has_many :project_members, -> { where(requested_at: nil) }, as: :source, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent alias_method :members, :project_members - has_many :users, -> { auto_include(false) }, through: :project_members + has_many :users, through: :project_members has_many :requesters, -> { where.not(requested_at: nil) }, as: :source, class_name: 'ProjectMember', dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_many :members_and_requesters, as: :source, class_name: 'ProjectMember' has_many :deploy_keys_projects - has_many :deploy_keys, -> { auto_include(false) }, through: :deploy_keys_projects + has_many :deploy_keys, through: :deploy_keys_projects has_many :users_star_projects - has_many :starrers, -> { auto_include(false) }, through: :users_star_projects, source: :user + has_many :starrers, through: :users_star_projects, source: :user has_many :releases has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :lfs_objects, -> { auto_include(false) }, through: :lfs_objects_projects + has_many :lfs_objects, through: :lfs_objects_projects has_many :lfs_file_locks has_many :project_group_links - has_many :invited_groups, -> { auto_include(false) }, through: :project_group_links, source: :group + has_many :invited_groups, through: :project_group_links, source: :group has_many :pages_domains has_many :todos has_many :notification_settings, as: :source, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent @@ -199,7 +199,7 @@ class Project < ActiveRecord::Base has_one :statistics, class_name: 'ProjectStatistics' has_one :cluster_project, class_name: 'Clusters::Project' - has_many :clusters, -> { auto_include(false) }, through: :cluster_project, class_name: 'Clusters::Cluster' + has_many :clusters, through: :cluster_project, class_name: 'Clusters::Cluster' # Container repositories need to remove data from the container registry, # which is not managed by the DB. Hence we're still using dependent: :destroy @@ -216,16 +216,16 @@ class Project < ActiveRecord::Base has_many :builds, class_name: 'Ci::Build', inverse_of: :project, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :build_trace_section_names, class_name: 'Ci::BuildTraceSectionName' has_many :runner_projects, class_name: 'Ci::RunnerProject' - has_many :runners, -> { auto_include(false) }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' + has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' has_many :variables, class_name: 'Ci::Variable' has_many :triggers, class_name: 'Ci::Trigger' has_many :environments has_many :deployments has_many :pipeline_schedules, class_name: 'Ci::PipelineSchedule' has_many :project_deploy_tokens - has_many :deploy_tokens, -> { auto_include(false) }, through: :project_deploy_tokens + has_many :deploy_tokens, through: :project_deploy_tokens - has_many :active_runners, -> { active.auto_include(false) }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' + has_many :active_runners, -> { active }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' has_one :auto_devops, class_name: 'ProjectAutoDevops' has_many :custom_attributes, class_name: 'ProjectCustomAttribute' diff --git a/app/models/todo.rb b/app/models/todo.rb index aad2c1dac4e..a2ab405fdbe 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -22,7 +22,7 @@ class Todo < ActiveRecord::Base belongs_to :author, class_name: "User" belongs_to :note belongs_to :project - belongs_to :target, -> { auto_include(false) }, polymorphic: true, touch: true # rubocop:disable Cop/PolymorphicAssociations + belongs_to :target, polymorphic: true, touch: true # rubocop:disable Cop/PolymorphicAssociations belongs_to :user delegate :name, :email, to: :author, prefix: true, allow_nil: true diff --git a/app/models/user.rb b/app/models/user.rb index d5c5c0964c5..7a42d546b9c 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -96,23 +96,23 @@ class User < ActiveRecord::Base # Groups has_many :members has_many :group_members, -> { where(requested_at: nil) }, source: 'GroupMember' - has_many :groups, -> { auto_include(false) }, through: :group_members - has_many :owned_groups, -> { where(members: { access_level: Gitlab::Access::OWNER }).auto_include(false) }, through: :group_members, source: :group - has_many :masters_groups, -> { where(members: { access_level: Gitlab::Access::MASTER }).auto_include(false) }, through: :group_members, source: :group + has_many :groups, through: :group_members + has_many :owned_groups, -> { where(members: { access_level: Gitlab::Access::OWNER }) }, through: :group_members, source: :group + has_many :masters_groups, -> { where(members: { access_level: Gitlab::Access::MASTER }) }, through: :group_members, source: :group # Projects - has_many :groups_projects, -> { auto_include(false) }, through: :groups, source: :projects - has_many :personal_projects, -> { auto_include(false) }, through: :namespace, source: :projects + has_many :groups_projects, through: :groups, source: :projects + has_many :personal_projects, through: :namespace, source: :projects has_many :project_members, -> { where(requested_at: nil) } - has_many :projects, -> { auto_include(false) }, through: :project_members - has_many :created_projects, foreign_key: :creator_id, class_name: 'Project' + has_many :projects, through: :project_members + has_many :created_projects, foreign_key: :creator_id, class_name: 'Project' has_many :users_star_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :starred_projects, -> { auto_include(false) }, through: :users_star_projects, source: :project + has_many :starred_projects, through: :users_star_projects, source: :project has_many :project_authorizations - has_many :authorized_projects, -> { auto_include(false) }, through: :project_authorizations, source: :project + has_many :authorized_projects, through: :project_authorizations, source: :project has_many :user_interacted_projects - has_many :project_interactions, -> { auto_include(false) }, through: :user_interacted_projects, source: :project, class_name: 'Project' + has_many :project_interactions, through: :user_interacted_projects, source: :project, class_name: 'Project' has_many :snippets, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent has_many :notes, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent @@ -132,7 +132,7 @@ class User < ActiveRecord::Base has_many :triggers, dependent: :destroy, class_name: 'Ci::Trigger', foreign_key: :owner_id # rubocop:disable Cop/ActiveRecordDependent has_many :issue_assignees - has_many :assigned_issues, -> { auto_include(false) }, class_name: "Issue", through: :issue_assignees, source: :issue + has_many :assigned_issues, class_name: "Issue", through: :issue_assignees, source: :issue has_many :assigned_merge_requests, dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" # rubocop:disable Cop/ActiveRecordDependent has_many :custom_attributes, class_name: 'UserCustomAttribute' -- cgit v1.2.3 From 1b0156204dd624f3645100d77f0eeed7e8b8e626 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Fri, 13 Apr 2018 18:52:28 +0200 Subject: Recover from errors when a parent is not preloaded --- app/models/concerns/group_descendant.rb | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/concerns/group_descendant.rb b/app/models/concerns/group_descendant.rb index 01957da0bf3..261ace57a17 100644 --- a/app/models/concerns/group_descendant.rb +++ b/app/models/concerns/group_descendant.rb @@ -37,7 +37,20 @@ module GroupDescendant parent ||= preloaded.detect { |possible_parent| possible_parent.is_a?(Group) && possible_parent.id == child.parent_id } if parent.nil? && !child.parent_id.nil? - raise ArgumentError.new('parent was not preloaded') + parent = child.parent + + exception = ArgumentError.new <<~MSG + parent: [GroupDescendant: #{parent.inspect}] was not preloaded for [#{child.inspect}]") + This error is not user facing, but causes a +1 query. + MSG + extras = { + parent: parent, + child: child, + preloaded: preloaded.map(&:full_path) + } + issue_url = 'https://gitlab.com/gitlab-org/gitlab-ce/issues/40785' + + Gitlab::Sentry.track_exception(exception, issue_url: issue_url, extra: extras) end if parent.nil? && hierarchy_top.present? -- cgit v1.2.3 From b32eabb153911b78d7ad1f6c8e3edfde482dd56a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Wed, 18 Apr 2018 17:32:33 +0200 Subject: Alias value to secret_value in Ci::PipelineVariable --- app/models/ci/pipeline_variable.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/pipeline_variable.rb b/app/models/ci/pipeline_variable.rb index de5aae17a15..38e14ffbc0c 100644 --- a/app/models/ci/pipeline_variable.rb +++ b/app/models/ci/pipeline_variable.rb @@ -5,6 +5,8 @@ module Ci belongs_to :pipeline + alias_attribute :secret_value, :value + validates :key, uniqueness: { scope: :pipeline_id } end end -- cgit v1.2.3 From 0d70dd6c48bed0f14b095521087c8b189b6b56fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Wed, 18 Apr 2018 17:35:39 +0200 Subject: Accept nested Variables in Ci::Pipeline --- app/models/ci/pipeline.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 434b9b64c65..52749c8bf7c 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -32,6 +32,8 @@ module Ci has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id' has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id' + accepts_nested_attributes_for :variables, reject_if: :persisted? + delegate :id, to: :project, prefix: true delegate :full_path, to: :project, prefix: true -- cgit v1.2.3 From e8a27a67fadfa8e62d1c72979281bcd74c39c489 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Javier=20L=C3=B3pez?= Date: Wed, 18 Apr 2018 17:50:56 +0000 Subject: Fix Custom hooks are not triggered by UI wiki edit --- app/models/project_wiki.rb | 6 +++++- app/models/wiki_page.rb | 18 +++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) (limited to 'app/models') diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index 52e067cb44c..b7e38ceb502 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -179,7 +179,11 @@ class ProjectWiki def commit_details(action, message = nil, title = nil) commit_message = message || default_message(action, title) - Gitlab::Git::Wiki::CommitDetails.new(@user.name, @user.email, commit_message) + Gitlab::Git::Wiki::CommitDetails.new(@user.id, + @user.username, + @user.name, + @user.email, + commit_message) end def default_message(action, title) diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index 0f5536415f7..cde79b95062 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -265,6 +265,15 @@ class WikiPage title.present? && self.class.unhyphenize(@page.url_path) != title end + # Updates the current @attributes hash by merging a hash of params + def update_attributes(attrs) + attrs[:title] = process_title(attrs[:title]) if attrs[:title].present? + + attrs.slice!(:content, :format, :message, :title) + + @attributes.merge!(attrs) + end + private # Process and format the title based on the user input. @@ -290,15 +299,6 @@ class WikiPage File.join(components) end - # Updates the current @attributes hash by merging a hash of params - def update_attributes(attrs) - attrs[:title] = process_title(attrs[:title]) if attrs[:title].present? - - attrs.slice!(:content, :format, :message, :title) - - @attributes.merge!(attrs) - end - def set_attributes attributes[:slug] = @page.url_path attributes[:title] = @page.title -- cgit v1.2.3 From 8b4e1d0ccd6c2624ac230597c4e63c8b51bc76ba Mon Sep 17 00:00:00 2001 From: Luke Bennett Date: Thu, 19 Apr 2018 01:20:38 +0100 Subject: Add/use Label#priority? and remove weird padding --- app/models/label.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'app/models') diff --git a/app/models/label.rb b/app/models/label.rb index f3496884cff..12e8c5695d4 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -137,6 +137,10 @@ class Label < ActiveRecord::Base priority.try(:priority) end + def priority? + priorities.present? + end + def template? template end -- cgit v1.2.3 From 90716733522691e964680539e231d3677bafbc51 Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Thu, 19 Apr 2018 15:42:50 +1100 Subject: [Rails5] Fix `User#manageable_groups` In `arel 7.0` (`7.1.4` version is used for rails5) there were introduced some changes that break our code in the `User#manageable_groups` method. The problem is that `arel_table[:id].in(Arel::Nodes::SqlLiteral)` generates wrong `IN ()` construction. The selection for `IN` is missing: => "\"namespaces\".\"id\" IN (0)" That caused such spec errors for the `rails5` branch: ``` 4) User groups with child groups #manageable_groups does not include duplicates if a membership was added for the subgroup Failure/Error: expect(user.manageable_groups).to contain_exactly(group, subgroup) expected collection contained: [#, #] actual collection contained: [] the missing elements were: [#, #] # ./spec/models/user_spec.rb:699:in `block (5 levels) in ' # ./spec/spec_helper.rb:188:in `block (2 levels) in ' # /var/lib/gems/2.3.0/gems/rspec-retry-0.4.6/lib/rspec/retry.rb:112:in `block in run' # /var/lib/gems/2.3.0/gems/rspec-retry-0.4.6/lib/rspec/retry.rb:101:in `loop' # /var/lib/gems/2.3.0/gems/rspec-retry-0.4.6/lib/rspec/retry.rb:101:in `run' # /var/lib/gems/2.3.0/gems/rspec-retry-0.4.6/lib/rspec_ext/rspec_ext.rb:12:in `run_with_retry' # /var/lib/gems/2.3.0/gems/rspec-retry-0.4.6/lib/rspec/retry.rb:30:in `block (2 levels) in setup' ``` This commit changes `User#manageable_groups` in the way to drop the usage of `Arel::Nodes::SqlLiteral` and adds usage of raw SQL query. This change should be updated when we're migrated to Rails 5.2 because arel was fixed in `9.0.0` (which is used in Rails 5.2). --- app/models/user.rb | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index d5c5c0964c5..cddc0b8d2e9 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -947,10 +947,13 @@ class User < ActiveRecord::Base end def manageable_groups - union = Gitlab::SQL::Union.new([owned_groups.select(:id), - masters_groups.select(:id)]) - arel_union = Arel::Nodes::SqlLiteral.new(union.to_sql) - owned_and_master_groups = Group.where(Group.arel_table[:id].in(arel_union)) + union_sql = Gitlab::SQL::Union.new([owned_groups.select(:id), masters_groups.select(:id)]).to_sql + + # Update this line to not use raw SQL when migrated to Rails 5.2. + # Either ActiveRecord or Arel constructions are fine. + # This was replaced with the raw SQL construction because of bugs in the arel gem. + # Bugs were fixed in arel 9.0.0 (Rails 5.2). + owned_and_master_groups = Group.where("namespaces.id IN (#{union_sql})") # rubocop:disable GitlabSecurity/SqlInjection Gitlab::GroupHierarchy.new(owned_and_master_groups).base_and_descendants end -- cgit v1.2.3 From 775211bc7076bba14d6e268fb324391124a2751f Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 18 Apr 2018 22:02:04 -0700 Subject: Fix N+1 queries when loading participants for a commit note We saw about 10,000 SQL queries for some commits in the NewNoteWorker, which stalled the Sidekiq queue for other new notes. The notification service took up to 8 minutes to process the commits. Avoiding this N+1 query brings the time down significantly. Closes #45526 --- app/models/commit.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/commit.rb b/app/models/commit.rb index de860df4b9c..9750e9298ec 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -248,7 +248,7 @@ class Commit end def notes_with_associations - notes.includes(:author) + notes.includes(:author, :award_emoji) end def merge_requests -- cgit v1.2.3 From 03b020f2e4a4e48dc4ddceb1656b84aaa7938149 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Wed, 21 Mar 2018 10:03:50 +1100 Subject: Update ProjectStatistics#build_artifacts_size synchronously without summing (#41059) Previously we scheduled a worker to just some this but we were running into performance issues when the build table was getting too large. So now we've updated the code such that this column is updated immediately and incremented/decremented by the correct amount whenever artifacts are created or deleted. We've also added the performance optimization that we do not update this statistic if a project is deleted because it could result in many updates for a project with many builds. --- app/models/ci/build.rb | 24 ++++++++++++++---------- app/models/ci/job_artifact.rb | 29 +++++++++++++++++++++++++---- app/models/project_statistics.rb | 20 +++++++++++--------- 3 files changed, 50 insertions(+), 23 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 4aa65bf4273..79408f69083 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -20,7 +20,7 @@ module Ci has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment' has_many :trace_sections, class_name: 'Ci::BuildTraceSection' - has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id @@ -95,8 +95,8 @@ module Ci run_after_commit { BuildHooksWorker.perform_async(build.id) } end - after_commit :update_project_statistics_after_save, on: [:create, :update] - after_commit :update_project_statistics, on: :destroy + after_save :update_project_statistics_after_save, if: :artifacts_size_changed? + after_destroy :update_project_statistics_after_destroy, unless: :project_destroyed? class << self # This is needed for url_for to work, @@ -664,16 +664,20 @@ module Ci pipeline.config_processor.build_attributes(name) end - def update_project_statistics - return unless project + def update_project_statistics_after_save + update_project_statistics(read_attribute(:artifacts_size).to_i - artifacts_size_was.to_i) + end - ProjectCacheWorker.perform_async(project_id, [], [:build_artifacts_size]) + def update_project_statistics_after_destroy + update_project_statistics(-artifacts_size) end - def update_project_statistics_after_save - if previous_changes.include?('artifacts_size') - update_project_statistics - end + def update_project_statistics(difference) + ProjectStatistics.increment_statistic(project_id, :build_artifacts_size, difference) + end + + def project_destroyed? + project.pending_delete? end end end diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index fbb95fe16df..f846482cdeb 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -9,6 +9,8 @@ module Ci before_save :update_file_store before_save :set_size, if: :file_changed? + after_save :update_project_statistics_after_save, if: :size_changed? + after_destroy :update_project_statistics_after_destroy, unless: :project_destroyed? scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) } @@ -34,10 +36,6 @@ module Ci [nil, ::JobArtifactUploader::Store::LOCAL].include?(self.file_store) end - def set_size - self.size = file.size - end - def expire_in expire_at - Time.now if expire_at end @@ -48,5 +46,28 @@ module Ci ChronicDuration.parse(value)&.seconds&.from_now end end + + private + + def set_size + self.size = file.size + end + + def update_project_statistics_after_save + update_project_statistics(size.to_i - size_was.to_i) + end + + def update_project_statistics_after_destroy + update_project_statistics(-self.size) + end + + def update_project_statistics(difference) + ProjectStatistics.increment_statistic(project_id, :build_artifacts_size, difference) + end + + def project_destroyed? + # Use job.project to avoid extra DB query for project + job.project.pending_delete? + end end end diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb index 87a4350f022..5d4e3c34b39 100644 --- a/app/models/project_statistics.rb +++ b/app/models/project_statistics.rb @@ -4,15 +4,15 @@ class ProjectStatistics < ActiveRecord::Base before_save :update_storage_size - STORAGE_COLUMNS = [:repository_size, :lfs_objects_size, :build_artifacts_size].freeze - STATISTICS_COLUMNS = [:commit_count] + STORAGE_COLUMNS + COLUMNS_TO_REFRESH = [:repository_size, :lfs_objects_size, :commit_count].freeze + INCREMENTABLE_COLUMNS = [:build_artifacts_size].freeze def total_repository_size repository_size + lfs_objects_size end def refresh!(only: nil) - STATISTICS_COLUMNS.each do |column, generator| + COLUMNS_TO_REFRESH.each do |column, generator| if only.blank? || only.include?(column) public_send("update_#{column}") # rubocop:disable GitlabSecurity/PublicSend end @@ -34,13 +34,15 @@ class ProjectStatistics < ActiveRecord::Base self.lfs_objects_size = project.lfs_objects.sum(:size) end - def update_build_artifacts_size - self.build_artifacts_size = - project.builds.sum(:artifacts_size) + - Ci::JobArtifact.artifacts_size_for(self.project) + def update_storage_size + self.storage_size = repository_size + lfs_objects_size + build_artifacts_size end - def update_storage_size - self.storage_size = STORAGE_COLUMNS.sum(&method(:read_attribute)) + def self.increment_statistic(project_id, key, amount) + raise ArgumentError, "Cannot increment attribute: #{key}" unless key.in?(INCREMENTABLE_COLUMNS) + return if amount == 0 + + where(project_id: project_id) + .update_all(["#{key} = COALESCE(#{key}, 0) + (?)", amount]) end end -- cgit v1.2.3 From a28f25b565c58088d80545ebe483f8cd25c22c10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Fri, 13 Apr 2018 10:20:07 +0200 Subject: Fix direct_upload when records with null file_store are used Old records have a null value of file_store column. This causes the problems with current direct_upload implementation, as this makes it to choose Store::REMOTE instead of Store::LOCAL. This change moves the store save when change saving the object. --- app/models/ci/job_artifact.rb | 11 +++++++---- app/models/lfs_object.rb | 6 ++++-- 2 files changed, 11 insertions(+), 6 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index f846482cdeb..39676efa08c 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -7,14 +7,15 @@ module Ci belongs_to :project belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id - before_save :update_file_store + mount_uploader :file, JobArtifactUploader + before_save :set_size, if: :file_changed? after_save :update_project_statistics_after_save, if: :size_changed? after_destroy :update_project_statistics_after_destroy, unless: :project_destroyed? - scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) } + after_save :update_file_store - mount_uploader :file, JobArtifactUploader + scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) } delegate :exists?, :open, to: :file @@ -25,7 +26,9 @@ module Ci } def update_file_store - self.file_store = file.object_store + # The file.object_store is set during `uploader.store!` + # which happens after object is inserted/updated + self.update_column(:file_store, file.object_store) end def self.artifacts_size_for(project) diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index b7de46fa202..6b7f280fb70 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -11,10 +11,12 @@ class LfsObject < ActiveRecord::Base mount_uploader :file, LfsObjectUploader - before_save :update_file_store + after_save :update_file_store def update_file_store - self.file_store = file.object_store + # The file.object_store is set during `uploader.store!` + # which happens after object is inserted/updated + self.update_column(:file_store, file.object_store) end def project_allowed_access?(project) -- cgit v1.2.3 From 5a29a304be5fc86a5bbe61764c47cdd8069e2103 Mon Sep 17 00:00:00 2001 From: Jacopo Date: Thu, 5 Apr 2018 17:17:02 +0200 Subject: Shows new branch/mr button even when branch exists --- app/models/issue.rb | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/issue.rb b/app/models/issue.rb index 7611e83647c..c34c35bcd34 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -194,6 +194,15 @@ class Issue < ActiveRecord::Base branches_with_iid - branches_with_merge_request end + def suggested_branch_name + return to_branch_name unless project.repository.branch_exists?(to_branch_name) + + index = 2 + index += 1 while project.repository.branch_exists?("#{to_branch_name}-#{index}") + + "#{to_branch_name}-#{index}" + end + # Returns boolean if a related branch exists for the current issue # ignores merge requests branchs def has_related_branch? @@ -248,11 +257,8 @@ class Issue < ActiveRecord::Base end end - def can_be_worked_on?(current_user) - !self.closed? && - !self.project.forked? && - self.related_branches(current_user).empty? && - self.closed_by_merge_requests(current_user).empty? + def can_be_worked_on? + !self.closed? && !self.project.forked? end # Returns `true` if the current issue can be viewed by either a logged in User -- cgit v1.2.3 From 6ae3098eb8f01406190942e8952866dd9af81dde Mon Sep 17 00:00:00 2001 From: Jacopo Date: Thu, 19 Apr 2018 08:59:37 +0200 Subject: Uses Uniquify to calculate Issue#suggested_branch_name --- app/models/concerns/uniquify.rb | 7 +++++-- app/models/issue.rb | 8 ++++---- 2 files changed, 9 insertions(+), 6 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/uniquify.rb b/app/models/concerns/uniquify.rb index a7fe5951b6e..db51ed2dbeb 100644 --- a/app/models/concerns/uniquify.rb +++ b/app/models/concerns/uniquify.rb @@ -3,11 +3,14 @@ class Uniquify # by appending a counter to it. Uniqueness is determined by # repeated calls to the passed block. # + # You can pass an initial value for the counter, if not given + # counting starts from 1. + # # If `base` is a function/proc, we expect that calling it with a # candidate counter returns a string to test/return. - def string(base) + def string(base, counter = nil) @base = base - @counter = nil + @counter = counter increment_counter! while yield(base_string) base_string diff --git a/app/models/issue.rb b/app/models/issue.rb index c34c35bcd34..51028a404c2 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -197,10 +197,10 @@ class Issue < ActiveRecord::Base def suggested_branch_name return to_branch_name unless project.repository.branch_exists?(to_branch_name) - index = 2 - index += 1 while project.repository.branch_exists?("#{to_branch_name}-#{index}") - - "#{to_branch_name}-#{index}" + start_counting_from = 2 + Uniquify.new.string(-> (counter) { "#{to_branch_name}-#{counter}" }, start_counting_from) do |suggested_branch_name| + project.repository.branch_exists?(suggested_branch_name) + end end # Returns boolean if a related branch exists for the current issue -- cgit v1.2.3 From 9000626a60c889c76ff1dfc8c6247f15953ca993 Mon Sep 17 00:00:00 2001 From: Jacopo Date: Thu, 19 Apr 2018 11:31:01 +0200 Subject: Moves Uniquify counter in the initializer --- app/models/concerns/uniquify.rb | 27 ++++++++++++++++----------- app/models/issue.rb | 2 +- 2 files changed, 17 insertions(+), 12 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/uniquify.rb b/app/models/concerns/uniquify.rb index db51ed2dbeb..549a76da20e 100644 --- a/app/models/concerns/uniquify.rb +++ b/app/models/concerns/uniquify.rb @@ -1,16 +1,21 @@ +# Uniquify +# +# Return a version of the given 'base' string that is unique +# by appending a counter to it. Uniqueness is determined by +# repeated calls to the passed block. +# +# You can pass an initial value for the counter, if not given +# counting starts from 1. +# +# If `base` is a function/proc, we expect that calling it with a +# candidate counter returns a string to test/return. class Uniquify - # Return a version of the given 'base' string that is unique - # by appending a counter to it. Uniqueness is determined by - # repeated calls to the passed block. - # - # You can pass an initial value for the counter, if not given - # counting starts from 1. - # - # If `base` is a function/proc, we expect that calling it with a - # candidate counter returns a string to test/return. - def string(base, counter = nil) - @base = base + def initialize(counter = nil) @counter = counter + end + + def string(base) + @base = base increment_counter! while yield(base_string) base_string diff --git a/app/models/issue.rb b/app/models/issue.rb index 51028a404c2..0332bfa9371 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -198,7 +198,7 @@ class Issue < ActiveRecord::Base return to_branch_name unless project.repository.branch_exists?(to_branch_name) start_counting_from = 2 - Uniquify.new.string(-> (counter) { "#{to_branch_name}-#{counter}" }, start_counting_from) do |suggested_branch_name| + Uniquify.new(start_counting_from).string(-> (counter) { "#{to_branch_name}-#{counter}" }) do |suggested_branch_name| project.repository.branch_exists?(suggested_branch_name) end end -- cgit v1.2.3 From 49b85262c5a1944d7fdb50e43900a4adb13aba6c Mon Sep 17 00:00:00 2001 From: Dennis Tang Date: Thu, 19 Apr 2018 14:43:20 +0000 Subject: Resolve "Improve tooltips of collapsed sidebars" --- app/models/concerns/milestoneish.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/milestoneish.rb b/app/models/concerns/milestoneish.rb index 5130ecec472..967fd9c5eea 100644 --- a/app/models/concerns/milestoneish.rb +++ b/app/models/concerns/milestoneish.rb @@ -102,14 +102,14 @@ module Milestoneish Gitlab::TimeTrackingFormatter.output(total_issue_time_estimate) end - private - def count_issues_by_state(user) memoize_per_user(user, :count_issues_by_state) do issues_visible_to_user(user).reorder(nil).group(:state).count end end + private + def memoize_per_user(user, method_name) memoized_users[method_name][user&.id] ||= yield end -- cgit v1.2.3 From 9ea2fc85a3387e7bbb10558b3f2302f05abe5b64 Mon Sep 17 00:00:00 2001 From: Andreas Brandl Date: Fri, 20 Apr 2018 14:00:15 +0000 Subject: Atomic internal ids for all models --- app/models/concerns/atomic_internal_id.rb | 5 +++-- app/models/concerns/nonatomic_internal_id.rb | 22 ---------------------- app/models/deployment.rb | 4 +++- app/models/internal_id.rb | 3 ++- app/models/merge_request.rb | 4 +++- app/models/milestone.rb | 5 ++++- 6 files changed, 15 insertions(+), 28 deletions(-) delete mode 100644 app/models/concerns/nonatomic_internal_id.rb (limited to 'app/models') diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb index 4b66725a3e6..22f516a172f 100644 --- a/app/models/concerns/atomic_internal_id.rb +++ b/app/models/concerns/atomic_internal_id.rb @@ -27,8 +27,9 @@ module AtomicInternalId module ClassMethods def has_internal_id(column, scope:, init:) # rubocop:disable Naming/PredicateName before_validation(on: :create) do - if read_attribute(column).blank? - scope_attrs = { scope => association(scope).reader } + scope_value = association(scope).reader + if read_attribute(column).blank? && scope_value + scope_attrs = { scope_value.class.table_name.singularize.to_sym => scope_value } usage = self.class.table_name.to_sym new_iid = InternalId.generate_next(self, scope_attrs, usage, init) diff --git a/app/models/concerns/nonatomic_internal_id.rb b/app/models/concerns/nonatomic_internal_id.rb deleted file mode 100644 index 9d0c9b8512f..00000000000 --- a/app/models/concerns/nonatomic_internal_id.rb +++ /dev/null @@ -1,22 +0,0 @@ -module NonatomicInternalId - extend ActiveSupport::Concern - - included do - validate :set_iid, on: :create - validates :iid, presence: true, numericality: true - end - - def set_iid - if iid.blank? - parent = project || group - records = parent.public_send(self.class.name.tableize) # rubocop:disable GitlabSecurity/PublicSend - max_iid = records.maximum(:iid) - - self.iid = max_iid.to_i + 1 - end - end - - def to_param - iid.to_s - end -end diff --git a/app/models/deployment.rb b/app/models/deployment.rb index e18ea8bfea4..254764eefde 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -1,11 +1,13 @@ class Deployment < ActiveRecord::Base - include NonatomicInternalId + include AtomicInternalId belongs_to :project, required: true belongs_to :environment, required: true belongs_to :user belongs_to :deployable, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations + has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.deployments&.maximum(:iid) } + validates :sha, presence: true validates :ref, presence: true diff --git a/app/models/internal_id.rb b/app/models/internal_id.rb index 96a43006642..189942c5ad8 100644 --- a/app/models/internal_id.rb +++ b/app/models/internal_id.rb @@ -12,8 +12,9 @@ # * (Optionally) add columns to `internal_ids` if needed for scope. class InternalId < ActiveRecord::Base belongs_to :project + belongs_to :namespace - enum usage: { issues: 0 } + enum usage: { issues: 0, merge_requests: 1, deployments: 2, milestones: 3, epics: 4 } validates :usage, presence: true diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 91d8be5559b..8f964a488aa 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -1,5 +1,5 @@ class MergeRequest < ActiveRecord::Base - include NonatomicInternalId + include AtomicInternalId include Issuable include Noteable include Referable @@ -18,6 +18,8 @@ class MergeRequest < ActiveRecord::Base belongs_to :source_project, class_name: "Project" belongs_to :merge_user, class_name: "User" + has_internal_id :iid, scope: :target_project, init: ->(s) { s&.target_project&.merge_requests&.maximum(:iid) } + has_many :merge_request_diffs has_one :merge_request_diff, diff --git a/app/models/milestone.rb b/app/models/milestone.rb index a66a0015827..d14e3a4ded5 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -8,7 +8,7 @@ class Milestone < ActiveRecord::Base Started = MilestoneStruct.new('Started', '#started', -3) include CacheMarkdownField - include NonatomicInternalId + include AtomicInternalId include Sortable include Referable include StripAttribute @@ -21,6 +21,9 @@ class Milestone < ActiveRecord::Base belongs_to :project belongs_to :group + has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.milestones&.maximum(:iid) } + has_internal_id :iid, scope: :group, init: ->(s) { s&.group&.milestones&.maximum(:iid) } + has_many :issues has_many :labels, -> { distinct.reorder('labels.title') }, through: :issues has_many :merge_requests -- cgit v1.2.3 From 0903456a0704bd5c4e594c423f0325b29cd99013 Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Mon, 16 Apr 2018 15:47:35 -0500 Subject: Expose deploy token to CI/CD jobs as environment variable - If a deploy token with a name 'gitlab-deploy-token' is exists for the project, CI_DEPLOY_USER and CI_DEPLOY_PASSWORD variables will be expose --- app/models/ci/build.rb | 8 ++++++++ app/models/deploy_token.rb | 1 + app/models/project.rb | 5 +++++ 3 files changed, 14 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index b0c02cdeec7..2a652b01313 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -624,6 +624,7 @@ module Ci variables.append(key: "CI_PIPELINE_TRIGGERED", value: 'true') if trigger_request variables.append(key: "CI_JOB_MANUAL", value: 'true') if action? variables.concat(legacy_variables) + variables.concat(deploy_token_variables) if project.gitlab_deploy_token end end @@ -654,6 +655,13 @@ module Ci end end + def deploy_token_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'CI_DEPLOY_USER', value: DeployToken::GITLAB_DEPLOY_TOKEN) + variables.append(key: 'CI_DEPLOY_PASSWORD', value: project.gitlab_deploy_token.token) + end + end + def environment_url options&.dig(:environment, :url) || persisted_environment&.external_url end diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index 979e9232fda..191f07c527f 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -4,6 +4,7 @@ class DeployToken < ActiveRecord::Base add_authentication_token_field :token AVAILABLE_SCOPES = %i(read_repository read_registry).freeze + GITLAB_DEPLOY_TOKEN = 'gitlab-deploy-token'.freeze default_value_for(:expires_at) { Forever.date } diff --git a/app/models/project.rb b/app/models/project.rb index cec1e705aa8..a594f2df662 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1879,6 +1879,11 @@ class Project < ActiveRecord::Base [] end + def gitlab_deploy_token + @gitlab_deploy_token ||= + deploy_tokens.active.find_by(name: DeployToken::GITLAB_DEPLOY_TOKEN) + end + private def storage -- cgit v1.2.3 From 0dd6d25c251beffca510094281ac8403fad6d8d0 Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Wed, 18 Apr 2018 12:25:57 -0500 Subject: Rename special deploy token to make it more descriptive Also: - Includes more specs - Improves a bit the documentation --- app/models/ci/build.rb | 2 +- app/models/deploy_token.rb | 2 +- app/models/project.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 2a652b01313..f3972e0cd26 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -657,7 +657,7 @@ module Ci def deploy_token_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| - variables.append(key: 'CI_DEPLOY_USER', value: DeployToken::GITLAB_DEPLOY_TOKEN) + variables.append(key: 'CI_DEPLOY_USER', value: DeployToken::GITLAB_DEPLOY_TOKEN_NAME) variables.append(key: 'CI_DEPLOY_PASSWORD', value: project.gitlab_deploy_token.token) end end diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index 191f07c527f..4e450d0bdc8 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -4,7 +4,7 @@ class DeployToken < ActiveRecord::Base add_authentication_token_field :token AVAILABLE_SCOPES = %i(read_repository read_registry).freeze - GITLAB_DEPLOY_TOKEN = 'gitlab-deploy-token'.freeze + GITLAB_DEPLOY_TOKEN_NAME = 'gitlab-deploy-token'.freeze default_value_for(:expires_at) { Forever.date } diff --git a/app/models/project.rb b/app/models/project.rb index a594f2df662..2684a02caba 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1881,7 +1881,7 @@ class Project < ActiveRecord::Base def gitlab_deploy_token @gitlab_deploy_token ||= - deploy_tokens.active.find_by(name: DeployToken::GITLAB_DEPLOY_TOKEN) + deploy_tokens.active.find_by(name: DeployToken::GITLAB_DEPLOY_TOKEN_NAME) end private -- cgit v1.2.3 From 800ee75aa5f65fc41f32c8d7f3519256cd37c645 Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Thu, 19 Apr 2018 10:31:46 -0500 Subject: Ensure deploy tokens variables are not available in the context of only/except --- app/models/ci/build.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index f3972e0cd26..8db07553665 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -27,6 +27,7 @@ module Ci has_one :metadata, class_name: 'Ci::BuildMetadata' delegate :timeout, to: :metadata, prefix: true, allow_nil: true + delegate :gitlab_deploy_token, to: :project ## # The "environment" field for builds is a String, and is the unexpanded name! @@ -604,6 +605,8 @@ module Ci .append(key: 'CI_REGISTRY_USER', value: CI_REGISTRY_USER) .append(key: 'CI_REGISTRY_PASSWORD', value: token, public: false) .append(key: 'CI_REPOSITORY_URL', value: repo_url, public: false) + + variables.concat(deploy_token_variables) if gitlab_deploy_token end end @@ -624,7 +627,6 @@ module Ci variables.append(key: "CI_PIPELINE_TRIGGERED", value: 'true') if trigger_request variables.append(key: "CI_JOB_MANUAL", value: 'true') if action? variables.concat(legacy_variables) - variables.concat(deploy_token_variables) if project.gitlab_deploy_token end end @@ -657,8 +659,8 @@ module Ci def deploy_token_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| - variables.append(key: 'CI_DEPLOY_USER', value: DeployToken::GITLAB_DEPLOY_TOKEN_NAME) - variables.append(key: 'CI_DEPLOY_PASSWORD', value: project.gitlab_deploy_token.token) + variables.append(key: 'CI_DEPLOY_USER', value: gitlab_deploy_token.name) + variables.append(key: 'CI_DEPLOY_PASSWORD', value: gitlab_deploy_token.token) end end -- cgit v1.2.3 From cdac54e2a2abe4b93bd5a96603b9d6d8745d277e Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Fri, 20 Apr 2018 10:05:16 -0500 Subject: Refactor deploy token methods on Ci::Build Also include a class method for retriving the gitlab_deploy_token on DeployTokens --- app/models/ci/build.rb | 7 ++++--- app/models/deploy_token.rb | 4 ++++ app/models/project.rb | 3 +-- 3 files changed, 9 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 8db07553665..9000ad860e9 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -605,8 +605,7 @@ module Ci .append(key: 'CI_REGISTRY_USER', value: CI_REGISTRY_USER) .append(key: 'CI_REGISTRY_PASSWORD', value: token, public: false) .append(key: 'CI_REPOSITORY_URL', value: repo_url, public: false) - - variables.concat(deploy_token_variables) if gitlab_deploy_token + .concat(deploy_token_variables) end end @@ -659,8 +658,10 @@ module Ci def deploy_token_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| + break variables unless gitlab_deploy_token + variables.append(key: 'CI_DEPLOY_USER', value: gitlab_deploy_token.name) - variables.append(key: 'CI_DEPLOY_PASSWORD', value: gitlab_deploy_token.token) + variables.append(key: 'CI_DEPLOY_PASSWORD', value: gitlab_deploy_token.token, public: false) end end diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index 4e450d0bdc8..5082dc45368 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -18,6 +18,10 @@ class DeployToken < ActiveRecord::Base scope :active, -> { where("revoked = false AND expires_at >= NOW()") } + def self.gitlab_deploy_token + active.find_by(name: GITLAB_DEPLOY_TOKEN_NAME) + end + def revoke! update!(revoked: true) end diff --git a/app/models/project.rb b/app/models/project.rb index 2684a02caba..c293b0b8cf4 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1880,8 +1880,7 @@ class Project < ActiveRecord::Base end def gitlab_deploy_token - @gitlab_deploy_token ||= - deploy_tokens.active.find_by(name: DeployToken::GITLAB_DEPLOY_TOKEN_NAME) + @gitlab_deploy_token ||= deploy_tokens.gitlab_deploy_token end private -- cgit v1.2.3 From 807b7a161d876e078b3f036465377da0c5de2a81 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Fri, 20 Apr 2018 13:02:42 +0200 Subject: Flowdock uses Gitaly, not Grit Prior to this change, Flowdock used Grit to get the difference between commits and post that to the remote service. This required direct path access, which doesn't work with Gitaly. Fixes gitlab-org/gitaly#1113 --- app/models/project_services/flowdock_service.rb | 48 ++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/project_services/flowdock_service.rb b/app/models/project_services/flowdock_service.rb index 4d23a17a545..da01ac1b7cf 100644 --- a/app/models/project_services/flowdock_service.rb +++ b/app/models/project_services/flowdock_service.rb @@ -1,5 +1,51 @@ require "flowdock-git-hook" +# Flow dock depends on Grit to compute the number of commits between two given +# commits. To make this depend on Gitaly, a monkey patch is applied +module Flowdock + class Git + # pass down a Repository all the way down + def repo + @options[:repo] + end + + def config + {} + end + + def messages + Git::Builder.new(repo: repo, + ref: @ref, + before: @from, + after: @to, + commit_url: @commit_url, + branch_url: @branch_url, + diff_url: @diff_url, + repo_url: @repo_url, + repo_name: @repo_name, + permanent_refs: @permanent_refs, + tags: tags + ).to_hashes + end + + class Builder + def commits + @repo.commits_between(@before, @after).map do |commit| + { + url: @opts[:commit_url] ? @opts[:commit_url] % [commit.sha] : nil, + id: commit.sha, + message: commit.message, + author: { + name: commit.author_name, + email: commit.author_email + } + } + end + end + end + end +end + class FlowdockService < Service prop_accessor :token validates :token, presence: true, if: :activated? @@ -34,7 +80,7 @@ class FlowdockService < Service data[:before], data[:after], token: token, - repo: project.repository.path_to_repo, + repo: project.repository, repo_url: "#{Gitlab.config.gitlab.url}/#{project.full_path}", commit_url: "#{Gitlab.config.gitlab.url}/#{project.full_path}/commit/%s", diff_url: "#{Gitlab.config.gitlab.url}/#{project.full_path}/compare/%s...%s" -- cgit v1.2.3 From 8a7654a52b75fadedbb374ef87e04668bef0ee48 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 23 Apr 2018 15:20:55 +0900 Subject: Fix statis analysys --- app/models/ci/job_trace_chunk.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb index 371417e7ff5..aeab4d0c87c 100644 --- a/app/models/ci/job_trace_chunk.rb +++ b/app/models/ci/job_trace_chunk.rb @@ -79,8 +79,8 @@ module Ci def use_database! in_lock do - return if db? - return unless size > 0 + break if db? + break unless size > 0 self.update!(raw_data: data, data_store: :db) redis_delete_data -- cgit v1.2.3 From 1549239849adf31a078be7503ab2288795e337cf Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Wed, 1 Mar 2017 13:06:35 +0100 Subject: add Ci::RunnerGroup join model --- app/models/ci/runner_group.rb | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 app/models/ci/runner_group.rb (limited to 'app/models') diff --git a/app/models/ci/runner_group.rb b/app/models/ci/runner_group.rb new file mode 100644 index 00000000000..87f3ba13bff --- /dev/null +++ b/app/models/ci/runner_group.rb @@ -0,0 +1,8 @@ +module Ci + class RunnerGroup < ActiveRecord::Base + extend Gitlab::Ci::Model + + belongs_to :runner + belongs_to :group, class_name: '::Group' + end +end -- cgit v1.2.3 From 295184f6a5ff0b98340c32e0cc715dafa4d9b60c Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Wed, 1 Mar 2017 17:19:12 +0100 Subject: include group runners in scope --- app/models/ci/runner.rb | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 5a4c56ec0dc..8f8dfbda412 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -14,6 +14,8 @@ module Ci has_many :builds has_many :runner_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :runner_projects + has_many :runner_groups + has_many :groups, through: :runner_groups has_one :last_build, ->() { order('id DESC') }, class_name: 'Ci::Build' @@ -27,8 +29,33 @@ module Ci scope :ordered, ->() { order(id: :desc) } scope :owned_or_shared, ->(project_id) do - joins('LEFT JOIN ci_runner_projects ON ci_runner_projects.runner_id = ci_runners.id') - .where("ci_runner_projects.project_id = :project_id OR ci_runners.is_shared = true", project_id: project_id) + joins( + %{ + -- project runners + LEFT JOIN ci_runner_projects ON ci_runner_projects.runner_id = ci_runners.id + + -- group runners + LEFT JOIN ci_runner_groups ON ci_runner_groups.runner_id = ci_runners.id + LEFT JOIN namespaces ON namespaces.id = ci_runner_groups.group_id + LEFT JOIN projects group_projects ON group_projects.namespace_id = namespaces.id + } + ).where( + %{ + -- project runners + ci_runner_projects.project_id = :project_id + + OR + + -- group runners + group_projects.id = :project_id + + OR + + -- shared runners + ci_runners.is_shared = true + }, + project_id: project_id + ) end scope :assignable_for, ->(project) do -- cgit v1.2.3 From 40b0f5406d97d2fff5019b87a2ab22468053af20 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Wed, 6 Sep 2017 10:53:07 +0200 Subject: use union instead of multiple joins the unions performs much better than the joins, and the execution time is constant with a growing number of records. --- app/models/ci/runner.rb | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 8f8dfbda412..cfed4a2eeea 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -29,33 +29,34 @@ module Ci scope :ordered, ->() { order(id: :desc) } scope :owned_or_shared, ->(project_id) do - joins( + project_runners = joins( %{ - -- project runners LEFT JOIN ci_runner_projects ON ci_runner_projects.runner_id = ci_runners.id + } + ).where( + %{ + ci_runner_projects.project_id = :project_id + }, + project_id: project_id + ) - -- group runners + group_runners = joins( + %{ LEFT JOIN ci_runner_groups ON ci_runner_groups.runner_id = ci_runners.id LEFT JOIN namespaces ON namespaces.id = ci_runner_groups.group_id LEFT JOIN projects group_projects ON group_projects.namespace_id = namespaces.id } ).where( %{ - -- project runners - ci_runner_projects.project_id = :project_id - - OR - - -- group runners group_projects.id = :project_id - - OR - - -- shared runners - ci_runners.is_shared = true }, project_id: project_id ) + + shared_runners = where(is_shared: true) + + union = Gitlab::SQL::Union.new([project_runners, group_runners, shared_runners]) + from("(#{union.to_sql}) ci_runners") end scope :assignable_for, ->(project) do -- cgit v1.2.3 From 9507f39459316719088722510a6ae11b79a4b442 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Wed, 6 Sep 2017 15:46:57 +0200 Subject: add runners_token column to namespaces --- app/models/group.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'app/models') diff --git a/app/models/group.rb b/app/models/group.rb index 8ff781059cc..ec27f757f46 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -9,6 +9,7 @@ class Group < Namespace include SelectForProjectAuthorization include LoadedInGroupList include GroupDescendant + include TokenAuthenticatable has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent alias_method :members, :group_members @@ -43,6 +44,9 @@ class Group < Namespace validates :two_factor_grace_period, presence: true, numericality: { greater_than_or_equal_to: 0 } + add_authentication_token_field :runners_token + before_save :ensure_runners_token + after_create :post_create_hook after_destroy :post_destroy_hook after_save :update_two_factor_requirement -- cgit v1.2.3 From 7fbdd17cbcd19086694f575884191a6d137838dc Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Thu, 7 Sep 2017 15:49:29 +0200 Subject: authorize group runners on user --- app/models/group.rb | 2 ++ app/models/user.rb | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/group.rb b/app/models/group.rb index ec27f757f46..c34c913a16b 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -29,6 +29,8 @@ class Group < Namespace has_many :labels, class_name: 'GroupLabel' has_many :variables, class_name: 'Ci::GroupVariable' has_many :custom_attributes, class_name: 'GroupCustomAttribute' + has_many :runner_groups, class_name: 'Ci::RunnerGroup' + has_many :runners, through: :runner_groups, source: :runner, class_name: 'Ci::Runner' has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent diff --git a/app/models/user.rb b/app/models/user.rb index b0668148972..0c5c0fef9d4 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -995,10 +995,17 @@ class User < ActiveRecord::Base def ci_authorized_runners @ci_authorized_runners ||= begin - runner_ids = Ci::RunnerProject + project_runner_ids = Ci::RunnerProject .where(project: authorized_projects(Gitlab::Access::MASTER)) .select(:runner_id) - Ci::Runner.specific.where(id: runner_ids) + + group_runner_ids = Ci::RunnerGroup + .where(group_id: owned_or_masters_groups.select(:id)) + .select(:runner_id) + + union = Gitlab::SQL::Union.new([project_runner_ids, group_runner_ids]) + + Ci::Runner.specific.where("ci_runners.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection end end @@ -1187,6 +1194,11 @@ class User < ActiveRecord::Base max_member_access_for_group_ids([group_id])[group_id] end + def owned_or_masters_groups + union = Gitlab::SQL::Union.new([owned_groups, masters_groups]) + Group.from("(#{union.to_sql}) namespaces") + end + protected # override, from Devise::Validatable -- cgit v1.2.3 From 850e327c70660a3935ca00c3d836f04695a408d3 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Tue, 12 Sep 2017 13:08:21 +0200 Subject: use INNER JOIN instead of LEFT JOIN as we're using UNION now we can use INNER JOIN. --- app/models/ci/runner.rb | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index cfed4a2eeea..a488d253f19 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -29,22 +29,13 @@ module Ci scope :ordered, ->() { order(id: :desc) } scope :owned_or_shared, ->(project_id) do - project_runners = joins( - %{ - LEFT JOIN ci_runner_projects ON ci_runner_projects.runner_id = ci_runners.id - } - ).where( - %{ - ci_runner_projects.project_id = :project_id - }, - project_id: project_id - ) + project_runners = joins(:runner_projects).where(ci_runner_projects: { project_id: project_id }) group_runners = joins( %{ - LEFT JOIN ci_runner_groups ON ci_runner_groups.runner_id = ci_runners.id - LEFT JOIN namespaces ON namespaces.id = ci_runner_groups.group_id - LEFT JOIN projects group_projects ON group_projects.namespace_id = namespaces.id + INNER JOIN ci_runner_groups ON ci_runner_groups.runner_id = ci_runners.id + INNER JOIN namespaces ON namespaces.id = ci_runner_groups.group_id + INNER JOIN projects group_projects ON group_projects.namespace_id = namespaces.id } ).where( %{ -- cgit v1.2.3 From 4b1b2f3b104df455d5d3265adca92dd09e079ee9 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Mon, 25 Sep 2017 15:28:33 +0200 Subject: add Ci::Runner#group? method --- app/models/ci/runner.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index a488d253f19..6ffa9372c6e 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -139,6 +139,10 @@ module Ci !shared? end + def group? + runner_groups.any? + end + def can_pick?(build) return false if self.ref_protected? && !build.protected? -- cgit v1.2.3 From 81c0c57acd0f065bc5b80902ee664256d4c3241f Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Mon, 25 Sep 2017 16:46:03 +0200 Subject: exclude group runners on projects that disabled it --- app/models/ci/runner.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 6ffa9372c6e..2f4342b79aa 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -35,13 +35,16 @@ module Ci %{ INNER JOIN ci_runner_groups ON ci_runner_groups.runner_id = ci_runners.id INNER JOIN namespaces ON namespaces.id = ci_runner_groups.group_id - INNER JOIN projects group_projects ON group_projects.namespace_id = namespaces.id + INNER JOIN projects ON projects.namespace_id = namespaces.id } ).where( %{ - group_projects.id = :project_id + projects.id = :project_id + AND + projects.group_runners_enabled = :true }, - project_id: project_id + project_id: project_id, + true: true ) shared_runners = where(is_shared: true) -- cgit v1.2.3 From d6167a9214b3a3c13850cdac9895c9d7577ddf25 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Wed, 4 Oct 2017 09:27:27 +0200 Subject: split up Ci::Runner.owned_or_shared scope --- app/models/ci/runner.rb | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 2f4342b79aa..b7af33c0480 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -21,17 +21,17 @@ module Ci before_validation :set_default_values - scope :specific, ->() { where(is_shared: false) } - scope :shared, ->() { where(is_shared: true) } - scope :active, ->() { where(active: true) } - scope :paused, ->() { where(active: false) } - scope :online, ->() { where('contacted_at > ?', contact_time_deadline) } - scope :ordered, ->() { order(id: :desc) } - - scope :owned_or_shared, ->(project_id) do - project_runners = joins(:runner_projects).where(ci_runner_projects: { project_id: project_id }) - - group_runners = joins( + scope :specific, -> { where(is_shared: false) } + scope :shared, -> { where(is_shared: true) } + scope :active, -> { where(active: true) } + scope :paused, -> { where(active: false) } + scope :online, -> { where('contacted_at > ?', contact_time_deadline) } + scope :ordered, -> { order(id: :desc) } + scope :belonging_to_project, -> (project_id) { + joins(:runner_projects).where(ci_runner_projects: { project_id: project_id }) + } + scope :belonging_to_group, -> (project_id) { + joins( %{ INNER JOIN ci_runner_groups ON ci_runner_groups.runner_id = ci_runners.id INNER JOIN namespaces ON namespaces.id = ci_runner_groups.group_id @@ -43,13 +43,13 @@ module Ci AND projects.group_runners_enabled = :true }, - project_id: project_id, - true: true + project_id: project_id, + true: true ) + } - shared_runners = where(is_shared: true) - - union = Gitlab::SQL::Union.new([project_runners, group_runners, shared_runners]) + scope :owned_or_shared, -> (project_id) do + union = Gitlab::SQL::Union.new([belonging_to_project(project_id), belonging_to_group(project_id), shared]) from("(#{union.to_sql}) ci_runners") end -- cgit v1.2.3 From 8dad45a82228a6f1c87f919063d96c8b20a567e2 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Wed, 4 Oct 2017 13:55:34 +0200 Subject: add method CI::Runner.project? --- app/models/ci/runner.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index b7af33c0480..586740a4a2a 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -146,6 +146,10 @@ module Ci runner_groups.any? end + def project? + runner_projects.any? + end + def can_pick?(build) return false if self.ref_protected? && !build.protected? -- cgit v1.2.3 From eba1a05f153335cb41bbf9396c7e88336a6b6be5 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Wed, 4 Oct 2017 13:57:50 +0200 Subject: ensure_runners_token on read instead of write 1. we don't want to migrate all existing groups 2. we generate the token when showing the runners page, as this is the first time that the token will be used. --- app/models/group.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/group.rb b/app/models/group.rb index c34c913a16b..95e2c3a8aab 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -47,7 +47,6 @@ class Group < Namespace validates :two_factor_grace_period, presence: true, numericality: { greater_than_or_equal_to: 0 } add_authentication_token_field :runners_token - before_save :ensure_runners_token after_create :post_create_hook after_destroy :post_destroy_hook @@ -296,6 +295,13 @@ class Group < Namespace refresh_members_authorized_projects(blocking: false) end + # each existing group needs to have a `runners_token`. + # we do this on read since migrating all existing groups is not a feasible + # solution. + def runners_token + ensure_runners_token! + end + private def update_two_factor_requirement -- cgit v1.2.3 From d588adff1a3ce87355c8b5ac09a77e6fc63fe89a Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Wed, 4 Oct 2017 15:51:23 +0200 Subject: don't filter group runners by project flag the scope `Ci::Runner.belonging_to_group` does not filter out the runners where the projects has `#group_runners_enabled` set to false anymore. it didn't show up in the runners UI anymore when group runners were disabled. this was confusing. the flag is only relevant when selecting appropriate runner for a build. --- app/models/ci/runner.rb | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 586740a4a2a..9efafa8681f 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -37,15 +37,7 @@ module Ci INNER JOIN namespaces ON namespaces.id = ci_runner_groups.group_id INNER JOIN projects ON projects.namespace_id = namespaces.id } - ).where( - %{ - projects.id = :project_id - AND - projects.group_runners_enabled = :true - }, - project_id: project_id, - true: true - ) + ).where('projects.id = :project_id', project_id: project_id) } scope :owned_or_shared, -> (project_id) do -- cgit v1.2.3 From 8d61d33d37f7b8f99eae73e4ba0b48fbe35a80dc Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Wed, 4 Oct 2017 17:11:53 +0200 Subject: use .owned_or_shared for #assignable_for? instead of having the explicit logic duplicated from the scope we can use the scope instead. --- app/models/ci/runner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 9efafa8681f..8a6de26164d 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -227,7 +227,7 @@ module Ci end def assignable_for?(project_id) - is_shared? || projects.exists?(id: project_id) + self.class.owned_or_shared(project_id).where(id: self.id).any? end def accepting_tags?(build) -- cgit v1.2.3 From a5f5a27df5d3055612fc8c2686ef3b9ab20bd85e Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Thu, 5 Oct 2017 13:53:18 +0200 Subject: include group runners in Project#any_runners? --- app/models/project.rb | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index cec1e705aa8..9a096ba1a7b 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1298,20 +1298,26 @@ class Project < ActiveRecord::Base project_feature.update_attribute(:builds_access_level, ProjectFeature::ENABLED) end - def shared_runners_available? - shared_runners_enabled? - end - def shared_runners - @shared_runners ||= shared_runners_available? ? Ci::Runner.shared : Ci::Runner.none + @shared_runners ||= shared_runners_enabled? ? Ci::Runner.shared : Ci::Runner.none end def active_shared_runners @active_shared_runners ||= shared_runners.active end + def group_runners + @group_runners ||= group_runners_enabled? ? Ci::Runner.belonging_to_group(self.id) : Ci::Runner.none + end + + def active_group_runners + @active_group_runners ||= group_runners.active + end + def any_runners?(&block) - active_runners.any?(&block) || active_shared_runners.any?(&block) + active_runners.any?(&block) || + active_shared_runners.any?(&block) || + active_group_runners.any?(&block) end def valid_runners_token?(token) -- cgit v1.2.3 From 9b836b83bcabb4b977afc0e06897c4f1509215b0 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Thu, 5 Oct 2017 20:53:24 +0200 Subject: support group hierarchies for group runners --- app/models/ci/runner.rb | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 8a6de26164d..b220ece3092 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -31,13 +31,10 @@ module Ci joins(:runner_projects).where(ci_runner_projects: { project_id: project_id }) } scope :belonging_to_group, -> (project_id) { - joins( - %{ - INNER JOIN ci_runner_groups ON ci_runner_groups.runner_id = ci_runners.id - INNER JOIN namespaces ON namespaces.id = ci_runner_groups.group_id - INNER JOIN projects ON projects.namespace_id = namespaces.id - } - ).where('projects.id = :project_id', project_id: project_id) + project_groups = ::Group.joins(:projects).where(projects: { id: project_id }) + hierarchy_groups = Gitlab::GroupHierarchy.new(project_groups).base_and_ancestors + + joins(:groups).where(namespaces: { id: hierarchy_groups }) } scope :owned_or_shared, -> (project_id) do -- cgit v1.2.3 From a2a7ad291f64a5db74c1bc21fb556e6e8862d0f3 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Wed, 8 Nov 2017 10:21:54 +0100 Subject: add project_settings (Project#settings) --- app/models/project.rb | 8 ++++++++ app/models/project_settings.rb | 3 +++ 2 files changed, 11 insertions(+) create mode 100644 app/models/project_settings.rb (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 9a096ba1a7b..dadd0755848 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -232,6 +232,14 @@ class Project < ActiveRecord::Base has_many :project_badges, class_name: 'ProjectBadge' + has_one :settings, -> (project) { + query = where(project_id: project) + query.presence || begin + ProjectSettings.create(project_id: project.id) + query + end + }, class_name: 'ProjectSettings' + accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature, update_only: true accepts_nested_attributes_for :import_data diff --git a/app/models/project_settings.rb b/app/models/project_settings.rb new file mode 100644 index 00000000000..b126f66fafa --- /dev/null +++ b/app/models/project_settings.rb @@ -0,0 +1,3 @@ +class ProjectSettings < ActiveRecord::Base + belongs_to :project +end -- cgit v1.2.3 From dd785467393610a73da6e9fd8413bca685d9356c Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Wed, 8 Nov 2017 13:08:13 +0100 Subject: project#group_runner_enabled -> project_settings --- app/models/project.rb | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index dadd0755848..3daedc2b47d 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -249,6 +249,7 @@ class Project < ActiveRecord::Base delegate :members, to: :team, prefix: true delegate :add_user, :add_users, to: :team delegate :add_guest, :add_reporter, :add_developer, :add_master, :add_role, to: :team + delegate :group_runners_enabled, :group_runners_enabled=, :group_runners_enabled?, to: :settings # Validations validates :creator, presence: true, on: :create @@ -1893,6 +1894,10 @@ class Project < ActiveRecord::Base [] end + def toggle_settings!(settings_attribute) + settings.toggle!(settings_attribute) + end + private def storage -- cgit v1.2.3 From d4bda7c1a5b4c11526cc0b4f62bd4e83eebfb01f Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Thu, 9 Nov 2017 15:19:25 +0100 Subject: use union for Project#any_runners? --- app/models/project.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 3daedc2b47d..220fd17fbc2 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1324,9 +1324,9 @@ class Project < ActiveRecord::Base end def any_runners?(&block) - active_runners.any?(&block) || - active_shared_runners.any?(&block) || - active_group_runners.any?(&block) + union = Gitlab::SQL::Union.new([active_runners, active_shared_runners, active_group_runners]) + runners = Ci::Runner.from("(#{union.to_sql}) ci_runners") + runners.any?(&block) end def valid_runners_token?(token) -- cgit v1.2.3 From 316ccb64a738d376de57b9df76dbec732f0d9eff Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Thu, 9 Nov 2017 15:30:08 +0100 Subject: add `active` scope only once, inline methods --- app/models/project.rb | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 220fd17fbc2..c5e61193c04 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -225,8 +225,6 @@ class Project < ActiveRecord::Base has_many :project_deploy_tokens has_many :deploy_tokens, through: :project_deploy_tokens - has_many :active_runners, -> { active }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' - has_one :auto_devops, class_name: 'ProjectAutoDevops' has_many :custom_attributes, class_name: 'ProjectCustomAttribute' @@ -1311,21 +1309,13 @@ class Project < ActiveRecord::Base @shared_runners ||= shared_runners_enabled? ? Ci::Runner.shared : Ci::Runner.none end - def active_shared_runners - @active_shared_runners ||= shared_runners.active - end - def group_runners @group_runners ||= group_runners_enabled? ? Ci::Runner.belonging_to_group(self.id) : Ci::Runner.none end - def active_group_runners - @active_group_runners ||= group_runners.active - end - def any_runners?(&block) - union = Gitlab::SQL::Union.new([active_runners, active_shared_runners, active_group_runners]) - runners = Ci::Runner.from("(#{union.to_sql}) ci_runners") + union = Gitlab::SQL::Union.new([runners, shared_runners, group_runners]) + runners = Ci::Runner.from("(#{union.to_sql}) ci_runners").active runners.any?(&block) end -- cgit v1.2.3 From 1acd8eb740dd070a5290d8a36c03e1b6f9691dba Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Thu, 7 Dec 2017 17:21:16 +0100 Subject: ci runners: assigned to either projects or group --- app/models/ci/runner.rb | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index b220ece3092..3a3f41cdf35 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -50,6 +50,7 @@ module Ci end validate :tag_constraints + validate :either_projects_or_group validates :access_level, presence: true acts_as_taggable @@ -227,6 +228,16 @@ module Ci self.class.owned_or_shared(project_id).where(id: self.id).any? end + def either_projects_or_group + if groups.length > 1 + errors.add(:runner, 'can only be assigned to one group') + end + + if groups.length > 0 && projects.length > 0 + errors.add(:runner, 'can only be assigned either to projects or to a group') + end + end + def accepting_tags?(build) (run_untagged? || build.has_tags?) && (build.tag_list - tag_list).empty? end -- cgit v1.2.3 From 9bed8de9100a394257a4a55e8b87bcfd015f0fbd Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Tue, 19 Dec 2017 15:12:21 +0100 Subject: simplify runner selection don't differentiate between the different runner types, instead we rely on the Runner model to provide the available projects. scheduling is now applied to all runners equally. --- app/models/ci/runner.rb | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 3a3f41cdf35..9139e5c830b 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -94,6 +94,24 @@ module Ci self.token = SecureRandom.hex(15) if self.token.blank? end + def accessible_projects + accessible_projects = + if shared? + Project.with_shared_runners + elsif project? + projects + elsif group? + hierarchy_groups = Gitlab::GroupHierarchy.new(groups).base_and_descendants + Project.where(namespace_id: hierarchy_groups) + else + Project.none + end + + accessible_projects + .with_builds_enabled + .without_deleted + end + def assign_to(project, current_user = nil) self.is_shared = false if shared? self.save -- cgit v1.2.3 From c585004b59e5fbd5e925dacb7259916240d1cf5a Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Tue, 19 Dec 2017 16:40:19 +0100 Subject: restrict projects ci controller to project runners --- app/models/ci/runner.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 9139e5c830b..42871163017 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -27,9 +27,13 @@ module Ci scope :paused, -> { where(active: false) } scope :online, -> { where('contacted_at > ?', contact_time_deadline) } scope :ordered, -> { order(id: :desc) } + scope :belonging_to_project, -> (project_id) { joins(:runner_projects).where(ci_runner_projects: { project_id: project_id }) } + + scope :belonging_to_any_project, -> { joins(:runner_projects) } + scope :belonging_to_group, -> (project_id) { project_groups = ::Group.joins(:projects).where(projects: { id: project_id }) hierarchy_groups = Gitlab::GroupHierarchy.new(project_groups).base_and_ancestors @@ -46,7 +50,8 @@ module Ci # FIXME: That `to_sql` is needed to workaround a weird Rails bug. # Without that, placeholders would miss one and couldn't match. where(locked: false) - .where.not("id IN (#{project.runners.select(:id).to_sql})").specific + .where.not("ci_runners.id IN (#{project.runners.select(:id).to_sql})") + .specific end validate :tag_constraints -- cgit v1.2.3 From e13026a378f186c3ef2b08720fa8420ead972f0f Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Mon, 26 Feb 2018 15:10:48 +0100 Subject: use more efficient AR length check methods --- app/models/ci/runner.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 42871163017..afd892658dc 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -252,11 +252,11 @@ module Ci end def either_projects_or_group - if groups.length > 1 + if groups.many? errors.add(:runner, 'can only be assigned to one group') end - if groups.length > 0 && projects.length > 0 + if group? && project? errors.add(:runner, 'can only be assigned either to projects or to a group') end end -- cgit v1.2.3 From 9447e5c27d8f840eaf4eee9635a5149ab36d93b6 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Mon, 26 Feb 2018 16:20:29 +0100 Subject: extract method to adhere to "tell, don't ask" --- app/models/ci/runner.rb | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index afd892658dc..d65395380d6 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -217,6 +217,12 @@ module Ci end end + def invalidate_build_cache!(build) + if can_pick?(build) + tick_runner_queue + end + end + private def cleanup_runner_queue -- cgit v1.2.3 From ab286656b22dd686a659afe908daade6e5a54ff3 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Mon, 23 Apr 2018 15:48:26 +0000 Subject: Resolve "Namespace factory is problematic" --- app/models/concerns/routable.rb | 7 ++++++- app/models/group.rb | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index dfd7d94450b..915ad6959be 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -102,7 +102,7 @@ module Routable # the route. Caching this per request ensures that even if we have multiple instances, # we will not have to duplicate work, avoiding N+1 queries in some cases. def full_path - return uncached_full_path unless RequestStore.active? + return uncached_full_path unless RequestStore.active? && persisted? RequestStore[full_path_key] ||= uncached_full_path end @@ -124,6 +124,11 @@ module Routable end end + # Group would override this to check from association + def owned_by?(user) + owner == user + end + private def set_path_errors diff --git a/app/models/group.rb b/app/models/group.rb index 8ff781059cc..9b42bbf99be 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -125,6 +125,10 @@ class Group < Namespace self[:lfs_enabled] end + def owned_by?(user) + owners.include?(user) + end + def add_users(users, access_level, current_user: nil, expires_at: nil) GroupMember.add_users( self, -- cgit v1.2.3 From 741f333d23cd09de328c9f4035c16210cb97aa10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mica=C3=ABl=20Bergeron?= Date: Mon, 23 Apr 2018 16:59:53 +0000 Subject: Resolve "Avatar URLs are wrong when using a CDN path and Object Storage" --- app/models/concerns/avatarable.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb index 7677891b9ce..13246a774e3 100644 --- a/app/models/concerns/avatarable.rb +++ b/app/models/concerns/avatarable.rb @@ -31,12 +31,13 @@ module Avatarable asset_host = ActionController::Base.asset_host use_asset_host = asset_host.present? + use_authentication = respond_to?(:public?) && !public? # Avatars for private and internal groups and projects require authentication to be viewed, # which means they can only be served by Rails, on the regular GitLab host. # If an asset host is configured, we need to return the fully qualified URL # instead of only the avatar path, so that Rails doesn't prefix it with the asset host. - if use_asset_host && respond_to?(:public?) && !public? + if use_asset_host && use_authentication use_asset_host = false only_path = false end @@ -49,6 +50,6 @@ module Avatarable url_base << gitlab_config.relative_url_root end - url_base + avatar.url + url_base + avatar.local_url end end -- cgit v1.2.3 From ac6eb51b806823c9e953b30f537014a4b329828b Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 24 Apr 2018 14:16:42 +0900 Subject: Rename SwapTraceChunkWorker to BuildTraceSwapChunkWorker --- app/models/ci/job_trace_chunk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb index aeab4d0c87c..47302265fb7 100644 --- a/app/models/ci/job_trace_chunk.rb +++ b/app/models/ci/job_trace_chunk.rb @@ -92,7 +92,7 @@ module Ci def schedule_to_db return if db? - SwapTraceChunkWorker.perform_async(id) + BuildTraceSwapChunkWorker.perform_async(id) end def fullfilled? -- cgit v1.2.3 From 94ce2233918b6b53411a015c245c2a8b2c6bfb96 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 24 Apr 2018 14:53:14 +0900 Subject: Swap redis chunks when build finished --- app/models/ci/build.rb | 2 +- app/models/ci/job_trace_chunk.rb | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 8126dd5256a..d7d24bc4201 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -25,7 +25,7 @@ module Ci has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id - has_many :chunks, class_name: 'Ci::JobTraceChunk', foreign_key: :job_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :chunks, class_name: 'Ci::JobTraceChunk', foreign_key: :job_id has_one :metadata, class_name: 'Ci::BuildMetadata' delegate :timeout, to: :metadata, prefix: true, allow_nil: true diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb index aeab4d0c87c..c5f10dccaaa 100644 --- a/app/models/ci/job_trace_chunk.rb +++ b/app/models/ci/job_trace_chunk.rb @@ -4,8 +4,6 @@ module Ci belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id - after_destroy :redis_delete_data, if: :redis? - default_value_for :data_store, :redis WriteError = Class.new(StandardError) -- cgit v1.2.3 From f0d59b95f1da0293345705fe0b2c629b911a5b5e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 17 Apr 2018 12:56:04 +0200 Subject: Validate presence of a stage index in the model --- app/models/ci/stage.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 75b8ea2a371..ea0b7d6655b 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -13,9 +13,12 @@ module Ci has_many :statuses, class_name: 'CommitStatus', foreign_key: :stage_id has_many :builds, foreign_key: :stage_id - validates :project, presence: true, unless: :importing? - validates :pipeline, presence: true, unless: :importing? - validates :name, presence: true, unless: :importing? + with_options unless: :importing? do + validates :project, presence: true + validates :pipeline, presence: true + validates :name, presence: true + validates :index, presence: true + end after_initialize do |stage| self.status = DEFAULT_STATUS if self.status.nil? -- cgit v1.2.3 From 6a2f8a9afb176600133939f61ccad587c9da7879 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 18 Apr 2018 14:56:08 +0200 Subject: Ensure that an imported pipeline stage can be updated --- app/models/ci/stage.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index ea0b7d6655b..a80c72d3452 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -20,10 +20,18 @@ module Ci validates :index, presence: true end - after_initialize do |stage| + after_initialize do self.status = DEFAULT_STATUS if self.status.nil? end + before_validation do + next unless index.nil? + + statuses.pluck(:stage_idx).tap do |indices| + self.index = indices.max_by { |index| indices.count(index) } + end + end + state_machine :status, initial: :created do event :enqueue do transition created: :pending -- cgit v1.2.3 From 5771972e8ca2a7ad06445ee539054b23cf228de8 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 24 Apr 2018 11:18:12 +0200 Subject: Use database query to calculate average stage position --- app/models/ci/stage.rb | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index a80c72d3452..5a77a909b9d 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -24,12 +24,14 @@ module Ci self.status = DEFAULT_STATUS if self.status.nil? end - before_validation do - next unless index.nil? + before_validation unless: :importing? do + next if index.present? - statuses.pluck(:stage_idx).tap do |indices| - self.index = indices.max_by { |index| indices.count(index) } - end + self.index = statuses.select(:stage_idx) + .where('stage_idx IS NOT NULL') + .group(:stage_idx) + .order('COUNT(*) DESC') + .first&.stage_idx.to_i end state_machine :status, initial: :created do -- cgit v1.2.3 From 35a49922e66ed9dc55685163126e1bee0a4e3dec Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Wed, 18 Apr 2018 15:52:55 +0200 Subject: Allow admins to push to empty repos --- app/models/concerns/protected_ref.rb | 2 +- app/models/project.rb | 10 ++-------- app/models/protected_branch.rb | 9 +++++++++ 3 files changed, 12 insertions(+), 9 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/protected_ref.rb b/app/models/concerns/protected_ref.rb index 454374121f3..94eef4ff7cd 100644 --- a/app/models/concerns/protected_ref.rb +++ b/app/models/concerns/protected_ref.rb @@ -31,7 +31,7 @@ module ProtectedRef end end - def protected_ref_accessible_to?(ref, user, action:, protected_refs: nil) + def protected_ref_accessible_to?(ref, user, project:, action:, protected_refs: nil) access_levels_for_ref(ref, action: action, protected_refs: protected_refs).any? do |access_level| access_level.check_access(user) end diff --git a/app/models/project.rb b/app/models/project.rb index c293b0b8cf4..645141ddbe5 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1041,13 +1041,6 @@ class Project < ActiveRecord::Base "#{web_url}.git" end - def user_can_push_to_empty_repo?(user) - return false unless empty_repo? - return false unless Ability.allowed?(user, :push_code, self) - - !ProtectedBranch.default_branch_protected? || team.max_member_access(user.id) > Gitlab::Access::DEVELOPER - end - def forked? return true if fork_network && fork_network.root_project != self @@ -2008,10 +2001,11 @@ class Project < ActiveRecord::Base def fetch_branch_allows_maintainer_push?(user, branch_name) check_access = -> do + next false if empty_repo? + merge_request = source_of_merge_requests.opened .where(allow_maintainer_to_push: true) .find_by(source_branch: branch_name) - merge_request&.can_be_merged_by?(user) end diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index 609780c5587..cb361a66591 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -4,6 +4,15 @@ class ProtectedBranch < ActiveRecord::Base protected_ref_access_levels :merge, :push + def self.protected_ref_accessible_to?(ref, user, project:, action:, protected_refs: nil) + # Masters, owners and admins are allowed to create the default branch + if default_branch_protected? && project.empty_repo? + return true if user.admin? || project.team.max_member_access(user.id) > Gitlab::Access::DEVELOPER + end + + super + end + # Check if branch name is marked as protected in the system def self.protected?(project, ref_name) return true if project.empty_repo? && default_branch_protected? -- cgit v1.2.3 From 12711de2564e1aecbd68f926e28d0d3e2241da41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Tue, 24 Apr 2018 15:13:46 +0200 Subject: Implement efficient destroy of job_trace_chunks --- app/models/ci/job_trace_chunk.rb | 24 ++++++++++++++++++++++++ app/models/project.rb | 9 +++++++++ 2 files changed, 33 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb index 1f18bb398ff..6f237300cc1 100644 --- a/app/models/ci/job_trace_chunk.rb +++ b/app/models/ci/job_trace_chunk.rb @@ -19,6 +19,30 @@ module Ci db: 2 } + def self.delayed_cleanup_blk + ids = all.redis.pluck(:job_id, :chunk_index).map do |data| + "gitlab:ci:trace:#{data.first}:chunks:#{data.second}:data" + end + + puts "before cleanup: #{ids.count}" + + Proc.new do + puts "after cleanup: #{ids.count}" + Gitlab::Redis::SharedState.with do |redis| + redis.del(ids) + end unless ids.empty? + + true + end + end + + def self.fast_destroy_all + delayed_cleanup_blk.tap do |cleanup| + delete_all + cleanup.call + end + end + def data if redis? redis_data diff --git a/app/models/project.rb b/app/models/project.rb index cec1e705aa8..9d6ffa42da2 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -209,12 +209,21 @@ class Project < ActiveRecord::Base has_many :commit_statuses has_many :pipelines, class_name: 'Ci::Pipeline', inverse_of: :project + # This has to be defined before `has_many :builds, depenedent: :destroy`, + # otherwise we will not delete any data, due to trace chunks + # going through :builds + before_destroy do + puts "destroying all chunks" + self.run_after_commit(&build_trace_chunks.delayed_cleanup_blk) + end + # Ci::Build objects store data on the file system such as artifact files and # build traces. Currently there's no efficient way of removing this data in # bulk that doesn't involve loading the rows into memory. As a result we're # still using `dependent: :destroy` here. has_many :builds, class_name: 'Ci::Build', inverse_of: :project, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :build_trace_section_names, class_name: 'Ci::BuildTraceSectionName' + has_many :build_trace_chunks, class_name: 'Ci::JobTraceChunk', foreign_key: :job_id, through: :builds, source: :chunks has_many :runner_projects, class_name: 'Ci::RunnerProject' has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' has_many :variables, class_name: 'Ci::Variable' -- cgit v1.2.3 From c9dc51111d53aba4e456afaade4bf9ad82c4b28c Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 24 Apr 2018 14:53:46 +0200 Subject: Rename stage index column name to priority column --- app/models/ci/stage.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 5a77a909b9d..9a913213bb9 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -17,7 +17,7 @@ module Ci validates :project, presence: true validates :pipeline, presence: true validates :name, presence: true - validates :index, presence: true + validates :priority, presence: true end after_initialize do @@ -25,9 +25,9 @@ module Ci end before_validation unless: :importing? do - next if index.present? + next if priority.present? - self.index = statuses.select(:stage_idx) + self.priority = statuses.select(:stage_idx) .where('stage_idx IS NOT NULL') .group(:stage_idx) .order('COUNT(*) DESC') -- cgit v1.2.3 From 5bb421ecbbfdf4658c8e3bfb05dc86a84c7e4725 Mon Sep 17 00:00:00 2001 From: Jan Provaznik Date: Tue, 6 Mar 2018 15:20:20 +0100 Subject: Cleanup after adding MR diff's commit_count (try 2) * processes any pending records which are not migrated yet * bumps import_export version because of new commits_count attribute * removes commits_count fallback method --- app/models/merge_request_diff.rb | 4 ---- 1 file changed, 4 deletions(-) (limited to 'app/models') diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index c1c27ccf3e5..06aa67c600f 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -197,10 +197,6 @@ class MergeRequestDiff < ActiveRecord::Base CompareService.new(project, head_commit_sha).execute(project, sha, straight: true) end - def commits_count - super || merge_request_diff_commits.size - end - private def create_merge_request_diff_files(diffs) -- cgit v1.2.3 From ec4423665cacfe2f0675fb8b9b5bd8dd17a77dd7 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Fri, 13 Apr 2018 12:57:19 +0200 Subject: Gitlab::Shell works on shard name, not path Direct disk access is done through Gitaly now, so the legacy path was deprecated. This path was used in Gitlab::Shell however. This required the refactoring in this commit. Added is the removal of direct path access on the project model, as that lookup wasn't needed anymore is most cases. Closes https://gitlab.com/gitlab-org/gitaly/issues/1111 --- app/models/concerns/storage/legacy_namespace.rb | 30 ++++++++++++------------- app/models/project.rb | 10 +++------ app/models/project_wiki.rb | 2 +- app/models/repository.rb | 15 +++++++------ app/models/storage/hashed_project.rb | 4 ++-- app/models/storage/legacy_project.rb | 8 +++---- 6 files changed, 33 insertions(+), 36 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/storage/legacy_namespace.rb b/app/models/concerns/storage/legacy_namespace.rb index f05e606995d..f66bdd529f1 100644 --- a/app/models/concerns/storage/legacy_namespace.rb +++ b/app/models/concerns/storage/legacy_namespace.rb @@ -45,25 +45,25 @@ module Storage # Hooks - # Save the storage paths before the projects are destroyed to use them on after destroy + # Save the storages before the projects are destroyed to use them on after destroy def prepare_for_destroy - old_repository_storage_paths + old_repository_storages end private def move_repositories - # Move the namespace directory in all storage paths used by member projects - repository_storage_paths.each do |repository_storage_path| + # Move the namespace directory in all storages used by member projects + repository_storages.each do |repository_storage| # Ensure old directory exists before moving it - gitlab_shell.add_namespace(repository_storage_path, full_path_was) + gitlab_shell.add_namespace(repository_storage, full_path_was) # Ensure new directory exists before moving it (if there's a parent) - gitlab_shell.add_namespace(repository_storage_path, parent.full_path) if parent + gitlab_shell.add_namespace(repository_storage, parent.full_path) if parent - unless gitlab_shell.mv_namespace(repository_storage_path, full_path_was, full_path) + unless gitlab_shell.mv_namespace(repository_storage, full_path_was, full_path) - Rails.logger.error "Exception moving path #{repository_storage_path} from #{full_path_was} to #{full_path}" + Rails.logger.error "Exception moving path #{repository_storage} from #{full_path_was} to #{full_path}" # if we cannot move namespace directory we should rollback # db changes in order to prevent out of sync between db and fs @@ -72,33 +72,33 @@ module Storage end end - def old_repository_storage_paths - @old_repository_storage_paths ||= repository_storage_paths + def old_repository_storages + @old_repository_storage_paths ||= repository_storages end - def repository_storage_paths + def repository_storages # We need to get the storage paths for all the projects, even the ones that are # pending delete. Unscoping also get rids of the default order, which causes # problems with SELECT DISTINCT. Project.unscoped do - all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path) + all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage) end end def rm_dir # Remove the namespace directory in all storages paths used by member projects - old_repository_storage_paths.each do |repository_storage_path| + old_repository_storages.each do |repository_storage| # Move namespace directory into trash. # We will remove it later async new_path = "#{full_path}+#{id}+deleted" - if gitlab_shell.mv_namespace(repository_storage_path, full_path, new_path) + if gitlab_shell.mv_namespace(repository_storage, full_path, new_path) Gitlab::AppLogger.info %Q(Namespace directory "#{full_path}" moved to "#{new_path}") # Remove namespace directroy async with delay so # GitLab has time to remove all projects first run_after_commit do - GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path) + GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage, new_path) end end end diff --git a/app/models/project.rb b/app/models/project.rb index cec1e705aa8..1eee8fa266a 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -512,10 +512,6 @@ class Project < ActiveRecord::Base repository.empty? end - def repository_storage_path - Gitlab.config.repositories.storages[repository_storage]&.legacy_disk_path - end - def team @team ||= ProjectTeam.new(self) end @@ -1106,7 +1102,7 @@ class Project < ActiveRecord::Base # Check if repository already exists on disk def check_repository_path_availability return true if skip_disk_validation - return false unless repository_storage_path + return false unless repository_storage expires_full_path_cache # we need to clear cache to validate renames correctly @@ -1907,14 +1903,14 @@ class Project < ActiveRecord::Base def check_repository_absence! return if skip_disk_validation - if repository_storage_path.blank? || repository_with_same_path_already_exists? + if repository_storage.blank? || repository_with_same_path_already_exists? errors.add(:base, 'There is already a repository with that name on disk') throw :abort end end def repository_with_same_path_already_exists? - gitlab_shell.exists?(repository_storage_path, "#{disk_path}.git") + gitlab_shell.exists?(repository_storage, "#{disk_path}.git") end # set last_activity_at to the same as created_at diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index b7e38ceb502..f799a0b4227 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -21,7 +21,7 @@ class ProjectWiki end delegate :empty?, to: :pages - delegate :repository_storage_path, :hashed_storage?, to: :project + delegate :repository_storage, :hashed_storage?, to: :project def path @project.path + '.wiki' diff --git a/app/models/repository.rb b/app/models/repository.rb index 5bdaa7f0720..6831305fb93 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -84,9 +84,14 @@ class Repository # Return absolute path to repository def path_to_repo - @path_to_repo ||= File.expand_path( - File.join(repository_storage_path, disk_path + '.git') - ) + @path_to_repo ||= + begin + storage = Gitlab.config.repositories.storages[@project.repository_storage] + + File.expand_path( + File.join(storage.legacy_disk_path, disk_path + '.git') + ) + end end def inspect @@ -915,10 +920,6 @@ class Repository raw_repository.fetch_ref(source_repository.raw_repository, source_ref: source_ref, target_ref: target_ref) end - def repository_storage_path - @project.repository_storage_path - end - def rebase(user, merge_request) raw.rebase(user, merge_request.id, branch: merge_request.source_branch, branch_sha: merge_request.source_branch_sha, diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb index fae1b64961a..26b4b78ac64 100644 --- a/app/models/storage/hashed_project.rb +++ b/app/models/storage/hashed_project.rb @@ -1,7 +1,7 @@ module Storage class HashedProject attr_accessor :project - delegate :gitlab_shell, :repository_storage_path, to: :project + delegate :gitlab_shell, :repository_storage, to: :project ROOT_PATH_PREFIX = '@hashed'.freeze @@ -24,7 +24,7 @@ module Storage end def ensure_storage_path_exists - gitlab_shell.add_namespace(repository_storage_path, base_dir) + gitlab_shell.add_namespace(repository_storage, base_dir) end def rename_repo diff --git a/app/models/storage/legacy_project.rb b/app/models/storage/legacy_project.rb index 9d9e5e1d352..27cb388c702 100644 --- a/app/models/storage/legacy_project.rb +++ b/app/models/storage/legacy_project.rb @@ -1,7 +1,7 @@ module Storage class LegacyProject attr_accessor :project - delegate :namespace, :gitlab_shell, :repository_storage_path, to: :project + delegate :namespace, :gitlab_shell, :repository_storage, to: :project def initialize(project) @project = project @@ -24,18 +24,18 @@ module Storage def ensure_storage_path_exists return unless namespace - gitlab_shell.add_namespace(repository_storage_path, base_dir) + gitlab_shell.add_namespace(repository_storage, base_dir) end def rename_repo new_full_path = project.build_full_path - if gitlab_shell.mv_repository(repository_storage_path, project.full_path_was, new_full_path) + if gitlab_shell.mv_repository(repository_storage, project.full_path_was, new_full_path) # If repository moved successfully we need to send update instructions to users. # However we cannot allow rollback since we moved repository # So we basically we mute exceptions in next actions begin - gitlab_shell.mv_repository(repository_storage_path, "#{project.full_path_was}.wiki", "#{new_full_path}.wiki") + gitlab_shell.mv_repository(repository_storage, "#{project.full_path_was}.wiki", "#{new_full_path}.wiki") return true rescue => e Rails.logger.error "Exception renaming #{project.full_path_was} -> #{new_full_path}: #{e}" -- cgit v1.2.3 From 92cbf9453a4435976bfe77cafd0f8c5f57833e59 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Thu, 26 Apr 2018 10:39:04 +0800 Subject: Switch to using ProjectCiCdSetting for group_runners_enabled and remove ProjectSettings --- app/models/project.rb | 14 +++----------- app/models/project_settings.rb | 3 --- 2 files changed, 3 insertions(+), 14 deletions(-) delete mode 100644 app/models/project_settings.rb (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index dddc7fb2b27..19024b4ea85 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -236,14 +236,6 @@ class Project < ActiveRecord::Base has_many :project_badges, class_name: 'ProjectBadge' has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting' - has_one :settings, -> (project) { - query = where(project_id: project) - query.presence || begin - ProjectSettings.create(project_id: project.id) - query - end - }, class_name: 'ProjectSettings' - accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature, update_only: true accepts_nested_attributes_for :import_data @@ -253,7 +245,7 @@ class Project < ActiveRecord::Base delegate :members, to: :team, prefix: true delegate :add_user, :add_users, to: :team delegate :add_guest, :add_reporter, :add_developer, :add_master, :add_role, to: :team - delegate :group_runners_enabled, :group_runners_enabled=, :group_runners_enabled?, to: :settings + delegate :group_runners_enabled, :group_runners_enabled=, :group_runners_enabled?, to: :ci_cd_settings # Validations validates :creator, presence: true, on: :create @@ -1879,8 +1871,8 @@ class Project < ActiveRecord::Base [] end - def toggle_settings!(settings_attribute) - settings.toggle!(settings_attribute) + def toggle_ci_cd_settings!(settings_attribute) + ci_cd_settings.toggle!(settings_attribute) end def gitlab_deploy_token diff --git a/app/models/project_settings.rb b/app/models/project_settings.rb deleted file mode 100644 index b126f66fafa..00000000000 --- a/app/models/project_settings.rb +++ /dev/null @@ -1,3 +0,0 @@ -class ProjectSettings < ActiveRecord::Base - belongs_to :project -end -- cgit v1.2.3 From 9d6fe7bfdf9ff3f68ee73baa0e3d0aa7df13c351 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 26 Apr 2018 15:06:04 +0900 Subject: Refactoring ci_job_trace to ci_build_trace --- app/models/ci/build.rb | 3 +- app/models/ci/build_trace_chunk.rb | 147 +++++++++++++++++++++++++++++++++++++ app/models/ci/job_trace_chunk.rb | 147 ------------------------------------- 3 files changed, 148 insertions(+), 149 deletions(-) create mode 100644 app/models/ci/build_trace_chunk.rb delete mode 100644 app/models/ci/job_trace_chunk.rb (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 56216093293..61a0299d4fb 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -19,14 +19,13 @@ module Ci has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment' has_many :trace_sections, class_name: 'Ci::BuildTraceSection' + has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id - has_many :chunks, class_name: 'Ci::JobTraceChunk', foreign_key: :job_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_one :metadata, class_name: 'Ci::BuildMetadata' delegate :timeout, to: :metadata, prefix: true, allow_nil: true delegate :gitlab_deploy_token, to: :project diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb new file mode 100644 index 00000000000..f3beb6d4156 --- /dev/null +++ b/app/models/ci/build_trace_chunk.rb @@ -0,0 +1,147 @@ +module Ci + class BuildTraceChunk < ActiveRecord::Base + extend Gitlab::Ci::Model + + belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id + + after_destroy :redis_delete_data, if: :redis? + + default_value_for :data_store, :redis + + WriteError = Class.new(StandardError) + + CHUNK_SIZE = 128.kilobytes + CHUNK_REDIS_TTL = 1.week + LOCK_RETRY = 100 + LOCK_SLEEP = 1 + LOCK_TTL = 5.minutes + + enum data_store: { + redis: 1, + db: 2 + } + + def data + if redis? + redis_data + elsif db? + raw_data + else + raise 'Unsupported data store' + end&.force_encoding(Encoding::BINARY) # Redis/Database return UTF-8 string as default + end + + def set_data(value) + raise ArgumentError, 'too much data' if value.bytesize > CHUNK_SIZE + + in_lock do + if redis? + redis_set_data(value) + elsif db? + self.raw_data = value + else + raise 'Unsupported data store' + end + + save! if changed? + end + + schedule_to_db if fullfilled? + end + + def truncate(offset = 0) + self.append("", offset) + end + + def append(new_data, offset) + current_data = self.data.to_s + raise ArgumentError, 'Offset is out of bound' if offset > current_data.bytesize || offset < 0 + raise ArgumentError, 'Outside of chunk size' if CHUNK_SIZE < offset + new_data.bytesize + + self.set_data(current_data.byteslice(0, offset) + new_data) + end + + def size + data&.bytesize.to_i + end + + def start_offset + chunk_index * CHUNK_SIZE + end + + def end_offset + start_offset + size + end + + def range + (start_offset...end_offset) + end + + def use_database! + in_lock do + break if db? + break unless size > 0 + + self.update!(raw_data: data, data_store: :db) + redis_delete_data + end + end + + private + + def schedule_to_db + return if db? + + BuildTraceSwapChunkWorker.perform_async(id) + end + + def fullfilled? + size == CHUNK_SIZE + end + + def redis_data + Gitlab::Redis::SharedState.with do |redis| + redis.get(redis_data_key) + end + end + + def redis_set_data(data) + Gitlab::Redis::SharedState.with do |redis| + redis.set(redis_data_key, data, ex: CHUNK_REDIS_TTL) + end + end + + def redis_delete_data + Gitlab::Redis::SharedState.with do |redis| + redis.del(redis_data_key) + end + end + + def redis_data_key + "gitlab:ci:trace:#{build_id}:chunks:#{chunk_index}:data" + end + + def redis_lock_key + "gitlab:ci:trace:#{build_id}:chunks:#{chunk_index}:lock" + end + + def in_lock + lease = Gitlab::ExclusiveLease.new(redis_lock_key, timeout: LOCK_TTL) + retry_count = 0 + + until uuid = lease.try_obtain + # Keep trying until we obtain the lease. To prevent hammering Redis too + # much we'll wait for a bit between retries. + sleep(LOCK_SLEEP) + break if LOCK_RETRY < (retry_count += 1) + end + + raise WriteError, 'Failed to obtain write lock' unless uuid + + self.reload if self.persisted? + return yield + ensure + Gitlab::ExclusiveLease.cancel(redis_lock_key, uuid) + end + end +end diff --git a/app/models/ci/job_trace_chunk.rb b/app/models/ci/job_trace_chunk.rb deleted file mode 100644 index 47302265fb7..00000000000 --- a/app/models/ci/job_trace_chunk.rb +++ /dev/null @@ -1,147 +0,0 @@ -module Ci - class JobTraceChunk < ActiveRecord::Base - extend Gitlab::Ci::Model - - belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id - - after_destroy :redis_delete_data, if: :redis? - - default_value_for :data_store, :redis - - WriteError = Class.new(StandardError) - - CHUNK_SIZE = 128.kilobytes - CHUNK_REDIS_TTL = 1.week - LOCK_RETRY = 100 - LOCK_SLEEP = 1 - LOCK_TTL = 5.minutes - - enum data_store: { - redis: 1, - db: 2 - } - - def data - if redis? - redis_data - elsif db? - raw_data - else - raise 'Unsupported data store' - end&.force_encoding(Encoding::BINARY) # Redis/Database return UTF-8 string as default - end - - def set_data(value) - raise ArgumentError, 'too much data' if value.bytesize > CHUNK_SIZE - - in_lock do - if redis? - redis_set_data(value) - elsif db? - self.raw_data = value - else - raise 'Unsupported data store' - end - - save! if changed? - end - - schedule_to_db if fullfilled? - end - - def truncate(offset = 0) - self.append("", offset) - end - - def append(new_data, offset) - current_data = self.data.to_s - raise ArgumentError, 'Offset is out of bound' if offset > current_data.bytesize || offset < 0 - raise ArgumentError, 'Outside of chunk size' if CHUNK_SIZE < offset + new_data.bytesize - - self.set_data(current_data.byteslice(0, offset) + new_data) - end - - def size - data&.bytesize.to_i - end - - def start_offset - chunk_index * CHUNK_SIZE - end - - def end_offset - start_offset + size - end - - def range - (start_offset...end_offset) - end - - def use_database! - in_lock do - break if db? - break unless size > 0 - - self.update!(raw_data: data, data_store: :db) - redis_delete_data - end - end - - private - - def schedule_to_db - return if db? - - BuildTraceSwapChunkWorker.perform_async(id) - end - - def fullfilled? - size == CHUNK_SIZE - end - - def redis_data - Gitlab::Redis::SharedState.with do |redis| - redis.get(redis_data_key) - end - end - - def redis_set_data(data) - Gitlab::Redis::SharedState.with do |redis| - redis.set(redis_data_key, data, ex: CHUNK_REDIS_TTL) - end - end - - def redis_delete_data - Gitlab::Redis::SharedState.with do |redis| - redis.del(redis_data_key) - end - end - - def redis_data_key - "gitlab:ci:trace:#{job_id}:chunks:#{chunk_index}:data" - end - - def redis_lock_key - "gitlab:ci:trace:#{job_id}:chunks:#{chunk_index}:lock" - end - - def in_lock - lease = Gitlab::ExclusiveLease.new(redis_lock_key, timeout: LOCK_TTL) - retry_count = 0 - - until uuid = lease.try_obtain - # Keep trying until we obtain the lease. To prevent hammering Redis too - # much we'll wait for a bit between retries. - sleep(LOCK_SLEEP) - break if LOCK_RETRY < (retry_count += 1) - end - - raise WriteError, 'Failed to obtain write lock' unless uuid - - self.reload if self.persisted? - return yield - ensure - Gitlab::ExclusiveLease.cancel(redis_lock_key, uuid) - end - end -end -- cgit v1.2.3 From c588dd843a6c68d0c10e5c447bfbe77ad7f3d9ea Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 26 Apr 2018 15:18:08 +0900 Subject: Rename ExclusiveLease for trace write locking --- app/models/ci/build_trace_chunk.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index f3beb6d4156..794db48e1dc 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -12,9 +12,9 @@ module Ci CHUNK_SIZE = 128.kilobytes CHUNK_REDIS_TTL = 1.week - LOCK_RETRY = 100 - LOCK_SLEEP = 1 - LOCK_TTL = 5.minutes + WRITE_LOCK_RETRY = 100 + WRITE_LOCK_SLEEP = 1 + WRITE_LOCK_TTL = 5.minutes enum data_store: { redis: 1, @@ -122,18 +122,18 @@ module Ci end def redis_lock_key - "gitlab:ci:trace:#{build_id}:chunks:#{chunk_index}:lock" + "trace_write:#{build_id}:chunks:#{chunk_index}" end def in_lock - lease = Gitlab::ExclusiveLease.new(redis_lock_key, timeout: LOCK_TTL) + lease = Gitlab::ExclusiveLease.new(redis_lock_key, timeout: WRITE_LOCK_TTL) retry_count = 0 until uuid = lease.try_obtain # Keep trying until we obtain the lease. To prevent hammering Redis too # much we'll wait for a bit between retries. - sleep(LOCK_SLEEP) - break if LOCK_RETRY < (retry_count += 1) + sleep(WRITE_LOCK_SLEEP) + break if WRITE_LOCK_RETRY < (retry_count += 1) end raise WriteError, 'Failed to obtain write lock' unless uuid -- cgit v1.2.3 From 8d381b359fd12874a91777cad0a54d58fcb2bf07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Mon, 23 Apr 2018 16:38:37 +0200 Subject: Ensure member notifications are sent after the member actual creation/update in the DB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- app/models/members/group_member.rb | 6 +++--- app/models/members/project_member.rb | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'app/models') diff --git a/app/models/members/group_member.rb b/app/models/members/group_member.rb index 661e668dbf9..5da739f9618 100644 --- a/app/models/members/group_member.rb +++ b/app/models/members/group_member.rb @@ -37,20 +37,20 @@ class GroupMember < Member private def send_invite - notification_service.invite_group_member(self, @raw_invite_token) + run_after_commit_or_now { notification_service.invite_group_member(self, @raw_invite_token) } super end def post_create_hook - notification_service.new_group_member(self) + run_after_commit_or_now { notification_service.new_group_member(self) } super end def post_update_hook if access_level_changed? - notification_service.update_group_member(self) + run_after_commit { notification_service.update_group_member(self) } end super diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb index 1c7ed4a96df..024106056b4 100644 --- a/app/models/members/project_member.rb +++ b/app/models/members/project_member.rb @@ -92,7 +92,7 @@ class ProjectMember < Member private def send_invite - notification_service.invite_project_member(self, @raw_invite_token) + run_after_commit_or_now { notification_service.invite_project_member(self, @raw_invite_token) } super end @@ -100,7 +100,7 @@ class ProjectMember < Member def post_create_hook unless owner? event_service.join_project(self.project, self.user) - notification_service.new_project_member(self) + run_after_commit_or_now { notification_service.new_project_member(self) } end super @@ -108,7 +108,7 @@ class ProjectMember < Member def post_update_hook if access_level_changed? - notification_service.update_project_member(self) + run_after_commit { notification_service.update_project_member(self) } end super -- cgit v1.2.3 From f819bb7270979cb47a3f5ca97826c9491c983e4d Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 26 Apr 2018 16:30:27 +0900 Subject: Optimize Trace#write/append/raw by caching data and avoiding unnecesary truncate --- app/models/ci/build_trace_chunk.rb | 62 +++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 27 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 794db48e1dc..9e7ebf41ee8 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -21,36 +21,14 @@ module Ci db: 2 } + ## + # Data is memoized for optimizing #size and #end_offset def data - if redis? - redis_data - elsif db? - raw_data - else - raise 'Unsupported data store' - end&.force_encoding(Encoding::BINARY) # Redis/Database return UTF-8 string as default - end - - def set_data(value) - raise ArgumentError, 'too much data' if value.bytesize > CHUNK_SIZE - - in_lock do - if redis? - redis_set_data(value) - elsif db? - self.raw_data = value - else - raise 'Unsupported data store' - end - - save! if changed? - end - - schedule_to_db if fullfilled? + @data ||= get_data end def truncate(offset = 0) - self.append("", offset) + self.append("", offset) if offset < size end def append(new_data, offset) @@ -58,7 +36,7 @@ module Ci raise ArgumentError, 'Offset is out of bound' if offset > current_data.bytesize || offset < 0 raise ArgumentError, 'Outside of chunk size' if CHUNK_SIZE < offset + new_data.bytesize - self.set_data(current_data.byteslice(0, offset) + new_data) + set_data(current_data.byteslice(0, offset) + new_data) end def size @@ -89,6 +67,36 @@ module Ci private + def get_data + if redis? + redis_data + elsif db? + raw_data + else + raise 'Unsupported data store' + end&.force_encoding(Encoding::BINARY) # Redis/Database return UTF-8 string as default + end + + def set_data(value) + raise ArgumentError, 'too much data' if value.bytesize > CHUNK_SIZE + + in_lock do + if redis? + redis_set_data(value) + elsif db? + self.raw_data = value + else + raise 'Unsupported data store' + end + + @data = value + + save! if changed? + end + + schedule_to_db if fullfilled? + end + def schedule_to_db return if db? -- cgit v1.2.3 From 645b404d899f3f9e6b1fb30a44c9a25d221b52c0 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 26 Apr 2018 17:25:20 +0900 Subject: Fix specs. Align with the new table name ci_build_trace_chunk --- app/models/ci/build.rb | 7 +------ app/models/ci/build_trace_chunk.rb | 5 +---- app/models/project.rb | 2 +- 3 files changed, 3 insertions(+), 11 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 77d275f304d..61c10c427dd 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -19,18 +19,13 @@ module Ci has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment' has_many :trace_sections, class_name: 'Ci::BuildTraceSection' - has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id -<<<<<<< HEAD - has_many :chunks, class_name: 'Ci::JobTraceChunk', foreign_key: :job_id - -======= ->>>>>>> live-trace-v2 has_one :metadata, class_name: 'Ci::BuildMetadata' delegate :timeout, to: :metadata, prefix: true, allow_nil: true delegate :gitlab_deploy_token, to: :project diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 00f7e1a121d..95a8a4eb1ad 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -19,9 +19,8 @@ module Ci db: 2 } -<<<<<<< HEAD:app/models/ci/job_trace_chunk.rb def self.delayed_cleanup_blk - ids = all.redis.pluck(:job_id, :chunk_index).map do |data| + ids = all.redis.pluck(:build_id, :chunk_index).map do |data| "gitlab:ci:trace:#{data.first}:chunks:#{data.second}:data" end @@ -44,10 +43,8 @@ module Ci end end -======= ## # Data is memoized for optimizing #size and #end_offset ->>>>>>> live-trace-v2:app/models/ci/build_trace_chunk.rb def data @data ||= get_data end diff --git a/app/models/project.rb b/app/models/project.rb index 512f16d2f05..48e30f0fd36 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -228,7 +228,7 @@ class Project < ActiveRecord::Base # still using `dependent: :destroy` here. has_many :builds, class_name: 'Ci::Build', inverse_of: :project, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :build_trace_section_names, class_name: 'Ci::BuildTraceSectionName' - has_many :build_trace_chunks, class_name: 'Ci::JobTraceChunk', foreign_key: :job_id, through: :builds, source: :chunks + has_many :build_trace_chunks, class_name: 'Ci::BuildTraceChunk', through: :builds, source: :trace_chunks has_many :runner_projects, class_name: 'Ci::RunnerProject' has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' has_many :variables, class_name: 'Ci::Variable' -- cgit v1.2.3 From 8d8139862aee97d7fadc0563e7df9842f5bd46ac Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Fri, 27 Apr 2018 09:15:54 +0800 Subject: Rename `runner.belonging_to_group(project.id) -> runner.belonging_to_parent_group_of_project(project.id)` --- app/models/ci/runner.rb | 4 ++-- app/models/project.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index d65395380d6..6904aca5e68 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -34,7 +34,7 @@ module Ci scope :belonging_to_any_project, -> { joins(:runner_projects) } - scope :belonging_to_group, -> (project_id) { + scope :belonging_to_parent_group_of_project, -> (project_id) { project_groups = ::Group.joins(:projects).where(projects: { id: project_id }) hierarchy_groups = Gitlab::GroupHierarchy.new(project_groups).base_and_ancestors @@ -42,7 +42,7 @@ module Ci } scope :owned_or_shared, -> (project_id) do - union = Gitlab::SQL::Union.new([belonging_to_project(project_id), belonging_to_group(project_id), shared]) + union = Gitlab::SQL::Union.new([belonging_to_project(project_id), belonging_to_parent_group_of_project(project_id), shared]) from("(#{union.to_sql}) ci_runners") end diff --git a/app/models/project.rb b/app/models/project.rb index 19024b4ea85..8e2145a4aa3 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1297,7 +1297,7 @@ class Project < ActiveRecord::Base end def group_runners - @group_runners ||= group_runners_enabled? ? Ci::Runner.belonging_to_group(self.id) : Ci::Runner.none + @group_runners ||= group_runners_enabled? ? Ci::Runner.belonging_to_parent_group_of_project(self.id) : Ci::Runner.none end def any_runners?(&block) -- cgit v1.2.3 From 0077679ae88793b200bdcf69d2ca7e4d15e6f7fa Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Fri, 27 Apr 2018 10:05:51 +0800 Subject: Bring back shared_runners_available? method since it is overriden in EE --- app/models/project.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 8e2145a4aa3..5c9bf8c61dd 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1292,8 +1292,12 @@ class Project < ActiveRecord::Base project_feature.update_attribute(:builds_access_level, ProjectFeature::ENABLED) end + def shared_runners_available? + shared_runners_enabled? + end + def shared_runners - @shared_runners ||= shared_runners_enabled? ? Ci::Runner.shared : Ci::Runner.none + @shared_runners ||= shared_runners_available? ? Ci::Runner.shared : Ci::Runner.none end def group_runners -- cgit v1.2.3 From 4a306796c108eb14f6025fecbe1ea878e8b0f0fc Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Sat, 28 Apr 2018 15:54:46 +1100 Subject: [Rails5] Fix `params` for DeleteUserWorker This commit fixes the error: ``` 1) Admin::UsersController DELETE #user with projects deletes the user and their contributions when hard delete is specified Failure/Error: Users::DestroyService.new(current_user).execute(delete_user, options.symbolize_keys) NoMethodError: undefined method `symbolize_keys' for "{\"hard_delete\"=>\"true\"}":String # ./app/workers/delete_user_worker.rb:8:in `perform' # ./lib/gitlab/sidekiq_status/server_middleware.rb:5:in `call' # ./config/initializers/forbid_sidekiq_in_transactions.rb:35:in `block (2 levels) in ' # ./app/models/user.rb:913:in `delete_async' # ./app/controllers/admin/users_controller.rb:148:in `destroy' # ./lib/gitlab/i18n.rb:50:in `with_locale' # ./lib/gitlab/i18n.rb:56:in `with_user_locale' # ./app/controllers/application_controller.rb:327:in `set_locale' # ./spec/controllers/admin/users_controller_spec.rb:28:in `block (3 levels) in ' Finished in 6.81 seconds (files took 13.9 seconds to load) 1 example, 1 failure ``` --- app/models/user.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index b0668148972..4a602ffbb05 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -910,7 +910,7 @@ class User < ActiveRecord::Base def delete_async(deleted_by:, params: {}) block if params[:hard_delete] - DeleteUserWorker.perform_async(deleted_by.id, id, params) + DeleteUserWorker.perform_async(deleted_by.id, id, params.to_h) end def notification_service -- cgit v1.2.3 From 68c75bc0f9f279602c5269569fb892b4ed243403 Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Sun, 29 Apr 2018 17:23:59 +1100 Subject: [Rails5] Add `touch_later` to `Commit` model This commit fixes errors like: ``` 1) API::Todos GET /todos when unauthenticated returns authentication error Failure/Error: @raw.__send__(method, *args, &block) # rubocop:disable GitlabSecurity/PublicSend NoMethodError: undefined method `touch_later' for # # ./app/models/commit.rb:259:in `method_missing' # ./spec/requests/api/todos_spec.rb:12:in `block (2 levels) in ' ``` --- app/models/commit.rb | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'app/models') diff --git a/app/models/commit.rb b/app/models/commit.rb index 9750e9298ec..70e5accfd1f 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -420,6 +420,12 @@ class Commit # no-op but needs to be defined since #persisted? is defined end + def touch_later + # No-op. + # This method is called by ActiveRecord. + # We don't want to do anything for `Commit` model, so this is empty. + end + WIP_REGEX = /\A\s*(((?i)(\[WIP\]|WIP:|WIP)\s|WIP$))|(fixup!|squash!)\s/.freeze def work_in_progress? -- cgit v1.2.3 From 23c8e198463d566d2e8d2351c315741903035a64 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 30 Apr 2018 14:52:29 +0900 Subject: Fix spec. Revert #truncate in stream (But still prevent redandant calls) --- app/models/ci/build_trace_chunk.rb | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 9e7ebf41ee8..a8f43fd549f 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -24,7 +24,7 @@ module Ci ## # Data is memoized for optimizing #size and #end_offset def data - @data ||= get_data + @data ||= get_data.to_s end def truncate(offset = 0) @@ -32,11 +32,10 @@ module Ci end def append(new_data, offset) - current_data = self.data.to_s - raise ArgumentError, 'Offset is out of bound' if offset > current_data.bytesize || offset < 0 - raise ArgumentError, 'Outside of chunk size' if CHUNK_SIZE < offset + new_data.bytesize + raise ArgumentError, 'Offset is out of range' if offset > data.bytesize || offset < 0 + raise ArgumentError, 'Chunk size overflow' if CHUNK_SIZE < (offset + new_data.bytesize) - set_data(current_data.byteslice(0, offset) + new_data) + set_data(data.byteslice(0, offset) + new_data) end def size -- cgit v1.2.3 From 87740df2ba7153439c30544f299b235632717738 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Mon, 30 Apr 2018 09:39:47 +0400 Subject: Revert fair scheduling for all builds Per discussion in https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/9646#note_65730532 this logic is being optimized elsewhere and it will simplify things if we make less changes to this code right now. --- app/models/ci/runner.rb | 18 ------------------ 1 file changed, 18 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 6904aca5e68..40d828b8414 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -99,24 +99,6 @@ module Ci self.token = SecureRandom.hex(15) if self.token.blank? end - def accessible_projects - accessible_projects = - if shared? - Project.with_shared_runners - elsif project? - projects - elsif group? - hierarchy_groups = Gitlab::GroupHierarchy.new(groups).base_and_descendants - Project.where(namespace_id: hierarchy_groups) - else - Project.none - end - - accessible_projects - .with_builds_enabled - .without_deleted - end - def assign_to(project, current_user = nil) self.is_shared = false if shared? self.save -- cgit v1.2.3 From d5344617a8b57e4d6d15f22ad3d09d5af82100fe Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 30 Apr 2018 15:27:05 +0900 Subject: Fix spec when parent record is destroyed --- app/models/ci/build_trace_chunk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index a8f43fd549f..b9b84104b33 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -125,7 +125,7 @@ module Ci end def redis_data_key - "gitlab:ci:trace:#{build_id}:chunks:#{chunk_index}:data" + "gitlab:ci:trace:#{build_id}:chunks:#{chunk_index}" end def redis_lock_key -- cgit v1.2.3 From 5652ff953cba9773edbcb677908fe3f18b103be3 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Mon, 30 Apr 2018 10:43:29 +0400 Subject: Rename Runner#group? -> #assigned_to_group? and Runner#project? -> #assigned_to_project? --- app/models/ci/runner.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 40d828b8414..da1107951bf 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -137,11 +137,11 @@ module Ci !shared? end - def group? + def assigned_to_group? runner_groups.any? end - def project? + def assigned_to_project? runner_projects.any? end @@ -244,7 +244,7 @@ module Ci errors.add(:runner, 'can only be assigned to one group') end - if group? && project? + if assigned_to_group? && assigned_to_project? errors.add(:runner, 'can only be assigned either to projects or to a group') end end -- cgit v1.2.3 From c46101e2ef5ff29a7257783038a8cfc58fa4ca48 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 30 Apr 2018 15:58:55 +0900 Subject: Added spec for trace archiving. --- app/models/ci/build_trace_chunk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 41c166aa804..5dc43c6e00f 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -21,7 +21,7 @@ module Ci def self.delayed_cleanup_blk ids = all.redis.pluck(:build_id, :chunk_index).map do |data| - "gitlab:ci:trace:#{data.first}:chunks:#{data.second}:data" + "gitlab:ci:trace:#{data.first}:chunks:#{data.second}" end puts "before cleanup: #{ids.count}" -- cgit v1.2.3 From 0d30b00de807df550bec947751c098317c5bb79f Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Mon, 30 Apr 2018 17:00:28 +0400 Subject: Start persisting runner_type when creating runners --- app/models/ci/runner.rb | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index da1107951bf..cdd28407172 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -67,6 +67,12 @@ module Ci ref_protected: 1 } + enum runner_type: { + instance_type: 1, + group_type: 2, + project_type: 3 + } + cached_attr_reader :version, :revision, :platform, :architecture, :contacted_at, :ip_address chronic_duration_attr :maximum_timeout_human_readable, :maximum_timeout -- cgit v1.2.3 From e98438d4efdf3aebabe5938979f4924cf7cb47a8 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Mon, 30 Apr 2018 17:10:07 +0400 Subject: Revert changes to User#ci_authorized_users to defer group runner API changes to later --- app/models/user.rb | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index 0c5c0fef9d4..b0668148972 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -995,17 +995,10 @@ class User < ActiveRecord::Base def ci_authorized_runners @ci_authorized_runners ||= begin - project_runner_ids = Ci::RunnerProject + runner_ids = Ci::RunnerProject .where(project: authorized_projects(Gitlab::Access::MASTER)) .select(:runner_id) - - group_runner_ids = Ci::RunnerGroup - .where(group_id: owned_or_masters_groups.select(:id)) - .select(:runner_id) - - union = Gitlab::SQL::Union.new([project_runner_ids, group_runner_ids]) - - Ci::Runner.specific.where("ci_runners.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection + Ci::Runner.specific.where(id: runner_ids) end end @@ -1194,11 +1187,6 @@ class User < ActiveRecord::Base max_member_access_for_group_ids([group_id])[group_id] end - def owned_or_masters_groups - union = Gitlab::SQL::Union.new([owned_groups, masters_groups]) - Group.from("(#{union.to_sql}) namespaces") - end - protected # override, from Devise::Validatable -- cgit v1.2.3 From d8dd25a60e7c96a24d33cfe1b93be09085a1b86c Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Mon, 30 Apr 2018 17:25:49 +0400 Subject: Introduce Project#all_runners and use in Ci::UpdateBuildQueueService --- app/models/project.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 5c9bf8c61dd..8abbb92da62 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1304,10 +1304,13 @@ class Project < ActiveRecord::Base @group_runners ||= group_runners_enabled? ? Ci::Runner.belonging_to_parent_group_of_project(self.id) : Ci::Runner.none end + def all_runners + union = Gitlab::SQL::Union.new([runners, group_runners, shared_runners]) + Ci::Runner.from("(#{union.to_sql}) ci_runners") + end + def any_runners?(&block) - union = Gitlab::SQL::Union.new([runners, shared_runners, group_runners]) - runners = Ci::Runner.from("(#{union.to_sql}) ci_runners").active - runners.any?(&block) + all_runners.active.any?(&block) end def valid_runners_token?(token) -- cgit v1.2.3 From be8a320bd8594fe42c2558d2eab471acdbdc7321 Mon Sep 17 00:00:00 2001 From: Oswaldo Ferreira Date: Mon, 30 Apr 2018 18:44:37 -0300 Subject: Use persisted diff data instead fetching Git on discussions Today, when fetching diffs of a note, we always go to Gitaly in order to diff between commits and return the diff of each discussion note. With this change we avoid doing that for notes on the "current version" of the MR. --- app/models/diff_note.rb | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb index 15122cbc693..616a626419b 100644 --- a/app/models/diff_note.rb +++ b/app/models/diff_note.rb @@ -54,7 +54,20 @@ class DiffNote < Note end def diff_file - @diff_file ||= self.original_position.diff_file(self.project.repository) + @diff_file ||= + begin + if created_at_diff?(noteable.diff_refs) + # We're able to use the already persisted diffs (Postgres) if we're + # presenting a "current version" of the MR discussion diff. + # So no need to make an extra Gitaly diff request for it. + # As an extra benefit, the returned `diff_file` already + # has `highlighted_diff_lines` data set from Redis on + # `Diff::FileCollection::MergeRequestDiff`. + noteable.diffs(paths: original_position.paths, expanded: true).diff_files.first + else + original_position.diff_file(self.project.repository) + end + end end def diff_line -- cgit v1.2.3 From 1d53918b62452f9758a837744bac6ca051801a6a Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 1 May 2018 17:06:44 +0900 Subject: Introduces `FastDestroyAll` module --- app/models/ci/build_trace_chunk.rb | 42 ++++++++++------------- app/models/concerns/fast_destroy_all.rb | 61 +++++++++++++++++++++++++++++++++ app/models/project.rb | 14 ++++---- 3 files changed, 85 insertions(+), 32 deletions(-) create mode 100644 app/models/concerns/fast_destroy_all.rb (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 5dc43c6e00f..3f4a901b626 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -1,10 +1,12 @@ module Ci class BuildTraceChunk < ActiveRecord::Base + include FastDestroyAll extend Gitlab::Ci::Model belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id default_value_for :data_store, :redis + fast_destroy_all_with :delete_all_redis_data, :redis_all_data_keys WriteError = Class.new(StandardError) @@ -19,27 +21,23 @@ module Ci db: 2 } - def self.delayed_cleanup_blk - ids = all.redis.pluck(:build_id, :chunk_index).map do |data| - "gitlab:ci:trace:#{data.first}:chunks:#{data.second}" + class << self + def redis_data_key(build_id, chunk_index) + "gitlab:ci:trace:#{build_id}:chunks:#{chunk_index}" end - puts "before cleanup: #{ids.count}" - - Proc.new do - puts "after cleanup: #{ids.count}" - Gitlab::Redis::SharedState.with do |redis| - redis.del(ids) - end unless ids.empty? - - true + def redis_all_data_keys + redis.pluck(:build_id, :chunk_index).map do |data| + redis_data_key(data.first, data.second) + end end - end - def self.fast_destroy_all - delayed_cleanup_blk.tap do |cleanup| - delete_all - cleanup.call + def delete_all_redis_data(redis_keys) + if redis_keys.any? + Gitlab::Redis::SharedState.with do |redis| + redis.del(redis_keys) + end + end end end @@ -130,26 +128,22 @@ module Ci def redis_data Gitlab::Redis::SharedState.with do |redis| - redis.get(redis_data_key) + redis.get(self.class.redis_data_key(build_id, chunk_index)) end end def redis_set_data(data) Gitlab::Redis::SharedState.with do |redis| - redis.set(redis_data_key, data, ex: CHUNK_REDIS_TTL) + redis.set(self.class.redis_data_key(build_id, chunk_index), data, ex: CHUNK_REDIS_TTL) end end def redis_delete_data Gitlab::Redis::SharedState.with do |redis| - redis.del(redis_data_key) + redis.del(self.class.redis_data_key(build_id, chunk_index)) end end - def redis_data_key - "gitlab:ci:trace:#{build_id}:chunks:#{chunk_index}" - end - def redis_lock_key "trace_write:#{build_id}:chunks:#{chunk_index}" end diff --git a/app/models/concerns/fast_destroy_all.rb b/app/models/concerns/fast_destroy_all.rb new file mode 100644 index 00000000000..28ab541dd13 --- /dev/null +++ b/app/models/concerns/fast_destroy_all.rb @@ -0,0 +1,61 @@ +## +# This module is for replacing `dependent: :destroy` and `before_destroy` hooks. +# +# In general, `destroy_all` is inefficient because it calls each callback with `DELETE` queries i.e. O(n), whereas, +# `delete_all` is efficient as it deletes all rows with a single `DELETE` query. +# +# It's better to use `delete_all` as much as possible, however, in the real cases, it's hard to adopt because +# in some cases, external data (in ObjectStorage/FileStorage/Redis) is assosiated with rows. +# +# This module introduces a protocol to support the adoption with easy way. +# You can use in the following scnenes +# - When calling `destroy_all` explicitly +# e.g. `job.trace_chunks.fast_destroy_all`` +# - When a parent record is deleted and all children are deleted subsequently (cascade delete) +# e.g. `before_destroy -> { run_after_commit(&build_trace_chunks.delete_all_external_data_proc) }`` +module FastDestroyAll + extend ActiveSupport::Concern + + included do + class_attribute :_delete_method, :_delete_params_generator + end + + class_methods do + ## + # This method is for registering :delete_method and :delete_params_generator + # You have to define the method if you want to use `FastDestroyAll`` module. + # + # e.g. fast_destroy_all_with :delete_all_redis_data, :redis_all_data_keys + def fast_destroy_all_with(delete_method, delete_params_generator) + self._delete_method = delete_method + self._delete_params_generator = delete_params_generator + end + + ## + # This method generates a proc to delete external data. + # It's useful especially when you want to hook parent record's deletion event. + # + # e.g. before_destroy -> { run_after_commit(&build_trace_chunks.delete_all_external_data_proc) } + def delete_all_external_data_proc + params = send self._delete_params_generator + callee = self # Preserve the callee class, otherwise `proc` uses a different class + + proc do + callee.send callee._delete_method, params + + true + end + end + + ## + # This method deletes all rows at first and delete all external data at second. + # Before deleting the rows, it generates a proc to delete external data. + # So it won't lose the track of deleting external data, even if it happened after rows had been deleted. + def fast_destroy_all + delete_all_external_data_proc.tap do |delete_all_external_data| + delete_all + delete_all_external_data.call + end + end + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 48e30f0fd36..fd46c7b1c70 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -78,6 +78,12 @@ class Project < ActiveRecord::Base after_update :update_forks_visibility_level before_destroy :remove_private_deploy_keys + + # This has to be defined before `has_many :builds, depenedent: :destroy`, + # otherwise we will not delete any data, due to trace chunks + # going through :builds + before_destroy -> { run_after_commit(&build_trace_chunks.delete_all_external_data_proc) } + after_destroy -> { run_after_commit { remove_pages } } after_destroy :remove_exports @@ -214,14 +220,6 @@ class Project < ActiveRecord::Base has_many :commit_statuses has_many :pipelines, class_name: 'Ci::Pipeline', inverse_of: :project - # This has to be defined before `has_many :builds, depenedent: :destroy`, - # otherwise we will not delete any data, due to trace chunks - # going through :builds - before_destroy do - puts "destroying all chunks" - self.run_after_commit(&build_trace_chunks.delayed_cleanup_blk) - end - # Ci::Build objects store data on the file system such as artifact files and # build traces. Currently there's no efficient way of removing this data in # bulk that doesn't involve loading the rows into memory. As a result we're -- cgit v1.2.3 From 0fd0b64be63c18bb216f15d887e3ce0955dcf269 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 1 May 2018 14:30:44 +0200 Subject: Use stages position column to track stage index --- app/models/ci/stage.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 9a913213bb9..5a1eeb966aa 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -17,7 +17,7 @@ module Ci validates :project, presence: true validates :pipeline, presence: true validates :name, presence: true - validates :priority, presence: true + validates :position, presence: true end after_initialize do @@ -25,9 +25,9 @@ module Ci end before_validation unless: :importing? do - next if priority.present? + next if position.present? - self.priority = statuses.select(:stage_idx) + self.position = statuses.select(:stage_idx) .where('stage_idx IS NOT NULL') .group(:stage_idx) .order('COUNT(*) DESC') -- cgit v1.2.3 From 7a76caa5a8d2c6be9aa4b698a8919e223df973d3 Mon Sep 17 00:00:00 2001 From: Jan Provaznik Date: Tue, 1 May 2018 12:39:44 +0000 Subject: Merge request and commit discussions API --- app/models/commit.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'app/models') diff --git a/app/models/commit.rb b/app/models/commit.rb index 9750e9298ec..32f3d90595a 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -105,6 +105,10 @@ class Commit end end end + + def parent_class + ::Project + end end attr_accessor :raw -- cgit v1.2.3 From 9bb7abedf57f5325a92bebc2d93b85cf7e1fc303 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Tue, 1 May 2018 22:27:54 +0200 Subject: Fix file_store for artifacts and lfs when saving --- app/models/ci/job_artifact.rb | 2 +- app/models/lfs_object.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 39676efa08c..3b952391b7e 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -13,7 +13,7 @@ module Ci after_save :update_project_statistics_after_save, if: :size_changed? after_destroy :update_project_statistics_after_destroy, unless: :project_destroyed? - after_save :update_file_store + after_save :update_file_store, if: :file_changed? scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) } diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index 6b7f280fb70..84487031ee5 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -11,7 +11,7 @@ class LfsObject < ActiveRecord::Base mount_uploader :file, LfsObjectUploader - after_save :update_file_store + after_save :update_file_store, if: :file_changed? def update_file_store # The file.object_store is set during `uploader.store!` -- cgit v1.2.3 From 9e65dc53776c39f285c97768555919e8d152b8a5 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 2 May 2018 15:55:55 +0900 Subject: Introduce `use_fast_destroy` helper for parent associations. Rename method names in build_trace_chunks. Forbid `destroy` method when `FastDestroyAll` included. --- app/models/ci/build_trace_chunk.rb | 23 +++++++---------- app/models/concerns/fast_destroy_all.rb | 44 +++++++++++++++++++++++++++------ app/models/project.rb | 9 ++++--- 3 files changed, 50 insertions(+), 26 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 3f4a901b626..8935c228566 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -6,7 +6,7 @@ module Ci belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id default_value_for :data_store, :redis - fast_destroy_all_with :delete_all_redis_data, :redis_all_data_keys + fast_destroy_all_with :redis_delete_data, :redis_data_keys WriteError = Class.new(StandardError) @@ -26,17 +26,17 @@ module Ci "gitlab:ci:trace:#{build_id}:chunks:#{chunk_index}" end - def redis_all_data_keys + def redis_data_keys redis.pluck(:build_id, :chunk_index).map do |data| redis_data_key(data.first, data.second) end end - def delete_all_redis_data(redis_keys) - if redis_keys.any? - Gitlab::Redis::SharedState.with do |redis| - redis.del(redis_keys) - end + def redis_delete_data(keys) + return if keys.empty? + + Gitlab::Redis::SharedState.with do |redis| + redis.del(keys) end end end @@ -80,7 +80,8 @@ module Ci break unless size > 0 self.update!(raw_data: data, data_store: :db) - redis_delete_data + key = self.class.redis_data_key(build_id, chunk_index) + self.class.redis_delete_data([key]) end end @@ -138,12 +139,6 @@ module Ci end end - def redis_delete_data - Gitlab::Redis::SharedState.with do |redis| - redis.del(self.class.redis_data_key(build_id, chunk_index)) - end - end - def redis_lock_key "trace_write:#{build_id}:chunks:#{chunk_index}" end diff --git a/app/models/concerns/fast_destroy_all.rb b/app/models/concerns/fast_destroy_all.rb index 28ab541dd13..3581822673e 100644 --- a/app/models/concerns/fast_destroy_all.rb +++ b/app/models/concerns/fast_destroy_all.rb @@ -4,7 +4,7 @@ # In general, `destroy_all` is inefficient because it calls each callback with `DELETE` queries i.e. O(n), whereas, # `delete_all` is efficient as it deletes all rows with a single `DELETE` query. # -# It's better to use `delete_all` as much as possible, however, in the real cases, it's hard to adopt because +# It's better to use `delete_all` as much as possible, however, in the real cases, it's hard to adopt because # in some cases, external data (in ObjectStorage/FileStorage/Redis) is assosiated with rows. # # This module introduces a protocol to support the adoption with easy way. @@ -12,12 +12,18 @@ # - When calling `destroy_all` explicitly # e.g. `job.trace_chunks.fast_destroy_all`` # - When a parent record is deleted and all children are deleted subsequently (cascade delete) -# e.g. `before_destroy -> { run_after_commit(&build_trace_chunks.delete_all_external_data_proc) }`` +# e.g. `before_destroy -> { run_after_commit(&build_trace_chunks.after_commit_cleanup_proc) }`` module FastDestroyAll extend ActiveSupport::Concern + ForbiddenActionError = Class.new(StandardError) + included do class_attribute :_delete_method, :_delete_params_generator + + before_destroy do + raise ForbiddenActionError, '`destroy` is forbbiden, please use `fast_destroy_all`' + end end class_methods do @@ -35,13 +41,13 @@ module FastDestroyAll # This method generates a proc to delete external data. # It's useful especially when you want to hook parent record's deletion event. # - # e.g. before_destroy -> { run_after_commit(&build_trace_chunks.delete_all_external_data_proc) } - def delete_all_external_data_proc - params = send self._delete_params_generator - callee = self # Preserve the callee class, otherwise `proc` uses a different class + # e.g. before_destroy -> { run_after_commit(&build_trace_chunks.after_commit_cleanup_proc) } + def after_commit_cleanup_proc + params = send self._delete_params_generator # rubocop:disable GitlabSecurity/PublicSend + subject = self # Preserve the subject class, otherwise `proc` uses a different class proc do - callee.send callee._delete_method, params + subject.send subject._delete_method, params # rubocop:disable GitlabSecurity/PublicSend true end @@ -52,10 +58,32 @@ module FastDestroyAll # Before deleting the rows, it generates a proc to delete external data. # So it won't lose the track of deleting external data, even if it happened after rows had been deleted. def fast_destroy_all - delete_all_external_data_proc.tap do |delete_all_external_data| + after_commit_cleanup_proc.tap do |delete_all_external_data| delete_all delete_all_external_data.call end end end + + module Helpers + extend ActiveSupport::Concern + + class_methods do + ## + # This is a helper method for performaning fast_destroy_all when parent relations are deleted + # Their children must include `FastDestroyAll` module. + # + # `use_fast_destroy` must be defined **before** `has_many` and `has_one`, such as `has_many :relation, depenedent: :destroy` + # Otherwise `use_fast_destroy` performs against **deleted** rows, which fails to get identifiers of external data + # + # e.g. use_fast_destroy :build_trace_chunks + def use_fast_destroy(relation) + before_destroy do + subject = public_send(relation) + after_commit_proc = subject.after_commit_cleanup_proc + run_after_commit(&after_commit_proc) + end + end + end + end end diff --git a/app/models/project.rb b/app/models/project.rb index fd46c7b1c70..e98b9fa1473 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -22,6 +22,7 @@ class Project < ActiveRecord::Base include DeploymentPlatform include ::Gitlab::Utils::StrongMemoize include ChronicDurationAttribute + include FastDestroyAll::Helpers extend Gitlab::ConfigHelper @@ -79,10 +80,10 @@ class Project < ActiveRecord::Base before_destroy :remove_private_deploy_keys - # This has to be defined before `has_many :builds, depenedent: :destroy`, - # otherwise we will not delete any data, due to trace chunks - # going through :builds - before_destroy -> { run_after_commit(&build_trace_chunks.delete_all_external_data_proc) } + ## + # `use_fast_destroy` must be defined **before** `has_many` and `has_one` such as `has_many :relation, depenedent: :destroy` + # Otherwise `use_fast_destroy` performs against **deleted** rows, which fails to get identifiers of external data + use_fast_destroy :build_trace_chunks after_destroy -> { run_after_commit { remove_pages } } after_destroy :remove_exports -- cgit v1.2.3 From 950df8babf07d7135b1f7cf95586d40f0ca36ff6 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 2 May 2018 16:52:54 +0900 Subject: Rename BuildTraceSwapChunkWorker to BuildTraceChunkFlushToDBWorker --- app/models/ci/build_trace_chunk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index b9b84104b33..4c1449cbb70 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -99,7 +99,7 @@ module Ci def schedule_to_db return if db? - BuildTraceSwapChunkWorker.perform_async(id) + BuildTraceChunkFlushToDBWorker.perform_async(id) end def fullfilled? -- cgit v1.2.3 From 9b33e3d36fcd46072b9fe83f1121fb0fd87c0fd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Reigel=20=28=20=F0=9F=8C=B4=20may=202nd=20-=20may?= =?UTF-8?q?=209th=20=F0=9F=8C=B4=20=29?= Date: Wed, 2 May 2018 08:08:16 +0000 Subject: Display and revoke active sessions --- app/models/active_session.rb | 110 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 app/models/active_session.rb (limited to 'app/models') diff --git a/app/models/active_session.rb b/app/models/active_session.rb new file mode 100644 index 00000000000..b4a86dbb331 --- /dev/null +++ b/app/models/active_session.rb @@ -0,0 +1,110 @@ +class ActiveSession + include ActiveModel::Model + + attr_accessor :created_at, :updated_at, + :session_id, :ip_address, + :browser, :os, :device_name, :device_type + + def current?(session) + return false if session_id.nil? || session.id.nil? + + session_id == session.id + end + + def human_device_type + device_type&.titleize + end + + def self.set(user, request) + Gitlab::Redis::SharedState.with do |redis| + session_id = request.session.id + client = DeviceDetector.new(request.user_agent) + timestamp = Time.current + + active_user_session = new( + ip_address: request.ip, + browser: client.name, + os: client.os_name, + device_name: client.device_name, + device_type: client.device_type, + created_at: user.current_sign_in_at || timestamp, + updated_at: timestamp, + session_id: session_id + ) + + redis.pipelined do + redis.setex( + key_name(user.id, session_id), + Settings.gitlab['session_expire_delay'] * 60, + Marshal.dump(active_user_session) + ) + + redis.sadd( + lookup_key_name(user.id), + session_id + ) + end + end + end + + def self.list(user) + Gitlab::Redis::SharedState.with do |redis| + cleaned_up_lookup_entries(redis, user.id).map do |entry| + # rubocop:disable Security/MarshalLoad + Marshal.load(entry) + # rubocop:enable Security/MarshalLoad + end + end + end + + def self.destroy(user, session_id) + Gitlab::Redis::SharedState.with do |redis| + redis.srem(lookup_key_name(user.id), session_id) + + deleted_keys = redis.del(key_name(user.id, session_id)) + + # only allow deleting the devise session if we could actually find a + # related active session. this prevents another user from deleting + # someone else's session. + if deleted_keys > 0 + redis.del("#{Gitlab::Redis::SharedState::SESSION_NAMESPACE}:#{session_id}") + end + end + end + + def self.cleanup(user) + Gitlab::Redis::SharedState.with do |redis| + cleaned_up_lookup_entries(redis, user.id) + end + end + + def self.key_name(user_id, session_id = '*') + "#{Gitlab::Redis::SharedState::USER_SESSIONS_NAMESPACE}:#{user_id}:#{session_id}" + end + + def self.lookup_key_name(user_id) + "#{Gitlab::Redis::SharedState::USER_SESSIONS_LOOKUP_NAMESPACE}:#{user_id}" + end + + def self.cleaned_up_lookup_entries(redis, user_id) + lookup_key = lookup_key_name(user_id) + + session_ids = redis.smembers(lookup_key) + + entry_keys = session_ids.map { |session_id| key_name(user_id, session_id) } + return [] if entry_keys.empty? + + entries = redis.mget(entry_keys) + + session_ids_and_entries = session_ids.zip(entries) + + # remove expired keys. + # only the single key entries are automatically expired by redis, the + # lookup entries in the set need to be removed manually. + session_ids_and_entries.reject { |_session_id, entry| entry }.each do |session_id, _entry| + redis.srem(lookup_key, session_id) + end + + session_ids_and_entries.select { |_session_id, entry| entry }.map { |_session_id, entry| entry } + end +end -- cgit v1.2.3 From 9e78b86dd48ee893c950643d6ca0188a4978eeaa Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Wed, 2 May 2018 08:09:19 +0000 Subject: [Rails5] Fix enum question mark methods --- app/models/ci/pipeline.rb | 11 +++++++++++ app/models/commit_status.rb | 7 +++++++ 2 files changed, 18 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 434b9b64c65..e1b9bc76475 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -530,6 +530,17 @@ module Ci @latest_builds_with_artifacts ||= builds.latest.with_artifacts_archive.to_a end + # Rails 5.0 autogenerated question mark enum methods return wrong result if enum value is nil. + # They always return `false`. + # These methods overwrite autogenerated ones to return correct results. + def unknown? + Gitlab.rails5? ? source.nil? : super + end + + def unknown_source? + Gitlab.rails5? ? config_source.nil? : super + end + private def ci_yaml_from_repo diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index b6276c2fb50..97d89422594 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -189,4 +189,11 @@ class CommitStatus < ActiveRecord::Base v =~ /\d+/ ? v.to_i : v end end + + # Rails 5.0 autogenerated question mark enum methods return wrong result if enum value is nil. + # They always return `false`. + # This method overwrites the autogenerated one to return correct result. + def unknown_failure? + Gitlab.rails5? ? failure_reason.nil? : super + end end -- cgit v1.2.3 From 7d626c41ccb4531dceeb1d1025c067bac3c63c4d Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 2 May 2018 21:22:08 +0900 Subject: Fix BuildTraceChunkFlushToDbWorker name --- app/models/ci/build_trace_chunk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 4c1449cbb70..ee999a3395d 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -99,7 +99,7 @@ module Ci def schedule_to_db return if db? - BuildTraceChunkFlushToDBWorker.perform_async(id) + BuildTraceChunkFlushToDbWorker.perform_async(id) end def fullfilled? -- cgit v1.2.3 From 2fde2d6524809e2083078626b9b43a4c11313fae Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 2 May 2018 21:30:39 +0900 Subject: Fix static analysys --- app/models/concerns/fast_destroy_all.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/fast_destroy_all.rb b/app/models/concerns/fast_destroy_all.rb index 3581822673e..7639af35b69 100644 --- a/app/models/concerns/fast_destroy_all.rb +++ b/app/models/concerns/fast_destroy_all.rb @@ -23,7 +23,7 @@ module FastDestroyAll before_destroy do raise ForbiddenActionError, '`destroy` is forbbiden, please use `fast_destroy_all`' - end + end end class_methods do @@ -79,7 +79,7 @@ module FastDestroyAll # e.g. use_fast_destroy :build_trace_chunks def use_fast_destroy(relation) before_destroy do - subject = public_send(relation) + subject = public_send(relation) # rubocop:disable GitlabSecurity/PublicSend after_commit_proc = subject.after_commit_cleanup_proc run_after_commit(&after_commit_proc) end -- cgit v1.2.3 From d14851f9a36ce4e41e94e21d251589a3d197c423 Mon Sep 17 00:00:00 2001 From: James Edwards-Jones Date: Sun, 22 Apr 2018 22:04:56 +0100 Subject: Backport Identity.uniqueness_scope from EE Originates in 'SamlProvider has many linked Identities' from EE --- app/models/identity.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/identity.rb b/app/models/identity.rb index 1011b9f1109..3fd0c5e751d 100644 --- a/app/models/identity.rb +++ b/app/models/identity.rb @@ -1,12 +1,16 @@ class Identity < ActiveRecord::Base + def self.uniqueness_scope + :provider + end + include Sortable include CaseSensitivity belongs_to :user validates :provider, presence: true - validates :extern_uid, allow_blank: true, uniqueness: { scope: :provider, case_sensitive: false } - validates :user_id, uniqueness: { scope: :provider } + validates :extern_uid, allow_blank: true, uniqueness: { scope: uniqueness_scope, case_sensitive: false } + validates :user_id, uniqueness: { scope: uniqueness_scope } before_save :ensure_normalized_extern_uid, if: :extern_uid_changed? after_destroy :clear_user_synced_attributes, if: :user_synced_attributes_metadata_from_provider? -- cgit v1.2.3 From 2261188f48dff25c5bfbbca739c5f570849155f4 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Wed, 2 May 2018 16:42:12 +0200 Subject: Rename Runner#invalidate_build_cache -> Runner#pick_build --- app/models/ci/runner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index cdd28407172..4d4aff4c830 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -205,7 +205,7 @@ module Ci end end - def invalidate_build_cache!(build) + def pick_build!(build) if can_pick?(build) tick_runner_queue end -- cgit v1.2.3 From 0970e7b9608d6ada1c0fe45242ea092ea91068aa Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Wed, 2 May 2018 17:43:20 +0200 Subject: Rename RunnerGroup -> RunnerNamespace --- app/models/ci/runner.rb | 6 +++--- app/models/ci/runner_group.rb | 8 -------- app/models/ci/runner_namespace.rb | 9 +++++++++ app/models/group.rb | 2 -- app/models/namespace.rb | 3 +++ 5 files changed, 15 insertions(+), 13 deletions(-) delete mode 100644 app/models/ci/runner_group.rb create mode 100644 app/models/ci/runner_namespace.rb (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 4d4aff4c830..0b47b71a267 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -14,8 +14,8 @@ module Ci has_many :builds has_many :runner_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :runner_projects - has_many :runner_groups - has_many :groups, through: :runner_groups + has_many :runner_namespaces + has_many :groups, through: :runner_namespaces has_one :last_build, ->() { order('id DESC') }, class_name: 'Ci::Build' @@ -144,7 +144,7 @@ module Ci end def assigned_to_group? - runner_groups.any? + runner_namespaces.any? end def assigned_to_project? diff --git a/app/models/ci/runner_group.rb b/app/models/ci/runner_group.rb deleted file mode 100644 index 87f3ba13bff..00000000000 --- a/app/models/ci/runner_group.rb +++ /dev/null @@ -1,8 +0,0 @@ -module Ci - class RunnerGroup < ActiveRecord::Base - extend Gitlab::Ci::Model - - belongs_to :runner - belongs_to :group, class_name: '::Group' - end -end diff --git a/app/models/ci/runner_namespace.rb b/app/models/ci/runner_namespace.rb new file mode 100644 index 00000000000..3269f86e8ca --- /dev/null +++ b/app/models/ci/runner_namespace.rb @@ -0,0 +1,9 @@ +module Ci + class RunnerNamespace < ActiveRecord::Base + extend Gitlab::Ci::Model + + belongs_to :runner + belongs_to :namespace, class_name: '::Namespace' + belongs_to :group, class_name: '::Group', foreign_key: :namespace_id + end +end diff --git a/app/models/group.rb b/app/models/group.rb index f21008e5f75..f493836a92e 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -29,8 +29,6 @@ class Group < Namespace has_many :labels, class_name: 'GroupLabel' has_many :variables, class_name: 'Ci::GroupVariable' has_many :custom_attributes, class_name: 'GroupCustomAttribute' - has_many :runner_groups, class_name: 'Ci::RunnerGroup' - has_many :runners, through: :runner_groups, source: :runner, class_name: 'Ci::Runner' has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent diff --git a/app/models/namespace.rb b/app/models/namespace.rb index c29a53e5ce7..5621eeba7c4 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -21,6 +21,9 @@ class Namespace < ActiveRecord::Base has_many :projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :project_statistics + has_many :runner_namespaces, class_name: 'Ci::RunnerNamespace' + has_many :runners, through: :runner_namespaces, source: :runner, class_name: 'Ci::Runner' + # This should _not_ be `inverse_of: :namespace`, because that would also set # `user.namespace` when this user creates a group with themselves as `owner`. belongs_to :owner, class_name: "User" -- cgit v1.2.3 From 45dc7ae2d22ceb2679a9c72367f3bb62051ce226 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 3 May 2018 14:57:15 +0900 Subject: Clean up build_trace_chunk --- app/models/ci/build_trace_chunk.rb | 17 +++++++++-------- app/models/concerns/fast_destroy_all.rb | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index bdba9e6ed3c..74b56ffed53 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -80,8 +80,7 @@ module Ci break unless size > 0 self.update!(raw_data: data, data_store: :db) - key = self.class.redis_data_key(build_id, chunk_index) - self.class.redis_delete_data([key]) + self.class.redis_delete_data([redis_data_key]) end end @@ -129,22 +128,24 @@ module Ci def redis_data Gitlab::Redis::SharedState.with do |redis| - redis.get(self.class.redis_data_key(build_id, chunk_index)) + redis.get(redis_data_key) end end def redis_set_data(data) Gitlab::Redis::SharedState.with do |redis| - redis.set(self.class.redis_data_key(build_id, chunk_index), data, ex: CHUNK_REDIS_TTL) + redis.set(redis_data_key, data, ex: CHUNK_REDIS_TTL) end end - def redis_lock_key - "trace_write:#{build_id}:chunks:#{chunk_index}" + def redis_data_key + self.class.redis_data_key(build_id, chunk_index) end def in_lock - lease = Gitlab::ExclusiveLease.new(redis_lock_key, timeout: WRITE_LOCK_TTL) + write_lock_key = "trace_write:#{build_id}:chunks:#{chunk_index}" + + lease = Gitlab::ExclusiveLease.new(write_lock_key, timeout: WRITE_LOCK_TTL) retry_count = 0 until uuid = lease.try_obtain @@ -159,7 +160,7 @@ module Ci self.reload if self.persisted? return yield ensure - Gitlab::ExclusiveLease.cancel(redis_lock_key, uuid) + Gitlab::ExclusiveLease.cancel(write_lock_key, uuid) end end end diff --git a/app/models/concerns/fast_destroy_all.rb b/app/models/concerns/fast_destroy_all.rb index 7639af35b69..0f6b57ff571 100644 --- a/app/models/concerns/fast_destroy_all.rb +++ b/app/models/concerns/fast_destroy_all.rb @@ -22,7 +22,7 @@ module FastDestroyAll class_attribute :_delete_method, :_delete_params_generator before_destroy do - raise ForbiddenActionError, '`destroy` is forbbiden, please use `fast_destroy_all`' + raise ForbiddenActionError, '`destroy` and `destroy_all` are forbbiden. Please use `fast_destroy_all`' end end -- cgit v1.2.3 From 18ba092aecb275b06e3482a079f0feca767624bf Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 3 May 2018 15:49:02 +0900 Subject: Fix typo for comments in fast_destroy_all --- app/models/concerns/fast_destroy_all.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/concerns/fast_destroy_all.rb b/app/models/concerns/fast_destroy_all.rb index 0f6b57ff571..5df23a74a41 100644 --- a/app/models/concerns/fast_destroy_all.rb +++ b/app/models/concerns/fast_destroy_all.rb @@ -73,7 +73,7 @@ module FastDestroyAll # This is a helper method for performaning fast_destroy_all when parent relations are deleted # Their children must include `FastDestroyAll` module. # - # `use_fast_destroy` must be defined **before** `has_many` and `has_one`, such as `has_many :relation, depenedent: :destroy` + # `use_fast_destroy` must be defined **before** `has_many` and `has_one`, such as `has_many :relation, dependent: :destroy` # Otherwise `use_fast_destroy` performs against **deleted** rows, which fails to get identifiers of external data # # e.g. use_fast_destroy :build_trace_chunks -- cgit v1.2.3 From b97d08494cab30a7b0168091f7b3fc17527cfda3 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 3 May 2018 15:50:35 +0900 Subject: Fix typo for comments in project.rb --- app/models/project.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index e98b9fa1473..dfe3a989c2c 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -81,7 +81,7 @@ class Project < ActiveRecord::Base before_destroy :remove_private_deploy_keys ## - # `use_fast_destroy` must be defined **before** `has_many` and `has_one` such as `has_many :relation, depenedent: :destroy` + # `use_fast_destroy` must be defined **before** `has_many` and `has_one` such as `has_many :relation, dependent: :destroy` # Otherwise `use_fast_destroy` performs against **deleted** rows, which fails to get identifiers of external data use_fast_destroy :build_trace_chunks -- cgit v1.2.3 From c80949fdcdb60003fa3d7dc6e1d7008c4cff5b5d Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 3 May 2018 17:08:05 +0900 Subject: Simplify FastDestroyAll module --- app/models/ci/build_trace_chunk.rb | 13 ++++- app/models/concerns/fast_destroy_all.rb | 94 +++++++++++++++++---------------- 2 files changed, 60 insertions(+), 47 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 74b56ffed53..406b4f91ce7 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -6,7 +6,6 @@ module Ci belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id default_value_for :data_store, :redis - fast_destroy_all_with :redis_delete_data, :redis_data_keys WriteError = Class.new(StandardError) @@ -39,6 +38,18 @@ module Ci redis.del(keys) end end + + ## + # FastDestroyAll concerns + def begin_fast_destroy + redis_data_keys + end + + ## + # FastDestroyAll concerns + def finalize_fast_destroy(keys) + redis_delete_data(keys) + end end ## diff --git a/app/models/concerns/fast_destroy_all.rb b/app/models/concerns/fast_destroy_all.rb index 5df23a74a41..6954db258bb 100644 --- a/app/models/concerns/fast_destroy_all.rb +++ b/app/models/concerns/fast_destroy_all.rb @@ -4,23 +4,33 @@ # In general, `destroy_all` is inefficient because it calls each callback with `DELETE` queries i.e. O(n), whereas, # `delete_all` is efficient as it deletes all rows with a single `DELETE` query. # -# It's better to use `delete_all` as much as possible, however, in the real cases, it's hard to adopt because -# in some cases, external data (in ObjectStorage/FileStorage/Redis) is assosiated with rows. +# It's better to use `delete_all` as our best practice, however, +# if external data (e.g. ObjectStorage, FileStorage or Redis) are assosiated with database records, +# it is difficult to accomplish it. # -# This module introduces a protocol to support the adoption with easy way. -# You can use in the following scnenes -# - When calling `destroy_all` explicitly -# e.g. `job.trace_chunks.fast_destroy_all`` -# - When a parent record is deleted and all children are deleted subsequently (cascade delete) -# e.g. `before_destroy -> { run_after_commit(&build_trace_chunks.after_commit_cleanup_proc) }`` +# This module defines a format to use `delete_all` and delete associated external data. +# Here is an exmaple +# +# Situation +# - `Project` has many `Ci::BuildTraceChunk` through `Ci::Build` +# - `Ci::BuildTraceChunk` stores associated data in Redis, so it relies on `dependent: :destroy` and `before_destroy` for the deletion +# +# How to use +# - Define `use_fast_destroy :build_trace_chunks` in `Project` model. +# - Define `begin_fast_destroy` and `finalize_fast_destroy(params)` in `Ci::BuildTraceChunk` model. +# - Use `fast_destroy_all` instead of `destroy` and `destroy_all` +# - Remove `dependent: :destroy` and `before_destroy` as it's no longer need +# +# Expectation +# - When a project is `destroy`ed, the associated trace_chunks will be deleted by `delete_all`, +# and the associated data will be removed, too. +# - When `fast_destroy_all` is called, it also performns as same. module FastDestroyAll extend ActiveSupport::Concern ForbiddenActionError = Class.new(StandardError) included do - class_attribute :_delete_method, :_delete_params_generator - before_destroy do raise ForbiddenActionError, '`destroy` and `destroy_all` are forbbiden. Please use `fast_destroy_all`' end @@ -28,40 +38,31 @@ module FastDestroyAll class_methods do ## - # This method is for registering :delete_method and :delete_params_generator - # You have to define the method if you want to use `FastDestroyAll`` module. + # This method delete rows and associated external data efficiently # - # e.g. fast_destroy_all_with :delete_all_redis_data, :redis_all_data_keys - def fast_destroy_all_with(delete_method, delete_params_generator) - self._delete_method = delete_method - self._delete_params_generator = delete_params_generator + # This method can replace `destroy` and `destroy_all` withtout having `after_destroy` hook + def fast_destroy_all + params = begin_fast_destroy + + delete_all + + finalize_fast_destroy(params) end ## - # This method generates a proc to delete external data. - # It's useful especially when you want to hook parent record's deletion event. + # This method returns identifiers to delete associated external data (e.g. file paths, redis keys) # - # e.g. before_destroy -> { run_after_commit(&build_trace_chunks.after_commit_cleanup_proc) } - def after_commit_cleanup_proc - params = send self._delete_params_generator # rubocop:disable GitlabSecurity/PublicSend - subject = self # Preserve the subject class, otherwise `proc` uses a different class - - proc do - subject.send subject._delete_method, params # rubocop:disable GitlabSecurity/PublicSend - - true - end + # This method must be defined in fast destroyable model + def begin_fast_destroy + raise NotImplementedError end ## - # This method deletes all rows at first and delete all external data at second. - # Before deleting the rows, it generates a proc to delete external data. - # So it won't lose the track of deleting external data, even if it happened after rows had been deleted. - def fast_destroy_all - after_commit_cleanup_proc.tap do |delete_all_external_data| - delete_all - delete_all_external_data.call - end + # This method deletes associated external data with the identifiers returned by `begin_fast_destroy` + # + # This method must be defined in fast destroyable model + def finalize_fast_destroy(params) + raise NotImplementedError end end @@ -70,20 +71,21 @@ module FastDestroyAll class_methods do ## - # This is a helper method for performaning fast_destroy_all when parent relations are deleted - # Their children must include `FastDestroyAll` module. - # - # `use_fast_destroy` must be defined **before** `has_many` and `has_one`, such as `has_many :relation, dependent: :destroy` - # Otherwise `use_fast_destroy` performs against **deleted** rows, which fails to get identifiers of external data - # - # e.g. use_fast_destroy :build_trace_chunks + # This method is to be defined on models which have fast destroyable models as children, + # and let us avoid to use `dependent: :destroy` hook def use_fast_destroy(relation) before_destroy do - subject = public_send(relation) # rubocop:disable GitlabSecurity/PublicSend - after_commit_proc = subject.after_commit_cleanup_proc - run_after_commit(&after_commit_proc) + perform_fast_destroy(public_send(relation)) # rubocop:disable GitlabSecurity/PublicSend end end end + + def perform_fast_destroy(subject) + params = subject.begin_fast_destroy + + run_after_commit do + subject.finalize_fast_destroy(params) + end + end end end -- cgit v1.2.3 From 3248ca0467a4bd816f281db1b30ca83df2865edd Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 24 Apr 2018 17:08:43 +0900 Subject: Add per-project pipeline id --- app/models/ci/pipeline.rb | 3 +++ app/models/internal_id.rb | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index e1b9bc76475..65b282ecec4 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -7,12 +7,15 @@ module Ci include Presentable include Gitlab::OptimisticLocking include Gitlab::Utils::StrongMemoize + include AtomicInternalId belongs_to :project, inverse_of: :pipelines belongs_to :user belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' + has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.pipelines.maximum(:iid) } + has_many :stages has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline diff --git a/app/models/internal_id.rb b/app/models/internal_id.rb index 189942c5ad8..dbd82dda06e 100644 --- a/app/models/internal_id.rb +++ b/app/models/internal_id.rb @@ -14,7 +14,7 @@ class InternalId < ActiveRecord::Base belongs_to :project belongs_to :namespace - enum usage: { issues: 0, merge_requests: 1, deployments: 2, milestones: 3, epics: 4 } + enum usage: { issues: 0, merge_requests: 1, deployments: 2, milestones: 3, epics: 4, ci_pipelines: 5 } validates :usage, presence: true -- cgit v1.2.3 From 8192cd70ed109e9c0b7f4670234af11f206694d4 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 25 Apr 2018 21:42:46 +0900 Subject: Expose CI_PIPELINE_IID_PER_PROJECT variable --- app/models/ci/pipeline.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 65b282ecec4..3cdb86e7b55 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -14,7 +14,7 @@ module Ci belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' - has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.pipelines.maximum(:iid) } + has_internal_id :iid_per_project, scope: :project, init: ->(s) { s&.project&.pipelines.count } has_many :stages has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline @@ -492,6 +492,7 @@ module Ci def predefined_variables Gitlab::Ci::Variables::Collection.new .append(key: 'CI_PIPELINE_ID', value: id.to_s) + .append(key: 'CI_PIPELINE_IID_PER_PROJECT', value: iid_per_project.to_s) .append(key: 'CI_CONFIG_PATH', value: ci_yaml_file_path) .append(key: 'CI_PIPELINE_SOURCE', value: source.to_s) end -- cgit v1.2.3 From 787eddd55d3699c73a69c99eb66e6a4b3b63b330 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 25 Apr 2018 22:00:11 +0900 Subject: Revert column name change --- app/models/ci/pipeline.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 3cdb86e7b55..efab6f4283a 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -14,7 +14,7 @@ module Ci belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' - has_internal_id :iid_per_project, scope: :project, init: ->(s) { s&.project&.pipelines.count } + has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.pipelines.count } has_many :stages has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline @@ -492,7 +492,7 @@ module Ci def predefined_variables Gitlab::Ci::Variables::Collection.new .append(key: 'CI_PIPELINE_ID', value: id.to_s) - .append(key: 'CI_PIPELINE_IID_PER_PROJECT', value: iid_per_project.to_s) + .append(key: 'CI_PIPELINE_IID_PER_PROJECT', value: iid.to_s) .append(key: 'CI_CONFIG_PATH', value: ci_yaml_file_path) .append(key: 'CI_PIPELINE_SOURCE', value: source.to_s) end -- cgit v1.2.3 From 58f229af992a9481c9eee3165dc5d14eb1dc7d00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Thu, 3 May 2018 10:48:23 +0200 Subject: Make Atomic Internal ID work for pipelines --- app/models/ci/pipeline.rb | 4 +++- app/models/concerns/atomic_internal_id.rb | 18 +++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index efab6f4283a..0622b9c6918 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -14,7 +14,9 @@ module Ci belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' - has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.pipelines.count } + has_internal_id :iid, scope: :project, presence: false, to_param: false, init: -> do |s| + s&.project&.pipelines&.maximum(:iid) || s&.project&.pipelines.count + end has_many :stages has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb index 22f516a172f..709589a9128 100644 --- a/app/models/concerns/atomic_internal_id.rb +++ b/app/models/concerns/atomic_internal_id.rb @@ -25,9 +25,13 @@ module AtomicInternalId extend ActiveSupport::Concern module ClassMethods - def has_internal_id(column, scope:, init:) # rubocop:disable Naming/PredicateName - before_validation(on: :create) do + def has_internal_id(column, scope:, init:, presence: true, to_param: true) # rubocop:disable Naming/PredicateName + before_validation :"ensure_#{column}!", on: :create + validates column, presence: presence, numericality: true + + define_method("ensure_#{column}!") do scope_value = association(scope).reader + if read_attribute(column).blank? && scope_value scope_attrs = { scope_value.class.table_name.singularize.to_sym => scope_value } usage = self.class.table_name.to_sym @@ -35,13 +39,13 @@ module AtomicInternalId new_iid = InternalId.generate_next(self, scope_attrs, usage, init) write_attribute(column, new_iid) end + + read_attribute(column) end - validates column, presence: true, numericality: true + define_method("to_param") do + read_attribute(column) + end if to_param end end - - def to_param - iid.to_s - end end -- cgit v1.2.3 From 07d1d8bd6730015e65bd5123f305bf35b4839237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Thu, 3 May 2018 10:50:57 +0200 Subject: Use **CI_PIPELINE_IID** --- app/models/ci/pipeline.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 0622b9c6918..84c5dae24bb 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -494,7 +494,7 @@ module Ci def predefined_variables Gitlab::Ci::Variables::Collection.new .append(key: 'CI_PIPELINE_ID', value: id.to_s) - .append(key: 'CI_PIPELINE_IID_PER_PROJECT', value: iid.to_s) + .append(key: 'CI_PIPELINE_IID', value: iid.to_s) .append(key: 'CI_CONFIG_PATH', value: ci_yaml_file_path) .append(key: 'CI_PIPELINE_SOURCE', value: source.to_s) end -- cgit v1.2.3 From 5a7fbb8d89404f27a4467db7f2c619cc56f91142 Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Thu, 3 May 2018 08:52:51 +0000 Subject: [Rails5] Fix params passed to MergeWorker --- app/models/merge_request.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 8f964a488aa..63c6ada86e1 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -323,7 +323,7 @@ class MergeRequest < ActiveRecord::Base # updates `merge_jid` with the MergeWorker#jid. # This helps tracking enqueued and ongoing merge jobs. def merge_async(user_id, params) - jid = MergeWorker.perform_async(id, user_id, params) + jid = MergeWorker.perform_async(id, user_id, params.to_h) update_column(:merge_jid, jid) end -- cgit v1.2.3 From 49cbe576229b7e4003575e04006cc4132c3c0060 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Thu, 3 May 2018 10:59:38 +0200 Subject: Remove Runner#belonging_to_any_project since this is no longer needed --- app/models/ci/runner.rb | 2 -- 1 file changed, 2 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 0b47b71a267..2dfd038d5a8 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -32,8 +32,6 @@ module Ci joins(:runner_projects).where(ci_runner_projects: { project_id: project_id }) } - scope :belonging_to_any_project, -> { joins(:runner_projects) } - scope :belonging_to_parent_group_of_project, -> (project_id) { project_groups = ::Group.joins(:projects).where(projects: { id: project_id }) hierarchy_groups = Gitlab::GroupHierarchy.new(project_groups).base_and_ancestors -- cgit v1.2.3 From dffc2cb7a33942173e7f49ac704f0cc211ca8048 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 3 May 2018 20:12:25 +0900 Subject: Fix typo in fast_destroy_all --- app/models/concerns/fast_destroy_all.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/concerns/fast_destroy_all.rb b/app/models/concerns/fast_destroy_all.rb index 6954db258bb..367a2df854f 100644 --- a/app/models/concerns/fast_destroy_all.rb +++ b/app/models/concerns/fast_destroy_all.rb @@ -40,7 +40,7 @@ module FastDestroyAll ## # This method delete rows and associated external data efficiently # - # This method can replace `destroy` and `destroy_all` withtout having `after_destroy` hook + # This method can replace `destroy` and `destroy_all` without having `after_destroy` hook def fast_destroy_all params = begin_fast_destroy -- cgit v1.2.3 From af15b6f0e144762a38591c53b970e312c35fe65f Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Thu, 3 May 2018 14:37:01 +0200 Subject: Fix Project#group_runners_enabled as it was doing nothing --- app/models/project.rb | 7 ++++++- app/models/project_ci_cd_setting.rb | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 8abbb92da62..50c404c300a 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -234,7 +234,7 @@ class Project < ActiveRecord::Base has_many :custom_attributes, class_name: 'ProjectCustomAttribute' has_many :project_badges, class_name: 'ProjectBadge' - has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting' + has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature, update_only: true @@ -331,6 +331,11 @@ class Project < ActiveRecord::Base scope :with_issues_available_for_user, ->(current_user) { with_feature_available_for_user(:issues, current_user) } scope :with_merge_requests_enabled, -> { with_feature_enabled(:merge_requests) } + scope :with_group_runners_enabled, -> do + joins(:ci_cd_settings) + .where(project_ci_cd_settings: { group_runners_enabled: true }) + end + enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 } chronic_duration_attr :build_timeout_human_readable, :build_timeout, default: 3600 diff --git a/app/models/project_ci_cd_setting.rb b/app/models/project_ci_cd_setting.rb index 9f10a93148c..588cced5781 100644 --- a/app/models/project_ci_cd_setting.rb +++ b/app/models/project_ci_cd_setting.rb @@ -1,5 +1,5 @@ class ProjectCiCdSetting < ActiveRecord::Base - belongs_to :project + belongs_to :project, inverse_of: :ci_cd_settings # The version of the schema that first introduced this model/table. MINIMUM_SCHEMA_VERSION = 20180403035759 -- cgit v1.2.3 From 15b10c344b1b909b2a90331926aa4082cd86045b Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Thu, 3 May 2018 17:15:32 +0200 Subject: Dont remove duplicates in Runner.owned_or_shared since its not necessary --- app/models/ci/runner.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 2dfd038d5a8..23078f1c3ed 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -40,7 +40,10 @@ module Ci } scope :owned_or_shared, -> (project_id) do - union = Gitlab::SQL::Union.new([belonging_to_project(project_id), belonging_to_parent_group_of_project(project_id), shared]) + union = Gitlab::SQL::Union.new( + [belonging_to_project(project_id), belonging_to_parent_group_of_project(project_id), shared], + remove_duplicates: false + ) from("(#{union.to_sql}) ci_runners") end -- cgit v1.2.3 From 1019dff2371b30979b33ce823abeadadad9cfab3 Mon Sep 17 00:00:00 2001 From: Chantal Rollison Date: Wed, 2 May 2018 14:34:05 -0700 Subject: Backport of 1481-changing-weight-values-should-trigger-system-notes --- app/models/system_note_metadata.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb index 29035480371..1c2161accc4 100644 --- a/app/models/system_note_metadata.rb +++ b/app/models/system_note_metadata.rb @@ -17,7 +17,11 @@ class SystemNoteMetadata < ActiveRecord::Base ].freeze validates :note, presence: true - validates :action, inclusion: ICON_TYPES, allow_nil: true + validates :action, inclusion: { in: :icon_types }, allow_nil: true belongs_to :note + + def icon_types + ICON_TYPES + end end -- cgit v1.2.3 From 812dd06d512ab7774b375ce45aa9235aafc99911 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 4 May 2018 17:02:08 +0900 Subject: Introduce Redis helpers. Rename BuildTraceChunkFlushToDbWorker to Ci::BuildTraceChunkFlushWorker. --- app/models/ci/build_trace_chunk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index ee999a3395d..870b4ae2033 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -99,7 +99,7 @@ module Ci def schedule_to_db return if db? - BuildTraceChunkFlushToDbWorker.perform_async(id) + Ci::BuildTraceChunkFlushWorker.perform_async(id) end def fullfilled? -- cgit v1.2.3 From 28284c14973a59d5a7f0f8d2862da7f61b101640 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 4 May 2018 17:42:37 +0900 Subject: Add validation and skip logic at #truncate --- app/models/ci/build_trace_chunk.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 870b4ae2033..08a4465821c 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -28,11 +28,14 @@ module Ci end def truncate(offset = 0) - self.append("", offset) if offset < size + raise ArgumentError, 'Offset is out of range' if offset > size || offset < 0 + return if offset == size # Skip the following process as it doesn't affect anything + + self.append("", offset) end def append(new_data, offset) - raise ArgumentError, 'Offset is out of range' if offset > data.bytesize || offset < 0 + raise ArgumentError, 'Offset is out of range' if offset > size || offset < 0 raise ArgumentError, 'Chunk size overflow' if CHUNK_SIZE < (offset + new_data.bytesize) set_data(data.byteslice(0, offset) + new_data) -- cgit v1.2.3 From ecf8287ce5f81aafefaa7b505f620c22527515d9 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 4 May 2018 20:47:23 +0900 Subject: Use prepend: true to evaluate always perform_fast_destroy than dependent: :destroy --- app/models/concerns/fast_destroy_all.rb | 2 +- app/models/project.rb | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/fast_destroy_all.rb b/app/models/concerns/fast_destroy_all.rb index 367a2df854f..7ea042c6742 100644 --- a/app/models/concerns/fast_destroy_all.rb +++ b/app/models/concerns/fast_destroy_all.rb @@ -74,7 +74,7 @@ module FastDestroyAll # This method is to be defined on models which have fast destroyable models as children, # and let us avoid to use `dependent: :destroy` hook def use_fast_destroy(relation) - before_destroy do + before_destroy(prepend: true) do perform_fast_destroy(public_send(relation)) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/app/models/project.rb b/app/models/project.rb index dfe3a989c2c..e48ecea1053 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -80,9 +80,6 @@ class Project < ActiveRecord::Base before_destroy :remove_private_deploy_keys - ## - # `use_fast_destroy` must be defined **before** `has_many` and `has_one` such as `has_many :relation, dependent: :destroy` - # Otherwise `use_fast_destroy` performs against **deleted** rows, which fails to get identifiers of external data use_fast_destroy :build_trace_chunks after_destroy -> { run_after_commit { remove_pages } } -- cgit v1.2.3 From cf37bef287d7dd5d2dce3e2276489767b8c0671f Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Tue, 24 Apr 2018 11:37:41 +0200 Subject: Add `Term` model to keep track of terms That way we can link a users acceptance of terms directly to a terms record. --- app/models/application_setting/term.rb | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 app/models/application_setting/term.rb (limited to 'app/models') diff --git a/app/models/application_setting/term.rb b/app/models/application_setting/term.rb new file mode 100644 index 00000000000..1f3d20e2b75 --- /dev/null +++ b/app/models/application_setting/term.rb @@ -0,0 +1,5 @@ +class ApplicationSetting + class Term < ActiveRecord::Base + validates :terms, presence: true + end +end -- cgit v1.2.3 From 3d6d0a09b65f032bbe1bd5ad4736dd764195bbe1 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Tue, 24 Apr 2018 18:28:04 +0200 Subject: Store application wide terms This allows admins to define terms in the application settings. Every time the terms are adjusted, a new version is stored and becomes the 'active' version. This allows tracking which specific version was accepted by a user. --- app/models/application_setting.rb | 19 +++++++++++++++++++ app/models/application_setting/term.rb | 8 ++++++++ 2 files changed, 27 insertions(+) (limited to 'app/models') diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 862933bf127..a734cc7a26b 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -220,12 +220,15 @@ class ApplicationSetting < ActiveRecord::Base end end + validate :terms_exist, if: :enforce_terms? + before_validation :ensure_uuid! before_save :ensure_runners_registration_token before_save :ensure_health_check_access_token after_commit do + reset_memoized_terms Rails.cache.write(CACHE_KEY, self) end @@ -507,6 +510,16 @@ class ApplicationSetting < ActiveRecord::Base password_authentication_enabled_for_web? || password_authentication_enabled_for_git? end + delegate :terms, to: :latest_terms, allow_nil: true + def latest_terms + @latest_terms ||= Term.latest + end + + def reset_memoized_terms + @latest_terms = nil + latest_terms + end + private def ensure_uuid! @@ -520,4 +533,10 @@ class ApplicationSetting < ActiveRecord::Base errors.add(:repository_storages, "can't include: #{invalid.join(", ")}") unless invalid.empty? end + + def terms_exist + return unless enforce_terms? + + errors.add(:terms, "You need to set terms to be enforced") unless terms.present? + end end diff --git a/app/models/application_setting/term.rb b/app/models/application_setting/term.rb index 1f3d20e2b75..e8ce0ccbb71 100644 --- a/app/models/application_setting/term.rb +++ b/app/models/application_setting/term.rb @@ -1,5 +1,13 @@ class ApplicationSetting class Term < ActiveRecord::Base + include CacheMarkdownField + validates :terms, presence: true + + cache_markdown_field :terms + + def self.latest + order(:id).last + end end end -- cgit v1.2.3 From 82eeb72c8c03727540b902d40e7e657d0a5ecb4c Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Wed, 25 Apr 2018 10:55:53 +0200 Subject: Add model to track users accepting agreements --- app/models/term_agreement.rb | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 app/models/term_agreement.rb (limited to 'app/models') diff --git a/app/models/term_agreement.rb b/app/models/term_agreement.rb new file mode 100644 index 00000000000..8458a231bbd --- /dev/null +++ b/app/models/term_agreement.rb @@ -0,0 +1,6 @@ +class TermAgreement < ActiveRecord::Base + belongs_to :term, class_name: 'ApplicationSetting::Term' + belongs_to :user + + validates :user, :term, presence: true +end -- cgit v1.2.3 From 10aa55a770c2985c22c92d17b8a7ea90b0a09085 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Fri, 27 Apr 2018 14:56:50 +0200 Subject: Allow a user to accept/decline terms When a user accepts, we store this in the agreements to keep track of which terms they accepted. We also update the flag on the user. --- app/models/user.rb | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index 4a602ffbb05..a9cfd39f604 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -138,6 +138,8 @@ class User < ActiveRecord::Base has_many :custom_attributes, class_name: 'UserCustomAttribute' has_many :callouts, class_name: 'UserCallout' has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :term_agreements + belongs_to :accepted_term, class_name: 'ApplicationSetting::Term' # # Validations @@ -1187,6 +1189,10 @@ class User < ActiveRecord::Base max_member_access_for_group_ids([group_id])[group_id] end + def terms_accepted? + accepted_term_id.present? + end + protected # override, from Devise::Validatable -- cgit v1.2.3 From bc7ea2d4386f5624676e7a2ffb196de19d52910b Mon Sep 17 00:00:00 2001 From: James Lopez Date: Fri, 4 May 2018 12:42:58 +0000 Subject: Add ci_cd_settings delete_all dependency on project --- app/models/project.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 50c404c300a..eb092ee742d 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -234,7 +234,7 @@ class Project < ActiveRecord::Base has_many :custom_attributes, class_name: 'ProjectCustomAttribute' has_many :project_badges, class_name: 'ProjectBadge' - has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true + has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature, update_only: true -- cgit v1.2.3 From bddbcaefc2389e4c61763472cecbea150f10cd75 Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Wed, 2 May 2018 14:35:04 +0100 Subject: Backports every CE related change from ee-44542 to CE --- app/models/project.rb | 154 ++++++++++++++++++++++++------------- app/models/project_import_state.rb | 55 +++++++++++++ 2 files changed, 154 insertions(+), 55 deletions(-) create mode 100644 app/models/project_import_state.rb (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 50c404c300a..41cad31ecf1 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -67,6 +67,9 @@ class Project < ActiveRecord::Base before_save :ensure_runners_token after_save :update_project_statistics, if: :namespace_id_changed? + + after_save :create_import_state, if: ->(project) { project.import? && project.import_state.nil? } + after_create :create_project_feature, unless: :project_feature after_create :create_ci_cd_settings, @@ -157,6 +160,8 @@ class Project < ActiveRecord::Base has_one :fork_network_member has_one :fork_network, through: :fork_network_member + has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project + # Merge Requests for target project should be removed with it has_many :merge_requests, foreign_key: 'target_project_id' has_many :source_of_merge_requests, foreign_key: 'source_project_id', class_name: 'MergeRequest' @@ -385,55 +390,9 @@ class Project < ActiveRecord::Base scope :abandoned, -> { where('projects.last_activity_at < ?', 6.months.ago) } scope :excluding_project, ->(project) { where.not(id: project) } - scope :import_started, -> { where(import_status: 'started') } - - state_machine :import_status, initial: :none do - event :import_schedule do - transition [:none, :finished, :failed] => :scheduled - end - - event :force_import_start do - transition [:none, :finished, :failed] => :started - end - - event :import_start do - transition scheduled: :started - end - - event :import_finish do - transition started: :finished - end - - event :import_fail do - transition [:scheduled, :started] => :failed - end - - event :import_retry do - transition failed: :started - end - - state :scheduled - state :started - state :finished - state :failed - - after_transition [:none, :finished, :failed] => :scheduled do |project, _| - project.run_after_commit do - job_id = add_import_job - update(import_jid: job_id) if job_id - end - end - after_transition started: :finished do |project, _| - project.reset_cache_and_import_attrs - - if Gitlab::ImportSources.importer_names.include?(project.import_type) && project.repo_exists? - project.run_after_commit do - Projects::AfterImportService.new(project).execute - end - end - end - end + scope :joins_import_state, -> { joins("LEFT JOIN project_mirror_data import_state ON import_state.project_id = projects.id") } + scope :import_started, -> { joins_import_state.where("import_state.status = 'started' OR projects.import_status = 'started'") } class << self # Searches for a list of projects based on the query given in `query`. @@ -663,10 +622,6 @@ class Project < ActiveRecord::Base external_import? || forked? || gitlab_project_import? || bare_repository_import? end - def no_import? - import_status == 'none' - end - def external_import? import_url.present? end @@ -679,6 +634,93 @@ class Project < ActiveRecord::Base import_started? || import_scheduled? end + def import_state_args + { + status: self[:import_status], + jid: self[:import_jid], + last_error: self[:import_error] + } + end + + def ensure_import_state + return if self[:import_status] == 'none' || self[:import_status].nil? + return unless import_state.nil? + + create_import_state(import_state_args) + + update_column(:import_status, 'none') + end + + def import_schedule + ensure_import_state + + import_state&.schedule + end + + def force_import_start + ensure_import_state + + import_state&.force_start + end + + def import_start + ensure_import_state + + import_state&.start + end + + def import_fail + ensure_import_state + + import_state&.fail_op + end + + def import_finish + ensure_import_state + + import_state&.finish + end + + def import_jid=(new_jid) + ensure_import_state + + import_state&.jid = new_jid + end + + def import_jid + ensure_import_state + + import_state&.jid + end + + def import_error=(new_error) + ensure_import_state + + import_state&.last_error = new_error + end + + def import_error + ensure_import_state + + import_state&.last_error + end + + def import_status=(new_status) + ensure_import_state + + import_state&.status = new_status + end + + def import_status + ensure_import_state + + import_state&.status || 'none' + end + + def no_import? + import_status == 'none' + end + def import_started? # import? does SQL work so only run it if it looks like there's an import running import_status == 'started' && import? @@ -1480,7 +1522,7 @@ class Project < ActiveRecord::Base def rename_repo_notify! # When we import a project overwriting the original project, there # is a move operation. In that case we don't want to send the instructions. - send_move_instructions(full_path_was) unless started? + send_move_instructions(full_path_was) unless import_started? expires_full_path_cache self.old_path_with_namespace = full_path_was @@ -1534,7 +1576,8 @@ class Project < ActiveRecord::Base return unless import_jid Gitlab::SidekiqStatus.unset(import_jid) - update_column(:import_jid, nil) + + import_state.update_column(:jid, nil) end def running_or_pending_build_count(force: false) @@ -1553,7 +1596,8 @@ class Project < ActiveRecord::Base sanitized_message = Gitlab::UrlSanitizer.sanitize(error_message) import_fail - update_column(:import_error, sanitized_message) + + import_state.update_column(:last_error, sanitized_message) rescue ActiveRecord::ActiveRecordError => e Rails.logger.error("Error setting import status to failed: #{e.message}. Original error: #{sanitized_message}") ensure diff --git a/app/models/project_import_state.rb b/app/models/project_import_state.rb new file mode 100644 index 00000000000..1605317ae14 --- /dev/null +++ b/app/models/project_import_state.rb @@ -0,0 +1,55 @@ +class ProjectImportState < ActiveRecord::Base + include AfterCommitQueue + + self.table_name = "project_mirror_data" + + belongs_to :project, inverse_of: :import_state + + validates :project, presence: true + + state_machine :status, initial: :none do + event :schedule do + transition [:none, :finished, :failed] => :scheduled + end + + event :force_start do + transition [:none, :finished, :failed] => :started + end + + event :start do + transition scheduled: :started + end + + event :finish do + transition started: :finished + end + + event :fail_op do + transition [:scheduled, :started] => :failed + end + + state :scheduled + state :started + state :finished + state :failed + + after_transition [:none, :finished, :failed] => :scheduled do |state, _| + state.run_after_commit do + job_id = project.add_import_job + update(jid: job_id) if job_id + end + end + + after_transition started: :finished do |state, _| + project = state.project + + project.reset_cache_and_import_attrs + + if Gitlab::ImportSources.importer_names.include?(project.import_type) && project.repo_exists? + state.run_after_commit do + Projects::AfterImportService.new(project).execute + end + end + end + end +end -- cgit v1.2.3 From 5c2078838bb9de710f9025513c4b6ec664bba313 Mon Sep 17 00:00:00 2001 From: Mario de la Ossa Date: Thu, 26 Apr 2018 13:53:13 -0600 Subject: Backport of 4084-epics-username-autocomplete --- app/models/ability.rb | 8 ++++++++ app/models/concerns/participable.rb | 4 ++++ app/models/group.rb | 31 +++++++++++++++++++++++++++++++ app/models/namespace.rb | 7 +++++++ 4 files changed, 50 insertions(+) (limited to 'app/models') diff --git a/app/models/ability.rb b/app/models/ability.rb index 618d4af4272..bb600eaccba 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -10,6 +10,14 @@ class Ability end end + # Given a list of users and a group this method returns the users that can + # read the given group. + def users_that_can_read_group(users, group) + DeclarativePolicy.subject_scope do + users.select { |u| allowed?(u, :read_group, group) } + end + end + # Given a list of users and a snippet this method returns the users that can # read the given snippet. def users_that_can_read_personal_snippet(users, snippet) diff --git a/app/models/concerns/participable.rb b/app/models/concerns/participable.rb index e48bc0be410..01b1ef9f82c 100644 --- a/app/models/concerns/participable.rb +++ b/app/models/concerns/participable.rb @@ -98,6 +98,10 @@ module Participable participants.merge(ext.users) + filter_by_ability(participants) + end + + def filter_by_ability(participants) case self when PersonalSnippet Ability.users_that_can_read_personal_snippet(participants.to_a, self) diff --git a/app/models/group.rb b/app/models/group.rb index 9b42bbf99be..66b4398858e 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -238,6 +238,13 @@ class Group < Namespace .where(source_id: self_and_descendants.reorder(nil).select(:id)) end + # Returns all members that are part of the group, it's subgroups, and ancestor groups + def direct_and_indirect_members + GroupMember + .active_without_invites_and_requests + .where(source_id: self_and_hierarchy.reorder(nil).select(:id)) + end + def users_with_parents User .where(id: members_with_parents.select(:user_id)) @@ -250,6 +257,30 @@ class Group < Namespace .reorder(nil) end + # Returns all users that are members of the group because: + # 1. They belong to the group + # 2. They belong to a project that belongs to the group + # 3. They belong to a sub-group or project in such sub-group + # 4. They belong to an ancestor group + def direct_and_indirect_users + union = Gitlab::SQL::Union.new([ + User + .where(id: direct_and_indirect_members.select(:user_id)) + .reorder(nil), + project_users_with_descendants + ]) + + User.from("(#{union.to_sql}) #{User.table_name}") + end + + # Returns all users that are members of projects + # belonging to the current group or sub-groups + def project_users_with_descendants + User + .joins(projects: :group) + .where(namespaces: { id: self_and_descendants.select(:id) }) + end + def max_member_access_for_user(user) return GroupMember::OWNER if user.admin? diff --git a/app/models/namespace.rb b/app/models/namespace.rb index c29a53e5ce7..637bcbc7a09 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -163,6 +163,13 @@ class Namespace < ActiveRecord::Base projects.with_shared_runners.any? end + # Returns all ancestors, self, and descendants of the current namespace. + def self_and_hierarchy + Gitlab::GroupHierarchy + .new(self.class.where(id: id)) + .all_groups + end + # Returns all the ancestors of the current namespaces. def ancestors return self.class.none unless parent_id -- cgit v1.2.3 From 2a03142c948f65311fd784d620c2d2082882bcd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Mon, 7 May 2018 10:34:47 +0200 Subject: Optimise write lock parameters --- app/models/ci/build_trace_chunk.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 08a4465821c..5715477266a 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -12,9 +12,9 @@ module Ci CHUNK_SIZE = 128.kilobytes CHUNK_REDIS_TTL = 1.week - WRITE_LOCK_RETRY = 100 - WRITE_LOCK_SLEEP = 1 - WRITE_LOCK_TTL = 5.minutes + WRITE_LOCK_RETRY = 10 + WRITE_LOCK_SLEEP = 5.milisecond + WRITE_LOCK_TTL = 1.minute enum data_store: { redis: 1, @@ -96,7 +96,7 @@ module Ci save! if changed? end - schedule_to_db if fullfilled? + schedule_to_db if full? end def schedule_to_db @@ -105,7 +105,7 @@ module Ci Ci::BuildTraceChunkFlushWorker.perform_async(id) end - def fullfilled? + def full? size == CHUNK_SIZE end -- cgit v1.2.3 From d6d25d2da67fce3bb1392317f4f08f5423ee687d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Mon, 7 May 2018 11:05:26 +0200 Subject: Fix WRITE_LOCK_SLEEP --- app/models/ci/build_trace_chunk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 5715477266a..779daa5fa45 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -13,7 +13,7 @@ module Ci CHUNK_SIZE = 128.kilobytes CHUNK_REDIS_TTL = 1.week WRITE_LOCK_RETRY = 10 - WRITE_LOCK_SLEEP = 5.milisecond + WRITE_LOCK_SLEEP = 0.01.second WRITE_LOCK_TTL = 1.minute enum data_store: { -- cgit v1.2.3 From e1d11cc64970d712352de0c5daadced7f274ea3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Mon, 7 May 2018 11:45:38 +0200 Subject: Fix rubocop --- app/models/ci/build_trace_chunk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 779daa5fa45..d9e923daa71 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -13,7 +13,7 @@ module Ci CHUNK_SIZE = 128.kilobytes CHUNK_REDIS_TTL = 1.week WRITE_LOCK_RETRY = 10 - WRITE_LOCK_SLEEP = 0.01.second + WRITE_LOCK_SLEEP = 0.01.seconds WRITE_LOCK_TTL = 1.minute enum data_store: { -- cgit v1.2.3 From 9a13059332a0c81b3a953f57bb9e40346eba951d Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Thu, 3 May 2018 13:55:14 +0100 Subject: Backports every CE related change from ee-5484 to CE --- app/models/project.rb | 41 +++++++++ app/models/remote_mirror.rb | 216 ++++++++++++++++++++++++++++++++++++++++++++ app/models/repository.rb | 14 +++ 3 files changed, 271 insertions(+) create mode 100644 app/models/remote_mirror.rb (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index b12b694aabd..4021e4e85bf 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -64,6 +64,9 @@ class Project < ActiveRecord::Base default_value_for :only_allow_merge_if_all_discussions_are_resolved, false add_authentication_token_field :runners_token + + before_validation :mark_remote_mirrors_for_removal + before_save :ensure_runners_token after_save :update_project_statistics, if: :namespace_id_changed? @@ -241,11 +244,17 @@ class Project < ActiveRecord::Base has_many :project_badges, class_name: 'ProjectBadge' has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :remote_mirrors, inverse_of: :project + accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature, update_only: true accepts_nested_attributes_for :import_data accepts_nested_attributes_for :auto_devops, update_only: true + accepts_nested_attributes_for :remote_mirrors, + allow_destroy: true, + reject_if: ->(attrs) { attrs[:id].blank? && attrs[:url].blank? } + delegate :name, to: :owner, allow_nil: true, prefix: true delegate :members, to: :team, prefix: true delegate :add_user, :add_users, to: :team @@ -335,6 +344,7 @@ class Project < ActiveRecord::Base scope :with_issues_enabled, -> { with_feature_enabled(:issues) } scope :with_issues_available_for_user, ->(current_user) { with_feature_available_for_user(:issues, current_user) } scope :with_merge_requests_enabled, -> { with_feature_enabled(:merge_requests) } + scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct } scope :with_group_runners_enabled, -> do joins(:ci_cd_settings) @@ -754,6 +764,37 @@ class Project < ActiveRecord::Base import_type == 'gitea' end + def has_remote_mirror? + remote_mirror_available? && remote_mirrors.enabled.exists? + end + + def updating_remote_mirror? + remote_mirrors.enabled.started.exists? + end + + def update_remote_mirrors + return unless remote_mirror_available? + + remote_mirrors.enabled.each(&:sync) + end + + def mark_stuck_remote_mirrors_as_failed! + remote_mirrors.stuck.update_all( + update_status: :failed, + last_error: 'The remote mirror took to long to complete.', + last_update_at: Time.now + ) + end + + def mark_remote_mirrors_for_removal + remote_mirrors.each(&:mark_for_delete_if_blank_url) + end + + def remote_mirror_available? + remote_mirror_available_overridden || + ::Gitlab::CurrentSettings.mirror_available + end + def check_limit unless creator.can_create_project? || namespace.kind == 'group' projects_limit = creator.projects_limit diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb new file mode 100644 index 00000000000..b7e45875830 --- /dev/null +++ b/app/models/remote_mirror.rb @@ -0,0 +1,216 @@ +class RemoteMirror < ActiveRecord::Base + include AfterCommitQueue + + PROTECTED_BACKOFF_DELAY = 1.minute + UNPROTECTED_BACKOFF_DELAY = 5.minutes + + attr_encrypted :credentials, + key: Gitlab::Application.secrets.db_key_base, + marshal: true, + encode: true, + mode: :per_attribute_iv_and_salt, + insecure_mode: true, + algorithm: 'aes-256-cbc' + + default_value_for :only_protected_branches, true + + belongs_to :project, inverse_of: :remote_mirrors + + validates :url, presence: true, url: { protocols: %w(ssh git http https), allow_blank: true } + validates :url, addressable_url: true, if: :url_changed? + + before_save :set_new_remote_name, if: :mirror_url_changed? + + after_save :set_override_remote_mirror_available, unless: -> { Gitlab::CurrentSettings.current_application_settings.mirror_available } + after_save :refresh_remote, if: :mirror_url_changed? + after_update :reset_fields, if: :mirror_url_changed? + + after_commit :remove_remote, on: :destroy + + scope :enabled, -> { where(enabled: true) } + scope :started, -> { with_update_status(:started) } + scope :stuck, -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.day.ago, 1.day.ago) } + + state_machine :update_status, initial: :none do + event :update_start do + transition [:none, :finished, :failed] => :started + end + + event :update_finish do + transition started: :finished + end + + event :update_fail do + transition started: :failed + end + + state :started + state :finished + state :failed + + after_transition any => :started do |remote_mirror, _| + Gitlab::Metrics.add_event(:remote_mirrors_running, path: remote_mirror.project.full_path) + + remote_mirror.update(last_update_started_at: Time.now) + end + + after_transition started: :finished do |remote_mirror, _| + Gitlab::Metrics.add_event(:remote_mirrors_finished, path: remote_mirror.project.full_path) + + timestamp = Time.now + remote_mirror.update_attributes!( + last_update_at: timestamp, last_successful_update_at: timestamp, last_error: nil + ) + end + + after_transition started: :failed do |remote_mirror, _| + Gitlab::Metrics.add_event(:remote_mirrors_failed, path: remote_mirror.project.full_path) + + remote_mirror.update(last_update_at: Time.now) + end + end + + def remote_name + super || fallback_remote_name + end + + def update_failed? + update_status == 'failed' + end + + def update_in_progress? + update_status == 'started' + end + + def update_repository(options) + raw.update(options) + end + + def sync + return unless enabled? + return if Gitlab::Geo.secondary? + + if recently_scheduled? + RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now) + else + RepositoryUpdateRemoteMirrorWorker.perform_async(self.id, Time.now) + end + end + + def enabled + return false unless project && super + return false unless project.remote_mirror_available? + return false unless project.repository_exists? + return false if project.pending_delete? + + true + end + alias_method :enabled?, :enabled + + def updated_since?(timestamp) + last_update_started_at && last_update_started_at > timestamp && !update_failed? + end + + def mark_for_delete_if_blank_url + mark_for_destruction if url.blank? + end + + def mark_as_failed(error_message) + update_fail + update_column(:last_error, Gitlab::UrlSanitizer.sanitize(error_message)) + end + + def url=(value) + super(value) && return unless Gitlab::UrlSanitizer.valid?(value) + + mirror_url = Gitlab::UrlSanitizer.new(value) + self.credentials = mirror_url.credentials + + super(mirror_url.sanitized_url) + end + + def url + if super + Gitlab::UrlSanitizer.new(super, credentials: credentials).full_url + end + rescue + super + end + + def safe_url + return if url.nil? + + result = URI.parse(url) + result.password = '*****' if result.password + result.user = '*****' if result.user && result.user != "git" # tokens or other data may be saved as user + result.to_s + end + + private + + def raw + @raw ||= Gitlab::Git::RemoteMirror.new(project.repository.raw, remote_name) + end + + def fallback_remote_name + return unless id + + "remote_mirror_#{id}" + end + + def recently_scheduled? + return false unless self.last_update_started_at + + self.last_update_started_at >= Time.now - backoff_delay + end + + def backoff_delay + if self.only_protected_branches + PROTECTED_BACKOFF_DELAY + else + UNPROTECTED_BACKOFF_DELAY + end + end + + def reset_fields + update_columns( + last_error: nil, + last_update_at: nil, + last_successful_update_at: nil, + update_status: 'finished' + ) + end + + def set_override_remote_mirror_available + enabled = read_attribute(:enabled) + + project.update(remote_mirror_available_overridden: enabled) + end + + def set_new_remote_name + self.remote_name = "remote_mirror_#{SecureRandom.hex}" + end + + def refresh_remote + return unless project + + # Before adding a new remote we have to delete the data from + # the previous remote name + prev_remote_name = remote_name_was || fallback_remote_name + run_after_commit do + project.repository.async_remove_remote(prev_remote_name) + end + + project.repository.add_remote(remote_name, url) + end + + def remove_remote + return unless project # could be pending to delete so don't need to touch the git repository + + project.repository.async_remove_remote(remote_name) + end + + def mirror_url_changed? + url_changed? || encrypted_credentials_changed? + end +end diff --git a/app/models/repository.rb b/app/models/repository.rb index 6831305fb93..2a0802482f5 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -861,6 +861,20 @@ class Repository gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) end + def async_remove_remote(remote_name) + return unless remote_name + + job_id = RepositoryRemoveRemoteWorker.perform_async(project.id, remote_name) + + if job_id + Rails.logger.info("Remove remote job scheduled for #{project.id} with remote name: #{remote_name} job ID #{job_id}.") + else + Rails.logger.info("Remove remote job failed to create for #{project.id} with remote name #{remote_name}.") + end + + job_id + end + def fetch_source_branch!(source_repository, source_branch, local_ref) raw_repository.fetch_source_branch!(source_repository.raw_repository, source_branch, local_ref) end -- cgit v1.2.3 From 961255b107370a1350f91d0835cf0e849d237f7d Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Thu, 3 May 2018 15:19:21 +0100 Subject: Adds remote mirror table migration --- app/models/application_setting.rb | 3 ++- app/models/project.rb | 2 +- app/models/remote_mirror.rb | 7 +++++-- app/models/repository.rb | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index a734cc7a26b..451e512aef7 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -334,7 +334,8 @@ class ApplicationSetting < ActiveRecord::Base gitaly_timeout_fast: 10, gitaly_timeout_medium: 30, gitaly_timeout_default: 55, - allow_local_requests_from_hooks_and_services: false + allow_local_requests_from_hooks_and_services: false, + mirror_available: true } end diff --git a/app/models/project.rb b/app/models/project.rb index 4021e4e85bf..3ad0d4f0c4c 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -65,7 +65,7 @@ class Project < ActiveRecord::Base add_authentication_token_field :runners_token - before_validation :mark_remote_mirrors_for_removal + before_validation :mark_remote_mirrors_for_removal, if: -> { ActiveRecord::Base.connection.table_exists?(:remote_mirrors) } before_save :ensure_runners_token diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb index b7e45875830..3b9fd6d710f 100644 --- a/app/models/remote_mirror.rb +++ b/app/models/remote_mirror.rb @@ -86,9 +86,12 @@ class RemoteMirror < ActiveRecord::Base raw.update(options) end + def sync? + !enabled? + end + def sync - return unless enabled? - return if Gitlab::Geo.secondary? + return if sync? if recently_scheduled? RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now) diff --git a/app/models/repository.rb b/app/models/repository.rb index 2a0802482f5..b75c4aca982 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -854,7 +854,7 @@ class Repository add_remote(remote_name, url, mirror_refmap: refmap) fetch_remote(remote_name, forced: forced, prune: prune) ensure - remove_remote(remote_name) if tmp_remote_name + async_remove_remote(remote_name) if tmp_remote_name end def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false, prune: true) -- cgit v1.2.3 From d12a29911771e794202ffc9d9ab64fddacfbd485 Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Mon, 7 May 2018 11:52:53 +0200 Subject: Adds changelog entry, changes RemoteMirror#sync? to be semantically sound and remove reference to pull mirrors in view --- app/models/remote_mirror.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb index 3b9fd6d710f..bbf8fd9c6a7 100644 --- a/app/models/remote_mirror.rb +++ b/app/models/remote_mirror.rb @@ -87,11 +87,11 @@ class RemoteMirror < ActiveRecord::Base end def sync? - !enabled? + enabled? end def sync - return if sync? + return unless sync? if recently_scheduled? RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now) -- cgit v1.2.3 From 698a0c479858b5d76f0ce18934a23c4ea5a1e332 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Tue, 1 May 2018 17:09:33 +0200 Subject: Add a CI_COMMIT_MESSAGE predefined variable --- app/models/ci/pipeline.rb | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 434b9b64c65..b44012ddc59 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -284,6 +284,14 @@ module Ci commit.try(:title) end + def git_commit_full_title + commit.try(:full_title) + end + + def git_commit_description + commit.try(:description) + end + def short_sha Ci::Pipeline.truncate_sha(sha) end @@ -491,6 +499,9 @@ module Ci .append(key: 'CI_PIPELINE_ID', value: id.to_s) .append(key: 'CI_CONFIG_PATH', value: ci_yaml_file_path) .append(key: 'CI_PIPELINE_SOURCE', value: source.to_s) + .append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message) + .append(key: 'CI_COMMIT_TITLE', value: git_commit_full_title) + .append(key: 'CI_COMMIT_DESCRIPTION', value: git_commit_description) end def queued_duration -- cgit v1.2.3 From b817e1d0f5fd1f35c1fe1107ffb0e267a30c468f Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Mon, 7 May 2018 17:13:03 +0200 Subject: Add memoization of commit related values in Ci::Pipeline --- app/models/ci/pipeline.rb | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index b44012ddc59..1d2db14ce36 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -269,27 +269,39 @@ module Ci end def git_author_name - commit.try(:author_name) + strong_memoize(:git_author_name) do + commit.try(:author_name) + end end def git_author_email - commit.try(:author_email) + strong_memoize(:git_author_email) do + commit.try(:author_email) + end end def git_commit_message - commit.try(:message) + strong_memoize(:git_commit_message) do + commit.try(:message) + end end def git_commit_title - commit.try(:title) + strong_memoize(:git_commit_title) do + commit.try(:title) + end end def git_commit_full_title - commit.try(:full_title) + strong_memoize(:git_commit_full_title) do + commit.try(:full_title) + end end def git_commit_description - commit.try(:description) + strong_memoize(:git_commit_description) do + commit.try(:description) + end end def short_sha -- cgit v1.2.3 From 3063225ca289e87c3f590e1722451a1c674e2ac9 Mon Sep 17 00:00:00 2001 From: Oswaldo Ferreira Date: Thu, 3 May 2018 17:28:00 -0300 Subject: Display merge commit SHA in merge widget after merge --- app/models/merge_request.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'app/models') diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 63c6ada86e1..628c61d5d69 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -1007,6 +1007,10 @@ class MergeRequest < ActiveRecord::Base @merge_commit ||= project.commit(merge_commit_sha) if merge_commit_sha end + def short_merge_commit_sha + Commit.truncate_sha(merge_commit_sha) if merge_commit_sha + end + def can_be_reverted?(current_user) return false unless merge_commit -- cgit v1.2.3 From a9496c27c9bb2876b9d94bfbe33f252a931caa8f Mon Sep 17 00:00:00 2001 From: Dennis Tang Date: Mon, 7 May 2018 18:06:02 +0000 Subject: Resolve "Inform users of up to $500 offer for GCP account" --- app/models/user_callout.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/user_callout.rb b/app/models/user_callout.rb index e4b69382626..9d461c6750a 100644 --- a/app/models/user_callout.rb +++ b/app/models/user_callout.rb @@ -2,7 +2,8 @@ class UserCallout < ActiveRecord::Base belongs_to :user enum feature_name: { - gke_cluster_integration: 1 + gke_cluster_integration: 1, + gcp_signup_offer: 2 } validates :user, presence: true -- cgit v1.2.3 From cfa798da5361128d9473ed170062586dea9525ae Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Mon, 7 May 2018 20:35:51 +0200 Subject: Force creation of new import_state when setting an import state field --- app/models/project.rb | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 37f1fcea280..1e3de535c18 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -647,8 +647,8 @@ class Project < ActiveRecord::Base } end - def ensure_import_state - return if self[:import_status] == 'none' || self[:import_status].nil? + def ensure_import_state(force: false) + return if !force && (self[:import_status] == 'none' || self[:import_status].nil?) return unless import_state.nil? create_import_state(import_state_args) @@ -657,39 +657,39 @@ class Project < ActiveRecord::Base end def import_schedule - ensure_import_state + ensure_import_state(force: true) - import_state&.schedule + import_state.schedule end def force_import_start - ensure_import_state + ensure_import_state(force: true) - import_state&.force_start + import_state.force_start end def import_start - ensure_import_state + ensure_import_state(force: true) - import_state&.start + import_state.start end def import_fail - ensure_import_state + ensure_import_state(force: true) - import_state&.fail_op + import_state.fail_op end def import_finish - ensure_import_state + ensure_import_state(force: true) - import_state&.finish + import_state.finish end def import_jid=(new_jid) - ensure_import_state + ensure_import_state(force: true) - import_state&.jid = new_jid + import_state.jid = new_jid end def import_jid @@ -699,9 +699,9 @@ class Project < ActiveRecord::Base end def import_error=(new_error) - ensure_import_state + ensure_import_state(force: true) - import_state&.last_error = new_error + import_state.last_error = new_error end def import_error @@ -711,9 +711,9 @@ class Project < ActiveRecord::Base end def import_status=(new_status) - ensure_import_state + ensure_import_state(force: true) - import_state&.status = new_status + import_state.status = new_status end def import_status -- cgit v1.2.3 From 02741ca4c58c625070d06c248125b2f510ac2c0b Mon Sep 17 00:00:00 2001 From: Mario de la Ossa Date: Thu, 3 May 2018 15:32:20 -0600 Subject: Backport 5480-epic-notifications from EE --- app/models/note.rb | 4 ---- app/models/sent_notification.rb | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) (limited to 'app/models') diff --git a/app/models/note.rb b/app/models/note.rb index e426f84832b..109405d3f17 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -317,10 +317,6 @@ class Note < ActiveRecord::Base !system? && !for_snippet? end - def can_create_notification? - true - end - def discussion_class(noteable = nil) # When commit notes are rendered on an MR's Discussion page, they are # displayed in one discussion instead of individually. diff --git a/app/models/sent_notification.rb b/app/models/sent_notification.rb index 6e311806be1..3da7c301d28 100644 --- a/app/models/sent_notification.rb +++ b/app/models/sent_notification.rb @@ -5,14 +5,14 @@ class SentNotification < ActiveRecord::Base belongs_to :noteable, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations belongs_to :recipient, class_name: "User" - validates :project, :recipient, presence: true + validates :recipient, presence: true validates :reply_key, presence: true, uniqueness: true validates :noteable_id, presence: true, unless: :for_commit? validates :commit_id, presence: true, if: :for_commit? validates :in_reply_to_discussion_id, format: { with: /\A\h{40}\z/, allow_nil: true } validate :note_valid - after_save :keep_around_commit + after_save :keep_around_commit, if: :for_commit? class << self def reply_key -- cgit v1.2.3 From 18094a17458aab8c9d629b86946b61e0e7bab251 Mon Sep 17 00:00:00 2001 From: Brett Walker Date: Tue, 8 May 2018 00:14:00 +0000 Subject: =?UTF-8?q?Backport:=20Keep=20ShaAttribute=20from=20halting=20star?= =?UTF-8?q?tup=20when=20we=20can=E2=80=99t=20connect=20to=20a=20database?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/concerns/sha_attribute.rb | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/sha_attribute.rb b/app/models/concerns/sha_attribute.rb index 703a72c355c..3340dc96e9f 100644 --- a/app/models/concerns/sha_attribute.rb +++ b/app/models/concerns/sha_attribute.rb @@ -4,18 +4,33 @@ module ShaAttribute module ClassMethods def sha_attribute(name) return if ENV['STATIC_VERIFICATION'] - return unless table_exists? + + validate_binary_column_exists!(name) unless Rails.env.production? + + attribute(name, Gitlab::Database::ShaAttribute.new) + end + + # This only gets executed in non-production environments as an additional check to ensure + # the column is the correct type. In production it should behave like any other attribute. + # See https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/5502 for more discussion + def validate_binary_column_exists!(name) + unless table_exists? + warn "WARNING: sha_attribute #{name.inspect} is invalid since the table doesn't exist - you may need to run database migrations" + return + end column = columns.find { |c| c.name == name.to_s } - # In case the table doesn't exist we won't be able to find the column, - # thus we will only check the type if the column is present. - if column && column.type != :binary - raise ArgumentError, - "sha_attribute #{name.inspect} is invalid since the column type is not :binary" + unless column + raise ArgumentError.new("sha_attribute #{name.inspect} is invalid since the column doesn't exist") end - attribute(name, Gitlab::Database::ShaAttribute.new) + unless column.type == :binary + raise ArgumentError.new("sha_attribute #{name.inspect} is invalid since the column type is not :binary") + end + rescue => error + Gitlab::AppLogger.error "ShaAttribute initialization: #{error.message}" + raise end end end -- cgit v1.2.3 From 0af2ab18fa7914380150c0403289a9d99ea45ded Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 8 May 2018 14:57:41 +0900 Subject: Decouple to_params from AtomicInternalId concern --- app/models/ci/pipeline.rb | 2 +- app/models/concerns/atomic_internal_id.rb | 6 +----- app/models/concerns/iid_routes.rb | 9 +++++++++ app/models/deployment.rb | 1 + app/models/issue.rb | 1 + app/models/merge_request.rb | 1 + app/models/milestone.rb | 1 + 7 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 app/models/concerns/iid_routes.rb (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 6bd2c42bbd3..d542868f01f 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -14,7 +14,7 @@ module Ci belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' - has_internal_id :iid, scope: :project, presence: false, to_param: false, init: -> do |s| + has_internal_id :iid, scope: :project, presence: false, init: -> do |s| s&.project&.pipelines&.maximum(:iid) || s&.project&.pipelines.count end diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb index 709589a9128..3d867df544f 100644 --- a/app/models/concerns/atomic_internal_id.rb +++ b/app/models/concerns/atomic_internal_id.rb @@ -25,7 +25,7 @@ module AtomicInternalId extend ActiveSupport::Concern module ClassMethods - def has_internal_id(column, scope:, init:, presence: true, to_param: true) # rubocop:disable Naming/PredicateName + def has_internal_id(column, scope:, init:, presence: true) # rubocop:disable Naming/PredicateName before_validation :"ensure_#{column}!", on: :create validates column, presence: presence, numericality: true @@ -42,10 +42,6 @@ module AtomicInternalId read_attribute(column) end - - define_method("to_param") do - read_attribute(column) - end if to_param end end end diff --git a/app/models/concerns/iid_routes.rb b/app/models/concerns/iid_routes.rb new file mode 100644 index 00000000000..50957e0230d --- /dev/null +++ b/app/models/concerns/iid_routes.rb @@ -0,0 +1,9 @@ +module IIDRoutes + ## + # This automagically enforces all related routes to use `iid` instead of `id` + # If you want to use `iid` for some routes and `id` for other routes, this module should not to be included, + # instead you should define `iid` or `id` explictly at each route generators. e.g. pipeline_path(project.id, pipeline.iid) + def to_param + iid.to_s + end +end diff --git a/app/models/deployment.rb b/app/models/deployment.rb index 254764eefde..dac97676348 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -1,5 +1,6 @@ class Deployment < ActiveRecord::Base include AtomicInternalId + include IIDRoutes belongs_to :project, required: true belongs_to :environment, required: true diff --git a/app/models/issue.rb b/app/models/issue.rb index 0332bfa9371..6d33a67845f 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -2,6 +2,7 @@ require 'carrierwave/orm/activerecord' class Issue < ActiveRecord::Base include AtomicInternalId + include IIDRoutes include Issuable include Noteable include Referable diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 628c61d5d69..a14681897fd 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -1,5 +1,6 @@ class MergeRequest < ActiveRecord::Base include AtomicInternalId + include IIDRoutes include Issuable include Noteable include Referable diff --git a/app/models/milestone.rb b/app/models/milestone.rb index d14e3a4ded5..f143b8452a2 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -9,6 +9,7 @@ class Milestone < ActiveRecord::Base include CacheMarkdownField include AtomicInternalId + include IIDRoutes include Sortable include Referable include StripAttribute -- cgit v1.2.3 From 1f898f1e20a74b67faa931384c3fec33b5fd956f Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Tue, 8 May 2018 07:07:48 +0100 Subject: Update commit status from external CI services less aggressively --- app/models/concerns/reactive_caching.rb | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/reactive_caching.rb b/app/models/concerns/reactive_caching.rb index 2589215ad19..eef9caf1c8e 100644 --- a/app/models/concerns/reactive_caching.rb +++ b/app/models/concerns/reactive_caching.rb @@ -60,13 +60,16 @@ module ReactiveCaching end def with_reactive_cache(*args, &blk) - within_reactive_cache_lifetime(*args) do + bootstrap = !within_reactive_cache_lifetime?(*args) + Rails.cache.write(alive_reactive_cache_key(*args), true, expires_in: self.class.reactive_cache_lifetime) + + if bootstrap + ReactiveCachingWorker.perform_async(self.class, id, *args) + nil + else data = Rails.cache.read(full_reactive_cache_key(*args)) yield data if data.present? end - ensure - Rails.cache.write(alive_reactive_cache_key(*args), true, expires_in: self.class.reactive_cache_lifetime) - ReactiveCachingWorker.perform_async(self.class, id, *args) end def clear_reactive_cache!(*args) @@ -75,7 +78,7 @@ module ReactiveCaching def exclusively_update_reactive_cache!(*args) locking_reactive_cache(*args) do - within_reactive_cache_lifetime(*args) do + if within_reactive_cache_lifetime?(*args) enqueuing_update(*args) do value = calculate_reactive_cache(*args) Rails.cache.write(full_reactive_cache_key(*args), value) @@ -105,8 +108,8 @@ module ReactiveCaching Gitlab::ExclusiveLease.cancel(full_reactive_cache_key(*args), uuid) end - def within_reactive_cache_lifetime(*args) - yield if Rails.cache.read(alive_reactive_cache_key(*args)) + def within_reactive_cache_lifetime?(*args) + !!Rails.cache.read(alive_reactive_cache_key(*args)) end def enqueuing_update(*args) -- cgit v1.2.3 From 04dc80dbb5cb991172ebeb69b9a20c7b6eef4dbf Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 8 May 2018 16:01:18 +0900 Subject: Fix spec --- app/models/ci/pipeline.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index d542868f01f..e9a56525fde 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -14,7 +14,7 @@ module Ci belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' - has_internal_id :iid, scope: :project, presence: false, init: -> do |s| + has_internal_id :iid, scope: :project, presence: false, init: ->(s) do s&.project&.pipelines&.maximum(:iid) || s&.project&.pipelines.count end -- cgit v1.2.3 From 1bd5896cd06e26130077632a4ac2d5cabd7be7d0 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Tue, 8 May 2018 09:46:29 +0200 Subject: Don't create import_state until project is persisted --- app/models/project.rb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index f6ac1802846..32d34f5e9b8 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -661,9 +661,15 @@ class Project < ActiveRecord::Base return if !force && (self[:import_status] == 'none' || self[:import_status].nil?) return unless import_state.nil? - create_import_state(import_state_args) + if persisted? + create_import_state(import_state_args) - update_column(:import_status, 'none') + update_column(:import_status, 'none') + else + build_import_state(import_state_args) + + self[:import_status] = 'none' + end end def import_schedule -- cgit v1.2.3 From 7be65f5d8fd7789b6f630ea04b7bcec8847ab436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Tue, 8 May 2018 12:20:54 +0200 Subject: Add cached_attr_time_reader to RedisCacheable --- app/models/concerns/redis_cacheable.rb | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'app/models') diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb index b889f4202dc..0dd15734eae 100644 --- a/app/models/concerns/redis_cacheable.rb +++ b/app/models/concerns/redis_cacheable.rb @@ -12,6 +12,15 @@ module RedisCacheable end end end + + def cached_attr_time_reader(*attributes) + attributes.each do |attribute| + define_method("#{attribute}") do + cached_value = cached_attribute(attribute) + cached_value ? Time.zone.parse(cached_value) : read_attribute(attribute) + end + end + end end def cached_attribute(attribute) -- cgit v1.2.3 From 71be7a1c224813e627c9d258a32189f1949338ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Tue, 8 May 2018 12:23:07 +0200 Subject: Move Runner#contacted_at to RedisCacheable#cached_attr_time_reader --- app/models/ci/runner.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 23078f1c3ed..ce3e595a2e1 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -74,7 +74,8 @@ module Ci project_type: 3 } - cached_attr_reader :version, :revision, :platform, :architecture, :contacted_at, :ip_address + cached_attr_reader :version, :revision, :platform, :architecture, :ip_address + cached_attr_time_reader :contacted_at chronic_duration_attr :maximum_timeout_human_readable, :maximum_timeout -- cgit v1.2.3 From a57c953e0e6ca03f929fb59f13307405822dfcb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Tue, 8 May 2018 22:39:28 +0200 Subject: Set `runner_type` for cluster/application --- app/models/clusters/applications/runner.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index 16efe90fa27..b881b4eaf36 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -43,12 +43,20 @@ module Clusters def create_and_assign_runner transaction do - project.runners.create!(name: 'kubernetes-cluster', tag_list: %w(kubernetes cluster)).tap do |runner| + project.runners.create!(runner_create_params).tap do |runner| update!(runner_id: runner.id) end end end + def runner_create_params + { + name: 'kubernetes-cluster', + runner_type: :project_type, + tag_list: %w(kubernetes cluster) + } + end + def gitlab_url Gitlab::Routing.url_helpers.root_url(only_path: false) end -- cgit v1.2.3 From 89ec120f61f8becb79403e7ad0a26fc8b942b57b Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Wed, 9 May 2018 09:57:16 +0200 Subject: Add validation Ci::Runner runner_type must be present --- app/models/ci/runner.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 23078f1c3ed..ed8b30dae49 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -58,6 +58,7 @@ module Ci validate :tag_constraints validate :either_projects_or_group validates :access_level, presence: true + validates :runner_type, presence: true acts_as_taggable -- cgit v1.2.3 From 890dc39ee0c92adb10a3221c2a564533e038a72c Mon Sep 17 00:00:00 2001 From: Jacopo Date: Tue, 8 May 2018 17:00:51 +0200 Subject: Updates updated_at on issue when using /spend quick action --- app/models/concerns/time_trackable.rb | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'app/models') diff --git a/app/models/concerns/time_trackable.rb b/app/models/concerns/time_trackable.rb index 5911b56c34c..73fc5048dcf 100644 --- a/app/models/concerns/time_trackable.rb +++ b/app/models/concerns/time_trackable.rb @@ -30,6 +30,8 @@ module TimeTrackable return if @time_spent == 0 + touch if touchable? + if @time_spent == :reset reset_spent_time else @@ -53,6 +55,10 @@ module TimeTrackable private + def touchable? + valid? && persisted? + end + def reset_spent_time timelogs.new(time_spent: total_time_spent * -1, user: @time_spent_user) # rubocop:disable Gitlab/ModuleWithInstanceVariables end -- cgit v1.2.3 From f6ecb72682000f7261219bbb6ce1ca9352aa473e Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 9 May 2018 16:54:47 +0200 Subject: Add Repository#xcode_project? method --- app/models/repository.rb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/repository.rb b/app/models/repository.rb index b75c4aca982..44c6bff6b66 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -37,7 +37,7 @@ class Repository changelog license_blob license_key gitignore koding_yml gitlab_ci_yml branch_names tag_names branch_count tag_count avatar exists? root_ref has_visible_content? - issue_template_names merge_request_template_names).freeze + issue_template_names merge_request_template_names xcode_project?).freeze # Methods that use cache_method but only memoize the value MEMOIZED_CACHED_METHODS = %i(license).freeze @@ -55,7 +55,8 @@ class Repository gitlab_ci: :gitlab_ci_yml, avatar: :avatar, issue_template: :issue_template_names, - merge_request_template: :merge_request_template_names + merge_request_template: :merge_request_template_names, + xcode_config: :xcode_project? }.freeze def initialize(full_path, project, disk_path: nil, is_wiki: false) @@ -594,6 +595,11 @@ class Repository end cache_method :gitlab_ci_yml + def xcode_project? + file_on_head(:xcode_config).present? + end + cache_method :xcode_project? + def head_commit @head_commit ||= commit(self.root_ref) end -- cgit v1.2.3 From 6a108b8fbd5f1b5c2d8e6b51bb34cda06fe0e5ee Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 10 May 2018 16:22:43 +0900 Subject: Fix ensure_iid! method override problem --- app/models/concerns/atomic_internal_id.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb index 3d867df544f..876dd0ee1f2 100644 --- a/app/models/concerns/atomic_internal_id.rb +++ b/app/models/concerns/atomic_internal_id.rb @@ -26,10 +26,10 @@ module AtomicInternalId module ClassMethods def has_internal_id(column, scope:, init:, presence: true) # rubocop:disable Naming/PredicateName - before_validation :"ensure_#{column}!", on: :create + before_validation :"ensure_#{scope}_#{column}!", on: :create validates column, presence: presence, numericality: true - define_method("ensure_#{column}!") do + define_method("ensure_#{scope}_#{column}!") do scope_value = association(scope).reader if read_attribute(column).blank? && scope_value -- cgit v1.2.3 From 61e9a3dcc4d8d33ab2a5c2773acfce03db08a039 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Fri, 20 Apr 2018 12:25:22 +0300 Subject: Add 2FA filter to group members page * Show 2fa badge on a group members page * Make group members page UI consistent with project members page * Fix ambiguous sql in User.with/without_two_factor methods Signed-off-by: Dmitriy Zaporozhets --- app/models/member.rb | 11 +++++++++++ app/models/user.rb | 10 +++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/member.rb b/app/models/member.rb index eac4a22a03f..68572f2e33a 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -96,6 +96,17 @@ class Member < ActiveRecord::Base joins(:user).merge(User.search(query)) end + def filter_by_2fa(value) + case value + when 'enabled' + left_join_users.merge(User.with_two_factor_indistinct) + when 'disabled' + left_join_users.merge(User.without_two_factor) + else + all + end + end + def sort_by_attribute(method) case method.to_s when 'access_level_asc' then reorder(access_level: :asc) diff --git a/app/models/user.rb b/app/models/user.rb index a9cfd39f604..d74d5aade5a 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -237,14 +237,18 @@ class User < ActiveRecord::Base scope :order_recent_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'DESC')) } scope :order_oldest_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'ASC')) } - def self.with_two_factor + def self.with_two_factor_indistinct joins("LEFT OUTER JOIN u2f_registrations AS u2f ON u2f.user_id = users.id") - .where("u2f.id IS NOT NULL OR otp_required_for_login = ?", true).distinct(arel_table[:id]) + .where("u2f.id IS NOT NULL OR users.otp_required_for_login = ?", true) + end + + def self.with_two_factor + with_two_factor_indistinct.distinct(arel_table[:id]) end def self.without_two_factor joins("LEFT OUTER JOIN u2f_registrations AS u2f ON u2f.user_id = users.id") - .where("u2f.id IS NULL AND otp_required_for_login = ?", false) + .where("u2f.id IS NULL AND users.otp_required_for_login = ?", false) end # -- cgit v1.2.3 From 84fa061086250b9f59eb51eb8728702dfedb7546 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Thu, 10 May 2018 11:16:26 +0200 Subject: Ensure runner_type is updated correctly when assigning shared runner to project --- app/models/ci/runner.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index ed8b30dae49..359a87a3f77 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -108,7 +108,11 @@ module Ci end def assign_to(project, current_user = nil) - self.is_shared = false if shared? + if shared? + self.is_shared = false if shared? + self.runner_type = :project_type + end + self.save project.runner_projects.create(runner_id: self.id) end -- cgit v1.2.3 From cd834b46a8b84eaf14891854925502800acff475 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Thu, 10 May 2018 12:40:30 +0200 Subject: Ensure Ci::Runner#assign_to errors for group runners --- app/models/ci/runner.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 359a87a3f77..bda69f85a78 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -111,6 +111,8 @@ module Ci if shared? self.is_shared = false if shared? self.runner_type = :project_type + elsif group_type? + raise ArgumentError, 'Transitioning a group runner to a project runner is not supported' end self.save -- cgit v1.2.3 From 8d49ec681ffe4638f4db3311879448958d34c6f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Thu, 10 May 2018 12:41:09 +0200 Subject: Use symbol instead of string in RedisCacheable attribute definitions --- app/models/concerns/redis_cacheable.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb index 0dd15734eae..bc86bbe6525 100644 --- a/app/models/concerns/redis_cacheable.rb +++ b/app/models/concerns/redis_cacheable.rb @@ -7,7 +7,7 @@ module RedisCacheable class_methods do def cached_attr_reader(*attributes) attributes.each do |attribute| - define_method("#{attribute}") do + define_method(attribute) do cached_attribute(attribute) || read_attribute(attribute) end end @@ -15,7 +15,7 @@ module RedisCacheable def cached_attr_time_reader(*attributes) attributes.each do |attribute| - define_method("#{attribute}") do + define_method(attribute) do cached_value = cached_attribute(attribute) cached_value ? Time.zone.parse(cached_value) : read_attribute(attribute) end -- cgit v1.2.3 From 42d27f0b43df544bab2ad5bc4e082728d86c7388 Mon Sep 17 00:00:00 2001 From: Brett Walker Date: Thu, 10 May 2018 10:00:32 -0500 Subject: only issue a warning if column doesn't exist --- app/models/concerns/sha_attribute.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/concerns/sha_attribute.rb b/app/models/concerns/sha_attribute.rb index 3340dc96e9f..3796737427a 100644 --- a/app/models/concerns/sha_attribute.rb +++ b/app/models/concerns/sha_attribute.rb @@ -22,7 +22,8 @@ module ShaAttribute column = columns.find { |c| c.name == name.to_s } unless column - raise ArgumentError.new("sha_attribute #{name.inspect} is invalid since the column doesn't exist") + warn "WARNING: sha_attribute #{name.inspect} is invalid since the column doesn't exist - you may need to run database migrations" + return end unless column.type == :binary -- cgit v1.2.3 From f7f13f9db0da92c7b43481dfe5559f317711e533 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Tue, 8 May 2018 15:07:55 +0200 Subject: Block access to API & git when terms are enforced When terms are enforced, but the user has not accepted the terms access to the API & git is rejected with a message directing the user to the web app to accept the terms. --- app/models/user.rb | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index a9cfd39f604..884f3bbb364 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1193,6 +1193,11 @@ class User < ActiveRecord::Base accepted_term_id.present? end + def required_terms_not_accepted? + Gitlab::CurrentSettings.current_application_settings.enforce_terms? && + !terms_accepted? + end + protected # override, from Devise::Validatable -- cgit v1.2.3 From c3e40ed8ff0b2d0d9260667ec7825453af0bd6f3 Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Fri, 11 May 2018 11:19:59 +1100 Subject: [Rails5] Fix `Route source can't be blank` In Rails 5.0 automatic inverse does not work for polymorphic relathionships. It was fixed in Rails 5.2: https://github.com/rails/rails/pull/28808 Until that the `inverse_of: :source` argument should be set explicitly. --- app/models/concerns/routable.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index 915ad6959be..0176a12a131 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -4,7 +4,9 @@ module Routable extend ActiveSupport::Concern included do - has_one :route, as: :source, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + # Remove `inverse_of: source` when upgraded to rails 5.2 + # See https://github.com/rails/rails/pull/28808 + has_one :route, as: :source, autosave: true, dependent: :destroy, inverse_of: :source # rubocop:disable Cop/ActiveRecordDependent has_many :redirect_routes, as: :source, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent validates :route, presence: true -- cgit v1.2.3 From 910a7d02a812b1203e320d843a77cad2c7069b63 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 11 May 2018 15:34:36 +0900 Subject: Remove numericality as it's redandant with integer column and validates nil IID --- app/models/concerns/atomic_internal_id.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb index 876dd0ee1f2..164c704260e 100644 --- a/app/models/concerns/atomic_internal_id.rb +++ b/app/models/concerns/atomic_internal_id.rb @@ -27,7 +27,7 @@ module AtomicInternalId module ClassMethods def has_internal_id(column, scope:, init:, presence: true) # rubocop:disable Naming/PredicateName before_validation :"ensure_#{scope}_#{column}!", on: :create - validates column, presence: presence, numericality: true + validates column, presence: presence define_method("ensure_#{scope}_#{column}!") do scope_value = association(scope).reader -- cgit v1.2.3 From a74184eb5e692ef77fe3be28b1f4a40549c8fcff Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 11 May 2018 16:52:48 +0900 Subject: Fix static analysys --- app/models/ci/pipeline.rb | 2 +- app/models/concerns/iid_routes.rb | 2 +- app/models/deployment.rb | 2 +- app/models/issue.rb | 2 +- app/models/merge_request.rb | 2 +- app/models/milestone.rb | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index e9a56525fde..accd0f11a9b 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -15,7 +15,7 @@ module Ci belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' has_internal_id :iid, scope: :project, presence: false, init: ->(s) do - s&.project&.pipelines&.maximum(:iid) || s&.project&.pipelines.count + s&.project&.pipelines&.maximum(:iid) || s&.project&.pipelines&.count end has_many :stages diff --git a/app/models/concerns/iid_routes.rb b/app/models/concerns/iid_routes.rb index 50957e0230d..246748cf52c 100644 --- a/app/models/concerns/iid_routes.rb +++ b/app/models/concerns/iid_routes.rb @@ -1,4 +1,4 @@ -module IIDRoutes +module IidRoutes ## # This automagically enforces all related routes to use `iid` instead of `id` # If you want to use `iid` for some routes and `id` for other routes, this module should not to be included, diff --git a/app/models/deployment.rb b/app/models/deployment.rb index dac97676348..ac86e9e8de0 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -1,6 +1,6 @@ class Deployment < ActiveRecord::Base include AtomicInternalId - include IIDRoutes + include IidRoutes belongs_to :project, required: true belongs_to :environment, required: true diff --git a/app/models/issue.rb b/app/models/issue.rb index 6d33a67845f..d0e8fe908e4 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -2,7 +2,7 @@ require 'carrierwave/orm/activerecord' class Issue < ActiveRecord::Base include AtomicInternalId - include IIDRoutes + include IidRoutes include Issuable include Noteable include Referable diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index a14681897fd..f42d432cc66 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -1,6 +1,6 @@ class MergeRequest < ActiveRecord::Base include AtomicInternalId - include IIDRoutes + include IidRoutes include Issuable include Noteable include Referable diff --git a/app/models/milestone.rb b/app/models/milestone.rb index f143b8452a2..d05dcfd083a 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -9,7 +9,7 @@ class Milestone < ActiveRecord::Base include CacheMarkdownField include AtomicInternalId - include IIDRoutes + include IidRoutes include Sortable include Referable include StripAttribute -- cgit v1.2.3 From 7f6691dde7be7c024d3ed9f9fee8b56813a45e51 Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Sat, 12 May 2018 00:07:23 +1100 Subject: [Rails5] Fix Ci::Pipeline validator for source In Rails 5 enum returns value instead of key. For this case, the `NilClass` is returned instead of `unknown` which breaks validation of the `source` attribute. This commit adds a custom validatior that returns the correct result for both rails4 and rails5. --- app/models/ci/pipeline.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 0b90834d415..1f49764e7cc 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -37,12 +37,16 @@ module Ci delegate :id, to: :project, prefix: true delegate :full_path, to: :project, prefix: true - validates :source, exclusion: { in: %w(unknown), unless: :importing? }, on: :create validates :sha, presence: { unless: :importing? } validates :ref, presence: { unless: :importing? } validates :status, presence: { unless: :importing? } validate :valid_commit_sha, unless: :importing? + # Replace validator below with + # `validates :source, presence: { unless: :importing? }, on: :create` + # when removing Gitlab.rails5? code. + validate :valid_source, unless: :importing?, on: :create + after_create :keep_around_commits, unless: :importing? enum source: { @@ -601,5 +605,11 @@ module Ci project.repository.keep_around(self.sha) project.repository.keep_around(self.before_sha) end + + def valid_source + if source.nil? || source == "unknown" + errors.add(:source, "invalid source") + end + end end end -- cgit v1.2.3 From 20cfc3fccec75562bdb514587d2c9f7b59554c07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Fri, 11 May 2018 18:36:16 +0200 Subject: Clear memoization after caching new values --- app/models/concerns/redis_cacheable.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'app/models') diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb index bc86bbe6525..8bd0df8dfbe 100644 --- a/app/models/concerns/redis_cacheable.rb +++ b/app/models/concerns/redis_cacheable.rb @@ -31,6 +31,8 @@ module RedisCacheable Gitlab::Redis::SharedState.with do |redis| redis.set(cache_attribute_key, values.to_json, ex: CACHED_ATTRIBUTES_EXPIRY_TIME) end + + clear_memoization(:cached_attributes) end private -- cgit v1.2.3 From e2e72fe5680e65bfed4d5fff5def829ff7ef9aeb Mon Sep 17 00:00:00 2001 From: Oswaldo Ferreira Date: Mon, 14 May 2018 17:40:48 -0300 Subject: Adjust board lists header text color Adjusts regression fixed by https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18786/. We were not returning the label text color correctly on the backend. --- app/models/list.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/list.rb b/app/models/list.rb index 918275be142..5daf35ef845 100644 --- a/app/models/list.rb +++ b/app/models/list.rb @@ -31,7 +31,8 @@ class List < ActiveRecord::Base if options.key?(:label) json[:label] = label.as_json( project: board.project, - only: [:id, :title, :description, :color] + only: [:id, :title, :description, :color], + methods: [:text_color] ) end end -- cgit v1.2.3 From 8e51c481b6a78cf34b4403d32111b15694adab2d Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 14 May 2018 20:23:40 -0300 Subject: Does not log failed sign-in attempts when database is in read-only mode --- app/models/user.rb | 3 +++ 1 file changed, 3 insertions(+) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index dfef065f094..173ab38e20c 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1097,8 +1097,11 @@ class User < ActiveRecord::Base # # def increment_failed_attempts! + return if ::Gitlab::Database.read_only? + self.failed_attempts ||= 0 self.failed_attempts += 1 + if attempts_exceeded? lock_access! unless access_locked? else -- cgit v1.2.3 From 46eeaafc255f9cc037f2656fcace67085519bf29 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Fri, 20 Apr 2018 14:17:00 +1000 Subject: Display help text below auto devops domain with nip.io domain name (#45561) --- app/models/project.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 32d34f5e9b8..534a0e630af 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -217,6 +217,7 @@ class Project < ActiveRecord::Base has_one :cluster_project, class_name: 'Clusters::Project' has_many :clusters, through: :cluster_project, class_name: 'Clusters::Cluster' + has_many :cluster_ingresses, through: :clusters, source: :application_ingress, class_name: 'Clusters::Applications::Ingress' # Container repositories need to remove data from the container registry, # which is not managed by the DB. Hence we're still using dependent: :destroy -- cgit v1.2.3 From 86ef8221778d2aff630feecfa15ba70514983dbc Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Tue, 15 May 2018 13:03:09 -0500 Subject: Makes CommitStatus and GenericCommitStatus respond to presentable methods Includes Presentable module into CommitStatus. This fixes presenter methods being called on those classes. Closes #46177 --- app/models/commit_status.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/models') diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 97d89422594..a7d05722287 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -2,6 +2,7 @@ class CommitStatus < ActiveRecord::Base include HasStatus include Importable include AfterCommitQueue + include Presentable self.table_name = 'ci_builds' -- cgit v1.2.3 From 6cc3a07dca635aa61d275eb6803cd986f4c9f967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Tue, 15 May 2018 21:55:16 +0200 Subject: Dynamically cast value from cache --- app/models/ci/runner.rb | 3 +-- app/models/concerns/redis_cacheable.rb | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 54326d0c42c..49db5f6ab7f 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -75,8 +75,7 @@ module Ci project_type: 3 } - cached_attr_reader :version, :revision, :platform, :architecture, :ip_address - cached_attr_time_reader :contacted_at + cached_attr_reader :version, :revision, :platform, :architecture, :ip_address, :contacted_at chronic_duration_attr :maximum_timeout_human_readable, :maximum_timeout diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb index 8bd0df8dfbe..4fdaaddeee7 100644 --- a/app/models/concerns/redis_cacheable.rb +++ b/app/models/concerns/redis_cacheable.rb @@ -6,18 +6,11 @@ module RedisCacheable class_methods do def cached_attr_reader(*attributes) - attributes.each do |attribute| - define_method(attribute) do - cached_attribute(attribute) || read_attribute(attribute) - end - end - end - - def cached_attr_time_reader(*attributes) attributes.each do |attribute| define_method(attribute) do cached_value = cached_attribute(attribute) - cached_value ? Time.zone.parse(cached_value) : read_attribute(attribute) + cached_value = cast_value_from_cache(attribute, cached_value) if cached_value + cached_value || read_attribute(attribute) end end end @@ -49,4 +42,12 @@ module RedisCacheable end end end + + def cast_value_from_cache(attribute, value) + if self.class.column_for_attribute(attribute).respond_to?(:type_cast_from_database) + self.class.column_for_attribute(attribute).type_cast_from_database(value) + else + self.class.type_for_attribute(attribute).cast(value) + end + end end -- cgit v1.2.3 From 7da3b2cdd09078984416aa03da108ad0a4a4e477 Mon Sep 17 00:00:00 2001 From: Jan Provaznik Date: Wed, 2 May 2018 18:21:42 +0200 Subject: Delete remote uploads ObjectStore uploader requires presence of associated `uploads` record when deleting the upload file (through the carrierwave's after_commit hook) because we keep info whether file is LOCAL or REMOTE in `upload` object. For this reason we can not destroy uploads as "dependent: :destroy" hook because these would be deleted too soon. Instead we rely on carrierwave's hook to destroy `uploads` in after_commit hook. But in before_destroy hook we still have to delete not-mounted uploads (which don't use carrierwave's destroy hook). This has to be done in before_Destroy instead of after_commit because `FileUpload` requires existence of model's object on destroy action. This is not ideal state of things, in a next step we should investigate how to unify model dependencies so we can use same workflow for all uploads. Related to #45425 --- app/models/appearance.rb | 3 +-- app/models/concerns/with_uploads.rb | 37 +++++++++++++++++++++++++++++++++++++ app/models/group.rb | 3 +-- app/models/project.rb | 3 +-- app/models/user.rb | 2 +- 5 files changed, 41 insertions(+), 7 deletions(-) create mode 100644 app/models/concerns/with_uploads.rb (limited to 'app/models') diff --git a/app/models/appearance.rb b/app/models/appearance.rb index fb66dd0b766..f3cfc8ccd1e 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -1,4 +1,5 @@ class Appearance < ActiveRecord::Base + include WithUploads include CacheMarkdownField include AfterCommitQueue include ObjectStorage::BackgroundMove @@ -14,8 +15,6 @@ class Appearance < ActiveRecord::Base mount_uploader :logo, AttachmentUploader mount_uploader :header_logo, AttachmentUploader - has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - CACHE_KEY = "current_appearance:#{Gitlab::VERSION}".freeze after_commit :flush_redis_cache diff --git a/app/models/concerns/with_uploads.rb b/app/models/concerns/with_uploads.rb new file mode 100644 index 00000000000..101d09f161d --- /dev/null +++ b/app/models/concerns/with_uploads.rb @@ -0,0 +1,37 @@ +# Mounted uploaders are destroyed by carrierwave's after_commit +# hook. This hook fetches upload location (local vs remote) from +# Upload model. So it's neccessary to make sure that during that +# after_commit hook model's associated uploads are not deleted yet. +# IOW we can not use denepdent => :destroy : +# has_many :uploads, as: :model, dependent: :destroy +# +# And because not-mounted uploads require presence of upload's +# object model when destroying them (FileUploader's `build_upload` method +# references `model` on delete), we can not use after_commit hook for these +# uploads. +# +# Instead FileUploads are destroyed in before_destroy hook and remaining uploads +# are destroyed by the carrierwave's after_commit hook. + +module WithUploads + extend ActiveSupport::Concern + + # Currently there is no simple way how to select only not-mounted + # uploads, it should be all FileUploaders so we select them by + # `uploader` class + FILE_UPLOADERS = %w(PersonalFileUploader NamespaceFileUploader FileUploader).freeze + + included do + has_many :uploads, as: :model + + before_destroy :destroy_file_uploads + end + + # mounted uploads are deleted in carrierwave's after_commit hook, + # but FileUploaders which are not mounted must be deleted explicitly and + # it can not be done in after_commit because FileUploader requires loads + # associated model on destroy (which is already deleted in after_commit) + def destroy_file_uploads + self.uploads.where(uploader: FILE_UPLOADERS).destroy_all + end +end diff --git a/app/models/group.rb b/app/models/group.rb index cefca316399..107711e3cc5 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -2,6 +2,7 @@ require 'carrierwave/orm/activerecord' class Group < Namespace include Gitlab::ConfigHelper + include WithUploads include AfterCommitQueue include AccessRequestable include Avatarable @@ -30,8 +31,6 @@ class Group < Namespace has_many :variables, class_name: 'Ci::GroupVariable' has_many :custom_attributes, class_name: 'GroupCustomAttribute' - has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :boards has_many :badges, class_name: 'GroupBadge' diff --git a/app/models/project.rb b/app/models/project.rb index 534a0e630af..0b0d653c4af 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -4,6 +4,7 @@ class Project < ActiveRecord::Base include Gitlab::ConfigHelper include Gitlab::ShellAdapter include Gitlab::VisibilityLevel + include WithUploads include AccessRequestable include Avatarable include CacheMarkdownField @@ -301,8 +302,6 @@ class Project < ActiveRecord::Base inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } } validates :variables, variable_duplicates: { scope: :environment_scope } - has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - # Scopes scope :pending_delete, -> { where(pending_delete: true) } scope :without_deleted, -> { where(pending_delete: false) } diff --git a/app/models/user.rb b/app/models/user.rb index 173ab38e20c..082ec76e86a 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -3,6 +3,7 @@ require 'carrierwave/orm/activerecord' class User < ActiveRecord::Base extend Gitlab::ConfigHelper + include WithUploads include Gitlab::ConfigHelper include Gitlab::SQL::Pattern include AfterCommitQueue @@ -137,7 +138,6 @@ class User < ActiveRecord::Base has_many :custom_attributes, class_name: 'UserCustomAttribute' has_many :callouts, class_name: 'UserCallout' - has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :term_agreements belongs_to :accepted_term, class_name: 'ApplicationSetting::Term' -- cgit v1.2.3 From c81a37c1d3f864cf0a00386dab29da78f222e3a5 Mon Sep 17 00:00:00 2001 From: Jan Provaznik Date: Fri, 4 May 2018 19:52:43 +0200 Subject: Use find_in_batches instead of destroy_all destroy_all loads all records at once --- app/models/concerns/with_uploads.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/concerns/with_uploads.rb b/app/models/concerns/with_uploads.rb index 101d09f161d..3d99889c131 100644 --- a/app/models/concerns/with_uploads.rb +++ b/app/models/concerns/with_uploads.rb @@ -32,6 +32,8 @@ module WithUploads # it can not be done in after_commit because FileUploader requires loads # associated model on destroy (which is already deleted in after_commit) def destroy_file_uploads - self.uploads.where(uploader: FILE_UPLOADERS).destroy_all + self.uploads.where(uploader: FILE_UPLOADERS).find_each do |upload| + upload.destroy + end end end -- cgit v1.2.3 From 32e22468300e3a52c82a855a01fc3983473107e0 Mon Sep 17 00:00:00 2001 From: Jan Provaznik Date: Mon, 7 May 2018 15:49:32 +0200 Subject: Changed order of include --- app/models/appearance.rb | 2 +- app/models/group.rb | 2 +- app/models/project.rb | 2 +- app/models/user.rb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) (limited to 'app/models') diff --git a/app/models/appearance.rb b/app/models/appearance.rb index f3cfc8ccd1e..f8713138a93 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -1,8 +1,8 @@ class Appearance < ActiveRecord::Base - include WithUploads include CacheMarkdownField include AfterCommitQueue include ObjectStorage::BackgroundMove + include WithUploads cache_markdown_field :description cache_markdown_field :new_project_guidelines diff --git a/app/models/group.rb b/app/models/group.rb index 107711e3cc5..8fb77a7869d 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -2,7 +2,6 @@ require 'carrierwave/orm/activerecord' class Group < Namespace include Gitlab::ConfigHelper - include WithUploads include AfterCommitQueue include AccessRequestable include Avatarable @@ -11,6 +10,7 @@ class Group < Namespace include LoadedInGroupList include GroupDescendant include TokenAuthenticatable + include WithUploads has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent alias_method :members, :group_members diff --git a/app/models/project.rb b/app/models/project.rb index 0b0d653c4af..0975e64e995 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -4,7 +4,6 @@ class Project < ActiveRecord::Base include Gitlab::ConfigHelper include Gitlab::ShellAdapter include Gitlab::VisibilityLevel - include WithUploads include AccessRequestable include Avatarable include CacheMarkdownField @@ -24,6 +23,7 @@ class Project < ActiveRecord::Base include ::Gitlab::Utils::StrongMemoize include ChronicDurationAttribute include FastDestroyAll::Helpers + include WithUploads extend Gitlab::ConfigHelper diff --git a/app/models/user.rb b/app/models/user.rb index 082ec76e86a..b90f5471071 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -3,7 +3,6 @@ require 'carrierwave/orm/activerecord' class User < ActiveRecord::Base extend Gitlab::ConfigHelper - include WithUploads include Gitlab::ConfigHelper include Gitlab::SQL::Pattern include AfterCommitQueue @@ -18,6 +17,7 @@ class User < ActiveRecord::Base include IgnorableColumn include BulkMemberAccessLoad include BlocksJsonSerialization + include WithUploads DEFAULT_NOTIFICATION_LEVEL = :participating -- cgit v1.2.3 From 8c0166f2b96e233cbf8ec84e4314094521dc0316 Mon Sep 17 00:00:00 2001 From: Jan Provaznik Date: Wed, 9 May 2018 21:43:23 +0200 Subject: Fixed typo --- app/models/concerns/with_uploads.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/concerns/with_uploads.rb b/app/models/concerns/with_uploads.rb index 3d99889c131..e7cfffb775b 100644 --- a/app/models/concerns/with_uploads.rb +++ b/app/models/concerns/with_uploads.rb @@ -2,7 +2,7 @@ # hook. This hook fetches upload location (local vs remote) from # Upload model. So it's neccessary to make sure that during that # after_commit hook model's associated uploads are not deleted yet. -# IOW we can not use denepdent => :destroy : +# IOW we can not use dependent: :destroy : # has_many :uploads, as: :model, dependent: :destroy # # And because not-mounted uploads require presence of upload's -- cgit v1.2.3 From 846f73b53b8a6d3bc1f18607630d7a7853cb9d13 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Wed, 9 May 2018 14:46:34 +0200 Subject: Allow group runners to be viewed/edited in API --- app/models/ci/runner.rb | 2 +- app/models/user.rb | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index bda69f85a78..e6f1ed519be 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -52,7 +52,7 @@ module Ci # Without that, placeholders would miss one and couldn't match. where(locked: false) .where.not("ci_runners.id IN (#{project.runners.select(:id).to_sql})") - .specific + .project_type end validate :tag_constraints diff --git a/app/models/user.rb b/app/models/user.rb index 173ab38e20c..2afe9ea77f9 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1001,10 +1001,17 @@ class User < ActiveRecord::Base def ci_authorized_runners @ci_authorized_runners ||= begin - runner_ids = Ci::RunnerProject + project_runner_ids = Ci::RunnerProject .where(project: authorized_projects(Gitlab::Access::MASTER)) .select(:runner_id) - Ci::Runner.specific.where(id: runner_ids) + + group_runner_ids = Ci::RunnerNamespace + .where(namespace_id: owned_or_masters_groups.select(:id)) + .select(:runner_id) + + union = Gitlab::SQL::Union.new([project_runner_ids, group_runner_ids]) + + Ci::Runner.specific.where("ci_runners.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection end end @@ -1205,6 +1212,11 @@ class User < ActiveRecord::Base !terms_accepted? end + def owned_or_masters_groups + union = Gitlab::SQL::Union.new([owned_groups, masters_groups]) + Group.from("(#{union.to_sql}) namespaces") + end + protected # override, from Devise::Validatable -- cgit v1.2.3 From c3f9d80a6e0950361e056ded4107015d3923f56d Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Thu, 10 May 2018 14:42:55 +0200 Subject: Rename User#ci_authorized_runners -> ci_owned_runners --- app/models/user.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index 2afe9ea77f9..226a4489261 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -999,8 +999,8 @@ class User < ActiveRecord::Base !solo_owned_groups.present? end - def ci_authorized_runners - @ci_authorized_runners ||= begin + def ci_owned_runners + @ci_owned_runners ||= begin project_runner_ids = Ci::RunnerProject .where(project: authorized_projects(Gitlab::Access::MASTER)) .select(:runner_id) -- cgit v1.2.3 From 3a94919e83f1e5b359f6772762392eeda2d7134d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 16 May 2018 14:02:04 +0200 Subject: Do not allow to trigger manual actions that were skipped --- app/models/ci/build.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 61c10c427dd..d9649e30edc 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -184,7 +184,7 @@ module Ci end def playable? - action? && (manual? || complete?) + action? && (manual? || retryable?) end def action? -- cgit v1.2.3 From 8d024ba79a2e2a2f2d34f4ee678b496f0fbe64f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Wed, 16 May 2018 20:39:29 +0200 Subject: Backport changes from EE to minimize the CE/EE diff in Projects::Settings::IntegrationsController MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- app/models/project.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 534a0e630af..107ee5f9a7e 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -998,7 +998,7 @@ class Project < ActiveRecord::Base available_services_names = Service.available_services_names - exceptions - available_services_names.map do |service_name| + available_services = available_services_names.map do |service_name| service = find_service(services, service_name) if service @@ -1015,6 +1015,14 @@ class Project < ActiveRecord::Base end end end + + available_services.reject do |service| + disabled_services.include?(service.to_param) + end + end + + def disabled_services + [] end def find_or_initialize_service(name) -- cgit v1.2.3 From 42ab6f8557505595f86604735d8805f879247da8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Wed, 16 May 2018 21:23:43 +0200 Subject: Move attribute casting to #cached_attribute --- app/models/concerns/redis_cacheable.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb index 4fdaaddeee7..d40df5ba2d4 100644 --- a/app/models/concerns/redis_cacheable.rb +++ b/app/models/concerns/redis_cacheable.rb @@ -8,16 +8,15 @@ module RedisCacheable def cached_attr_reader(*attributes) attributes.each do |attribute| define_method(attribute) do - cached_value = cached_attribute(attribute) - cached_value = cast_value_from_cache(attribute, cached_value) if cached_value - cached_value || read_attribute(attribute) + cached_attribute(attribute) || read_attribute(attribute) end end end end def cached_attribute(attribute) - (cached_attributes || {})[attribute] + cached_value = (cached_attributes || {})[attribute] + cast_value_from_cache(attribute, cached_value) if cached_value end def cache_attributes(values) -- cgit v1.2.3 From a4b0876b391f0717365cabd78cf9715b64649797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Wed, 16 May 2018 21:36:20 +0200 Subject: Add attribute check in cached getter --- app/models/concerns/redis_cacheable.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'app/models') diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb index d40df5ba2d4..a8d96f63d7a 100644 --- a/app/models/concerns/redis_cacheable.rb +++ b/app/models/concerns/redis_cacheable.rb @@ -8,6 +8,8 @@ module RedisCacheable def cached_attr_reader(*attributes) attributes.each do |attribute| define_method(attribute) do + raise ArgumentError, "Not a database attribute" unless self.has_attribute?(attribute) + cached_attribute(attribute) || read_attribute(attribute) end end -- cgit v1.2.3 From 4e1bb1d1014237df79db6b3cc2beb24228a4b228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Wed, 16 May 2018 21:49:09 +0200 Subject: Move argument check to cached getter definition class method --- app/models/concerns/redis_cacheable.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb index a8d96f63d7a..bf046c0e333 100644 --- a/app/models/concerns/redis_cacheable.rb +++ b/app/models/concerns/redis_cacheable.rb @@ -7,9 +7,9 @@ module RedisCacheable class_methods do def cached_attr_reader(*attributes) attributes.each do |attribute| - define_method(attribute) do - raise ArgumentError, "Not a database attribute" unless self.has_attribute?(attribute) + raise ArgumentError, "Not a database attribute" unless self.attribute_names.include?(attribute.to_s) + define_method(attribute) do cached_attribute(attribute) || read_attribute(attribute) end end -- cgit v1.2.3 From eeb955a66d2410d7117737ecd1b33b0cfe67327b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Wed, 16 May 2018 22:43:38 +0200 Subject: Revert "Move argument check to cached getter definition class method" This reverts commit 4e1bb1d1014237df79db6b3cc2beb24228a4b228. --- app/models/concerns/redis_cacheable.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb index bf046c0e333..a8d96f63d7a 100644 --- a/app/models/concerns/redis_cacheable.rb +++ b/app/models/concerns/redis_cacheable.rb @@ -7,9 +7,9 @@ module RedisCacheable class_methods do def cached_attr_reader(*attributes) attributes.each do |attribute| - raise ArgumentError, "Not a database attribute" unless self.attribute_names.include?(attribute.to_s) - define_method(attribute) do + raise ArgumentError, "Not a database attribute" unless self.has_attribute?(attribute) + cached_attribute(attribute) || read_attribute(attribute) end end -- cgit v1.2.3 From 51f5ee33c448ee8a157ad12dac80482a11a7c72f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Thu, 17 May 2018 01:12:37 +0200 Subject: Use Gitlab.rails5? for checking if on rails5 --- app/models/concerns/redis_cacheable.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb index a8d96f63d7a..3d24f0dfdc1 100644 --- a/app/models/concerns/redis_cacheable.rb +++ b/app/models/concerns/redis_cacheable.rb @@ -45,10 +45,10 @@ module RedisCacheable end def cast_value_from_cache(attribute, value) - if self.class.column_for_attribute(attribute).respond_to?(:type_cast_from_database) - self.class.column_for_attribute(attribute).type_cast_from_database(value) - else + if Gitlab.rails5? self.class.type_for_attribute(attribute).cast(value) + else + self.class.column_for_attribute(attribute).type_cast_from_database(value) end end end -- cgit v1.2.3 From 7517105303c264484d8677c81268f9f43ecc5593 Mon Sep 17 00:00:00 2001 From: lulalala Date: Wed, 28 Mar 2018 11:20:12 +0800 Subject: Add cannot_be_merged_recheck merge_status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First, transitions between can_be_merged & cannot_be_merged are removed, as they are currently blocked in `check_if_can_be_merged`. `can_be_merge` always returns to `unchecked` first, before it can transition to `cannot_be_merged` (and vice versa). We want to avoid repeated notification triggered by repeated transition between `cannot_be_merged` & `unchecked`. So we added `cannot_be_merged_recheck` state, similar to `unchecked`, but as a mean to remember it’s from cannot_be_merged. See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18042/#note_65945407 Since `unchecked` and `cannot_be_merged_recheck` both mean “we are in the middle of checking if it is mergeable”, quite often we need to see if merge_status is in either one of them, so `check_state?` is added to achieve this. --- app/models/merge_request.rb | 14 ++++++++++---- app/models/project_services/drone_ci_service.rb | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 628c61d5d69..ea5057c2a78 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -104,24 +104,30 @@ class MergeRequest < ActiveRecord::Base state_machine :merge_status, initial: :unchecked do event :mark_as_unchecked do - transition [:can_be_merged, :cannot_be_merged] => :unchecked + transition [:can_be_merged] => :unchecked + transition [:cannot_be_merged] => :cannot_be_merged_recheck end event :mark_as_mergeable do - transition [:unchecked, :cannot_be_merged] => :can_be_merged + transition [:unchecked, :cannot_be_merged_recheck] => :can_be_merged end event :mark_as_unmergeable do - transition [:unchecked, :can_be_merged] => :cannot_be_merged + transition [:unchecked, :cannot_be_merged_recheck] => :cannot_be_merged end state :unchecked + state :cannot_be_merged_recheck state :can_be_merged state :cannot_be_merged around_transition do |merge_request, transition, block| Gitlab::Timeless.timeless(merge_request, &block) end + + def check_state?(merge_status) + [:unchecked, :cannot_be_merged_recheck].include?(merge_status.to_sym) + end end validates :source_project, presence: true, unless: [:allow_broken, :importing?, :closed_without_fork?] @@ -602,7 +608,7 @@ class MergeRequest < ActiveRecord::Base end def check_if_can_be_merged - return unless unchecked? && Gitlab::Database.read_write? + return unless self.class.state_machines[:merge_status].check_state?(merge_status) && Gitlab::Database.read_write? can_be_merged = !broken? && project.repository.can_be_merged?(diff_head_sha, target_branch) diff --git a/app/models/project_services/drone_ci_service.rb b/app/models/project_services/drone_ci_service.rb index 71b10fc6bc1..a4bf427ac0b 100644 --- a/app/models/project_services/drone_ci_service.rb +++ b/app/models/project_services/drone_ci_service.rb @@ -115,6 +115,6 @@ class DroneCiService < CiService def merge_request_valid?(data) data[:object_attributes][:state] == 'opened' && - data[:object_attributes][:merge_status] == 'unchecked' + MergeRequest.state_machines[:merge_status].check_state?(data[:object_attributes][:merge_status]) end end -- cgit v1.2.3 From 179a1ee7c669e758f8dc9fb5e9c8f2563012a6e1 Mon Sep 17 00:00:00 2001 From: lulalala Date: Wed, 9 May 2018 17:43:47 +0800 Subject: Add MergeRequest#merge_participants For notifying via todo or email. --- app/models/merge_request.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'app/models') diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index ea5057c2a78..dd4daf3711b 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -333,6 +333,16 @@ class MergeRequest < ActiveRecord::Base update_column(:merge_jid, jid) end + def merge_participants + participants = [author] + + if merge_when_pipeline_succeeds? && !participants.include?(merge_user) + participants << merge_user + end + + participants + end + def first_commit merge_request_diff ? merge_request_diff.first_commit : compare_commits.first end -- cgit v1.2.3 From dc174e9655267e89e1b7c63f8c9f4dac069069c7 Mon Sep 17 00:00:00 2001 From: lulalala Date: Wed, 9 May 2018 17:49:33 +0800 Subject: Notify with email when merge request became unmergeable Display MR unmergeable reasons --- app/models/merge_request.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'app/models') diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index dd4daf3711b..99687d305e7 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -125,6 +125,10 @@ class MergeRequest < ActiveRecord::Base Gitlab::Timeless.timeless(merge_request, &block) end + after_transition unchecked: :cannot_be_merged do |merge_request, transition| + NotificationService.new.merge_request_unmergeable(merge_request) + end + def check_state?(merge_status) [:unchecked, :cannot_be_merged_recheck].include?(merge_status.to_sym) end -- cgit v1.2.3 From 953bb41f25270c07ab12c17472ef0fe8ab848301 Mon Sep 17 00:00:00 2001 From: lulalala Date: Wed, 9 May 2018 17:55:00 +0800 Subject: Create TODO when MR became unmergeable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Old behavior of creating TODO when “Merge When Pipeline Succeeds” service fails, is generalized to: Create a TODO whenever MR became unmergeable (and similar to notification, MR author and merge_user are both applicable) --- app/models/merge_request.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/models') diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 99687d305e7..a0ad7f3c609 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -127,6 +127,7 @@ class MergeRequest < ActiveRecord::Base after_transition unchecked: :cannot_be_merged do |merge_request, transition| NotificationService.new.merge_request_unmergeable(merge_request) + TodoService.new.merge_request_became_unmergeable(merge_request) end def check_state?(merge_status) -- cgit v1.2.3 From 01275667e323d4702cc396f6f756305b06cba726 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=99=88=20=20jacopo=20beschi=20=F0=9F=99=89?= Date: Thu, 17 May 2018 09:19:47 +0000 Subject: Resolve "Opening Project with invite but without accepting leads to 404 error page" --- app/models/user.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index 474fde36c02..8ef3c3ceff0 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -860,6 +860,16 @@ class User < ActiveRecord::Base confirmed? && !temp_oauth_email? end + def accept_pending_invitations! + pending_invitations.select do |member| + member.accept_invite!(self) + end + end + + def pending_invitations + Member.where(invite_email: verified_emails).invite + end + def all_emails all_emails = [] all_emails << email unless temp_oauth_email? -- cgit v1.2.3 From a63ada5e77c4d817b05552d066dc6004003aaf98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Thu, 17 May 2018 11:57:23 +0200 Subject: Include class name and argument name in argument error --- app/models/concerns/redis_cacheable.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb index 3d24f0dfdc1..b5425295130 100644 --- a/app/models/concerns/redis_cacheable.rb +++ b/app/models/concerns/redis_cacheable.rb @@ -8,7 +8,9 @@ module RedisCacheable def cached_attr_reader(*attributes) attributes.each do |attribute| define_method(attribute) do - raise ArgumentError, "Not a database attribute" unless self.has_attribute?(attribute) + unless self.has_attribute?(attribute) + raise ArgumentError, "`cached_attr_reader` requires the #{self.class.name}\##{attribute} attribute to have a database column" + end cached_attribute(attribute) || read_attribute(attribute) end -- cgit v1.2.3 From 19428e800895ba20eacb3357285acef8d69f6d8c Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 7 May 2018 18:22:07 +0200 Subject: Preload pipeline data for project pipelines When displaying the pipelines of a project we now preload the following data: 1. Authors of the commits that belong to these pipelines 2. The number of warnings per pipeline, which is used by Ci::Pipeline#has_warnings? == Commit Authors Previously this data was queried for every Commit separately, leading to 20 SQL queries being executed in the worst case. With an average of 3 to 5 milliseconds per SQL query this could result in 100 milliseconds being spent in _just_ getting Commit authors. To preload this data Commit#author now uses BatchLoader (through Commit#lazy_author), and a separate module Gitlab::Ci::Pipeline::Preloader is used to ensure all authors are loaded before they are used. == Number of warnings This changes Ci::Pipeline#has_warnings? so it supports preloading of the number of warnings per pipeline. This removes the need for executing a COUNT(*) query for every pipeline just to see if it has any warnings or not. --- app/models/ci/pipeline.rb | 13 ++++++++++++- app/models/commit.rb | 28 +++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 1f49764e7cc..c26f0b6dcdc 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -406,7 +406,18 @@ module Ci end def has_warnings? - builds.latest.failed_but_allowed.any? + number_of_warnings.positive? + end + + def number_of_warnings + BatchLoader.for(id).batch(default_value: 0) do |pipeline_ids, loader| + Build.where(commit_id: pipeline_ids) + .latest + .failed_but_allowed + .group(:commit_id) + .count + .each { |id, amount| loader.call(id, amount) } + end end def set_config_source diff --git a/app/models/commit.rb b/app/models/commit.rb index b46f9f34689..56d4c86774e 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -224,8 +224,34 @@ class Commit Gitlab::ClosingIssueExtractor.new(project, current_user).closed_by_message(safe_message) end + def lazy_author + BatchLoader.for(author_email.downcase).batch do |emails, loader| + # A Hash that maps user Emails to the corresponding User objects. The + # Emails at this point are the _primary_ Emails of the Users. + users_for_emails = User + .by_any_email(emails) + .each_with_object({}) { |user, hash| hash[user.email] = user } + + users_for_ids = users_for_emails + .values + .each_with_object({}) { |user, hash| hash[user.id] = user } + + # Some commits may have used an alternative Email address. In this case we + # need to query the "emails" table to map those addresses to User objects. + Email + .where(email: emails - users_for_emails.keys) + .pluck(:email, :user_id) + .each { |(email, id)| users_for_emails[email] = users_for_ids[id] } + + users_for_emails.each { |email, user| loader.call(email, user) } + end + end + def author - User.find_by_any_email(author_email.downcase) + # We use __sync so that we get the actual objects back (including an actual + # nil), instead of a wrapper, as returning a wrapped nil breaks a lot of + # code. + lazy_author.__sync end request_cache(:author) { author_email.downcase } -- cgit v1.2.3 From b11c218ad9d4635c2230e7f8d105236ca32e5072 Mon Sep 17 00:00:00 2001 From: Harish Ved Date: Thu, 17 May 2018 16:20:41 +0000 Subject: Fix: Use case in-sensitive ordering by name for groups --- app/models/concerns/sortable.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/sortable.rb b/app/models/concerns/sortable.rb index cefa5c13c5f..db7254c27e0 100644 --- a/app/models/concerns/sortable.rb +++ b/app/models/concerns/sortable.rb @@ -12,8 +12,8 @@ module Sortable scope :order_created_asc, -> { reorder(created_at: :asc) } scope :order_updated_desc, -> { reorder(updated_at: :desc) } scope :order_updated_asc, -> { reorder(updated_at: :asc) } - scope :order_name_asc, -> { reorder(name: :asc) } - scope :order_name_desc, -> { reorder(name: :desc) } + scope :order_name_asc, -> { reorder("lower(name) asc") } + scope :order_name_desc, -> { reorder("lower(name) desc") } end module ClassMethods -- cgit v1.2.3 From e4adf0150b58d0b7f8437cbb9a3cb3ac8aa31bec Mon Sep 17 00:00:00 2001 From: Jacopo Date: Tue, 15 May 2018 12:40:17 +0200 Subject: Fixes 500 error on /estimate BIG_VALUE --- app/models/concerns/time_trackable.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'app/models') diff --git a/app/models/concerns/time_trackable.rb b/app/models/concerns/time_trackable.rb index 73fc5048dcf..1caf47072bc 100644 --- a/app/models/concerns/time_trackable.rb +++ b/app/models/concerns/time_trackable.rb @@ -53,6 +53,10 @@ module TimeTrackable Gitlab::TimeTrackingFormatter.output(time_estimate) end + def time_estimate=(val) + val.is_a?(Integer) ? super([val, Gitlab::Database::MAX_INT_VALUE].min) : super(val) + end + private def touchable? -- cgit v1.2.3 From b1ab1609b66f6cb47bb7d7b3babdd29cd0aae3c4 Mon Sep 17 00:00:00 2001 From: Mark Chao Date: Fri, 18 May 2018 17:02:35 +0800 Subject: Avoid race condition of re-triggering mark_as_unchecked --- app/models/merge_request.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index a0ad7f3c609..9c4384a6e42 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -104,8 +104,8 @@ class MergeRequest < ActiveRecord::Base state_machine :merge_status, initial: :unchecked do event :mark_as_unchecked do - transition [:can_be_merged] => :unchecked - transition [:cannot_be_merged] => :cannot_be_merged_recheck + transition [:can_be_merged, :unchecked] => :unchecked + transition [:cannot_be_merged, :cannot_be_merged_recheck] => :cannot_be_merged_recheck end event :mark_as_mergeable do -- cgit v1.2.3 From 9f7deb85b5d6937e6cf7068b864f49693b4a2623 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 18 May 2018 11:28:32 +0200 Subject: Do not allow to use `CI_PIPELINE_ID` in environment name --- app/models/ci/pipeline.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index c26f0b6dcdc..7d7349b04bc 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -523,9 +523,14 @@ module Ci strong_memoize(:legacy_trigger) { trigger_requests.first } end + def persisted_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'CI_PIPELINE_ID', value: id.to_s) if persisted? + end + end + def predefined_variables - Gitlab::Ci::Variables::Collection.new - .append(key: 'CI_PIPELINE_ID', value: id.to_s) + persisted_variables .append(key: 'CI_CONFIG_PATH', value: ci_yaml_file_path) .append(key: 'CI_PIPELINE_SOURCE', value: source.to_s) .append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message) -- cgit v1.2.3 From d9a3f020be570f135c0ce7c7676b4c1ed332ce1d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 18 May 2018 14:05:29 +0200 Subject: Separate persisted and runtime pipeline variables --- app/models/ci/build.rb | 1 + app/models/ci/pipeline.rb | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 61c10c427dd..78f054a6527 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -599,6 +599,7 @@ module Ci break variables unless persisted? variables + .concat(pipeline.persisted_variables) .append(key: 'CI_JOB_ID', value: id.to_s) .append(key: 'CI_JOB_TOKEN', value: token, public: false) .append(key: 'CI_BUILD_ID', value: id.to_s) diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 7d7349b04bc..53af87a271a 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -530,7 +530,7 @@ module Ci end def predefined_variables - persisted_variables + Gitlab::Ci::Variables::Collection.new .append(key: 'CI_CONFIG_PATH', value: ci_yaml_file_path) .append(key: 'CI_PIPELINE_SOURCE', value: source.to_s) .append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message) -- cgit v1.2.3 From 2ebafdfb2f026580153fd2cf50f4b6b7ab3a0344 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Fri, 18 May 2018 15:33:54 +0200 Subject: Improve cacheable module --- app/models/concerns/redis_cacheable.rb | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb index b5425295130..53f022e2b35 100644 --- a/app/models/concerns/redis_cacheable.rb +++ b/app/models/concerns/redis_cacheable.rb @@ -7,11 +7,11 @@ module RedisCacheable class_methods do def cached_attr_reader(*attributes) attributes.each do |attribute| - define_method(attribute) do - unless self.has_attribute?(attribute) - raise ArgumentError, "`cached_attr_reader` requires the #{self.class.name}\##{attribute} attribute to have a database column" - end + unless self.column_names.include?(attribute.to_s) + raise ArgumentError, "`cached_attr_reader` requires the #{self.name}##{attribute} to be a database attribute" + end + define_method(attribute) do cached_attribute(attribute) || read_attribute(attribute) end end @@ -50,7 +50,9 @@ module RedisCacheable if Gitlab.rails5? self.class.type_for_attribute(attribute).cast(value) else - self.class.column_for_attribute(attribute).type_cast_from_database(value) + ActiveSupport::Deprecation.silence do + self.class.column_for_attribute(attribute).type_cast_from_database(value) + end end end end -- cgit v1.2.3 From 55e2ce762d52e680b45c9b87a238f993485f2866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Fri, 18 May 2018 14:34:08 +0000 Subject: Revert "Improve cacheable module" This reverts commit 2ebafdfb2f026580153fd2cf50f4b6b7ab3a0344 --- app/models/concerns/redis_cacheable.rb | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb index 53f022e2b35..b5425295130 100644 --- a/app/models/concerns/redis_cacheable.rb +++ b/app/models/concerns/redis_cacheable.rb @@ -7,11 +7,11 @@ module RedisCacheable class_methods do def cached_attr_reader(*attributes) attributes.each do |attribute| - unless self.column_names.include?(attribute.to_s) - raise ArgumentError, "`cached_attr_reader` requires the #{self.name}##{attribute} to be a database attribute" - end - define_method(attribute) do + unless self.has_attribute?(attribute) + raise ArgumentError, "`cached_attr_reader` requires the #{self.class.name}\##{attribute} attribute to have a database column" + end + cached_attribute(attribute) || read_attribute(attribute) end end @@ -50,9 +50,7 @@ module RedisCacheable if Gitlab.rails5? self.class.type_for_attribute(attribute).cast(value) else - ActiveSupport::Deprecation.silence do - self.class.column_for_attribute(attribute).type_cast_from_database(value) - end + self.class.column_for_attribute(attribute).type_cast_from_database(value) end end end -- cgit v1.2.3 From dbe0839396f56e30780350e840a1ded303dfbb81 Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Fri, 18 May 2018 10:29:20 -0500 Subject: Fixes deploy tokens build variables It was using name, instead of username. Fixes documentation as well Closes #46454 --- app/models/ci/build.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index d9649e30edc..1c42ed4d3e5 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -661,7 +661,7 @@ module Ci Gitlab::Ci::Variables::Collection.new.tap do |variables| break variables unless gitlab_deploy_token - variables.append(key: 'CI_DEPLOY_USER', value: gitlab_deploy_token.name) + variables.append(key: 'CI_DEPLOY_USER', value: gitlab_deploy_token.username) variables.append(key: 'CI_DEPLOY_PASSWORD', value: gitlab_deploy_token.token, public: false) end end -- cgit v1.2.3 From c1e89492375b696b25ff63257c3cbe58ce86cc18 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 16 May 2018 21:35:24 +0900 Subject: Rescue Kubeclient::HttpError when generating prometheus_client --- app/models/clusters/applications/prometheus.rb | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'app/models') diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb index 7b25d8c4089..c702c4ee807 100644 --- a/app/models/clusters/applications/prometheus.rb +++ b/app/models/clusters/applications/prometheus.rb @@ -49,6 +49,11 @@ module Clusters # ensures headers containing auth data are appended to original k8s client options options = kube_client.rest_client.options.merge(headers: kube_client.headers) RestClient::Resource.new(proxy_url, options) + rescue Kubeclient::HttpError + # If users have mistakenly set parameters or removed the depended clusters, + # `proxy_url` could raise an exception because gitlab can not communicate with the cluster. + # Since `PrometheusAdapter#can_query?` is eargely loaded on environement pages in gitlab, + # we need to silence the exceptions end private -- cgit v1.2.3 From 631bd9bf082c396059867d512fcfbdc80445c65e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 21 May 2018 11:39:46 +0200 Subject: Use persisted stages to load pipelines index table --- app/models/ci/pipeline.rb | 2 +- app/models/ci/stage.rb | 18 ++++++++++++++++++ app/models/project.rb | 1 + 3 files changed, 20 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index c26f0b6dcdc..e7569e0d31e 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -13,7 +13,7 @@ module Ci belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' - has_many :stages + has_many :stages, inverse_of: :pipeline # -> { order(position: :asc) }, inverse_of: :pipeline has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 5a1eeb966aa..4fc8a00d9c3 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -79,5 +79,23 @@ module Ci end end end + + def groups + @groups ||= statuses.ordered.latest + .sort_by(&:sortable_name).group_by(&:group_name) + .map do |group_name, grouped_statuses| + Ci::Group.new(self, name: group_name, jobs: grouped_statuses) + end + end + + def has_warnings? + statuses.latest.failed_but_allowed.any? + end + + def detailed_status(current_user) + Gitlab::Ci::Status::Stage::Factory + .new(self, current_user) + .fabricate! + end end end diff --git a/app/models/project.rb b/app/models/project.rb index 35c873830a7..50b8a9a95ea 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -227,6 +227,7 @@ class Project < ActiveRecord::Base has_many :commit_statuses has_many :pipelines, class_name: 'Ci::Pipeline', inverse_of: :project + has_many :stages, class_name: 'Ci::Stage', inverse_of: :project # Ci::Build objects store data on the file system such as artifact files and # build traces. Currently there's no efficient way of removing this data in -- cgit v1.2.3 From a0c79f9d7025872fc2aa91805058739b26093989 Mon Sep 17 00:00:00 2001 From: Andreas Brandl Date: Thu, 17 May 2018 10:28:41 +0200 Subject: Add NOT NULL constraints to project_authorizations. Closes #32258. --- app/models/user.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index 8ef3c3ceff0..5a4c373705b 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -109,7 +109,7 @@ class User < ActiveRecord::Base has_many :created_projects, foreign_key: :creator_id, class_name: 'Project' has_many :users_star_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :starred_projects, through: :users_star_projects, source: :project - has_many :project_authorizations + has_many :project_authorizations, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_many :authorized_projects, through: :project_authorizations, source: :project has_many :user_interacted_projects -- cgit v1.2.3 From 83cda43a00652d992cb2a47920db81cc0dae23e4 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Fri, 18 May 2018 11:01:27 +0000 Subject: Fix Xcode project detection by looking for dirs instead of files --- app/models/repository.rb | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/repository.rb b/app/models/repository.rb index 44c6bff6b66..0e1bf11d7c0 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -596,7 +596,7 @@ class Repository cache_method :gitlab_ci_yml def xcode_project? - file_on_head(:xcode_config).present? + file_on_head(:xcode_config, :tree).present? end cache_method :xcode_project? @@ -920,11 +920,21 @@ class Repository end end - def file_on_head(type) - if head = tree(:head) - head.blobs.find do |blob| - Gitlab::FileDetector.type_of(blob.path) == type + def file_on_head(type, object_type = :blob) + return unless head = tree(:head) + + objects = + case object_type + when :blob + head.blobs + when :tree + head.trees + else + raise ArgumentError, "Object type #{object_type} is not supported" end + + objects.find do |object| + Gitlab::FileDetector.type_of(object.path) == type end end -- cgit v1.2.3 From 9520d2ff278f12cf2e01a755b1ea12213fd22edb Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 22 May 2018 12:29:52 +0200 Subject: Memoize project active runners to avoid N+1 queries --- app/models/project.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 50b8a9a95ea..5943a929364 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1418,7 +1418,9 @@ class Project < ActiveRecord::Base end def any_runners?(&block) - all_runners.active.any?(&block) + @active_runners ||= all_runners.active + + @active_runners.any?(&block) end def valid_runners_token?(token) -- cgit v1.2.3 From 6c63f96e0a26fa046fb0cc4664240e37db8b37d8 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 22 May 2018 12:30:45 +0200 Subject: Preload number of warnings in every stage in a pipeline This makes it possible to avoid N+1 queries when loading pipelines table. --- app/models/ci/pipeline.rb | 2 +- app/models/ci/stage.rb | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index e7569e0d31e..9462afe9e71 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -411,7 +411,7 @@ module Ci def number_of_warnings BatchLoader.for(id).batch(default_value: 0) do |pipeline_ids, loader| - Build.where(commit_id: pipeline_ids) + ::Ci::Build.where(commit_id: pipeline_ids) .latest .failed_but_allowed .group(:commit_id) diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 4fc8a00d9c3..faedb9c29fd 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -89,7 +89,18 @@ module Ci end def has_warnings? - statuses.latest.failed_but_allowed.any? + number_of_warnings.positive? + end + + def number_of_warnings + BatchLoader.for(id).batch(default_value: 0) do |stage_ids, loader| + ::Ci::Build.where(stage_id: stage_ids) + .latest + .failed_but_allowed + .group(:stage_id) + .count + .each { |id, amount| loader.call(id, amount) } + end end def detailed_status(current_user) -- cgit v1.2.3 From 3f73b6bee07b81814a623776338abe7b74885302 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Thu, 19 Apr 2018 11:20:53 +0200 Subject: Don't set the notification_email when only unconfirmed_email is changed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- app/models/user.rb | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) (limited to 'app/models') diff --git a/app/models/user.rb b/app/models/user.rb index 8ef3c3ceff0..e937375feb4 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -165,8 +165,7 @@ class User < ActiveRecord::Base validate :signup_domain_valid?, on: :create, if: ->(user) { !user.created_by_id } before_validation :sanitize_attrs - before_validation :set_notification_email, if: :email_changed? - before_save :set_notification_email, if: :email_changed? # in case validation is skipped + before_validation :set_notification_email, if: :new_record? before_validation :set_public_email, if: :public_email_changed? before_save :set_public_email, if: :public_email_changed? # in case validation is skipped before_save :ensure_incoming_email_token @@ -179,8 +178,21 @@ class User < ActiveRecord::Base after_update :username_changed_hook, if: :username_changed? after_destroy :post_destroy_hook after_destroy :remove_key_cache - after_commit :update_emails_with_primary_email, on: :update, if: -> { previous_changes.key?('email') } - after_commit :update_invalid_gpg_signatures, on: :update, if: -> { previous_changes.key?('email') } + after_commit(on: :update) do + if previous_changes.key?('email') + # Grab previous_email here since previous_changes changes after + # #update_emails_with_primary_email and #update_notification_email are called + previous_email = previous_changes[:email][0] + + update_emails_with_primary_email(previous_email) + update_invalid_gpg_signatures + + if previous_email == notification_email + self.notification_email = email + save + end + end + end after_initialize :set_projects_limit @@ -546,8 +558,7 @@ class User < ActiveRecord::Base # hash and `_was` variables getting munged. # By using an `after_commit` instead of `after_update`, we avoid the recursive callback # scenario, though it then requires us to use the `previous_changes` hash - def update_emails_with_primary_email - previous_email = previous_changes[:email][0] # grab this before the DestroyService is called + def update_emails_with_primary_email(previous_email) primary_email_record = emails.find_by(email: email) Emails::DestroyService.new(self, user: self).execute(primary_email_record) if primary_email_record @@ -772,13 +783,13 @@ class User < ActiveRecord::Base end def set_notification_email - if notification_email.blank? || !all_emails.include?(notification_email) + if notification_email.blank? || all_emails.exclude?(notification_email) self.notification_email = email end end def set_public_email - if public_email.blank? || !all_emails.include?(public_email) + if public_email.blank? || all_emails.exclude?(public_email) self.public_email = '' end end -- cgit v1.2.3 From 10237d458d0266787aa15852b525469d0dcb5362 Mon Sep 17 00:00:00 2001 From: Imre Farkas Date: Tue, 22 May 2018 10:53:37 +0000 Subject: Expose readme url in Project API --- app/models/project.rb | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 35c873830a7..0e727664d39 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -894,6 +894,13 @@ class Project < ActiveRecord::Base Gitlab::Routing.url_helpers.project_url(self) end + def readme_url + readme = repository.readme + if readme + Gitlab::Routing.url_helpers.project_blob_url(self, File.join(default_branch, readme.path)) + end + end + def new_issuable_address(author, address_type) return unless Gitlab::IncomingEmail.supports_issue_creation? && author -- cgit v1.2.3 From fb706d69abb2bb8f07f78b2a14206027f062cc30 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 22 May 2018 13:03:42 +0200 Subject: Order pipeline stages by a position in a relation --- app/models/ci/pipeline.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 9462afe9e71..774d5eb8125 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -13,7 +13,7 @@ module Ci belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' - has_many :stages, inverse_of: :pipeline # -> { order(position: :asc) }, inverse_of: :pipeline + has_many :stages, -> { order(position: :asc) }, inverse_of: :pipeline has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent -- cgit v1.2.3 From 76a7157c76c47fd4f835a27eeeda7b42012936be Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 22 May 2018 13:04:07 +0200 Subject: Abstract persisted/legacy stages in pipeline model --- app/models/ci/pipeline.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 774d5eb8125..9dac56fcd57 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -249,6 +249,16 @@ module Ci stage unless stage.statuses_count.zero? end + ## + # TODO consider switching to persisted stages only in pipelines table + # (not necessairly in the show pipeline page because of #23257. + # Hide this behind two feature flags - enabled / disabled and only + # gitlab-ce / everywhere. + # + def stages + super + end + def legacy_stages # TODO, this needs refactoring, see gitlab-ce#26481. -- cgit v1.2.3 From be3a0377dad4da54f5dbc51b88e39394c1b920be Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 22 May 2018 14:03:02 +0200 Subject: Fix memoization-related rubocop offense in project --- app/models/project.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 5943a929364..e0f6c856f00 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1417,10 +1417,14 @@ class Project < ActiveRecord::Base Ci::Runner.from("(#{union.to_sql}) ci_runners") end - def any_runners?(&block) - @active_runners ||= all_runners.active + def active_runners + strong_memoize(:active_runners) do + all_runners.active + end + end - @active_runners.any?(&block) + def any_runners?(&block) + active_runners.any?(&block) end def valid_runners_token?(token) -- cgit v1.2.3 From 8cca6c83a99100fcf3a0a3a56f10eebe3c9b7716 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 23 May 2018 10:37:01 +0200 Subject: DRY creating groups of common builds in a stage --- app/models/ci/group.rb | 8 ++++++++ app/models/ci/legacy_stage.rb | 6 +----- app/models/ci/stage.rb | 6 +----- 3 files changed, 10 insertions(+), 10 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/group.rb b/app/models/ci/group.rb index 87898b086c6..9c1046e8715 100644 --- a/app/models/ci/group.rb +++ b/app/models/ci/group.rb @@ -31,6 +31,14 @@ module Ci end end + def self.fabricate(stage) + stage.statuses.ordered.latest + .sort_by(&:sortable_name).group_by(&:group_name) + .map do |group_name, grouped_statuses| + self.new(stage, name: group_name, jobs: grouped_statuses) + end + end + private def commit_statuses diff --git a/app/models/ci/legacy_stage.rb b/app/models/ci/legacy_stage.rb index 9b536af672b..ce691875e42 100644 --- a/app/models/ci/legacy_stage.rb +++ b/app/models/ci/legacy_stage.rb @@ -16,11 +16,7 @@ module Ci end def groups - @groups ||= statuses.ordered.latest - .sort_by(&:sortable_name).group_by(&:group_name) - .map do |group_name, grouped_statuses| - Ci::Group.new(self, name: group_name, jobs: grouped_statuses) - end + @groups ||= Ci::Group.fabricate(self) end def to_param diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index faedb9c29fd..0044d1af106 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -81,11 +81,7 @@ module Ci end def groups - @groups ||= statuses.ordered.latest - .sort_by(&:sortable_name).group_by(&:group_name) - .map do |group_name, grouped_statuses| - Ci::Group.new(self, name: group_name, jobs: grouped_statuses) - end + @groups ||= Ci::Group.fabricate(self) end def has_warnings? -- cgit v1.2.3 From 40dd12c7758c90ad124e68bd549027eac1973fe6 Mon Sep 17 00:00:00 2001 From: Olivier Gonzalez Date: Wed, 23 May 2018 09:40:45 +0000 Subject: Resolve "Deprecate Gemnasium project service" --- app/models/project_services/gemnasium_service.rb | 13 +++++++++++++ app/models/service.rb | 1 - 2 files changed, 13 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/project_services/gemnasium_service.rb b/app/models/project_services/gemnasium_service.rb index 26cbfd784ad..84248f9590b 100644 --- a/app/models/project_services/gemnasium_service.rb +++ b/app/models/project_services/gemnasium_service.rb @@ -3,6 +3,7 @@ require "gemnasium/gitlab_service" class GemnasiumService < Service prop_accessor :token, :api_key validates :token, :api_key, presence: true, if: :activated? + validate :deprecation_validation def title 'Gemnasium' @@ -27,6 +28,18 @@ class GemnasiumService < Service %w(push) end + def deprecated? + true + end + + def deprecation_message + "Gemnasium has been acquired by GitLab in January 2018. Since May 15, 2018, the service provided by Gemnasium is no longer available." + end + + def deprecation_validation + errors[:base] << deprecation_message + end + def execute(data) return unless supported_events.include?(data[:object_kind]) diff --git a/app/models/service.rb b/app/models/service.rb index f7e3f7590ad..831c2ea1141 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -253,7 +253,6 @@ class Service < ActiveRecord::Base emails_on_push external_wiki flowdock - gemnasium hipchat irker jira -- cgit v1.2.3 From 02f17a0988acfdc60d7e1f920e8f750cb81f09d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Fri, 4 May 2018 19:23:08 +0200 Subject: Introduce a new CacheableAttributes concern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- app/models/concerns/cacheable_attributes.rb | 54 +++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 app/models/concerns/cacheable_attributes.rb (limited to 'app/models') diff --git a/app/models/concerns/cacheable_attributes.rb b/app/models/concerns/cacheable_attributes.rb new file mode 100644 index 00000000000..b32459fdabf --- /dev/null +++ b/app/models/concerns/cacheable_attributes.rb @@ -0,0 +1,54 @@ +module CacheableAttributes + extend ActiveSupport::Concern + + included do + after_commit { self.class.expire } + end + + class_methods do + # Can be overriden + def current_without_cache + last + end + + def cache_key + "#{name}:#{Gitlab::VERSION}:#{Gitlab.migrations_hash}:json".freeze + end + + def defaults + {} + end + + def build_from_defaults(attributes = {}) + new(defaults.merge(attributes)) + end + + def cached + json_attributes = Rails.cache.read(cache_key) + return nil unless json_attributes.present? + + build_from_defaults(JSON.parse(json_attributes)) + end + + def current + cached_record = cached + return cached_record if cached_record.present? + + current_without_cache.tap { |current_record| current_record&.cache! } + rescue + # Fall back to an uncached value if there are any problems (e.g. Redis down) + current_without_cache + end + + def expire + Rails.cache.delete(cache_key) + rescue + # Gracefully handle when Redis is not available. For example, + # omnibus may fail here during gitlab:assets:compile. + end + end + + def cache! + Rails.cache.write(self.class.cache_key, attributes.to_json) + end +end -- cgit v1.2.3 From e531a0c11cebc282029ba735d69c44c39660ca34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Fri, 4 May 2018 19:23:50 +0200 Subject: Use the new CacheableAttributes concern in the ApplicationSetting and Appearance models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- app/models/appearance.rb | 15 ++++----------- app/models/application_setting.rb | 36 +----------------------------------- 2 files changed, 5 insertions(+), 46 deletions(-) (limited to 'app/models') diff --git a/app/models/appearance.rb b/app/models/appearance.rb index f8713138a93..67cc84a9140 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -1,6 +1,6 @@ class Appearance < ActiveRecord::Base + include CacheableAttributes include CacheMarkdownField - include AfterCommitQueue include ObjectStorage::BackgroundMove include WithUploads @@ -15,16 +15,9 @@ class Appearance < ActiveRecord::Base mount_uploader :logo, AttachmentUploader mount_uploader :header_logo, AttachmentUploader - CACHE_KEY = "current_appearance:#{Gitlab::VERSION}".freeze - - after_commit :flush_redis_cache - - def self.current - Rails.cache.fetch(CACHE_KEY) { first } - end - - def flush_redis_cache - Rails.cache.delete(CACHE_KEY) + # Overrides CacheableAttributes.current_without_cache + def self.current_without_cache + first end def single_appearance_row diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 451e512aef7..e8ccb320fae 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -1,11 +1,11 @@ class ApplicationSetting < ActiveRecord::Base + include CacheableAttributes include CacheMarkdownField include TokenAuthenticatable add_authentication_token_field :runners_registration_token add_authentication_token_field :health_check_access_token - CACHE_KEY = 'application_setting.last'.freeze DOMAIN_LIST_SEPARATOR = %r{\s*[,;]\s* # comma or semicolon, optionally surrounded by whitespace | # or \s # any whitespace character @@ -229,40 +229,6 @@ class ApplicationSetting < ActiveRecord::Base after_commit do reset_memoized_terms - Rails.cache.write(CACHE_KEY, self) - end - - def self.current - ensure_cache_setup - - Rails.cache.fetch(CACHE_KEY) do - ApplicationSetting.last.tap do |settings| - # do not cache nils - raise 'missing settings' unless settings - end - end - rescue - # Fall back to an uncached value if there are any problems (e.g. redis down) - ApplicationSetting.last - end - - def self.expire - Rails.cache.delete(CACHE_KEY) - rescue - # Gracefully handle when Redis is not available. For example, - # omnibus may fail here during gitlab:assets:compile. - end - - def self.cached - value = Rails.cache.read(CACHE_KEY) - ensure_cache_setup if value.present? - value - end - - def self.ensure_cache_setup - # This is a workaround for a Rails bug that causes attribute methods not - # to be loaded when read from cache: https://github.com/rails/rails/issues/27348 - ApplicationSetting.define_attribute_methods end def self.defaults -- cgit v1.2.3 From 0230ca839a2ec8debcbe4476c3deed8b422bf451 Mon Sep 17 00:00:00 2001 From: Jan Provaznik Date: Tue, 22 May 2018 12:24:14 +0200 Subject: Check if note's noteable is not nil when checking resolvability This can occur when a note is added to a commit and then this commit is deleted. --- app/models/concerns/resolvable_note.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/concerns/resolvable_note.rb b/app/models/concerns/resolvable_note.rb index 668c5a079e3..4a0f8b92b3a 100644 --- a/app/models/concerns/resolvable_note.rb +++ b/app/models/concerns/resolvable_note.rb @@ -32,7 +32,7 @@ module ResolvableNote # Keep this method in sync with the `potentially_resolvable` scope def potentially_resolvable? - RESOLVABLE_TYPES.include?(self.class.name) && noteable.supports_resolvable_notes? + RESOLVABLE_TYPES.include?(self.class.name) && noteable&.supports_resolvable_notes? end # Keep this method in sync with the `resolvable` scope -- cgit v1.2.3 From d50a80a282b2ca145841774f4be154a977cccfd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Thu, 24 May 2018 09:57:54 +0200 Subject: Replace Gitlab::REVISION with Gitlab.revision and handle installations without a .git directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- app/models/ci/build.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 495430931aa..75fd55a8f7b 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -618,7 +618,7 @@ module Ci variables.append(key: 'GITLAB_FEATURES', value: project.licensed_features.join(',')) variables.append(key: 'CI_SERVER_NAME', value: 'GitLab') variables.append(key: 'CI_SERVER_VERSION', value: Gitlab::VERSION) - variables.append(key: 'CI_SERVER_REVISION', value: Gitlab::REVISION) + variables.append(key: 'CI_SERVER_REVISION', value: Gitlab.revision) variables.append(key: 'CI_JOB_NAME', value: name) variables.append(key: 'CI_JOB_STAGE', value: stage) variables.append(key: 'CI_COMMIT_SHA', value: sha) -- cgit v1.2.3 From aee2b235499f9ea179aed82d6beb8ea955986aa3 Mon Sep 17 00:00:00 2001 From: Felipe Artur Date: Thu, 17 May 2018 16:39:48 -0300 Subject: Fix sidebar issue count --- app/models/project.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 534a0e630af..f227ee11cb0 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1417,8 +1417,8 @@ class Project < ActiveRecord::Base self.runners_token && ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.runners_token) end - def open_issues_count - Projects::OpenIssuesCountService.new(self).count + def open_issues_count(current_user = nil) + Projects::OpenIssuesCountService.new(self, current_user).count end def open_merge_requests_count -- cgit v1.2.3 From bb8f2520b4254c9dabe377df48e29c5f17894a1d Mon Sep 17 00:00:00 2001 From: Oswaldo Ferreira Date: Wed, 16 May 2018 12:46:18 -0300 Subject: Persist truncated note diffs on a new table We request Gitaly in a N+1 manner to build discussion diffs. Once the diffs are from different revisions, it's hard to make a single request to the service in order to build the whole response. With this change we solve this problem and simplify a lot fetching this piece of info. --- app/models/concerns/diff_file.rb | 9 +++++ app/models/diff_note.rb | 67 +++++++++++++++++++++++++++-------- app/models/merge_request_diff_file.rb | 7 +--- app/models/note.rb | 4 ++- app/models/note_diff_file.rb | 7 ++++ 5 files changed, 72 insertions(+), 22 deletions(-) create mode 100644 app/models/concerns/diff_file.rb create mode 100644 app/models/note_diff_file.rb (limited to 'app/models') diff --git a/app/models/concerns/diff_file.rb b/app/models/concerns/diff_file.rb new file mode 100644 index 00000000000..72332072012 --- /dev/null +++ b/app/models/concerns/diff_file.rb @@ -0,0 +1,9 @@ +module DiffFile + extend ActiveSupport::Concern + + def to_hash + keys = Gitlab::Git::Diff::SERIALIZE_KEYS - [:diff] + + as_json(only: keys).merge(diff: diff).with_indifferent_access + end +end diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb index 616a626419b..d752d5bcdee 100644 --- a/app/models/diff_note.rb +++ b/app/models/diff_note.rb @@ -3,6 +3,7 @@ # A note of this type can be resolvable. class DiffNote < Note include NoteOnDiff + include Gitlab::Utils::StrongMemoize NOTEABLE_TYPES = %w(MergeRequest Commit).freeze @@ -12,7 +13,6 @@ class DiffNote < Note validates :original_position, presence: true validates :position, presence: true - validates :diff_line, presence: true, if: :on_text? validates :line_code, presence: true, line_code: true, if: :on_text? validates :noteable_type, inclusion: { in: NOTEABLE_TYPES } validate :positions_complete @@ -23,6 +23,7 @@ class DiffNote < Note before_validation :update_position, on: :create, if: :on_text? before_validation :set_line_code, if: :on_text? after_save :keep_around_commits + after_commit :create_diff_file, on: :create def discussion_class(*) DiffDiscussion @@ -53,21 +54,25 @@ class DiffNote < Note position.position_type == "image" end + def create_diff_file + return unless should_create_diff_file? + + diff_file = fetch_diff_file + diff_line = diff_file.line_for_position(self.original_position) + + creation_params = diff_file.diff.to_hash + .except(:too_large) + .merge(diff: diff_file.diff_hunk(diff_line)) + + create_note_diff_file(creation_params) + end + def diff_file - @diff_file ||= - begin - if created_at_diff?(noteable.diff_refs) - # We're able to use the already persisted diffs (Postgres) if we're - # presenting a "current version" of the MR discussion diff. - # So no need to make an extra Gitaly diff request for it. - # As an extra benefit, the returned `diff_file` already - # has `highlighted_diff_lines` data set from Redis on - # `Diff::FileCollection::MergeRequestDiff`. - noteable.diffs(paths: original_position.paths, expanded: true).diff_files.first - else - original_position.diff_file(self.project.repository) - end - end + strong_memoize(:diff_file) do + enqueue_diff_file_creation_job if should_create_diff_file? + + fetch_diff_file + end end def diff_line @@ -98,6 +103,38 @@ class DiffNote < Note private + def enqueue_diff_file_creation_job + # Avoid enqueuing multiple file creation jobs at once for a note (i.e. + # parallel calls to `DiffNote#diff_file`). + lease = Gitlab::ExclusiveLease.new("note_diff_file_creation:#{id}", timeout: 1.hour.to_i) + return unless lease.try_obtain + + CreateNoteDiffFileWorker.perform_async(id) + end + + def should_create_diff_file? + on_text? && note_diff_file.nil? && self == discussion.first_note + end + + def fetch_diff_file + if note_diff_file + diff = Gitlab::Git::Diff.new(note_diff_file.to_hash) + Gitlab::Diff::File.new(diff, + repository: project.repository, + diff_refs: original_position.diff_refs) + elsif created_at_diff?(noteable.diff_refs) + # We're able to use the already persisted diffs (Postgres) if we're + # presenting a "current version" of the MR discussion diff. + # So no need to make an extra Gitaly diff request for it. + # As an extra benefit, the returned `diff_file` already + # has `highlighted_diff_lines` data set from Redis on + # `Diff::FileCollection::MergeRequestDiff`. + noteable.diffs(paths: original_position.paths, expanded: true).diff_files.first + else + original_position.diff_file(self.project.repository) + end + end + def supported? for_commit? || self.noteable.has_complete_diff_refs? end diff --git a/app/models/merge_request_diff_file.rb b/app/models/merge_request_diff_file.rb index 1199ff5af22..cd8ba6b904d 100644 --- a/app/models/merge_request_diff_file.rb +++ b/app/models/merge_request_diff_file.rb @@ -1,5 +1,6 @@ class MergeRequestDiffFile < ActiveRecord::Base include Gitlab::EncodingHelper + include DiffFile belongs_to :merge_request_diff @@ -12,10 +13,4 @@ class MergeRequestDiffFile < ActiveRecord::Base def diff binary? ? super.unpack('m0').first : super end - - def to_hash - keys = Gitlab::Git::Diff::SERIALIZE_KEYS - [:diff] - - as_json(only: keys).merge(diff: diff).with_indifferent_access - end end diff --git a/app/models/note.rb b/app/models/note.rb index 109405d3f17..02f7a9b1e4f 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -63,6 +63,7 @@ class Note < ActiveRecord::Base has_many :todos has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_one :system_note_metadata + has_one :note_diff_file, inverse_of: :diff_note, foreign_key: :diff_note_id delegate :gfm_reference, :local_reference, to: :noteable delegate :name, to: :project, prefix: true @@ -100,7 +101,8 @@ class Note < ActiveRecord::Base scope :inc_author_project, -> { includes(:project, :author) } scope :inc_author, -> { includes(:author) } scope :inc_relations_for_view, -> do - includes(:project, :author, :updated_by, :resolved_by, :award_emoji, :system_note_metadata) + includes(:project, :author, :updated_by, :resolved_by, :award_emoji, + :system_note_metadata, :note_diff_file) end scope :diff_notes, -> { where(type: %w(LegacyDiffNote DiffNote)) } diff --git a/app/models/note_diff_file.rb b/app/models/note_diff_file.rb new file mode 100644 index 00000000000..e688018a6d9 --- /dev/null +++ b/app/models/note_diff_file.rb @@ -0,0 +1,7 @@ +class NoteDiffFile < ActiveRecord::Base + include DiffFile + + belongs_to :diff_note, inverse_of: :note_diff_file + + validates :diff_note, presence: true +end -- cgit v1.2.3 From 760fdd1dd31d30d5ab407a0c42e864040d79504c Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 26 Apr 2018 19:45:22 -0700 Subject: Fix project destruction failing due to idle in transaction timeouts When deleting associated records, Rails loads all associations into memory (https://github.com/rails/rails/issues/22510) before destroying them. This can cause a surge in memory and cause destruction of objects to fail due to idle in transaction database timeouts. This fix is inspired from https://github.com/thisismydesign to destroy `has_many` relationships in batches. Closes #44610 --- .../batch_destroy_dependent_associations.rb | 28 ++++++++++++++++++++++ app/models/project.rb | 1 + 2 files changed, 29 insertions(+) create mode 100644 app/models/concerns/batch_destroy_dependent_associations.rb (limited to 'app/models') diff --git a/app/models/concerns/batch_destroy_dependent_associations.rb b/app/models/concerns/batch_destroy_dependent_associations.rb new file mode 100644 index 00000000000..353ee2e73d0 --- /dev/null +++ b/app/models/concerns/batch_destroy_dependent_associations.rb @@ -0,0 +1,28 @@ +# Provides a way to work around Rails issue where dependent objects are all +# loaded into memory before destroyed: https://github.com/rails/rails/issues/22510. +# +# This concern allows an ActiveRecord module to destroy all its dependent +# associations in batches. The idea is borrowed from https://github.com/thisismydesign/batch_dependent_associations. +# +# The differences here with that gem: +# +# 1. We allow excluding certain associations. +# 2. We don't need to support delete_all since we can use the EachBatch concern. +module BatchDestroyDependentAssociations + extend ActiveSupport::Concern + + DEPENDENT_ASSOCIATIONS_BATCH_SIZE = 1000 + + def dependent_associations_to_destroy + self.class.reflect_on_all_associations(:has_many).select { |assoc| assoc.options[:dependent] == :destroy } + end + + def destroy_dependent_associations_in_batches(exclude: []) + dependent_associations_to_destroy.each do |association| + next if exclude.include?(association.name) + + # rubocop:disable GitlabSecurity/PublicSend + public_send(association.name).find_each(batch_size: DEPENDENT_ASSOCIATIONS_BATCH_SIZE, &:destroy) + end + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 0fe9f8880b4..e275ac4dc6f 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -24,6 +24,7 @@ class Project < ActiveRecord::Base include ChronicDurationAttribute include FastDestroyAll::Helpers include WithUploads + include BatchDestroyDependentAssociations extend Gitlab::ConfigHelper -- cgit v1.2.3 From 887818d3cb58406555bc038ed36b75b7a4ce2631 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 25 May 2018 12:43:27 +0200 Subject: Do not update stage status when it is just created --- app/models/ci/stage.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 0044d1af106..abf73c00539 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -74,8 +74,7 @@ module Ci when 'failed' then drop when 'canceled' then cancel when 'manual' then block - when 'skipped' then skip - else skip + when 'skipped', nil then skip end end end -- cgit v1.2.3 From 8a1ac8f4ce0d8e96234ef32cd032adaf7cc57b1a Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Fri, 11 May 2018 16:29:55 +0300 Subject: Add Applications::Jupyter class sceleton Signed-off-by: Dmitriy Zaporozhets --- app/models/clusters/applications/jupyter.rb | 28 ++++++++++++++++++++++++++++ app/models/clusters/cluster.rb | 3 ++- 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 app/models/clusters/applications/jupyter.rb (limited to 'app/models') diff --git a/app/models/clusters/applications/jupyter.rb b/app/models/clusters/applications/jupyter.rb new file mode 100644 index 00000000000..ec75c120dac --- /dev/null +++ b/app/models/clusters/applications/jupyter.rb @@ -0,0 +1,28 @@ +module Clusters + module Applications + class Jupyter < ActiveRecord::Base + VERSION = '0.0.1'.freeze + + self.table_name = 'clusters_applications_jupyters' + + include ::Clusters::Concerns::ApplicationCore + include ::Clusters::Concerns::ApplicationStatus + include ::Clusters::Concerns::ApplicationData + + default_value_for :version, VERSION + + def chart + # TODO: publish jupyterhub charts that we can use for our installation + # and provide path to it here. + end + + def install_command + Gitlab::Kubernetes::Helm::InstallCommand.new( + name, + chart: chart, + values: values + ) + end + end + end +end diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 77947d515c1..92e5da77066 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -8,7 +8,8 @@ module Clusters Applications::Helm.application_name => Applications::Helm, Applications::Ingress.application_name => Applications::Ingress, Applications::Prometheus.application_name => Applications::Prometheus, - Applications::Runner.application_name => Applications::Runner + Applications::Runner.application_name => Applications::Runner, + Applications::Jupyter.application_name => Applications::Jupyter }.freeze DEFAULT_ENVIRONMENT = '*'.freeze -- cgit v1.2.3 From 4220e914db356f4a55c771a7ad7f559e2507dd56 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Wed, 16 May 2018 12:01:13 +0300 Subject: Add support for Jupyter in GitLab via Kubernetes Signed-off-by: Dmitriy Zaporozhets --- app/models/clusters/applications/jupyter.rb | 28 +++++++++++++++++++++++++--- app/models/clusters/cluster.rb | 4 +++- 2 files changed, 28 insertions(+), 4 deletions(-) (limited to 'app/models') diff --git a/app/models/clusters/applications/jupyter.rb b/app/models/clusters/applications/jupyter.rb index ec75c120dac..ef62be34abd 100644 --- a/app/models/clusters/applications/jupyter.rb +++ b/app/models/clusters/applications/jupyter.rb @@ -12,17 +12,39 @@ module Clusters default_value_for :version, VERSION def chart - # TODO: publish jupyterhub charts that we can use for our installation - # and provide path to it here. + "#{name}/jupyterhub" + end + + def repository + 'https://jupyterhub.github.io/helm-chart/' + end + + def values + content_values.to_yaml end def install_command Gitlab::Kubernetes::Helm::InstallCommand.new( name, chart: chart, - values: values + values: values, + repository: repository ) end + + private + + def specification + { + "ingress" => { "hosts" => [hostname] }, + "hub" => { "cookieSecret" => SecureRandom.hex(32) }, + "proxy" => { "secretToken" => SecureRandom.hex(32) } + } + end + + def content_values + YAML.load_file(chart_values_file).deep_merge!(specification) + end end end end diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 92e5da77066..d99f858e0c0 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -27,6 +27,7 @@ module Clusters has_one :application_ingress, class_name: 'Clusters::Applications::Ingress' has_one :application_prometheus, class_name: 'Clusters::Applications::Prometheus' has_one :application_runner, class_name: 'Clusters::Applications::Runner' + has_one :application_jupyter, class_name: 'Clusters::Applications::Jupyter' accepts_nested_attributes_for :provider_gcp, update_only: true accepts_nested_attributes_for :platform_kubernetes, update_only: true @@ -75,7 +76,8 @@ module Clusters application_helm || build_application_helm, application_ingress || build_application_ingress, application_prometheus || build_application_prometheus, - application_runner || build_application_runner + application_runner || build_application_runner, + application_jupyter || build_application_jupyter ] end -- cgit v1.2.3 From b3cf1530829755411a7057031c2e8ea36ee31b62 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Thu, 24 May 2018 18:11:30 +0300 Subject: Add oauth reference to jupyter cluster app Signed-off-by: Dmitriy Zaporozhets --- app/models/clusters/applications/jupyter.rb | 32 ++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/clusters/applications/jupyter.rb b/app/models/clusters/applications/jupyter.rb index ef62be34abd..1f9cb1aade2 100644 --- a/app/models/clusters/applications/jupyter.rb +++ b/app/models/clusters/applications/jupyter.rb @@ -9,6 +9,8 @@ module Clusters include ::Clusters::Concerns::ApplicationStatus include ::Clusters::Concerns::ApplicationData + belongs_to :oauth_application, class_name: 'Doorkeeper::Application' + default_value_for :version, VERSION def chart @@ -32,16 +34,40 @@ module Clusters ) end + def callback_url + "http://#{hostname}/hub/oauth_callback" + end + private def specification { - "ingress" => { "hosts" => [hostname] }, - "hub" => { "cookieSecret" => SecureRandom.hex(32) }, - "proxy" => { "secretToken" => SecureRandom.hex(32) } + "ingress" => { + "hosts" => [hostname] + }, + "hub" => { + "extraEnv" => { + "GITLAB_HOST" => gitlab_url + }, + "cookieSecret" => SecureRandom.hex(32) + }, + "proxy" => { + "secretToken" => SecureRandom.hex(32) + }, + "auth" => { + "gitlab" => { + "clientId" => oauth_application.uid, + "clientSecret" => oauth_application.secret, + "callbackUrl" => callback_url + } + } } end + def gitlab_url + Gitlab.config.gitlab.url + end + def content_values YAML.load_file(chart_values_file).deep_merge!(specification) end -- cgit v1.2.3 From 2ca00a360507055f49819ec2c6db7e9a2de15b5f Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Fri, 25 May 2018 15:44:37 +0300 Subject: Don't allow jupyter install unless ingress external ip is assigned Signed-off-by: Dmitriy Zaporozhets --- app/models/clusters/applications/jupyter.rb | 8 ++++++++ app/models/clusters/cluster.rb | 1 + 2 files changed, 9 insertions(+) (limited to 'app/models') diff --git a/app/models/clusters/applications/jupyter.rb b/app/models/clusters/applications/jupyter.rb index 1f9cb1aade2..c26e984f9eb 100644 --- a/app/models/clusters/applications/jupyter.rb +++ b/app/models/clusters/applications/jupyter.rb @@ -13,6 +13,14 @@ module Clusters default_value_for :version, VERSION + def set_initial_status + return unless not_installable? + + if cluster&.application_ingress_installed? && cluster.application_ingress.external_ip + self.status = 'installable' + end + end + def chart "#{name}/jupyterhub" end diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index d99f858e0c0..b426b1bf8a1 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -41,6 +41,7 @@ module Clusters delegate :active?, to: :platform_kubernetes, prefix: true, allow_nil: true delegate :installed?, to: :application_helm, prefix: true, allow_nil: true + delegate :installed?, to: :application_ingress, prefix: true, allow_nil: true enum platform_type: { kubernetes: 1 -- cgit v1.2.3 From b5b72ed18bcdfe296eaa645c47f670832a4081e4 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Fri, 25 May 2018 17:09:26 +0300 Subject: Add some tests for jupyter app AR model Signed-off-by: Dmitriy Zaporozhets --- app/models/clusters/applications/jupyter.rb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/clusters/applications/jupyter.rb b/app/models/clusters/applications/jupyter.rb index c26e984f9eb..64217b6240f 100644 --- a/app/models/clusters/applications/jupyter.rb +++ b/app/models/clusters/applications/jupyter.rb @@ -57,10 +57,10 @@ module Clusters "extraEnv" => { "GITLAB_HOST" => gitlab_url }, - "cookieSecret" => SecureRandom.hex(32) + "cookieSecret" => cookie_secret }, "proxy" => { - "secretToken" => SecureRandom.hex(32) + "secretToken" => secret_token }, "auth" => { "gitlab" => { @@ -79,6 +79,14 @@ module Clusters def content_values YAML.load_file(chart_values_file).deep_merge!(specification) end + + def secret_token + @secret_token ||= SecureRandom.hex(32) + end + + def cookie_secret + @cookie_secret ||= SecureRandom.hex(32) + end end end end -- cgit v1.2.3 From f2cc1169e8c4e4ed209df0c6b50c1f5b69f5fdb0 Mon Sep 17 00:00:00 2001 From: Andreas Brandl Date: Mon, 28 May 2018 12:44:07 +0200 Subject: Remove double-checked internal id generation. This was needed for a transition phase only. For details see #45389. Closes #45389. --- app/models/internal_id.rb | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) (limited to 'app/models') diff --git a/app/models/internal_id.rb b/app/models/internal_id.rb index 189942c5ad8..f7f930e86ed 100644 --- a/app/models/internal_id.rb +++ b/app/models/internal_id.rb @@ -24,12 +24,9 @@ class InternalId < ActiveRecord::Base # # The operation locks the record and gathers a `ROW SHARE` lock (in PostgreSQL). # As such, the increment is atomic and safe to be called concurrently. - # - # If a `maximum_iid` is passed in, this overrides the incremented value if it's - # greater than that. This can be used to correct the increment value if necessary. - def increment_and_save!(maximum_iid) + def increment_and_save! lock! - self.last_value = [(last_value || 0) + 1, (maximum_iid || 0) + 1].max + self.last_value = (last_value || 0) + 1 save! last_value end @@ -93,16 +90,7 @@ class InternalId < ActiveRecord::Base # and increment its last value # # Note this will acquire a ROW SHARE lock on the InternalId record - - # Note we always calculate the maximum iid present here and - # pass it in to correct the InternalId entry if it's last_value is off. - # - # This can happen in a transition phase where both `AtomicInternalId` and - # `NonatomicInternalId` code runs (e.g. during a deploy). - # - # This is subject to be cleaned up with the 10.8 release: - # https://gitlab.com/gitlab-org/gitlab-ce/issues/45389. - (lookup || create_record).increment_and_save!(maximum_iid) + (lookup || create_record).increment_and_save! end end @@ -128,15 +116,11 @@ class InternalId < ActiveRecord::Base InternalId.create!( **scope, usage: usage_value, - last_value: maximum_iid + last_value: init.call(subject) || 0 ) end rescue ActiveRecord::RecordNotUnique lookup end - - def maximum_iid - @maximum_iid ||= init.call(subject) || 0 - end end end -- cgit v1.2.3 From 2b8eb7273efee84180b322a0aac33e1f4ee30d2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Fri, 25 May 2018 18:44:15 +0200 Subject: Ensure ApplicationSetting#performance_bar_allowed_group_id is properly set when retrieved from cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- app/models/application_setting.rb | 39 +++++---------------------------------- 1 file changed, 5 insertions(+), 34 deletions(-) (limited to 'app/models') diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index e8ccb320fae..b12f7a2c83f 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -230,6 +230,7 @@ class ApplicationSetting < ActiveRecord::Base after_commit do reset_memoized_terms end + after_commit :expire_performance_bar_allowed_user_ids_cache, if: -> { previous_changes.key?('performance_bar_allowed_group_id') } def self.defaults { @@ -386,31 +387,6 @@ class ApplicationSetting < ActiveRecord::Base super(levels.map { |level| Gitlab::VisibilityLevel.level_value(level) }) end - def performance_bar_allowed_group_id=(group_full_path) - group_full_path = nil if group_full_path.blank? - - if group_full_path.nil? - if group_full_path != performance_bar_allowed_group_id - super(group_full_path) - Gitlab::PerformanceBar.expire_allowed_user_ids_cache - end - - return - end - - group = Group.find_by_full_path(group_full_path) - - if group - if group.id != performance_bar_allowed_group_id - super(group.id) - Gitlab::PerformanceBar.expire_allowed_user_ids_cache - end - else - super(nil) - Gitlab::PerformanceBar.expire_allowed_user_ids_cache - end - end - def performance_bar_allowed_group Group.find_by_id(performance_bar_allowed_group_id) end @@ -420,15 +396,6 @@ class ApplicationSetting < ActiveRecord::Base performance_bar_allowed_group_id.present? end - # - If `enable` is true, we early return since the actual attribute that holds - # the enabling/disabling is `performance_bar_allowed_group_id` - # - If `enable` is false, we set `performance_bar_allowed_group_id` to `nil` - def performance_bar_enabled=(enable) - return if Gitlab::Utils.to_boolean(enable) - - self.performance_bar_allowed_group_id = nil - end - # Choose one of the available repository storage options. Currently all have # equal weighting. def pick_repository_storage @@ -506,4 +473,8 @@ class ApplicationSetting < ActiveRecord::Base errors.add(:terms, "You need to set terms to be enforced") unless terms.present? end + + def expire_performance_bar_allowed_user_ids_cache + Gitlab::PerformanceBar.expire_allowed_user_ids_cache + end end -- cgit v1.2.3 From 4cff66a6c46361e8d775ea3f5a80bf147d4020b3 Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Tue, 29 May 2018 20:51:43 +1100 Subject: Add 'squash and rebase' feature to CE --- app/models/merge_request.rb | 7 +++++++ app/models/repository.rb | 8 ++++++++ 2 files changed, 15 insertions(+) (limited to 'app/models') diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 9c4384a6e42..bc97fc3a5d9 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -1140,4 +1140,11 @@ class MergeRequest < ActiveRecord::Base maintainer_push_possible? && Ability.allowed?(user, :push_code, source_project) end + + def squash_in_progress? + # The source project can be deleted + return false unless source_project + + source_project.repository.squash_in_progress?(id) + end end diff --git a/app/models/repository.rb b/app/models/repository.rb index 0e1bf11d7c0..6165808cd9a 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -957,6 +957,14 @@ class Repository remote_branch: merge_request.target_branch) end + def squash(user, merge_request) + raw.squash(user, merge_request.id, branch: merge_request.target_branch, + start_sha: merge_request.diff_start_sha, + end_sha: merge_request.diff_head_sha, + author: merge_request.author, + message: merge_request.title) + end + private # TODO Generice finder, later split this on finders by Ref or Oid -- cgit v1.2.3 From 61ea2f52bd32d5038b9eec1caf29a4d4531d03c4 Mon Sep 17 00:00:00 2001 From: Andreas Brandl Date: Mon, 28 May 2018 14:58:23 +0200 Subject: Throttle updates to Project#last_repository_updated_at. Closes #35364. --- app/models/event.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'app/models') diff --git a/app/models/event.rb b/app/models/event.rb index 741a84194e2..ac0b1c7b27c 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -40,6 +40,7 @@ class Event < ActiveRecord::Base ).freeze RESET_PROJECT_ACTIVITY_INTERVAL = 1.hour + REPOSITORY_UPDATED_AT_INTERVAL = 5.minutes delegate :name, :email, :public_email, :username, to: :author, prefix: true, allow_nil: true delegate :title, to: :issue, prefix: true, allow_nil: true @@ -391,6 +392,7 @@ class Event < ActiveRecord::Base def set_last_repository_updated_at Project.unscoped.where(id: project_id) + .where("last_repository_updated_at < ? OR last_repository_updated_at IS NULL", REPOSITORY_UPDATED_AT_INTERVAL.ago) .update_all(last_repository_updated_at: created_at) end -- cgit v1.2.3 From 884fbf1d058ac00bc5c107f327475fa6eeda1f9f Mon Sep 17 00:00:00 2001 From: Jan Provaznik Date: Tue, 29 May 2018 14:46:42 +0200 Subject: Fix boolean casting for nil value `nil` value is not included in `ActiveModel::Type::Boolean::FALSE_VALUES` which caused that in Rails 5 the boolean_accessor converted `nil` to `true` instead of `false`. --- app/models/service.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/service.rb b/app/models/service.rb index 831c2ea1141..1d259bcfec7 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -206,10 +206,11 @@ class Service < ActiveRecord::Base args.each do |arg| class_eval %{ def #{arg}? + # '!!' is used because nil or empty string is converted to nil if Gitlab.rails5? - !ActiveModel::Type::Boolean::FALSE_VALUES.include?(#{arg}) + !!ActiveRecord::Type::Boolean.new.cast(#{arg}) else - ActiveRecord::ConnectionAdapters::Column::TRUE_VALUES.include?(#{arg}) + !!ActiveRecord::Type::Boolean.new.type_cast_from_database(#{arg}) end end } -- cgit v1.2.3 From fc3d214130f2038b3c8bdf142506c9116f373244 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 29 May 2018 15:20:10 +0200 Subject: Add a feature flag for switching pipeline stages --- app/models/ci/pipeline.rb | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 9dac56fcd57..4b8c23a393d 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -250,13 +250,17 @@ module Ci end ## - # TODO consider switching to persisted stages only in pipelines table - # (not necessairly in the show pipeline page because of #23257. - # Hide this behind two feature flags - enabled / disabled and only - # gitlab-ce / everywhere. + # TODO We do not completely switch to persisted stages because of + # race conditions with setting statuses gitlab-ce#23257. # - def stages - super + def ordered_stages + return legacy_stages unless complete? + + if Feature.enabled?('ci_pipeline_persisted_stages') + stages + else + legacy_stages + end end def legacy_stages -- cgit v1.2.3 From b5c706326ada2c0d213dd512842c5f677d9d94f9 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sat, 19 May 2018 06:03:29 -0700 Subject: Upgrade to Ruby 2.4.4 Fixes that make this work: * A change in Ruby (https://github.com/ruby/ruby/commit/ce635262f53b760284d56bb1027baebaaec175d1) requires passing in the exact required length for OpenSSL keys and IVs. * Ensure the secrets.yml is generated before any prepended modules are loaded. This is done by renaming the `secret_token.rb` initializer to `01_secret_token.rb`, which is a bit ugly but involves the least impact on other files. --- app/models/clusters/platforms/kubernetes.rb | 4 ++-- app/models/clusters/providers/gcp.rb | 2 +- app/models/concerns/has_variable.rb | 2 +- app/models/pages_domain.rb | 2 +- app/models/project_import_data.rb | 2 +- app/models/remote_mirror.rb | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) (limited to 'app/models') diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index ba6552f238f..25eac5160f1 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -11,12 +11,12 @@ module Clusters attr_encrypted :password, mode: :per_attribute_iv, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base, algorithm: 'aes-256-cbc' attr_encrypted :token, mode: :per_attribute_iv, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base, algorithm: 'aes-256-cbc' before_validation :enforce_namespace_to_lower_case diff --git a/app/models/clusters/providers/gcp.rb b/app/models/clusters/providers/gcp.rb index 7fac32466ab..eb2e42fd3fe 100644 --- a/app/models/clusters/providers/gcp.rb +++ b/app/models/clusters/providers/gcp.rb @@ -11,7 +11,7 @@ module Clusters attr_encrypted :access_token, mode: :per_attribute_iv, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base, algorithm: 'aes-256-cbc' validates :gcp_project_id, diff --git a/app/models/concerns/has_variable.rb b/app/models/concerns/has_variable.rb index 8a241e4374a..c8e20c0ab81 100644 --- a/app/models/concerns/has_variable.rb +++ b/app/models/concerns/has_variable.rb @@ -13,7 +13,7 @@ module HasVariable attr_encrypted :value, mode: :per_attribute_iv_and_salt, insecure_mode: true, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base, algorithm: 'aes-256-cbc' def key=(new_key) diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index 2e478a24778..bfea64c3759 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -19,7 +19,7 @@ class PagesDomain < ActiveRecord::Base attr_encrypted :key, mode: :per_attribute_iv_and_salt, insecure_mode: true, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base, algorithm: 'aes-256-cbc' after_initialize :set_verification_code diff --git a/app/models/project_import_data.rb b/app/models/project_import_data.rb index 6da6632f4f2..1d7089ccfc7 100644 --- a/app/models/project_import_data.rb +++ b/app/models/project_import_data.rb @@ -3,7 +3,7 @@ require 'carrierwave/orm/activerecord' class ProjectImportData < ActiveRecord::Base belongs_to :project, inverse_of: :import_data attr_encrypted :credentials, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base, marshal: true, encode: true, mode: :per_attribute_iv_and_salt, diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb index bbf8fd9c6a7..aba1f2f384f 100644 --- a/app/models/remote_mirror.rb +++ b/app/models/remote_mirror.rb @@ -5,7 +5,7 @@ class RemoteMirror < ActiveRecord::Base UNPROTECTED_BACKOFF_DELAY = 5.minutes attr_encrypted :credentials, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base, marshal: true, encode: true, mode: :per_attribute_iv_and_salt, -- cgit v1.2.3 From db9f765852d9fef464e69c0bf47a382f2ab7219d Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Wed, 30 May 2018 12:09:12 +0300 Subject: Rename clusters_applications_jupyters to uncountable Signed-off-by: Dmitriy Zaporozhets --- app/models/clusters/applications/jupyter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/clusters/applications/jupyter.rb b/app/models/clusters/applications/jupyter.rb index 64217b6240f..975d434e1a4 100644 --- a/app/models/clusters/applications/jupyter.rb +++ b/app/models/clusters/applications/jupyter.rb @@ -3,7 +3,7 @@ module Clusters class Jupyter < ActiveRecord::Base VERSION = '0.0.1'.freeze - self.table_name = 'clusters_applications_jupyters' + self.table_name = 'clusters_applications_jupyter' include ::Clusters::Concerns::ApplicationCore include ::Clusters::Concerns::ApplicationStatus -- cgit v1.2.3 From db926729660d66098db717ae9f64fab0cd2f601f Mon Sep 17 00:00:00 2001 From: Oswaldo Ferreira Date: Wed, 30 May 2018 12:14:56 -0300 Subject: Ensure metrics regardless of being imported on MRs --- app/models/concerns/issuable.rb | 2 -- app/models/issue.rb | 1 + app/models/merge_request.rb | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index b45395343cc..44150b37708 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -97,8 +97,6 @@ module Issuable strip_attributes :title - after_save :ensure_metrics, unless: :imported? - # We want to use optimistic lock for cases when only title or description are involved # http://api.rubyonrails.org/classes/ActiveRecord/Locking/Optimistic.html def locking_enabled? diff --git a/app/models/issue.rb b/app/models/issue.rb index 0332bfa9371..57787815fae 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -59,6 +59,7 @@ class Issue < ActiveRecord::Base scope :public_only, -> { where(confidential: false) } after_save :expire_etag_cache + after_save :ensure_metrics, unless: :imported? attr_spammable :title, spam_title: true attr_spammable :description, spam_description: true diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index bc97fc3a5d9..79fc155fd3c 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -58,6 +58,7 @@ class MergeRequest < ActiveRecord::Base after_create :ensure_merge_request_diff, unless: :importing? after_update :clear_memoized_shas after_update :reload_diff_if_branch_changed + after_save :ensure_metrics # When this attribute is true some MR validation is ignored # It allows us to close or modify broken merge requests -- cgit v1.2.3 From 4d3f7ae1ef5881869140f0c4a5865f65569db26a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Javier=20L=C3=B3pez?= Date: Tue, 15 May 2018 15:39:33 +0200 Subject: Removed API endpoint and specs --- app/models/application_setting.rb | 11 ----------- app/models/concerns/project_features_compatibility.rb | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) (limited to 'app/models') diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index b12f7a2c83f..3d58a14882f 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -360,17 +360,6 @@ class ApplicationSetting < ActiveRecord::Base Array(read_attribute(:repository_storages)) end - # DEPRECATED - # repository_storage is still required in the API. Remove in 9.0 - # Still used in API v3 - def repository_storage - repository_storages.first - end - - def repository_storage=(value) - self.repository_storages = [value] - end - def default_project_visibility=(level) super(Gitlab::VisibilityLevel.level_value(level)) end diff --git a/app/models/concerns/project_features_compatibility.rb b/app/models/concerns/project_features_compatibility.rb index b3fec99c816..1f7d78a2efe 100644 --- a/app/models/concerns/project_features_compatibility.rb +++ b/app/models/concerns/project_features_compatibility.rb @@ -1,4 +1,4 @@ -# Makes api V3 compatible with old project features permissions methods +# Makes api V4 compatible with old project features permissions methods # # After migrating issues_enabled merge_requests_enabled builds_enabled snippets_enabled and wiki_enabled # fields to a new table "project_features", support for the old fields is still needed in the API. -- cgit v1.2.3 From 2346dda469965ce4ab109daf707e0570c0521a11 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 30 May 2018 09:35:23 -0700 Subject: Make Repository#blob_data_at a public method This reduces conflicts with EE, where it is public because it is called in ee/lib/gitlab/ci/external/file/local.rb. --- app/models/repository.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'app/models') diff --git a/app/models/repository.rb b/app/models/repository.rb index 6165808cd9a..82cf47ba04e 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -957,6 +957,14 @@ class Repository remote_branch: merge_request.target_branch) end + def blob_data_at(sha, path) + blob = blob_at(sha, path) + return unless blob + + blob.load_all_data! + blob.data + end + def squash(user, merge_request) raw.squash(user, merge_request.id, branch: merge_request.target_branch, start_sha: merge_request.diff_start_sha, @@ -979,14 +987,6 @@ class Repository ::Commit.new(commit, @project) if commit end - def blob_data_at(sha, path) - blob = blob_at(sha, path) - return unless blob - - blob.load_all_data! - blob.data - end - def cache @cache ||= Gitlab::RepositoryCache.new(self) end -- cgit v1.2.3 From ab489d293d6ee3e30673817ce4652c7b413988c0 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Fri, 11 May 2018 12:03:40 +0200 Subject: Improve runner_type validations for Ci::Runner --- app/models/ci/runner.rb | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 530eacf4be0..2119b70c18c 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -56,7 +56,9 @@ module Ci end validate :tag_constraints - validate :either_projects_or_group + validate :no_projects, unless: :project_type? + validate :no_groups, unless: :group_type? + validate :only_one_group, if: :group_type? validates :access_level, presence: true validates :runner_type, presence: true @@ -253,13 +255,21 @@ module Ci self.class.owned_or_shared(project_id).where(id: self.id).any? end - def either_projects_or_group - if groups.many? - errors.add(:runner, 'can only be assigned to one group') + def no_projects + if projects.any? + errors.add(:runner, 'cannot assign project to a non-project runner') + end + end + + def no_groups + if groups.any? + errors.add(:runner, 'cannot assign group to a non-group runner') end + end - if assigned_to_group? && assigned_to_project? - errors.add(:runner, 'can only be assigned either to projects or to a group') + def only_one_group + if groups.many? + errors.add(:runner, 'can only be assigned to one group') end end -- cgit v1.2.3 From 0a2f5065f26f259ed66359f3754ecc21505fea0d Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Mon, 14 May 2018 15:54:47 +0200 Subject: Ensure we validate Runner#runner_type when persisting RunnerNamespace --- app/models/ci/runner_namespace.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/runner_namespace.rb b/app/models/ci/runner_namespace.rb index 3269f86e8ca..420e34df091 100644 --- a/app/models/ci/runner_namespace.rb +++ b/app/models/ci/runner_namespace.rb @@ -2,7 +2,7 @@ module Ci class RunnerNamespace < ActiveRecord::Base extend Gitlab::Ci::Model - belongs_to :runner + belongs_to :runner, validate: true belongs_to :namespace, class_name: '::Namespace' belongs_to :group, class_name: '::Group', foreign_key: :namespace_id end -- cgit v1.2.3 From 051f385e7e82130e6978cd3956e5c48fbdc83b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Wed, 23 May 2018 13:23:49 +0200 Subject: Refactor validations and make runner factory by default to be instance-wide runner --- app/models/ci/runner.rb | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 2119b70c18c..ac9f04bb3d4 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -56,12 +56,15 @@ module Ci end validate :tag_constraints - validate :no_projects, unless: :project_type? - validate :no_groups, unless: :group_type? - validate :only_one_group, if: :group_type? validates :access_level, presence: true validates :runner_type, presence: true + validate :no_projects, unless: :project_type? + validate :no_groups, unless: :group_type? + validate :any_project, if: :project_type? + validate :exactly_one_group, if: :group_type? + validate :is_shared_is_valid + acts_as_taggable after_destroy :cleanup_runner_queue @@ -117,8 +120,8 @@ module Ci raise ArgumentError, 'Transitioning a group runner to a project runner is not supported' end - self.save - project.runner_projects.create(runner_id: self.id) + self.projects << project + self.save! end def display_name @@ -257,19 +260,31 @@ module Ci def no_projects if projects.any? - errors.add(:runner, 'cannot assign project to a non-project runner') + errors.add(:runner, 'cannot have projects assigned') end end def no_groups if groups.any? - errors.add(:runner, 'cannot assign group to a non-group runner') + errors.add(:runner, 'cannot have groups assigned') + end + end + + def any_project + unless projects.any? + errors.add(:runner, 'needs to be assigned to at least one project') + end + end + + def exactly_one_group + unless groups.one? + errors.add(:runner, 'needs to be assigned to exactly one group') end end - def only_one_group - if groups.many? - errors.add(:runner, 'can only be assigned to one group') + def is_shared_is_valid + unless is_shared? == instance_type? + errors.add(:is_shared, 'is not equal to instance_type?') end end -- cgit v1.2.3 From 03ff1da278117ce36aaec4e0af267bbc07dc571c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Wed, 23 May 2018 15:29:48 +0200 Subject: Fix some failures --- app/models/clusters/applications/runner.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index b881b4eaf36..e6f795f3e0b 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -43,7 +43,7 @@ module Clusters def create_and_assign_runner transaction do - project.runners.create!(runner_create_params).tap do |runner| + Ci::Runner.create!(runner_create_params).tap do |runner| update!(runner_id: runner.id) end end @@ -53,7 +53,8 @@ module Clusters { name: 'kubernetes-cluster', runner_type: :project_type, - tag_list: %w(kubernetes cluster) + tag_list: %w(kubernetes cluster), + projects: [project] } end -- cgit v1.2.3 From da75618bebe57132f847df733625cc4757f8084d Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Thu, 24 May 2018 12:24:41 +0200 Subject: Make Ci::Runner#assign_to keep returning RunnerProject --- app/models/ci/runner.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index ac9f04bb3d4..5cec88660f8 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -120,8 +120,9 @@ module Ci raise ArgumentError, 'Transitioning a group runner to a project runner is not supported' end - self.projects << project + runner_project = project.runner_projects.create(runner_id: self.id) self.save! + runner_project end def display_name -- cgit v1.2.3 From c6e95b04405f1e07f76505b03c6c096f4c4d084b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Mon, 28 May 2018 12:49:08 +0200 Subject: Improve `Ci::Runner#assign_to` to return a flag whether it succeeded or not --- app/models/ci/runner.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 5cec88660f8..e2a1c9fb929 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -120,9 +120,8 @@ module Ci raise ArgumentError, 'Transitioning a group runner to a project runner is not supported' end - runner_project = project.runner_projects.create(runner_id: self.id) - self.save! - runner_project + self.projects << project + self.save end def display_name -- cgit v1.2.3 From 53ef14c6765c09601e0ef2bd632741a78f84ae32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Mon, 28 May 2018 12:56:45 +0200 Subject: Fix static analysis --- app/models/ci/runner.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index e2a1c9fb929..5017c983425 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -63,7 +63,7 @@ module Ci validate :no_groups, unless: :group_type? validate :any_project, if: :project_type? validate :exactly_one_group, if: :group_type? - validate :is_shared_is_valid + validate :validate_is_shared acts_as_taggable @@ -282,7 +282,7 @@ module Ci end end - def is_shared_is_valid + def validate_is_shared unless is_shared? == instance_type? errors.add(:is_shared, 'is not equal to instance_type?') end -- cgit v1.2.3 From cfee3e0c4ef3a8b3bf6aafd325c791d0d89d68df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Mon, 28 May 2018 13:07:28 +0200 Subject: Add `Ci::Runner` inverse_of's --- app/models/ci/runner.rb | 4 ++-- app/models/ci/runner_namespace.rb | 4 ++-- app/models/ci/runner_project.rb | 4 ++-- app/models/namespace.rb | 2 +- app/models/project.rb | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 5017c983425..3f0476fb7cf 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -12,9 +12,9 @@ module Ci FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level maximum_timeout_human_readable].freeze has_many :builds - has_many :runner_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :runner_projects, inverse_of: :runner, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :runner_projects - has_many :runner_namespaces + has_many :runner_namespaces, inverse_of: :runner has_many :groups, through: :runner_namespaces has_one :last_build, ->() { order('id DESC') }, class_name: 'Ci::Build' diff --git a/app/models/ci/runner_namespace.rb b/app/models/ci/runner_namespace.rb index 420e34df091..26ca8bbdeeb 100644 --- a/app/models/ci/runner_namespace.rb +++ b/app/models/ci/runner_namespace.rb @@ -2,8 +2,8 @@ module Ci class RunnerNamespace < ActiveRecord::Base extend Gitlab::Ci::Model - belongs_to :runner, validate: true - belongs_to :namespace, class_name: '::Namespace' + belongs_to :runner, inverse_of: :runner_namespaces, validate: true + belongs_to :namespace, inverse_of: :runner_namespaces, class_name: '::Namespace' belongs_to :group, class_name: '::Group', foreign_key: :namespace_id end end diff --git a/app/models/ci/runner_project.rb b/app/models/ci/runner_project.rb index 505d178ba8e..52437047300 100644 --- a/app/models/ci/runner_project.rb +++ b/app/models/ci/runner_project.rb @@ -2,8 +2,8 @@ module Ci class RunnerProject < ActiveRecord::Base extend Gitlab::Ci::Model - belongs_to :runner - belongs_to :project + belongs_to :runner, inverse_of: :runner_projects + belongs_to :project, inverse_of: :runner_projects validates :runner_id, uniqueness: { scope: :project_id } end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 3dad4277713..52fe529c016 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -21,7 +21,7 @@ class Namespace < ActiveRecord::Base has_many :projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :project_statistics - has_many :runner_namespaces, class_name: 'Ci::RunnerNamespace' + has_many :runner_namespaces, inverse_of: :namespace, class_name: 'Ci::RunnerNamespace' has_many :runners, through: :runner_namespaces, source: :runner, class_name: 'Ci::Runner' # This should _not_ be `inverse_of: :namespace`, because that would also set diff --git a/app/models/project.rb b/app/models/project.rb index e275ac4dc6f..d2f360e4915 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -236,7 +236,7 @@ class Project < ActiveRecord::Base has_many :builds, class_name: 'Ci::Build', inverse_of: :project, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :build_trace_section_names, class_name: 'Ci::BuildTraceSectionName' has_many :build_trace_chunks, class_name: 'Ci::BuildTraceChunk', through: :builds, source: :trace_chunks - has_many :runner_projects, class_name: 'Ci::RunnerProject' + has_many :runner_projects, class_name: 'Ci::RunnerProject', inverse_of: :project has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' has_many :variables, class_name: 'Ci::Variable' has_many :triggers, class_name: 'Ci::Trigger' -- cgit v1.2.3 From bf74e69eeb3d7881fe46ee34a26ba03a1744e8d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Mon, 28 May 2018 14:36:47 +0200 Subject: Add uniqueness for RunnerNamespace --- app/models/ci/runner_namespace.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/runner_namespace.rb b/app/models/ci/runner_namespace.rb index 26ca8bbdeeb..29508fdd326 100644 --- a/app/models/ci/runner_namespace.rb +++ b/app/models/ci/runner_namespace.rb @@ -5,5 +5,7 @@ module Ci belongs_to :runner, inverse_of: :runner_namespaces, validate: true belongs_to :namespace, inverse_of: :runner_namespaces, class_name: '::Namespace' belongs_to :group, class_name: '::Group', foreign_key: :namespace_id + + validates :runner_id, uniqueness: { scope: :namespace_id } end end -- cgit v1.2.3 From 2ccbe4fd341254c1d3146c81888881ace5f997c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Mon, 28 May 2018 15:31:46 +0200 Subject: Run `Ci::Runner#assign_to` in transaction --- app/models/ci/runner.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 3f0476fb7cf..57edd6a4956 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -120,8 +120,15 @@ module Ci raise ArgumentError, 'Transitioning a group runner to a project runner is not supported' end - self.projects << project - self.save + begin + transaction do + self.projects << project + self.save! + end + rescue ActiveRecord::RecordInvalid => e + self.errors.add(:assign_to, e.message) + false + end end def display_name -- cgit v1.2.3 From 20dfe25c151cc883ce0d38b67125b5ca41e6d422 Mon Sep 17 00:00:00 2001 From: Imre Farkas Date: Thu, 31 May 2018 14:01:04 +0000 Subject: Export assigned issues in iCalendar feed --- app/models/issue.rb | 16 ++++++++++------ app/models/user.rb | 8 ++++---- 2 files changed, 14 insertions(+), 10 deletions(-) (limited to 'app/models') diff --git a/app/models/issue.rb b/app/models/issue.rb index 0332bfa9371..559770fa442 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -14,12 +14,13 @@ class Issue < ActiveRecord::Base ignore_column :assignee_id, :branch_name, :deleted_at - DueDateStruct = Struct.new(:title, :name).freeze - NoDueDate = DueDateStruct.new('No Due Date', '0').freeze - AnyDueDate = DueDateStruct.new('Any Due Date', '').freeze - Overdue = DueDateStruct.new('Overdue', 'overdue').freeze - DueThisWeek = DueDateStruct.new('Due This Week', 'week').freeze - DueThisMonth = DueDateStruct.new('Due This Month', 'month').freeze + DueDateStruct = Struct.new(:title, :name).freeze + NoDueDate = DueDateStruct.new('No Due Date', '0').freeze + AnyDueDate = DueDateStruct.new('Any Due Date', '').freeze + Overdue = DueDateStruct.new('Overdue', 'overdue').freeze + DueThisWeek = DueDateStruct.new('Due This Week', 'week').freeze + DueThisMonth = DueDateStruct.new('Due This Month', 'month').freeze + DueNextMonthAndPreviousTwoWeeks = DueDateStruct.new('Due Next Month And Previous Two Weeks', 'next_month_and_previous_two_weeks').freeze belongs_to :project belongs_to :moved_to, class_name: 'Issue' @@ -46,6 +47,7 @@ class Issue < ActiveRecord::Base scope :unassigned, -> { where('NOT EXISTS (SELECT TRUE FROM issue_assignees WHERE issue_id = issues.id)') } scope :assigned_to, ->(u) { where('EXISTS (SELECT TRUE FROM issue_assignees WHERE user_id = ? AND issue_id = issues.id)', u.id)} + scope :with_due_date, -> { where('due_date IS NOT NULL') } scope :without_due_date, -> { where(due_date: nil) } scope :due_before, ->(date) { where('issues.due_date < ?', date) } scope :due_between, ->(from_date, to_date) { where('issues.due_date >= ?', from_date).where('issues.due_date <= ?', to_date) } @@ -53,6 +55,7 @@ class Issue < ActiveRecord::Base scope :order_due_date_asc, -> { reorder('issues.due_date IS NULL, issues.due_date ASC') } scope :order_due_date_desc, -> { reorder('issues.due_date IS NULL, issues.due_date DESC') } + scope :order_closest_future_date, -> { reorder('CASE WHEN due_date >= CURRENT_DATE THEN 0 ELSE 1 END ASC, ABS(CURRENT_DATE - due_date) ASC') } scope :preload_associations, -> { preload(:labels, project: :namespace) } @@ -119,6 +122,7 @@ class Issue < ActiveRecord::Base def self.sort_by_attribute(method, excluded_labels: []) case method.to_s + when 'closest_future_date' then order_closest_future_date when 'due_date' then order_due_date_asc when 'due_date_asc' then order_due_date_asc when 'due_date_desc' then order_due_date_desc diff --git a/app/models/user.rb b/app/models/user.rb index 0a838d34054..e219ab800ad 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -26,7 +26,7 @@ class User < ActiveRecord::Base ignore_column :authentication_token add_authentication_token_field :incoming_email_token - add_authentication_token_field :rss_token + add_authentication_token_field :feed_token default_value_for :admin, false default_value_for(:external) { Gitlab::CurrentSettings.user_default_external } @@ -1167,11 +1167,11 @@ class User < ActiveRecord::Base save end - # each existing user needs to have an `rss_token`. + # each existing user needs to have an `feed_token`. # we do this on read since migrating all existing users is not a feasible # solution. - def rss_token - ensure_rss_token! + def feed_token + ensure_feed_token! end def sync_attribute?(attribute) -- cgit v1.2.3 From 0d44f4d50ef175997fe1f90de9e622a4f3b867e3 Mon Sep 17 00:00:00 2001 From: Mark Chao Date: Wed, 23 May 2018 09:54:57 +0800 Subject: Rephrase "maintainer" to more precise "members who can merge to the target branch" "Maintainer" will be freed to be used for #42751 --- app/models/merge_request.rb | 12 ++++++------ app/models/project.rb | 14 +++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) (limited to 'app/models') diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 79fc155fd3c..86437594279 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -1124,21 +1124,21 @@ class MergeRequest < ActiveRecord::Base project.merge_requests.merged.where(author_id: author_id).empty? end - def allow_maintainer_to_push - maintainer_push_possible? && super + def allow_collaboration + collaborative_push_possible? && super end - alias_method :allow_maintainer_to_push?, :allow_maintainer_to_push + alias_method :allow_collaboration?, :allow_collaboration - def maintainer_push_possible? + def collaborative_push_possible? source_project.present? && for_fork? && target_project.visibility_level > Gitlab::VisibilityLevel::PRIVATE && source_project.visibility_level > Gitlab::VisibilityLevel::PRIVATE && !ProtectedBranch.protected?(source_project, source_branch) end - def can_allow_maintainer_to_push?(user) - maintainer_push_possible? && + def can_allow_collaboration?(user) + collaborative_push_possible? && Ability.allowed?(user, :push_code, source_project) end diff --git a/app/models/project.rb b/app/models/project.rb index e275ac4dc6f..73b808b4cb5 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1973,18 +1973,18 @@ class Project < ActiveRecord::Base .limit(1) .select(1) source_of_merge_requests.opened - .where(allow_maintainer_to_push: true) + .where(allow_collaboration: true) .where('EXISTS (?)', developer_access_exists) end - def branch_allows_maintainer_push?(user, branch_name) + def branch_allows_collaboration?(user, branch_name) return false unless user cache_key = "user:#{user.id}:#{branch_name}:branch_allows_push" - memoized_results = strong_memoize(:branch_allows_maintainer_push) do + memoized_results = strong_memoize(:branch_allows_collaboration) do Hash.new do |result, cache_key| - result[cache_key] = fetch_branch_allows_maintainer_push?(user, branch_name) + result[cache_key] = fetch_branch_allows_collaboration?(user, branch_name) end end @@ -2126,18 +2126,18 @@ class Project < ActiveRecord::Base raise ex end - def fetch_branch_allows_maintainer_push?(user, branch_name) + def fetch_branch_allows_collaboration?(user, branch_name) check_access = -> do next false if empty_repo? merge_request = source_of_merge_requests.opened - .where(allow_maintainer_to_push: true) + .where(allow_collaboration: true) .find_by(source_branch: branch_name) merge_request&.can_be_merged_by?(user) end if RequestStore.active? - RequestStore.fetch("project-#{id}:branch-#{branch_name}:user-#{user.id}:branch_allows_maintainer_push") do + RequestStore.fetch("project-#{id}:branch-#{branch_name}:user-#{user.id}:branch_allows_collaboration") do check_access.call end else -- cgit v1.2.3 From 840f80d48b7d8363f171f6137cd9f1fbafb52bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Javier=20L=C3=B3pez?= Date: Fri, 1 Jun 2018 11:43:53 +0000 Subject: Add validation to webhook and service URLs to ensure they are not blocked because of SSRF --- app/models/badge.rb | 2 +- app/models/environment.rb | 2 +- app/models/generic_commit_status.rb | 2 +- app/models/hooks/system_hook.rb | 5 +++++ app/models/hooks/web_hook.rb | 9 ++++++++- app/models/project.rb | 5 +++-- app/models/project_services/bamboo_service.rb | 2 +- app/models/project_services/bugzilla_service.rb | 2 +- app/models/project_services/buildkite_service.rb | 2 +- app/models/project_services/chat_notification_service.rb | 2 +- app/models/project_services/custom_issue_tracker_service.rb | 2 +- app/models/project_services/drone_ci_service.rb | 2 +- app/models/project_services/external_wiki_service.rb | 2 +- app/models/project_services/gitlab_issue_tracker_service.rb | 2 +- app/models/project_services/jira_service.rb | 4 ++-- app/models/project_services/kubernetes_service.rb | 2 +- app/models/project_services/mock_ci_service.rb | 2 +- app/models/project_services/prometheus_service.rb | 2 +- app/models/project_services/redmine_service.rb | 2 +- app/models/project_services/teamcity_service.rb | 2 +- app/models/remote_mirror.rb | 1 - 21 files changed, 34 insertions(+), 22 deletions(-) (limited to 'app/models') diff --git a/app/models/badge.rb b/app/models/badge.rb index f7e10c2ebfc..265c5d872d4 100644 --- a/app/models/badge.rb +++ b/app/models/badge.rb @@ -18,7 +18,7 @@ class Badge < ActiveRecord::Base scope :order_created_at_asc, -> { reorder(created_at: :asc) } - validates :link_url, :image_url, url_placeholder: { protocols: %w(http https), placeholder_regex: PLACEHOLDERS_REGEX } + validates :link_url, :image_url, url: { protocols: %w(http https) } validates :type, presence: true def rendered_link_url(project = nil) diff --git a/app/models/environment.rb b/app/models/environment.rb index fddb269af4b..8d523dae324 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -32,7 +32,7 @@ class Environment < ActiveRecord::Base validates :external_url, length: { maximum: 255 }, allow_nil: true, - addressable_url: true + url: true delegate :stop_action, :manual_actions, to: :last_deployment, allow_nil: true diff --git a/app/models/generic_commit_status.rb b/app/models/generic_commit_status.rb index 532b8f4ad69..5ac8bde44cd 100644 --- a/app/models/generic_commit_status.rb +++ b/app/models/generic_commit_status.rb @@ -1,7 +1,7 @@ class GenericCommitStatus < CommitStatus before_validation :set_default_values - validates :target_url, addressable_url: true, + validates :target_url, url: true, length: { maximum: 255 }, allow_nil: true diff --git a/app/models/hooks/system_hook.rb b/app/models/hooks/system_hook.rb index 0528266e5b3..6bef00f26ea 100644 --- a/app/models/hooks/system_hook.rb +++ b/app/models/hooks/system_hook.rb @@ -11,4 +11,9 @@ class SystemHook < WebHook default_value_for :push_events, false default_value_for :repository_update_events, true default_value_for :merge_requests_events, false + + # Allow urls pointing localhost and the local network + def allow_local_requests? + true + end end diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb index 27729deeac9..e353abdda9c 100644 --- a/app/models/hooks/web_hook.rb +++ b/app/models/hooks/web_hook.rb @@ -3,7 +3,9 @@ class WebHook < ActiveRecord::Base has_many :web_hook_logs, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - validates :url, presence: true, url: true + validates :url, presence: true, public_url: { allow_localhost: lambda(&:allow_local_requests?), + allow_local_network: lambda(&:allow_local_requests?) } + validates :token, format: { without: /\n/ } def execute(data, hook_name) @@ -13,4 +15,9 @@ class WebHook < ActiveRecord::Base def async_execute(data, hook_name) WebHookService.new(self, data, hook_name).async_execute end + + # Allow urls pointing localhost and the local network + def allow_local_requests? + false + end end diff --git a/app/models/project.rb b/app/models/project.rb index e275ac4dc6f..af9fca62dc3 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -289,8 +289,9 @@ class Project < ActiveRecord::Base validates :namespace, presence: true validates :name, uniqueness: { scope: :namespace_id } - validates :import_url, addressable_url: true, if: :external_import? - validates :import_url, importable_url: true, if: [:external_import?, :import_url_changed?] + validates :import_url, url: { protocols: %w(http https ssh git), + allow_localhost: false, + ports: VALID_IMPORT_PORTS }, if: [:external_import?, :import_url_changed?] validates :star_count, numericality: { greater_than_or_equal_to: 0 } validate :check_limit, on: :create validate :check_repository_path_availability, on: :update, if: ->(project) { project.renamed? } diff --git a/app/models/project_services/bamboo_service.rb b/app/models/project_services/bamboo_service.rb index 54e4b3278db..7f4c47a6d14 100644 --- a/app/models/project_services/bamboo_service.rb +++ b/app/models/project_services/bamboo_service.rb @@ -3,7 +3,7 @@ class BambooService < CiService prop_accessor :bamboo_url, :build_key, :username, :password - validates :bamboo_url, presence: true, url: true, if: :activated? + validates :bamboo_url, presence: true, public_url: true, if: :activated? validates :build_key, presence: true, if: :activated? validates :username, presence: true, diff --git a/app/models/project_services/bugzilla_service.rb b/app/models/project_services/bugzilla_service.rb index 046e2809f45..e4e3a80976b 100644 --- a/app/models/project_services/bugzilla_service.rb +++ b/app/models/project_services/bugzilla_service.rb @@ -1,5 +1,5 @@ class BugzillaService < IssueTrackerService - validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated? + validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url diff --git a/app/models/project_services/buildkite_service.rb b/app/models/project_services/buildkite_service.rb index d2aaff8817a..35884c4560c 100644 --- a/app/models/project_services/buildkite_service.rb +++ b/app/models/project_services/buildkite_service.rb @@ -8,7 +8,7 @@ class BuildkiteService < CiService prop_accessor :project_url, :token boolean_accessor :enable_ssl_verification - validates :project_url, presence: true, url: true, if: :activated? + validates :project_url, presence: true, public_url: true, if: :activated? validates :token, presence: true, if: :activated? after_save :compose_service_hook, if: :activated? diff --git a/app/models/project_services/chat_notification_service.rb b/app/models/project_services/chat_notification_service.rb index 7591ab4f478..ae0debbd3ac 100644 --- a/app/models/project_services/chat_notification_service.rb +++ b/app/models/project_services/chat_notification_service.rb @@ -8,7 +8,7 @@ class ChatNotificationService < Service prop_accessor :webhook, :username, :channel boolean_accessor :notify_only_broken_pipelines, :notify_only_default_branch - validates :webhook, presence: true, url: true, if: :activated? + validates :webhook, presence: true, public_url: true, if: :activated? def initialize_properties # Custom serialized properties initialization diff --git a/app/models/project_services/custom_issue_tracker_service.rb b/app/models/project_services/custom_issue_tracker_service.rb index b9e3e982b64..456c7f5cee2 100644 --- a/app/models/project_services/custom_issue_tracker_service.rb +++ b/app/models/project_services/custom_issue_tracker_service.rb @@ -1,5 +1,5 @@ class CustomIssueTrackerService < IssueTrackerService - validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated? + validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url diff --git a/app/models/project_services/drone_ci_service.rb b/app/models/project_services/drone_ci_service.rb index a4bf427ac0b..ab4e46da89f 100644 --- a/app/models/project_services/drone_ci_service.rb +++ b/app/models/project_services/drone_ci_service.rb @@ -4,7 +4,7 @@ class DroneCiService < CiService prop_accessor :drone_url, :token boolean_accessor :enable_ssl_verification - validates :drone_url, presence: true, url: true, if: :activated? + validates :drone_url, presence: true, public_url: true, if: :activated? validates :token, presence: true, if: :activated? after_save :compose_service_hook, if: :activated? diff --git a/app/models/project_services/external_wiki_service.rb b/app/models/project_services/external_wiki_service.rb index 1553f169827..a4b1ef09e93 100644 --- a/app/models/project_services/external_wiki_service.rb +++ b/app/models/project_services/external_wiki_service.rb @@ -1,7 +1,7 @@ class ExternalWikiService < Service prop_accessor :external_wiki_url - validates :external_wiki_url, presence: true, url: true, if: :activated? + validates :external_wiki_url, presence: true, public_url: true, if: :activated? def title 'External Wiki' diff --git a/app/models/project_services/gitlab_issue_tracker_service.rb b/app/models/project_services/gitlab_issue_tracker_service.rb index 88c428b4aae..16e32a4139e 100644 --- a/app/models/project_services/gitlab_issue_tracker_service.rb +++ b/app/models/project_services/gitlab_issue_tracker_service.rb @@ -1,7 +1,7 @@ class GitlabIssueTrackerService < IssueTrackerService include Gitlab::Routing - validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated? + validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index ed4bbfb6cfc..eb3261c902f 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -3,8 +3,8 @@ class JiraService < IssueTrackerService include ApplicationHelper include ActionView::Helpers::AssetUrlHelper - validates :url, url: true, presence: true, if: :activated? - validates :api_url, url: true, allow_blank: true + validates :url, public_url: true, presence: true, if: :activated? + validates :api_url, public_url: true, allow_blank: true validates :username, presence: true, if: :activated? validates :password, presence: true, if: :activated? diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb index 20fed432e55..ddd4026019b 100644 --- a/app/models/project_services/kubernetes_service.rb +++ b/app/models/project_services/kubernetes_service.rb @@ -24,7 +24,7 @@ class KubernetesService < DeploymentService prop_accessor :ca_pem with_options presence: true, if: :activated? do - validates :api_url, url: true + validates :api_url, public_url: true validates :token end diff --git a/app/models/project_services/mock_ci_service.rb b/app/models/project_services/mock_ci_service.rb index 2221459c90b..b89dc07a73e 100644 --- a/app/models/project_services/mock_ci_service.rb +++ b/app/models/project_services/mock_ci_service.rb @@ -3,7 +3,7 @@ class MockCiService < CiService ALLOWED_STATES = %w[failed canceled running pending success success_with_warnings skipped not_found].freeze prop_accessor :mock_service_url - validates :mock_service_url, presence: true, url: true, if: :activated? + validates :mock_service_url, presence: true, public_url: true, if: :activated? def title 'MockCI' diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb index dcaeb65dc32..df4254e0523 100644 --- a/app/models/project_services/prometheus_service.rb +++ b/app/models/project_services/prometheus_service.rb @@ -6,7 +6,7 @@ class PrometheusService < MonitoringService boolean_accessor :manual_configuration with_options presence: true, if: :manual_configuration? do - validates :api_url, url: true + validates :api_url, public_url: true end before_save :synchronize_service_state diff --git a/app/models/project_services/redmine_service.rb b/app/models/project_services/redmine_service.rb index 6acf611eba5..3721093a6d1 100644 --- a/app/models/project_services/redmine_service.rb +++ b/app/models/project_services/redmine_service.rb @@ -1,5 +1,5 @@ class RedmineService < IssueTrackerService - validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated? + validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url diff --git a/app/models/project_services/teamcity_service.rb b/app/models/project_services/teamcity_service.rb index 145313b8e71..802678147cf 100644 --- a/app/models/project_services/teamcity_service.rb +++ b/app/models/project_services/teamcity_service.rb @@ -3,7 +3,7 @@ class TeamcityService < CiService prop_accessor :teamcity_url, :build_type, :username, :password - validates :teamcity_url, presence: true, url: true, if: :activated? + validates :teamcity_url, presence: true, public_url: true, if: :activated? validates :build_type, presence: true, if: :activated? validates :username, presence: true, diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb index bbf8fd9c6a7..9722cbb2b7c 100644 --- a/app/models/remote_mirror.rb +++ b/app/models/remote_mirror.rb @@ -17,7 +17,6 @@ class RemoteMirror < ActiveRecord::Base belongs_to :project, inverse_of: :remote_mirrors validates :url, presence: true, url: { protocols: %w(ssh git http https), allow_blank: true } - validates :url, addressable_url: true, if: :url_changed? before_save :set_new_remote_name, if: :mirror_url_changed? -- cgit v1.2.3 From a886532cc04bfa7ec6885ea883889f6d138961bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Tue, 29 May 2018 17:49:52 +0200 Subject: Revert to caching the AR object in CacheableAttributes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Caching the attributes as JSON and manually instantiating the record ended-up very complex since the edge-cases such as upload fields, serialized fields, and fields with custom accessors had to be handled. To ensure 3 points out of 4 are checked from https://gitlab.com/gitlab-org/gitlab-ce/issues/45175 we now include the Rails version in the cache key. Signed-off-by: Rémy Coutable --- app/models/concerns/cacheable_attributes.rb | 34 +++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 9 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/cacheable_attributes.rb b/app/models/concerns/cacheable_attributes.rb index b32459fdabf..dd4fccc811d 100644 --- a/app/models/concerns/cacheable_attributes.rb +++ b/app/models/concerns/cacheable_attributes.rb @@ -6,15 +6,16 @@ module CacheableAttributes end class_methods do + def cache_key + "#{name}:#{Gitlab::VERSION}:#{Gitlab.migrations_hash}:#{Rails.version}".freeze + end + # Can be overriden def current_without_cache last end - def cache_key - "#{name}:#{Gitlab::VERSION}:#{Gitlab.migrations_hash}:json".freeze - end - + # Can be overriden def defaults {} end @@ -24,10 +25,14 @@ module CacheableAttributes end def cached - json_attributes = Rails.cache.read(cache_key) - return nil unless json_attributes.present? + retrieve_from_cache + end + + def retrieve_from_cache + record = Rails.cache.read(cache_key) + ensure_cache_setup if record.present? - build_from_defaults(JSON.parse(json_attributes)) + record end def current @@ -35,7 +40,12 @@ module CacheableAttributes return cached_record if cached_record.present? current_without_cache.tap { |current_record| current_record&.cache! } - rescue + rescue => e + if Rails.env.production? + Rails.logger.warn("Cached record for #{name} couldn't be loaded, falling back to uncached record: #{e}") + else + raise e + end # Fall back to an uncached value if there are any problems (e.g. Redis down) current_without_cache end @@ -46,9 +56,15 @@ module CacheableAttributes # Gracefully handle when Redis is not available. For example, # omnibus may fail here during gitlab:assets:compile. end + + def ensure_cache_setup + # This is a workaround for a Rails bug that causes attribute methods not + # to be loaded when read from cache: https://github.com/rails/rails/issues/27348 + define_attribute_methods + end end def cache! - Rails.cache.write(self.class.cache_key, attributes.to_json) + Rails.cache.write(self.class.cache_key, self) end end -- cgit v1.2.3 From 4eda09e3fbfe82fd1467e97cbc5bd085b91f257d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Tue, 29 May 2018 18:39:03 +0200 Subject: Use RequestStore in CacheableAttributes.cached for greater performance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- app/models/concerns/cacheable_attributes.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/concerns/cacheable_attributes.rb b/app/models/concerns/cacheable_attributes.rb index dd4fccc811d..d58d7165969 100644 --- a/app/models/concerns/cacheable_attributes.rb +++ b/app/models/concerns/cacheable_attributes.rb @@ -25,7 +25,11 @@ module CacheableAttributes end def cached - retrieve_from_cache + if RequestStore.active? + RequestStore[:"#{name}_cached_attributes"] ||= retrieve_from_cache + else + retrieve_from_cache + end end def retrieve_from_cache -- cgit v1.2.3 From 2535834f8ad35e7691384454013b5938a0a2af5d Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Fri, 1 Jun 2018 14:15:13 +0100 Subject: ReactiveCaching#clear_reactive_cache! should clear the not keep the cache alive --- app/models/concerns/reactive_caching.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/models') diff --git a/app/models/concerns/reactive_caching.rb b/app/models/concerns/reactive_caching.rb index eef9caf1c8e..be0a5b49012 100644 --- a/app/models/concerns/reactive_caching.rb +++ b/app/models/concerns/reactive_caching.rb @@ -74,6 +74,7 @@ module ReactiveCaching def clear_reactive_cache!(*args) Rails.cache.delete(full_reactive_cache_key(*args)) + Rails.cache.delete(alive_reactive_cache_key(*args)) end def exclusively_update_reactive_cache!(*args) -- cgit v1.2.3 From 4c8783636cdc279aea802760146d58e6259bed57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=99=88=20=20jacopo=20beschi=20=F0=9F=99=89?= Date: Fri, 1 Jun 2018 15:09:08 +0000 Subject: Resolve "Update `updated_at` on an issue/mr on every issue/mr changes" --- app/models/concerns/time_trackable.rb | 6 ------ app/models/timelog.rb | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/time_trackable.rb b/app/models/concerns/time_trackable.rb index 1caf47072bc..0fc321c52bc 100644 --- a/app/models/concerns/time_trackable.rb +++ b/app/models/concerns/time_trackable.rb @@ -30,8 +30,6 @@ module TimeTrackable return if @time_spent == 0 - touch if touchable? - if @time_spent == :reset reset_spent_time else @@ -59,10 +57,6 @@ module TimeTrackable private - def touchable? - valid? && persisted? - end - def reset_spent_time timelogs.new(time_spent: total_time_spent * -1, user: @time_spent_user) # rubocop:disable Gitlab/ModuleWithInstanceVariables end diff --git a/app/models/timelog.rb b/app/models/timelog.rb index e166cf69703..f4c5c581a11 100644 --- a/app/models/timelog.rb +++ b/app/models/timelog.rb @@ -2,8 +2,8 @@ class Timelog < ActiveRecord::Base validates :time_spent, :user, presence: true validate :issuable_id_is_present - belongs_to :issue - belongs_to :merge_request + belongs_to :issue, touch: true + belongs_to :merge_request, touch: true belongs_to :user def issuable -- cgit v1.2.3 From 5e9687a198f205131d8e65c747f6b4ac2ae092f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Fri, 1 Jun 2018 23:39:24 +0200 Subject: Add deploy_strategy to ProjectAutoDevops --- app/models/project_auto_devops.rb | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'app/models') diff --git a/app/models/project_auto_devops.rb b/app/models/project_auto_devops.rb index ed6c1eddbc1..8464b1d9157 100644 --- a/app/models/project_auto_devops.rb +++ b/app/models/project_auto_devops.rb @@ -1,6 +1,11 @@ class ProjectAutoDevops < ActiveRecord::Base belongs_to :project + enum deploy_strategy: { + manual: 1, + continuous: 2 + } + scope :enabled, -> { where(enabled: true) } scope :disabled, -> { where(enabled: false) } -- cgit v1.2.3 From 39412d0a163625bd91632267a1dd36c1ebb23906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Fri, 1 Jun 2018 23:47:38 +0200 Subject: Add deploy strategy related predefined variables --- app/models/project_auto_devops.rb | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'app/models') diff --git a/app/models/project_auto_devops.rb b/app/models/project_auto_devops.rb index 8464b1d9157..d5e00f84cf7 100644 --- a/app/models/project_auto_devops.rb +++ b/app/models/project_auto_devops.rb @@ -25,6 +25,11 @@ class ProjectAutoDevops < ActiveRecord::Base variables.append(key: 'AUTO_DEVOPS_DOMAIN', value: domain.presence || instance_domain) end + + if continuous? + variables.append(key: 'STAGING_ENABLED', value: 1) + variables.append(key: 'INCREMENTAL_ROLLOUT_ENABLED', value: 1) + end end end end -- cgit v1.2.3 From 61df812ac688cb0848752f9f26f77d65eadf160a Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sat, 2 Jun 2018 02:32:30 -0700 Subject: Fix attr_encryption key settings attr_encrypted does different things with `key` depending on what mode you are using: 1. In `:per_attribute_iv_and_salt` mode, it generates a hash with the salt: https://github.com/attr-encrypted/encryptor/blob/c3a62c4a9e74686dd95e0548f9dc2a361fdc95d1/lib/encryptor.rb#L77. There is no need to truncate the key to 32 bytes here. 2. In `:per_attribute_iv` mode, it sets the key directly to the password, so truncation to 32 bytes is necessary. Closes #47166 --- app/models/clusters/platforms/kubernetes.rb | 4 ++-- app/models/clusters/providers/gcp.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index 25eac5160f1..36631d57ad1 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -11,12 +11,12 @@ module Clusters attr_encrypted :password, mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base, + key: Settings.attr_encrypted_db_key_base_truncated, algorithm: 'aes-256-cbc' attr_encrypted :token, mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base, + key: Settings.attr_encrypted_db_key_base_truncated, algorithm: 'aes-256-cbc' before_validation :enforce_namespace_to_lower_case diff --git a/app/models/clusters/providers/gcp.rb b/app/models/clusters/providers/gcp.rb index eb2e42fd3fe..4db1bb35c12 100644 --- a/app/models/clusters/providers/gcp.rb +++ b/app/models/clusters/providers/gcp.rb @@ -11,7 +11,7 @@ module Clusters attr_encrypted :access_token, mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base, + key: Settings.attr_encrypted_db_key_base_truncated, algorithm: 'aes-256-cbc' validates :gcp_project_id, -- cgit v1.2.3 From 0d00d02e842a0c4b22d213e00143a08d97597000 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sun, 3 Jun 2018 14:21:50 +0900 Subject: Directly refer application code from migration code --- app/models/ci/build.rb | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 75fd55a8f7b..6cbf9108244 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -55,6 +55,11 @@ module Ci where('(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)', '', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').archive) end + + scope :without_archived_trace, ->() do + where('NOT EXISTS (?)', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').trace) + end + scope :with_artifacts_stored_locally, -> { with_artifacts_archive.where(artifacts_file_store: [nil, LegacyArtifactUploader::Store::LOCAL]) } scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) } -- cgit v1.2.3 From 114c26ccf0f10788271c6108774e72809a7f93e1 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 4 Jun 2018 11:29:39 +0200 Subject: Raise error if pipeline / stage hits unknown status --- app/models/ci/pipeline.rb | 6 +++++- app/models/ci/stage.rb | 4 ++++ app/models/concerns/has_status.rb | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 94a548181d9..1274d2a5f16 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -517,7 +517,8 @@ module Ci def update_status retry_optimistic_lock(self) do - case latest_builds_status + case latest_builds_status.to_s + when 'created' then nil when 'pending' then enqueue when 'running' then run when 'success' then succeed @@ -525,6 +526,9 @@ module Ci when 'canceled' then cancel when 'skipped' then skip when 'manual' then block + else + raise HasStatus::UnknownStatusError, + "Unknown status `#{latest_builds_status}`" end end end diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index abf73c00539..ea07f37e6c1 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -68,6 +68,7 @@ module Ci def update_status retry_optimistic_lock(self) do case statuses.latest.status + when 'created' then nil when 'pending' then enqueue when 'running' then run when 'success' then succeed @@ -75,6 +76,9 @@ module Ci when 'canceled' then cancel when 'manual' then block when 'skipped', nil then skip + else + raise HasStatus::UnknownStatusError, + "Unknown status `#{statuses.latest.status}`" end end end diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index 7c3ed96bc28..72c236a0fc7 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -11,6 +11,8 @@ module HasStatus STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3, failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze + UnknownStatusError = Class.new(StandardError) + class_methods do def status_sql scope_relevant = respond_to?(:exclude_ignored) ? exclude_ignored : all -- cgit v1.2.3 From 7350eb1fa83662d4aaa7541acb387b3742ba9788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Javier=20L=C3=B3pez?= Date: Mon, 4 Jun 2018 11:41:37 +0000 Subject: Add ability to search wiki titles --- app/models/project_wiki.rb | 4 ---- 1 file changed, 4 deletions(-) (limited to 'app/models') diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index f799a0b4227..a6f94b3e3b0 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -140,10 +140,6 @@ class ProjectWiki [title, title_array.join("/")] end - def search_files(query) - repository.search_files_by_content(query, default_branch) - end - def repository @repository ||= Repository.new(full_path, @project, disk_path: disk_path, is_wiki: true) end -- cgit v1.2.3 From f12ee2a2f490e6d126ac6345a5ad7cbf12833791 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Mon, 4 Jun 2018 21:34:11 +0200 Subject: Remove unused running_or_pending_build_count --- app/models/ci/build.rb | 1 - app/models/project.rb | 6 ------ 2 files changed, 7 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 75fd55a8f7b..7cf4dda178a 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -403,7 +403,6 @@ module Ci project.execute_hooks(build_data.dup, :job_hooks) project.execute_services(build_data.dup, :job_hooks) PagesService.new(build_data).execute - project.running_or_pending_build_count(force: true) end def browsable_artifacts? diff --git a/app/models/project.rb b/app/models/project.rb index 32298fc7f5c..a094dbcb747 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1649,12 +1649,6 @@ class Project < ActiveRecord::Base import_state.update_column(:jid, nil) end - def running_or_pending_build_count(force: false) - Rails.cache.fetch(['projects', id, 'running_or_pending_build_count'], force: force) do - builds.running_or_pending.count(:all) - end - end - # Lazy loading of the `pipeline_status` attribute def pipeline_status @pipeline_status ||= Gitlab::Cache::Ci::ProjectPipelineStatus.load_for_project(self) -- cgit v1.2.3 From 285ffb223896f2226531be2ba10933414194155c Mon Sep 17 00:00:00 2001 From: tauriedavis Date: Fri, 25 May 2018 16:17:57 -0700 Subject: Messaging on terms page when user already accepted We show a blue flash banner if the user already accepted, and show a button allowing them to continue to the application. --- app/models/application_setting/term.rb | 6 ++++++ app/models/term_agreement.rb | 2 ++ 2 files changed, 8 insertions(+) (limited to 'app/models') diff --git a/app/models/application_setting/term.rb b/app/models/application_setting/term.rb index e8ce0ccbb71..3b1dfe7e4ef 100644 --- a/app/models/application_setting/term.rb +++ b/app/models/application_setting/term.rb @@ -1,6 +1,7 @@ class ApplicationSetting class Term < ActiveRecord::Base include CacheMarkdownField + has_many :term_agreements validates :terms, presence: true @@ -9,5 +10,10 @@ class ApplicationSetting def self.latest order(:id).last end + + def accepted_by_user?(user) + user.accepted_term_id == id || + term_agreements.accepted.where(user: user).exists? + end end end diff --git a/app/models/term_agreement.rb b/app/models/term_agreement.rb index 8458a231bbd..c317bd0c90b 100644 --- a/app/models/term_agreement.rb +++ b/app/models/term_agreement.rb @@ -2,5 +2,7 @@ class TermAgreement < ActiveRecord::Base belongs_to :term, class_name: 'ApplicationSetting::Term' belongs_to :user + scope :accepted, -> { where(accepted: true) } + validates :user, :term, presence: true end -- cgit v1.2.3 From 36fbe1422666cb5553ccde9e534bb2a504f0f2c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Mon, 28 May 2018 18:17:18 +0200 Subject: Do not validate cached data --- app/models/ci/runner.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 57edd6a4956..8c9aacca8de 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -219,10 +219,8 @@ module Ci cache_attributes(values) - if persist_cached_data? - self.assign_attributes(values) - self.save if self.changed? - end + # We save data without validation, it will always change due to `contacted_at` + self.update_columns(values) if persist_cached_data? end def pick_build!(build) -- cgit v1.2.3 From b009a0084c67877ba6a808c4c8a81c568598d624 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Mon, 4 Jun 2018 21:49:39 +0200 Subject: Remove PagesService and instead make it explicit that we call PagesWorker --- app/models/ci/build.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 7cf4dda178a..746464d0e21 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -144,6 +144,7 @@ module Ci after_transition any => [:success] do |build| build.run_after_commit do BuildSuccessWorker.perform_async(id) + PagesWorker.perform_async(:deploy, id) if build.pages_generator? end end @@ -183,6 +184,11 @@ module Ci pipeline.manual_actions.where.not(name: name) end + def pages_generator? + Gitlab.config.pages.enabled && + self.name == 'pages' + end + def playable? action? && (manual? || retryable?) end @@ -402,7 +408,6 @@ module Ci build_data = Gitlab::DataBuilder::Build.build(self) project.execute_hooks(build_data.dup, :job_hooks) project.execute_services(build_data.dup, :job_hooks) - PagesService.new(build_data).execute end def browsable_artifacts? -- cgit v1.2.3 From 6ecf819f73ae28547e1b816780de0d85c3a653cf Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Thu, 19 Apr 2018 17:22:23 +0100 Subject: Fix an N+1 in avatar URLs This is tricky: the query was being run in `ObjectStorage::Extension::RecordsUploads#retrieve_from_store!`, but we can't just add batch loading there, because the `#upload=` method there would use the result immediately, making the batch only have one item. Instead, we can pre-emptively add an item to the batch whenever an avatarable object is initialized, and then reuse that batch item in `#retrieve_from_store!`. However, this also has problems: 1. There is a lot of logic in `Avatarable#retrieve_upload_from_batch`. 2. Some of that logic constructs a 'fake' model for the batch key. This should be fine, because of ActiveRecord's override of `#==`, but it relies on that staying the same. --- app/models/concerns/avatarable.rb | 47 +++++++++++++++++++++++++++++++++++++ app/models/concerns/with_uploads.rb | 4 ++++ app/models/note.rb | 4 ++++ app/models/personal_snippet.rb | 1 + 4 files changed, 56 insertions(+) (limited to 'app/models') diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb index 13246a774e3..095897b08e3 100644 --- a/app/models/concerns/avatarable.rb +++ b/app/models/concerns/avatarable.rb @@ -4,11 +4,14 @@ module Avatarable included do prepend ShadowMethods include ObjectStorage::BackgroundMove + include Gitlab::Utils::StrongMemoize validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? } validates :avatar, file_size: { maximum: 200.kilobytes.to_i } mount_uploader :avatar, AvatarUploader + + after_initialize :add_avatar_to_batch end module ShadowMethods @@ -18,6 +21,17 @@ module Avatarable avatar_path(only_path: args.fetch(:only_path, true)) || super end + + def retrieve_upload(identifier, paths) + upload = retrieve_upload_from_batch(identifier) + + # This fallback is needed when deleting an upload, because we may have + # already been removed from the DB. We have to check an explicit `#nil?` + # because it's a BatchLoader instance. + upload = super if upload.nil? + + upload + end end def avatar_type @@ -52,4 +66,37 @@ module Avatarable url_base + avatar.local_url end + + # Path that is persisted in the tracking Upload model. Used to fetch the + # upload from the model. + def upload_paths(identifier) + avatar_mounter.blank_uploader.store_dirs.map { |store, path| File.join(path, identifier) } + end + + private + + def retrieve_upload_from_batch(identifier) + BatchLoader.for(identifier: identifier, model: self).batch(key: self.class) do |upload_params, loader, args| + model_class = args[:key] + paths = upload_params.flat_map do |params| + params[:model].upload_paths(params[:identifier]) + end + + Upload.where(uploader: AvatarUploader, path: paths).find_each do |upload| + model = model_class.instantiate('id' => upload.model_id) + + loader.call({ model: model, identifier: File.basename(upload.path) }, upload) + end + end + end + + def add_avatar_to_batch + return unless avatar_mounter + + avatar_mounter.read_identifiers.each { |identifier| retrieve_upload_from_batch(identifier) } + end + + def avatar_mounter + strong_memoize(:avatar_mounter) { _mounter(:avatar) } + end end diff --git a/app/models/concerns/with_uploads.rb b/app/models/concerns/with_uploads.rb index e7cfffb775b..4245d083a49 100644 --- a/app/models/concerns/with_uploads.rb +++ b/app/models/concerns/with_uploads.rb @@ -36,4 +36,8 @@ module WithUploads upload.destroy end end + + def retrieve_upload(_identifier, paths) + uploads.find_by(path: paths) + end end diff --git a/app/models/note.rb b/app/models/note.rb index 02f7a9b1e4f..41c04ae0571 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -435,6 +435,10 @@ class Note < ActiveRecord::Base super.merge(noteable: noteable) end + def retrieve_upload(_identifier, paths) + Upload.find_by(model: self, path: paths) + end + private def keep_around_commit diff --git a/app/models/personal_snippet.rb b/app/models/personal_snippet.rb index 82c1c4de3a0..355624fd552 100644 --- a/app/models/personal_snippet.rb +++ b/app/models/personal_snippet.rb @@ -1,2 +1,3 @@ class PersonalSnippet < Snippet + include WithUploads end -- cgit v1.2.3 From ce6172e863f982c73fedc9f4a73877543c30cb1c Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Tue, 26 Sep 2017 09:54:32 +0200 Subject: allow uploading favicon in appearance settings --- app/models/appearance.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/models') diff --git a/app/models/appearance.rb b/app/models/appearance.rb index 67cc84a9140..b770aadef0e 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -14,6 +14,7 @@ class Appearance < ActiveRecord::Base mount_uploader :logo, AttachmentUploader mount_uploader :header_logo, AttachmentUploader + mount_uploader :favicon, FaviconUploader # Overrides CacheableAttributes.current_without_cache def self.current_without_cache -- cgit v1.2.3 From 949c30d42b91a0dd3959a3ca303b8f76158a2556 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Thu, 7 Dec 2017 13:15:49 +0100 Subject: remove all .ico favicon variations, use png always the ci status icons are generated client side, wo we don't need the static files anymore. --- app/models/project_services/jira_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index eb3261c902f..2edfe7d81f8 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -265,7 +265,7 @@ class JiraService < IssueTrackerService title: title, status: status, icon: { - title: 'GitLab', url16x16: asset_url('favicon.ico', host: gitlab_config.url) + title: 'GitLab', url16x16: asset_url('favicon.png', host: gitlab_config.url) } } } -- cgit v1.2.3 From 7706b3a471de37240c0da6fcc14c7d38c6d62eb9 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Fri, 13 Apr 2018 22:30:08 +0200 Subject: use Gitlab::Favicon for jira service --- app/models/project_services/jira_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index 2edfe7d81f8..412d62388f0 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -265,7 +265,7 @@ class JiraService < IssueTrackerService title: title, status: status, icon: { - title: 'GitLab', url16x16: asset_url('favicon.png', host: gitlab_config.url) + title: 'GitLab', url16x16: asset_url(Gitlab::Favicon.main, host: gitlab_config.url) } } } -- cgit v1.2.3 From 65dbead7967937eb87ffdc987c319d7006662889 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Tue, 5 Jun 2018 19:22:55 +0100 Subject: Fix repository archive generation when hashed storage is enabled --- app/models/repository.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'app/models') diff --git a/app/models/repository.rb b/app/models/repository.rb index 82cf47ba04e..0784891d1bf 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -270,6 +270,16 @@ class Repository end end + def archive_metadata(ref, storage_path, format = "tar.gz", append_sha:) + raw_repository.archive_metadata( + ref, + storage_path, + project.path, + format, + append_sha: append_sha + ) + end + def expire_tags_cache expire_method_caches(%i(tag_names tag_count)) @tags = nil -- cgit v1.2.3 From 3ed66d4abde28c9f586342fe8e6481360825b823 Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Tue, 5 Jun 2018 19:12:26 +0100 Subject: Adds #human_import_status_name to make it comply with ProjectImportState#human_status_name --- app/models/project.rb | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index a094dbcb747..3ae445ed200 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -674,6 +674,12 @@ class Project < ActiveRecord::Base end end + def human_import_status_name + ensure_import_state + + import_state.human_status_name + end + def import_schedule ensure_import_state(force: true) -- cgit v1.2.3 From 3b569fceff09db6471a054128d2cbf268e63964d Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 31 May 2018 16:12:20 +0900 Subject: Clean up worker --- app/models/ci/build_trace_chunk.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 4856f10846c..4884e355c36 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -50,6 +50,21 @@ module Ci def finalize_fast_destroy(keys) redis_delete_data(keys) end + + # Find stale live traces and return their build ids + def find_stale(finished_before: 1.hour.ago) + include(EachBatch) + .select(:build_id) + .group(:build_id) + .joins(:build) + .merge(Ci::Build.finished) + .where('ci_builds.finished_at < ?', finished_before) + .each_batch(column: :build_id) do |chunks| + build_ids = chunks.map { |chunk| [chunk.build_id] } + + yield build_ids + end + end end ## -- cgit v1.2.3 From 10acdc30df99ede1356493c14a57b94ed1ae7950 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 31 May 2018 17:11:50 +0900 Subject: Rename find_stale. Fix worker name in declaration. --- app/models/ci/build_trace_chunk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 4884e355c36..f17d8a07b13 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -52,7 +52,7 @@ module Ci end # Find stale live traces and return their build ids - def find_stale(finished_before: 1.hour.ago) + def find_stale_in_batches(finished_before: 1.hour.ago) include(EachBatch) .select(:build_id) .group(:build_id) -- cgit v1.2.3 From 32f825c648cb5fc829cd32b3392530613c62e983 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 1 Jun 2018 14:22:31 +0900 Subject: Add tests for each new code --- app/models/ci/build_trace_chunk.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index f17d8a07b13..7940f7c2ee7 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -60,7 +60,7 @@ module Ci .merge(Ci::Build.finished) .where('ci_builds.finished_at < ?', finished_before) .each_batch(column: :build_id) do |chunks| - build_ids = chunks.map { |chunk| [chunk.build_id] } + build_ids = chunks.map { |chunk| chunk.build_id } yield build_ids end -- cgit v1.2.3 From 4064481501a24d31872914c845f5d8c2cfc08040 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 1 Jun 2018 16:25:17 +0900 Subject: Rename find_stale_in_batches to find_builds_from_stale_live_trace. Fix comments --- app/models/ci/build_trace_chunk.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 7940f7c2ee7..80a63434283 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -52,13 +52,13 @@ module Ci end # Find stale live traces and return their build ids - def find_stale_in_batches(finished_before: 1.hour.ago) + def find_builds_from_stale_live_trace include(EachBatch) .select(:build_id) .group(:build_id) .joins(:build) .merge(Ci::Build.finished) - .where('ci_builds.finished_at < ?', finished_before) + .where('ci_builds.finished_at < ?', 1.hour.ago) .each_batch(column: :build_id) do |chunks| build_ids = chunks.map { |chunk| chunk.build_id } -- cgit v1.2.3 From 2084e7ab9aad92d4a8196af01b9b8e02ffacb0a4 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 1 Jun 2018 17:09:46 +0900 Subject: Move find_builds_from_stale_live_traces method to Ci::Build --- app/models/ci/build.rb | 15 +++++++++++++++ app/models/ci/build_trace_chunk.rb | 15 --------------- 2 files changed, 15 insertions(+), 15 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index d93e7cb896f..f8fcab0e2f0 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -583,6 +583,21 @@ module Ci super(options).merge(when: read_attribute(:when)) end + # Find stale live traces and return their build ids + def self.find_builds_from_stale_live_traces + binding.pry + + Ci::BuildTraceChunk + .include(EachBatch).select(:build_id).group(:build_id).joins(:build) + .merge(Ci::Build.finished).where('ci_builds.finished_at < ?', 1.hour.ago) + .each_batch(column: :build_id) do |chunks| + build_ids = chunks.map { |chunk| chunk.build_id } + + binding.pry + yield where(id: build_ids) + end + end + private def update_artifacts_size diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 80a63434283..4856f10846c 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -50,21 +50,6 @@ module Ci def finalize_fast_destroy(keys) redis_delete_data(keys) end - - # Find stale live traces and return their build ids - def find_builds_from_stale_live_trace - include(EachBatch) - .select(:build_id) - .group(:build_id) - .joins(:build) - .merge(Ci::Build.finished) - .where('ci_builds.finished_at < ?', 1.hour.ago) - .each_batch(column: :build_id) do |chunks| - build_ids = chunks.map { |chunk| chunk.build_id } - - yield build_ids - end - end end ## -- cgit v1.2.3 From 5a1ee0c391a06a323d960eab6ffb33dfcf2edca7 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 2 Jun 2018 13:08:34 +0900 Subject: Fix the query to select stale live traces --- app/models/ci/build.rb | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index f8fcab0e2f0..2d675726939 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -66,6 +66,7 @@ module Ci scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) } scope :ref_protected, -> { where(protected: true) } + scope :with_live_trace, -> { where('EXISTS (?)', Ci::BuildTraceChunk.where('ci_builds.id = ci_build_trace_chunks.build_id').select(1)) } scope :matches_tag_ids, -> (tag_ids) do matcher = ::ActsAsTaggableOn::Tagging @@ -583,21 +584,6 @@ module Ci super(options).merge(when: read_attribute(:when)) end - # Find stale live traces and return their build ids - def self.find_builds_from_stale_live_traces - binding.pry - - Ci::BuildTraceChunk - .include(EachBatch).select(:build_id).group(:build_id).joins(:build) - .merge(Ci::Build.finished).where('ci_builds.finished_at < ?', 1.hour.ago) - .each_batch(column: :build_id) do |chunks| - build_ids = chunks.map { |chunk| chunk.build_id } - - binding.pry - yield where(id: build_ids) - end - end - private def update_artifacts_size -- cgit v1.2.3 From a8b570d8bced794dcb8f6f3a1155360038d76a66 Mon Sep 17 00:00:00 2001 From: Mark Chao Date: Tue, 22 May 2018 16:51:45 +0800 Subject: Rename master to maintainer --- app/models/protected_branch.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index cb361a66591..dff99cfca35 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -5,7 +5,7 @@ class ProtectedBranch < ActiveRecord::Base protected_ref_access_levels :merge, :push def self.protected_ref_accessible_to?(ref, user, project:, action:, protected_refs: nil) - # Masters, owners and admins are allowed to create the default branch + # Maintainers, owners and admins are allowed to create the default branch if default_branch_protected? && project.empty_repo? return true if user.admin? || project.team.max_member_access(user.id) > Gitlab::Access::DEVELOPER end -- cgit v1.2.3 From cfcc7043fe614cbc5fc82f3021a0c4cd8519e24b Mon Sep 17 00:00:00 2001 From: Mark Chao Date: Tue, 22 May 2018 19:54:50 +0800 Subject: =?UTF-8?q?Rename=20=E2=80=9CDevelopers=20+=20Masters=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/concerns/protected_ref_access.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/protected_ref_access.rb b/app/models/concerns/protected_ref_access.rb index bfda5b1678b..e3a7f2d5498 100644 --- a/app/models/concerns/protected_ref_access.rb +++ b/app/models/concerns/protected_ref_access.rb @@ -8,8 +8,8 @@ module ProtectedRefAccess ].freeze HUMAN_ACCESS_LEVELS = { - Gitlab::Access::MASTER => "Masters".freeze, - Gitlab::Access::DEVELOPER => "Developers + Masters".freeze, + Gitlab::Access::MASTER => "Maintainers".freeze, + Gitlab::Access::DEVELOPER => "Developers + Maintainers".freeze, Gitlab::Access::NO_ACCESS => "No one".freeze }.freeze -- cgit v1.2.3 From 52e62f718f56852daa541553acf28ec75d8d5ff9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Thu, 31 May 2018 14:00:36 +0200 Subject: Reduce CE/EE diff in app/views/admin/groups/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- app/models/group.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/group.rb b/app/models/group.rb index 8fb77a7869d..aeb50fb9bb7 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -141,13 +141,14 @@ class Group < Namespace ) end - def add_user(user, access_level, current_user: nil, expires_at: nil) + def add_user(user, access_level, current_user: nil, expires_at: nil, ldap: false) GroupMember.add_user( self, user, access_level, current_user: current_user, - expires_at: expires_at + expires_at: expires_at, + ldap: ldap ) end @@ -195,6 +196,10 @@ class Group < Namespace owners.include?(user) && owners.size == 1 end + def ldap_synced? + false + end + def post_create_hook Gitlab::AppLogger.info("Group \"#{name}\" was created") -- cgit v1.2.3 From e8f49b4bee8d803953b852685889a2912609ae84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Javier=20L=C3=B3pez?= Date: Wed, 6 Jun 2018 16:42:18 +0000 Subject: Support LFS objects when creating a project by import --- app/models/repository.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'app/models') diff --git a/app/models/repository.rb b/app/models/repository.rb index 0784891d1bf..e4202505634 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -956,6 +956,10 @@ class Repository blob_data_at(sha, path) end + def lfsconfig_for(sha) + blob_data_at(sha, '.lfsconfig') + end + def fetch_ref(source_repository, source_ref:, target_ref:) raw_repository.fetch_ref(source_repository.raw_repository, source_ref: source_ref, target_ref: target_ref) end -- cgit v1.2.3 From 9738b0a42bdfca5b8fbf0a0ef2ade4d46be38c75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Wed, 6 Jun 2018 18:58:26 +0200 Subject: Make deploy_strategy zero based --- app/models/project_auto_devops.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/project_auto_devops.rb b/app/models/project_auto_devops.rb index d5e00f84cf7..3c468831e91 100644 --- a/app/models/project_auto_devops.rb +++ b/app/models/project_auto_devops.rb @@ -2,8 +2,8 @@ class ProjectAutoDevops < ActiveRecord::Base belongs_to :project enum deploy_strategy: { - manual: 1, - continuous: 2 + manual: 0, + continuous: 1 } scope :enabled, -> { where(enabled: true) } -- cgit v1.2.3 From 78ae23045bae8a75775a11675afd5836fb8f318b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Wed, 6 Jun 2018 19:09:46 +0200 Subject: Reverse logic of manual and continuous deploy strategies --- app/models/project_auto_devops.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/project_auto_devops.rb b/app/models/project_auto_devops.rb index 3c468831e91..ed0428615a2 100644 --- a/app/models/project_auto_devops.rb +++ b/app/models/project_auto_devops.rb @@ -26,7 +26,7 @@ class ProjectAutoDevops < ActiveRecord::Base value: domain.presence || instance_domain) end - if continuous? + if manual? variables.append(key: 'STAGING_ENABLED', value: 1) variables.append(key: 'INCREMENTAL_ROLLOUT_ENABLED', value: 1) end -- cgit v1.2.3 From 0665a8f730e19e180f2b4d240ca7e93408d61b12 Mon Sep 17 00:00:00 2001 From: Jan Provaznik Date: Thu, 31 May 2018 20:32:36 +0200 Subject: Enable mapping to nil in enums Enum in Rails 5 does not map nil values - IOW nil value remains nil, even if there is a key with nil value in the enum definition. This commit overrides the underlying Enum methods so nil value is still mapped. This solution is far from being ideal: it uses dynamic definition of methods which introduces more magic/confusion into the codebase. It would be better to get rid of the nil value in enums. --- app/models/ci/pipeline.rb | 16 +++------------- app/models/commit_status.rb | 10 ++-------- app/models/concerns/enum_with_nil.rb | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 21 deletions(-) create mode 100644 app/models/concerns/enum_with_nil.rb (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index eecd86349e4..0878356e87a 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -8,6 +8,7 @@ module Ci include Gitlab::OptimisticLocking include Gitlab::Utils::StrongMemoize include AtomicInternalId + include EnumWithNil belongs_to :project, inverse_of: :pipelines belongs_to :user @@ -54,7 +55,7 @@ module Ci after_create :keep_around_commits, unless: :importing? - enum source: { + enum_with_nil source: { unknown: nil, push: 1, web: 2, @@ -64,7 +65,7 @@ module Ci external: 6 } - enum config_source: { + enum_with_nil config_source: { unknown_source: nil, repository_source: 1, auto_devops_source: 2 @@ -599,17 +600,6 @@ module Ci @latest_builds_with_artifacts ||= builds.latest.with_artifacts_archive.to_a end - # Rails 5.0 autogenerated question mark enum methods return wrong result if enum value is nil. - # They always return `false`. - # These methods overwrite autogenerated ones to return correct results. - def unknown? - Gitlab.rails5? ? source.nil? : super - end - - def unknown_source? - Gitlab.rails5? ? config_source.nil? : super - end - private def ci_yaml_from_repo diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index a7d05722287..97516079b66 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -3,6 +3,7 @@ class CommitStatus < ActiveRecord::Base include Importable include AfterCommitQueue include Presentable + include EnumWithNil self.table_name = 'ci_builds' @@ -39,7 +40,7 @@ class CommitStatus < ActiveRecord::Base scope :retried_ordered, -> { retried.ordered.includes(project: :namespace) } scope :after_stage, -> (index) { where('stage_idx > ?', index) } - enum failure_reason: { + enum_with_nil failure_reason: { unknown_failure: nil, script_failure: 1, api_failure: 2, @@ -190,11 +191,4 @@ class CommitStatus < ActiveRecord::Base v =~ /\d+/ ? v.to_i : v end end - - # Rails 5.0 autogenerated question mark enum methods return wrong result if enum value is nil. - # They always return `false`. - # This method overwrites the autogenerated one to return correct result. - def unknown_failure? - Gitlab.rails5? ? failure_reason.nil? : super - end end diff --git a/app/models/concerns/enum_with_nil.rb b/app/models/concerns/enum_with_nil.rb new file mode 100644 index 00000000000..6b37903da20 --- /dev/null +++ b/app/models/concerns/enum_with_nil.rb @@ -0,0 +1,33 @@ +module EnumWithNil + extend ActiveSupport::Concern + + included do + def self.enum_with_nil(definitions) + # use original `enum` to auto-define all methods + enum(definitions) + + # override auto-defined methods only for the + # key which uses nil value + definitions.each do |name, values| + next unless key_with_nil = values.key(nil) + + # E.g. for enum_with_nil failure_reason: { unknown_failure: nil } + # this overrides auto-generated method `unknown_failure?` + define_method("#{key_with_nil}?") do + Gitlab.rails5? ? self[name].nil? : super() + end + + # E.g. for enum_with_nil failure_reason: { unknown_failure: nil } + # this overrides auto-generated method `failure_reason` + define_method(name) do + orig = super() + + return orig unless Gitlab.rails5? + return orig unless orig.nil? + + self.class.public_send(name.to_s.pluralize).key(nil) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + end +end -- cgit v1.2.3 From 0206476ae2a2d659fd0fb42338050253a9a91439 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Thu, 7 Jun 2018 12:14:27 +0100 Subject: Fix some N+1s when calculating notification recipients First N+1: we may have loaded a user's notification settings already, but not have loaded their sources. Because we're iterating through, we'd potentially load sources that are completely unrelated, just because they belong to this user. Second N+1: we do a separate query for each user who could be subscribed to or unsubcribed from the target. It's actually more efficient in this case to get all subscriptions at once, as we will need to check most of them. We can fix both by the slightly unpleasant means of checking IDs manually, rather than object equality. --- app/models/notification_recipient.rb | 2 +- app/models/user.rb | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/notification_recipient.rb b/app/models/notification_recipient.rb index 2c3580bbdc6..79458bd048a 100644 --- a/app/models/notification_recipient.rb +++ b/app/models/notification_recipient.rb @@ -64,7 +64,7 @@ class NotificationRecipient return false unless @target return false unless @target.respond_to?(:subscriptions) - subscription = @target.subscriptions.find_by_user_id(@user.id) + subscription = @target.subscriptions.find { |subscription| subscription.user_id == @user.id } subscription && !subscription.subscribed end diff --git a/app/models/user.rb b/app/models/user.rb index e219ab800ad..8e0dc91b2a7 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1038,7 +1038,10 @@ class User < ActiveRecord::Base def notification_settings_for(source) if notification_settings.loaded? - notification_settings.find { |notification| notification.source == source } + notification_settings.find do |notification| + notification.source_type == source.class.base_class.name && + notification.source_id == source.id + end else notification_settings.find_or_initialize_by(source: source) end -- cgit v1.2.3 From 2bc1835597446a1240cfb364d1943feb4080e675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matija=20=C4=8Cupi=C4=87?= Date: Thu, 7 Jun 2018 18:11:08 +0200 Subject: Use 0 continuous default for deploy strategy --- app/models/project_auto_devops.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/project_auto_devops.rb b/app/models/project_auto_devops.rb index ed0428615a2..a6f728e478e 100644 --- a/app/models/project_auto_devops.rb +++ b/app/models/project_auto_devops.rb @@ -2,8 +2,8 @@ class ProjectAutoDevops < ActiveRecord::Base belongs_to :project enum deploy_strategy: { - manual: 0, - continuous: 1 + continuous: 0, + manual: 1 } scope :enabled, -> { where(enabled: true) } -- cgit v1.2.3 From 5370c442dfc0f1009b557cea9ce4dafbfe821569 Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Thu, 7 Jun 2018 18:09:14 +0000 Subject: Resolve "Automatically provide a Deploy Token to projects when Auto DevOps is enabled" --- app/models/project_auto_devops.rb | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'app/models') diff --git a/app/models/project_auto_devops.rb b/app/models/project_auto_devops.rb index ed6c1eddbc1..c6c990dfa00 100644 --- a/app/models/project_auto_devops.rb +++ b/app/models/project_auto_devops.rb @@ -6,6 +6,8 @@ class ProjectAutoDevops < ActiveRecord::Base validates :domain, allow_blank: true, hostname: { allow_numeric_hostname: true } + after_save :create_gitlab_deploy_token, if: :needs_to_create_deploy_token? + def instance_domain Gitlab::CurrentSettings.auto_devops_domain end @@ -22,4 +24,23 @@ class ProjectAutoDevops < ActiveRecord::Base end end end + + private + + def create_gitlab_deploy_token + project.deploy_tokens.create!( + name: DeployToken::GITLAB_DEPLOY_TOKEN_NAME, + read_registry: true + ) + end + + def needs_to_create_deploy_token? + auto_devops_enabled? && + !project.public? && + !project.deploy_tokens.find_by(name: DeployToken::GITLAB_DEPLOY_TOKEN_NAME).present? + end + + def auto_devops_enabled? + Gitlab::CurrentSettings.auto_devops_enabled? || enabled? + end end -- cgit v1.2.3 From afe5d7d56ee771dac6e4a97d23e69d678c03da2d Mon Sep 17 00:00:00 2001 From: Felipe Artur Date: Mon, 28 May 2018 15:43:46 -0300 Subject: Apply notification settings level of groups to all child objects --- app/models/group.rb | 20 ++++++++++++++++++++ app/models/notification_recipient.rb | 29 +++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/group.rb b/app/models/group.rb index 8fb77a7869d..1bd718d0a92 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -11,6 +11,7 @@ class Group < Namespace include GroupDescendant include TokenAuthenticatable include WithUploads + include Gitlab::Utils::StrongMemoize has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent alias_method :members, :group_members @@ -26,7 +27,11 @@ class Group < Namespace has_many :milestones has_many :project_group_links, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :shared_projects, through: :project_group_links, source: :project + + # Overridden on another method + # Left here just to be dependent: :destroy has_many :notification_settings, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent + has_many :labels, class_name: 'GroupLabel' has_many :variables, class_name: 'Ci::GroupVariable' has_many :custom_attributes, class_name: 'GroupCustomAttribute' @@ -88,6 +93,15 @@ class Group < Namespace end end + # Overrides notification_settings has_many association + # This allows to apply notification settings from parent groups + # to child groups and projects. + def notification_settings + source_type = self.class.base_class.name + + NotificationSetting.where(source_type: source_type, source_id: self_and_ancestors_ids) + end + def to_reference(_from = nil, full: nil) "#{self.class.reference_prefix}#{full_path}" end @@ -220,6 +234,12 @@ class Group < Namespace members_with_parents.pluck(:user_id) end + def self_and_ancestors_ids + strong_memoize(:self_and_ancestors_ids) do + self_and_ancestors.pluck(:id) + end + end + def members_with_parents # Avoids an unnecessary SELECT when the group has no parents source_ids = diff --git a/app/models/notification_recipient.rb b/app/models/notification_recipient.rb index 2c3580bbdc6..ee497579e60 100644 --- a/app/models/notification_recipient.rb +++ b/app/models/notification_recipient.rb @@ -1,4 +1,6 @@ class NotificationRecipient + include Gitlab::Utils::StrongMemoize + attr_reader :user, :type, :reason def initialize(user, type, **opts) unless NotificationSetting.levels.key?(type) || type == :subscription @@ -142,10 +144,33 @@ class NotificationRecipient return project_setting unless project_setting.nil? || project_setting.global? - group_setting = @group && user.notification_settings_for(@group) + group_setting = closest_non_global_group_notification_settting - return group_setting unless group_setting.nil? || group_setting.global? + return group_setting unless group_setting.nil? user.global_notification_setting end + + # Returns the notificaton_setting of the lowest group in hierarchy with non global level + def closest_non_global_group_notification_settting + return unless @group + return if indexed_group_notification_settings.empty? + + notification_setting = nil + + @group.self_and_ancestors_ids.each do |id| + notification_setting = indexed_group_notification_settings[id] + break if notification_setting + end + + notification_setting + end + + def indexed_group_notification_settings + strong_memoize(:indexed_group_notification_settings) do + @group.notification_settings.where(user_id: user.id) + .where.not(level: NotificationSetting.levels[:global]) + .index_by(&:source_id) + end + end end -- cgit v1.2.3 From cf41aaba5ab9fb1d229807f77b2b77585d3550b0 Mon Sep 17 00:00:00 2001 From: Mario de la Ossa Date: Thu, 7 Jun 2018 20:54:24 +0000 Subject: Backport of "Add assignee lists to boards" --- app/models/list.rb | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) (limited to 'app/models') diff --git a/app/models/list.rb b/app/models/list.rb index 5daf35ef845..4edcfa78835 100644 --- a/app/models/list.rb +++ b/app/models/list.rb @@ -2,17 +2,27 @@ class List < ActiveRecord::Base belongs_to :board belongs_to :label - enum list_type: { backlog: 0, label: 1, closed: 2 } + enum list_type: { backlog: 0, label: 1, closed: 2, assignee: 3 } validates :board, :list_type, presence: true validates :label, :position, presence: true, if: :label? validates :label_id, uniqueness: { scope: :board_id }, if: :label? - validates :position, numericality: { only_integer: true, greater_than_or_equal_to: 0 }, if: :label? + validates :position, numericality: { only_integer: true, greater_than_or_equal_to: 0 }, if: :movable? before_destroy :can_be_destroyed - scope :destroyable, -> { where(list_type: list_types[:label]) } - scope :movable, -> { where(list_type: list_types[:label]) } + scope :destroyable, -> { where(list_type: list_types.slice(*destroyable_types).values) } + scope :movable, -> { where(list_type: list_types.slice(*movable_types).values) } + + class << self + def destroyable_types + [:label] + end + + def movable_types + [:label] + end + end def destroyable? label? -- cgit v1.2.3 From 68cb1c2651fde0ae009beed10f81a74172ceeab1 Mon Sep 17 00:00:00 2001 From: Mark Chao Date: Fri, 8 Jun 2018 13:12:51 +0000 Subject: Revert rename allow collaboration column --- app/models/merge_request.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 535a2c362f2..324065c1162 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -1125,8 +1125,11 @@ class MergeRequest < ActiveRecord::Base project.merge_requests.merged.where(author_id: author_id).empty? end + # TODO: remove once production database rename completes + alias_attribute :allow_collaboration, :allow_maintainer_to_push + def allow_collaboration - collaborative_push_possible? && super + collaborative_push_possible? && allow_maintainer_to_push end alias_method :allow_collaboration?, :allow_collaboration -- cgit v1.2.3 From 1418afc2d6e7699f08a1fc5f33b78ea847ac1451 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Javier=20L=C3=B3pez?= Date: Mon, 11 Jun 2018 13:29:37 +0000 Subject: Avoid checking the user format in every url validation --- app/models/project.rb | 1 + app/models/remote_mirror.rb | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 60cd13b371f..9ca733ecd98 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -292,6 +292,7 @@ class Project < ActiveRecord::Base validates :name, uniqueness: { scope: :namespace_id } validates :import_url, url: { protocols: %w(http https ssh git), allow_localhost: false, + enforce_user: true, ports: VALID_IMPORT_PORTS }, if: [:external_import?, :import_url_changed?] validates :star_count, numericality: { greater_than_or_equal_to: 0 } validate :check_limit, on: :create diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb index 5cd222e18a4..c4b5dd2dc96 100644 --- a/app/models/remote_mirror.rb +++ b/app/models/remote_mirror.rb @@ -16,7 +16,7 @@ class RemoteMirror < ActiveRecord::Base belongs_to :project, inverse_of: :remote_mirrors - validates :url, presence: true, url: { protocols: %w(ssh git http https), allow_blank: true } + validates :url, presence: true, url: { protocols: %w(ssh git http https), allow_blank: true, enforce_user: true } before_save :set_new_remote_name, if: :mirror_url_changed? -- cgit v1.2.3 From 6defeb0a7d6928ad32d4d7a2fa35d0d71dbb9dea Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Sat, 9 Jun 2018 22:35:06 -0700 Subject: Expire Wiki content cache after importing a repository The cache state for Wikis that were imported via GitHub or Bitbucket does not appear to have been flushed after a successful import. Closes #47546 --- app/models/project.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 60cd13b371f..bc91b0aa1b0 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1615,6 +1615,7 @@ class Project < ActiveRecord::Base def after_import repository.after_import + wiki.repository.after_import import_finish remove_import_jid update_project_counter_caches -- cgit v1.2.3 From 698515313fe38fb3f85fdeec1efa15e2c8b54cfd Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Tue, 15 May 2018 12:25:51 +0200 Subject: Fixes rejected pushes from maintainers Before the push git would make a call to `/:namespace/:project/git-receive-pack`. This would perform an access check without a ref. So the `Project#branch_allows_maintainer_push?` would return false. This adjusts `Project#branch_allows_maintainer_push?` to return true when passing no branch name if there are merge requests open that would allow the user to push. The actual check then happens when a call to `/api/v4/internal/allowed` is made from a git hook. --- app/models/project.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 9ca733ecd98..57da42adc28 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -2139,10 +2139,14 @@ class Project < ActiveRecord::Base check_access = -> do next false if empty_repo? - merge_request = source_of_merge_requests.opened - .where(allow_collaboration: true) - .find_by(source_branch: branch_name) - merge_request&.can_be_merged_by?(user) + merge_requests = source_of_merge_requests.opened + .where(allow_collaboration: true) + + if branch_name + merge_requests.find_by(source_branch: branch_name)&.can_be_merged_by?(user) + else + merge_requests.any? { |merge_request| merge_request.can_be_merged_by?(user) } + end end if RequestStore.active? -- cgit v1.2.3 From 6633fbef4e6c7adaca9b1f1633640f09c3bc90d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Sun, 20 May 2018 14:18:50 +0200 Subject: Add CI_{PIPELINE,JOB}_URL --- app/models/ci/build.rb | 1 + app/models/ci/pipeline.rb | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 2d675726939..41446946a5e 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -611,6 +611,7 @@ module Ci variables .concat(pipeline.persisted_variables) .append(key: 'CI_JOB_ID', value: id.to_s) + .append(key: 'CI_JOB_URL', value: Gitlab::Routing.url_helpers.project_job_url(project, self)) .append(key: 'CI_JOB_TOKEN', value: token, public: false) .append(key: 'CI_BUILD_ID', value: id.to_s) .append(key: 'CI_BUILD_TOKEN', value: token, public: false) diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index eecd86349e4..689e0ac0ef5 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -548,7 +548,10 @@ module Ci def persisted_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| - variables.append(key: 'CI_PIPELINE_ID', value: id.to_s) if persisted? + break variables unless persisted? + + variables.append(key: 'CI_PIPELINE_ID', value: id.to_s) + variables.append(key: 'CI_PIPELINE_URL', value: Gitlab::Routing.url_helpers.project_pipeline_url(project, self)) end end -- cgit v1.2.3 From 9aad5ed0a23a6386bf48f166a6b07b3a41d9c2c1 Mon Sep 17 00:00:00 2001 From: Jeff Brown Date: Mon, 11 Jun 2018 10:28:30 +0200 Subject: Fixes Microsoft Teams notifications for pipeline events Closes #42342 --- app/models/project_services/chat_message/base_message.rb | 7 ++++++- app/models/project_services/chat_message/pipeline_message.rb | 4 ---- app/models/project_services/microsoft_teams_service.rb | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) (limited to 'app/models') diff --git a/app/models/project_services/chat_message/base_message.rb b/app/models/project_services/chat_message/base_message.rb index 22a65b5145e..f710fa85b5d 100644 --- a/app/models/project_services/chat_message/base_message.rb +++ b/app/models/project_services/chat_message/base_message.rb @@ -26,13 +26,18 @@ module ChatMessage end end - def pretext + def summary return message if markdown format(message) end + def pretext + summary + end + def fallback + format(message) end def attachments diff --git a/app/models/project_services/chat_message/pipeline_message.rb b/app/models/project_services/chat_message/pipeline_message.rb index 2135122278a..96fd23aede3 100644 --- a/app/models/project_services/chat_message/pipeline_message.rb +++ b/app/models/project_services/chat_message/pipeline_message.rb @@ -23,10 +23,6 @@ module ChatMessage '' end - def fallback - format(message) - end - def attachments return message if markdown diff --git a/app/models/project_services/microsoft_teams_service.rb b/app/models/project_services/microsoft_teams_service.rb index 2facff53e26..99500caec0e 100644 --- a/app/models/project_services/microsoft_teams_service.rb +++ b/app/models/project_services/microsoft_teams_service.rb @@ -44,7 +44,7 @@ class MicrosoftTeamsService < ChatNotificationService def notify(message, opts) MicrosoftTeams::Notifier.new(webhook).ping( title: message.project_name, - pretext: message.pretext, + summary: message.summary, activity: message.activity, attachments: message.attachments ) -- cgit v1.2.3 From be16f54d9d167c6338367c8175aaf135c44dab94 Mon Sep 17 00:00:00 2001 From: Jasper Maes Date: Wed, 13 Jun 2018 07:48:05 +0200 Subject: Rails5 fix expected `issuable.reload.updated_at` to have changed --- app/models/timelog.rb | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'app/models') diff --git a/app/models/timelog.rb b/app/models/timelog.rb index f4c5c581a11..659146f43e4 100644 --- a/app/models/timelog.rb +++ b/app/models/timelog.rb @@ -19,4 +19,9 @@ class Timelog < ActiveRecord::Base errors.add(:base, 'Issue or Merge Request ID is required') end end + + # Rails5 defaults to :touch_later, overwrite for normal touch + def belongs_to_touch_method + :touch + end end -- cgit v1.2.3 From fa36101a7fc8679d98198942f15dd6285673594d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarka=20Kadlecov=C3=A1?= Date: Tue, 5 Jun 2018 11:29:33 +0200 Subject: Use data_source_exists? instead of table_exists? Use data_source_exists? where possible instead of table_exists? in order to be Rails5 compatible --- app/models/project.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 562198e2369..fb6e9fb2676 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -68,7 +68,7 @@ class Project < ActiveRecord::Base add_authentication_token_field :runners_token - before_validation :mark_remote_mirrors_for_removal, if: -> { ActiveRecord::Base.connection.table_exists?(:remote_mirrors) } + before_validation :mark_remote_mirrors_for_removal, if: -> { RemoteMirror.table_exists? } before_save :ensure_runners_token -- cgit v1.2.3 From f79410fe1772798d7fa1b288880e14e806483c3c Mon Sep 17 00:00:00 2001 From: Brett Walker Date: Thu, 14 Jun 2018 08:30:16 +0000 Subject: enable CommonMark as the default --- app/models/concerns/cache_markdown_field.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index db8cf322ef7..9f6358cecbe 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -114,7 +114,7 @@ module CacheMarkdownField end def latest_cached_markdown_version - return CacheMarkdownField::CACHE_REDCARPET_VERSION unless cached_markdown_version + return CacheMarkdownField::CACHE_COMMONMARK_VERSION unless cached_markdown_version if cached_markdown_version < CacheMarkdownField::CACHE_COMMONMARK_VERSION_START CacheMarkdownField::CACHE_REDCARPET_VERSION -- cgit v1.2.3 From 5cf5680f9c8ce251570efce7dd4c348fb68efccf Mon Sep 17 00:00:00 2001 From: "Jacob Vosmaer (GitLab)" Date: Thu, 14 Jun 2018 11:18:25 +0000 Subject: Deny repository disk access in development and test --- app/models/project_services/gemnasium_service.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/project_services/gemnasium_service.rb b/app/models/project_services/gemnasium_service.rb index 84248f9590b..8a6b0ed1a5f 100644 --- a/app/models/project_services/gemnasium_service.rb +++ b/app/models/project_services/gemnasium_service.rb @@ -43,13 +43,18 @@ class GemnasiumService < Service def execute(data) return unless supported_events.include?(data[:object_kind]) + # Gitaly: this class will be removed https://gitlab.com/gitlab-org/gitlab-ee/issues/6010 + repo_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + project.repository.path_to_repo + end + Gemnasium::GitlabService.execute( ref: data[:ref], before: data[:before], after: data[:after], token: token, api_key: api_key, - repo: project.repository.path_to_repo # Gitaly: fixed by https://gitlab.com/gitlab-org/security-products/gemnasium-migration/issues/9 + repo: repo_path ) end end -- cgit v1.2.3 From 2fec4183368aebf848e3014598081948ddce49ca Mon Sep 17 00:00:00 2001 From: Mario de la Ossa Date: Thu, 14 Jun 2018 11:40:35 -0600 Subject: ChatNotificationService - fix sending tag notifications when "only default branch" enabled --- app/models/project_services/chat_notification_service.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/models') diff --git a/app/models/project_services/chat_notification_service.rb b/app/models/project_services/chat_notification_service.rb index ae0debbd3ac..a60b4c7fd0d 100644 --- a/app/models/project_services/chat_notification_service.rb +++ b/app/models/project_services/chat_notification_service.rb @@ -155,6 +155,7 @@ class ChatNotificationService < Service end def notify_for_ref?(data) + return true if data[:object_kind] == 'tag_push' return true if data.dig(:object_attributes, :tag) return true unless notify_only_default_branch? -- cgit v1.2.3 From 96f8ad0683619bb852931f4005f0b83c8337e8e6 Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Mon, 18 Jun 2018 17:47:20 +0200 Subject: Properly detect label reference if followed by period or question mark --- app/models/label.rb | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/label.rb b/app/models/label.rb index 1cf04976602..7bbcaa121ca 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -85,11 +85,16 @@ class Label < ActiveRecord::Base (#{Project.reference_pattern})? #{Regexp.escape(reference_prefix)} (?: - (?\d+(?!\S\w)\b) | # Integer-based label ID, or - (? - [A-Za-z0-9_\-\?\.&]+ | # String-based single-word label title, or - ".+?" # String-based multi-word label surrounded in quotes - ) + (?\d+(?!\S\w)\b) + | # Integer-based label ID, or + (? + # String-based single-word label title, or + [A-Za-z0-9_\-\?\.&]+ + (? Date: Mon, 18 Jun 2018 16:38:34 -0700 Subject: Eliminate N+1 queries in LFS file locks checks during a push This significantly improves performance when a user pushes many references. project.path_locks.any? doesn't cache the output and runs `SELECT 1 AS one FROM "path_locks" WHERE project_id = N` each time. When there are thousands of refs being pushed, this can time out the unicorn worker. CE port for https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/6159. --- app/models/project.rb | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index e5fa1c4db7b..0d777515536 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -25,6 +25,7 @@ class Project < ActiveRecord::Base include FastDestroyAll::Helpers include WithUploads include BatchDestroyDependentAssociations + extend Gitlab::Cache::RequestCache extend Gitlab::ConfigHelper @@ -2013,6 +2014,11 @@ class Project < ActiveRecord::Base @gitlab_deploy_token ||= deploy_tokens.gitlab_deploy_token end + def any_lfs_file_locks? + lfs_file_locks.any? + end + request_cache(:any_lfs_file_locks?) { self.id } + private def storage -- cgit v1.2.3 From 9d5f5c43119197b9c522ae121af5564f43d06574 Mon Sep 17 00:00:00 2001 From: Mario de la Ossa Date: Tue, 19 Jun 2018 09:27:09 -0600 Subject: Fix redis_cacheable deserialization --- app/models/concerns/redis_cacheable.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb index b5425295130..3bdc1330d23 100644 --- a/app/models/concerns/redis_cacheable.rb +++ b/app/models/concerns/redis_cacheable.rb @@ -48,7 +48,7 @@ module RedisCacheable def cast_value_from_cache(attribute, value) if Gitlab.rails5? - self.class.type_for_attribute(attribute).cast(value) + self.class.type_for_attribute(attribute.to_s).cast(value) else self.class.column_for_attribute(attribute).type_cast_from_database(value) end -- cgit v1.2.3 From 46b56b18f5bf94662a8500e2b8789f504b5d6ea8 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Tue, 19 Jun 2018 17:55:02 +0200 Subject: Move mergablility check to Gitaly Closes https://gitlab.com/gitlab-org/gitaly/issues/889 --- app/models/repository.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/repository.rb b/app/models/repository.rb index e4202505634..de889d18729 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -847,7 +847,7 @@ class Repository @root_ref_sha ||= commit(root_ref).sha end - delegate :merged_branch_names, :can_be_merged?, to: :raw_repository + delegate :merged_branch_names, to: :raw_repository def merge_base(first_commit_id, second_commit_id) first_commit_id = commit(first_commit_id).try(:id) || first_commit_id -- cgit v1.2.3 From e9e9da822ac49cc857158c4ecc777b077a963280 Mon Sep 17 00:00:00 2001 From: "Jacob Vosmaer (GitLab)" Date: Wed, 20 Jun 2018 09:33:50 +0000 Subject: More gitaly disk access blocks --- app/models/repository.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/repository.rb b/app/models/repository.rb index e4202505634..c2f62badbcb 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -154,7 +154,10 @@ class Repository # Returns a list of commits that are not present in any reference def new_commits(newrev) - refs = ::Gitlab::Git::RevList.new(raw, newrev: newrev).new_refs + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/1233 + refs = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + ::Gitlab::Git::RevList.new(raw, newrev: newrev).new_refs + end refs.map { |sha| commit(sha.strip) } end -- cgit v1.2.3 From 5841e92390051065d89a5b0602fa326eda018cd9 Mon Sep 17 00:00:00 2001 From: Mayra Cabrera Date: Wed, 20 Jun 2018 12:10:14 +0000 Subject: Resolve "Unable to install Prometheus on Clusters: 'Error: Chart incompatible with Tiller v2.7.0'" --- app/models/clusters/applications/prometheus.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb index c702c4ee807..48137c2ed68 100644 --- a/app/models/clusters/applications/prometheus.rb +++ b/app/models/clusters/applications/prometheus.rb @@ -3,7 +3,7 @@ module Clusters class Prometheus < ActiveRecord::Base include PrometheusAdapter - VERSION = "2.0.0".freeze + VERSION = '6.7.3'.freeze self.table_name = 'clusters_applications_prometheus' @@ -37,6 +37,7 @@ module Clusters Gitlab::Kubernetes::Helm::InstallCommand.new( name, chart: chart, + version: version, values: values ) end -- cgit v1.2.3 From 5b994b8199a3679b6f5ef0d00edce22ea9662664 Mon Sep 17 00:00:00 2001 From: Mark Chao Date: Thu, 7 Jun 2018 23:56:59 +0900 Subject: Notify only when unmergeable due to conflict There is still the edge case when 'no commits' changes to 'conflict' would not trigger notification, which we ignore for now. Calling can_be_merged? can cause exception (e.g. non-UTF8) Ignore those by rescueing. Remove unmergeable_reason as now only conflict is notified Update spec --- app/models/merge_request.rb | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 324065c1162..4b78ba1029f 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -128,8 +128,17 @@ class MergeRequest < ActiveRecord::Base end after_transition unchecked: :cannot_be_merged do |merge_request, transition| - NotificationService.new.merge_request_unmergeable(merge_request) - TodoService.new.merge_request_became_unmergeable(merge_request) + begin + # Merge request can become unmergeable due to many reasons. + # We only notify if it is due to conflict. + unless merge_request.project.repository.can_be_merged?(merge_request.diff_head_sha, merge_request.target_branch) + NotificationService.new.merge_request_unmergeable(merge_request) + TodoService.new.merge_request_became_unmergeable(merge_request) + end + rescue Gitlab::Git::CommandError + # Checking mergeability can trigger exception, e.g. non-utf8 + # We ignore this type of errors. + end end def check_state?(merge_status) -- cgit v1.2.3 From 3e66795ef1ff1228906239763910b051d8afcc37 Mon Sep 17 00:00:00 2001 From: Felipe Artur Date: Thu, 21 Jun 2018 12:22:40 +0000 Subject: Changes tab VUE refactoring --- app/models/concerns/issuable.rb | 4 ++++ app/models/discussion.rb | 4 ++++ app/models/issue.rb | 4 ++++ app/models/merge_request.rb | 4 ++++ app/models/note.rb | 1 + 5 files changed, 17 insertions(+) (limited to 'app/models') diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 44150b37708..b93c1145f82 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -107,6 +107,10 @@ module Issuable false end + def etag_caching_enabled? + false + end + def has_multiple_assignees? assignees.count > 1 end diff --git a/app/models/discussion.rb b/app/models/discussion.rb index 92482a1a875..35a0ef00856 100644 --- a/app/models/discussion.rb +++ b/app/models/discussion.rb @@ -17,6 +17,10 @@ class Discussion to: :first_note + def project_id + project&.id + end + def self.build(notes, context_noteable = nil) notes.first.discussion_class(context_noteable).new(notes, context_noteable) end diff --git a/app/models/issue.rb b/app/models/issue.rb index d136700836d..d3df2da14e2 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -308,6 +308,10 @@ class Issue < ActiveRecord::Base end end + def etag_caching_enabled? + true + end + def discussions_rendered_on_frontend? true end diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 4b78ba1029f..3df1130a6e2 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -1124,6 +1124,10 @@ class MergeRequest < ActiveRecord::Base true end + def discussions_rendered_on_frontend? + true + end + def update_project_counter_caches Projects::OpenMergeRequestsCountService.new(target_project).refresh_cache end diff --git a/app/models/note.rb b/app/models/note.rb index 41c04ae0571..abc40d9016e 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -384,6 +384,7 @@ class Note < ActiveRecord::Base def expire_etag_cache return unless noteable&.discussions_rendered_on_frontend? + return unless noteable&.etag_caching_enabled? Gitlab::EtagCaching::Store.new.touch(etag_key) end -- cgit v1.2.3 From f5ed18e1e3b5e69fa7bd650f3144fcfe26ac315f Mon Sep 17 00:00:00 2001 From: Oswaldo Ferreira Date: Mon, 11 Jun 2018 17:45:16 -0300 Subject: Delete non-latest merge request diff files upon diffs reload --- app/models/merge_request.rb | 17 +++++------------ app/models/merge_request_diff.rb | 27 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 12 deletions(-) (limited to 'app/models') diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 3df1130a6e2..f112c06e26f 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -378,6 +378,10 @@ class MergeRequest < ActiveRecord::Base end end + def non_latest_diffs + merge_request_diffs.where.not(id: merge_request_diff.id) + end + def diff_size # Calling `merge_request_diff.diffs.real_size` will also perform # highlighting, which we don't need here. @@ -619,18 +623,7 @@ class MergeRequest < ActiveRecord::Base def reload_diff(current_user = nil) return unless open? - old_diff_refs = self.diff_refs - new_diff = create_merge_request_diff - - MergeRequests::MergeRequestDiffCacheService.new.execute(self, new_diff) - - new_diff_refs = self.diff_refs - - update_diff_discussion_positions( - old_diff_refs: old_diff_refs, - new_diff_refs: new_diff_refs, - current_user: current_user - ) + MergeRequests::ReloadDiffsService.new(self, current_user).execute end def check_if_can_be_merged diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index 06aa67c600f..3d72c447b4b 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -3,6 +3,7 @@ class MergeRequestDiff < ActiveRecord::Base include Importable include ManualInverseAssociation include IgnorableColumn + include EachBatch # Don't display more than 100 commits at once COMMITS_SAFE_SIZE = 100 @@ -17,8 +18,14 @@ class MergeRequestDiff < ActiveRecord::Base has_many :merge_request_diff_commits, -> { order(:merge_request_diff_id, :relative_order) } state_machine :state, initial: :empty do + event :clean do + transition any => :without_files + end + state :collected state :overflow + # Diff files have been deleted by the system + state :without_files # Deprecated states: these are no longer used but these values may still occur # in the database. state :timeout @@ -27,6 +34,7 @@ class MergeRequestDiff < ActiveRecord::Base state :overflow_diff_lines_limit end + scope :with_files, -> { without_states(:without_files, :empty) } scope :viewable, -> { without_state(:empty) } scope :by_commit_sha, ->(sha) do joins(:merge_request_diff_commits).where(merge_request_diff_commits: { sha: sha }).reorder(nil) @@ -42,6 +50,10 @@ class MergeRequestDiff < ActiveRecord::Base find_by(start_commit_sha: diff_refs.start_sha, head_commit_sha: diff_refs.head_sha, base_commit_sha: diff_refs.base_sha) end + def viewable? + collected? || without_files? || overflow? + end + # Collect information about commits and diff from repository # and save it to the database as serialized data def save_git_content @@ -170,6 +182,21 @@ class MergeRequestDiff < ActiveRecord::Base end def diffs(diff_options = nil) + if without_files? && comparison = diff_refs.compare_in(project) + # It should fetch the repository when diffs are cleaned by the system. + # We don't keep these for storage overload purposes. + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/37639 + comparison.diffs(diff_options) + else + diffs_collection(diff_options) + end + end + + # Should always return the DB persisted diffs collection + # (e.g. Gitlab::Diff::FileCollection::MergeRequestDiff. + # It's useful when trying to invalidate old caches through + # FileCollection::MergeRequestDiff#clear_cache! + def diffs_collection(diff_options = nil) Gitlab::Diff::FileCollection::MergeRequestDiff.new(self, diff_options: diff_options) end -- cgit v1.2.3 From face19997d2356b49a8a991d764415886a83c37a Mon Sep 17 00:00:00 2001 From: Jan Provaznik Date: Mon, 25 Jun 2018 08:22:42 +0000 Subject: Fix auto_cancel_pending_pipelines check --- app/models/project.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'app/models') diff --git a/app/models/project.rb b/app/models/project.rb index 0d777515536..d91d7dcfe9a 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -2019,6 +2019,10 @@ class Project < ActiveRecord::Base end request_cache(:any_lfs_file_locks?) { self.id } + def auto_cancel_pending_pipelines? + auto_cancel_pending_pipelines == 'enabled' + end + private def storage -- cgit v1.2.3 From 156339569d52bef66fe41c85d29ab3c9865f7362 Mon Sep 17 00:00:00 2001 From: Tomasz Maczukin Date: Thu, 21 Jun 2018 13:17:35 +0200 Subject: Enforce setting string as value of the CI/CD variable --- app/models/ci/pipeline.rb | 6 +++--- app/models/project_auto_devops.rb | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index f430f18ca9a..e5caa3ffa41 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -561,9 +561,9 @@ module Ci .append(key: 'CI_PIPELINE_IID', value: iid.to_s) .append(key: 'CI_CONFIG_PATH', value: ci_yaml_file_path) .append(key: 'CI_PIPELINE_SOURCE', value: source.to_s) - .append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message) - .append(key: 'CI_COMMIT_TITLE', value: git_commit_full_title) - .append(key: 'CI_COMMIT_DESCRIPTION', value: git_commit_description) + .append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message.to_s) + .append(key: 'CI_COMMIT_TITLE', value: git_commit_full_title.to_s) + .append(key: 'CI_COMMIT_DESCRIPTION', value: git_commit_description.to_s) end def queued_duration diff --git a/app/models/project_auto_devops.rb b/app/models/project_auto_devops.rb index d7d6aaceb27..faa831b1949 100644 --- a/app/models/project_auto_devops.rb +++ b/app/models/project_auto_devops.rb @@ -29,8 +29,8 @@ class ProjectAutoDevops < ActiveRecord::Base end if manual? - variables.append(key: 'STAGING_ENABLED', value: 1) - variables.append(key: 'INCREMENTAL_ROLLOUT_ENABLED', value: 1) + variables.append(key: 'STAGING_ENABLED', value: '1') + variables.append(key: 'INCREMENTAL_ROLLOUT_ENABLED', value: '1') end end end -- cgit v1.2.3 From 29b4d657bec11ed8715f4e0ddeead61b9066e9a4 Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Mon, 25 Jun 2018 14:56:27 +0200 Subject: Moves another RPC to mandatory This specific one isn't used on most machines, therefor low risk. Closes https://gitlab.com/gitlab-org/gitaly/issues/944 --- app/models/repository.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/repository.rb b/app/models/repository.rb index 3089d0162ee..3056c20516a 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -21,7 +21,7 @@ class Repository attr_accessor :full_path, :disk_path, :project, :is_wiki delegate :ref_name_for_sha, to: :raw_repository - delegate :bundle_to_disk, :create_from_bundle, to: :raw_repository + delegate :bundle_to_disk, to: :raw_repository CreateTreeError = Class.new(StandardError) -- cgit v1.2.3 From ea25fbb8f594fb615571056c38d38a2a759c4e7b Mon Sep 17 00:00:00 2001 From: Mark Chao Date: Mon, 25 Jun 2018 15:20:29 +0000 Subject: Notify conflict only for opened/locked merge requests --- app/models/merge_request.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'app/models') diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index f112c06e26f..6c96c8ca391 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -129,9 +129,7 @@ class MergeRequest < ActiveRecord::Base after_transition unchecked: :cannot_be_merged do |merge_request, transition| begin - # Merge request can become unmergeable due to many reasons. - # We only notify if it is due to conflict. - unless merge_request.project.repository.can_be_merged?(merge_request.diff_head_sha, merge_request.target_branch) + if merge_request.notify_conflict? NotificationService.new.merge_request_unmergeable(merge_request) TodoService.new.merge_request_became_unmergeable(merge_request) end @@ -708,6 +706,10 @@ class MergeRequest < ActiveRecord::Base should_remove_source_branch? || force_remove_source_branch? end + def notify_conflict? + (opened? || locked?) && !project.repository.can_be_merged?(diff_head_sha, target_branch) + end + def related_notes # Fetch comments only from last 100 commits commits_for_notes_limit = 100 -- cgit v1.2.3 From 2024522b51ae672df3b04ec0ed3d17eefcb45264 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Tue, 26 Jun 2018 02:58:41 +0800 Subject: Bring changes from EE --- app/models/project_team.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/project_team.rb b/app/models/project_team.rb index 33280eda0b9..9a38806baab 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -24,7 +24,7 @@ class ProjectTeam end def add_role(user, role, current_user: nil) - send(:"add_#{role}", user, current_user: current_user) # rubocop:disable GitlabSecurity/PublicSend + public_send(:"add_#{role}", user, current_user: current_user) # rubocop:disable GitlabSecurity/PublicSend end def find_member(user_id) -- cgit v1.2.3 From e8edf620f10fb03edfefc32af0b9a9b780ab107a Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 25 Jun 2018 16:46:45 -0300 Subject: Fix sorting by name on explore projects page --- app/models/concerns/sortable.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/concerns/sortable.rb b/app/models/concerns/sortable.rb index db7254c27e0..cb76ae971d4 100644 --- a/app/models/concerns/sortable.rb +++ b/app/models/concerns/sortable.rb @@ -12,8 +12,8 @@ module Sortable scope :order_created_asc, -> { reorder(created_at: :asc) } scope :order_updated_desc, -> { reorder(updated_at: :desc) } scope :order_updated_asc, -> { reorder(updated_at: :asc) } - scope :order_name_asc, -> { reorder("lower(name) asc") } - scope :order_name_desc, -> { reorder("lower(name) desc") } + scope :order_name_asc, -> { reorder(Arel::Nodes::Ascending.new(arel_table[:name].lower)) } + scope :order_name_desc, -> { reorder(Arel::Nodes::Descending.new(arel_table[:name].lower)) } end module ClassMethods -- cgit v1.2.3 From c2a48fd163bf9e345ad7baf4707f6bb50de5be78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Mon, 25 Jun 2018 10:28:19 +0200 Subject: Ignore unknown OAuth sources in ApplicationSetting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- app/models/application_setting.rb | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'app/models') diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 3d58a14882f..bddeb8b0352 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -212,14 +212,6 @@ class ApplicationSetting < ActiveRecord::Base end end - validates_each :disabled_oauth_sign_in_sources do |record, attr, value| - value&.each do |source| - unless Devise.omniauth_providers.include?(source.to_sym) - record.errors.add(attr, "'#{source}' is not an OAuth sign-in source") - end - end - end - validate :terms_exist, if: :enforce_terms? before_validation :ensure_uuid! @@ -330,6 +322,11 @@ class ApplicationSetting < ActiveRecord::Base ::Gitlab::Database.cached_column_exists?(:application_settings, :sidekiq_throttling_enabled) end + def disabled_oauth_sign_in_sources=(sources) + sources = (sources || []).map(&:to_s) & Devise.omniauth_providers.map(&:to_s) + super(sources) + end + def domain_whitelist_raw self.domain_whitelist&.join("\n") end -- cgit v1.2.3 From d5a1910f0ce24cf88bc7f0279904a8d5088e2ffa Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Tue, 26 Jun 2018 17:58:51 +0800 Subject: Port Namespace#root_ancestor to CE --- app/models/namespace.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'app/models') diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 52fe529c016..7034c633268 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -228,6 +228,10 @@ class Namespace < ActiveRecord::Base parent.present? end + def root_ancestor + ancestors.reorder(nil).find_by(parent_id: nil) + end + def subgroup? has_parent? end -- cgit v1.2.3