diff options
| author | Lin Jen-Shin <godfat@godfat.org> | 2018-06-27 11:15:06 +0300 |
|---|---|---|
| committer | Lin Jen-Shin <godfat@godfat.org> | 2018-06-27 11:15:06 +0300 |
| commit | 849f9995d97c85d88b36a40ee563f7dd51fdc3f1 (patch) | |
| tree | 3523089d253b001f7e3b029266399e497407441b /app/models | |
| parent | ef6b3e0271d226462bed5f899f3964cf5652978c (diff) | |
| parent | 87f7597a4fb7852fc81f830158cdfd5fdec8fac4 (diff) | |
Merge remote-tracking branch 'upstream/master' into 14995-custom_wiki_sidebar
* upstream/master: (4180 commits)
Enable frozen string literals for app/workers/*.rb
Resolve "Search dropdown hides & shows when typing"
Revert merge request widget button max height
Update CHANGELOG.md for 11.0.2
Update external link icon in header user dropdown
Added Diff Viewer to new VUE based MR page
Fixed eslint failure in IDE spec helpers
Use refs instead of querySelector.
Show file in tree on WebIDE open
Resolve "Remove unused bootstrap component CSS"
Resolve "Explain what Groups are in the New Group page"
[QA] Make sure we wait for the deploy key list to load
Update _scopes_form.html.haml to remove duplicate information
Use the branch instead of the tag to install
port the EE changes
Add index on deployable_type/id for deployments
Add a helper to rename a column using a background migration
Fix performance bottleneck when rendering large wiki pages
Port Namespace#root_ancestor to CE
Remove duplicate spec
...
Diffstat (limited to 'app/models')
144 files changed, 3265 insertions, 811 deletions
diff --git a/app/models/ability.rb b/app/models/ability.rb index 6dae49f38dc..bb600eaccba 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -10,6 +10,14 @@ class Ability end end + # Given a list of users and a group this method returns the users that can + # read the given group. + def users_that_can_read_group(users, group) + DeclarativePolicy.subject_scope do + users.select { |u| allowed?(u, :read_group, group) } + end + end + # Given a list of users and a snippet this method returns the users that can # read the given snippet. def users_that_can_read_personal_snippet(users, snippet) @@ -46,10 +54,6 @@ class Ability end end - def can_edit_note?(user, note) - allowed?(user, :edit_note, note) - end - def allowed?(user, action, subject = :global, opts = {}) if subject.is_a?(Hash) opts, subject = subject, :global diff --git a/app/models/active_session.rb b/app/models/active_session.rb new file mode 100644 index 00000000000..b4a86dbb331 --- /dev/null +++ b/app/models/active_session.rb @@ -0,0 +1,110 @@ +class ActiveSession + include ActiveModel::Model + + attr_accessor :created_at, :updated_at, + :session_id, :ip_address, + :browser, :os, :device_name, :device_type + + def current?(session) + return false if session_id.nil? || session.id.nil? + + session_id == session.id + end + + def human_device_type + device_type&.titleize + end + + def self.set(user, request) + Gitlab::Redis::SharedState.with do |redis| + session_id = request.session.id + client = DeviceDetector.new(request.user_agent) + timestamp = Time.current + + active_user_session = new( + ip_address: request.ip, + browser: client.name, + os: client.os_name, + device_name: client.device_name, + device_type: client.device_type, + created_at: user.current_sign_in_at || timestamp, + updated_at: timestamp, + session_id: session_id + ) + + redis.pipelined do + redis.setex( + key_name(user.id, session_id), + Settings.gitlab['session_expire_delay'] * 60, + Marshal.dump(active_user_session) + ) + + redis.sadd( + lookup_key_name(user.id), + session_id + ) + end + end + end + + def self.list(user) + Gitlab::Redis::SharedState.with do |redis| + cleaned_up_lookup_entries(redis, user.id).map do |entry| + # rubocop:disable Security/MarshalLoad + Marshal.load(entry) + # rubocop:enable Security/MarshalLoad + end + end + end + + def self.destroy(user, session_id) + Gitlab::Redis::SharedState.with do |redis| + redis.srem(lookup_key_name(user.id), session_id) + + deleted_keys = redis.del(key_name(user.id, session_id)) + + # only allow deleting the devise session if we could actually find a + # related active session. this prevents another user from deleting + # someone else's session. + if deleted_keys > 0 + redis.del("#{Gitlab::Redis::SharedState::SESSION_NAMESPACE}:#{session_id}") + end + end + end + + def self.cleanup(user) + Gitlab::Redis::SharedState.with do |redis| + cleaned_up_lookup_entries(redis, user.id) + end + end + + def self.key_name(user_id, session_id = '*') + "#{Gitlab::Redis::SharedState::USER_SESSIONS_NAMESPACE}:#{user_id}:#{session_id}" + end + + def self.lookup_key_name(user_id) + "#{Gitlab::Redis::SharedState::USER_SESSIONS_LOOKUP_NAMESPACE}:#{user_id}" + end + + def self.cleaned_up_lookup_entries(redis, user_id) + lookup_key = lookup_key_name(user_id) + + session_ids = redis.smembers(lookup_key) + + entry_keys = session_ids.map { |session_id| key_name(user_id, session_id) } + return [] if entry_keys.empty? + + entries = redis.mget(entry_keys) + + session_ids_and_entries = session_ids.zip(entries) + + # remove expired keys. + # only the single key entries are automatically expired by redis, the + # lookup entries in the set need to be removed manually. + session_ids_and_entries.reject { |_session_id, entry| entry }.each do |session_id, _entry| + redis.srem(lookup_key, session_id) + end + + session_ids_and_entries.select { |_session_id, entry| entry }.map { |_session_id, entry| entry } + end +end diff --git a/app/models/appearance.rb b/app/models/appearance.rb index dcd14c08f3c..b770aadef0e 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -1,5 +1,8 @@ class Appearance < ActiveRecord::Base + include CacheableAttributes include CacheMarkdownField + include ObjectStorage::BackgroundMove + include WithUploads cache_markdown_field :description cache_markdown_field :new_project_guidelines @@ -11,19 +14,11 @@ class Appearance < ActiveRecord::Base mount_uploader :logo, AttachmentUploader mount_uploader :header_logo, AttachmentUploader + mount_uploader :favicon, FaviconUploader - has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - - CACHE_KEY = 'current_appearance'.freeze - - after_commit :flush_redis_cache - - def self.current - Rails.cache.fetch(CACHE_KEY) { first } - end - - def flush_redis_cache - Rails.cache.delete(CACHE_KEY) + # Overrides CacheableAttributes.current_without_cache + def self.current_without_cache + first end def single_appearance_row diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 862933bf127..bddeb8b0352 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -1,11 +1,11 @@ class ApplicationSetting < ActiveRecord::Base + include CacheableAttributes include CacheMarkdownField include TokenAuthenticatable add_authentication_token_field :runners_registration_token add_authentication_token_field :health_check_access_token - CACHE_KEY = 'application_setting.last'.freeze DOMAIN_LIST_SEPARATOR = %r{\s*[,;]\s* # comma or semicolon, optionally surrounded by whitespace | # or \s # any whitespace character @@ -212,13 +212,7 @@ class ApplicationSetting < ActiveRecord::Base end end - validates_each :disabled_oauth_sign_in_sources do |record, attr, value| - value&.each do |source| - unless Devise.omniauth_providers.include?(source.to_sym) - record.errors.add(attr, "'#{source}' is not an OAuth sign-in source") - end - end - end + validate :terms_exist, if: :enforce_terms? before_validation :ensure_uuid! @@ -226,41 +220,9 @@ class ApplicationSetting < ActiveRecord::Base before_save :ensure_health_check_access_token after_commit do - Rails.cache.write(CACHE_KEY, self) - end - - def self.current - ensure_cache_setup - - Rails.cache.fetch(CACHE_KEY) do - ApplicationSetting.last.tap do |settings| - # do not cache nils - raise 'missing settings' unless settings - end - end - rescue - # Fall back to an uncached value if there are any problems (e.g. redis down) - ApplicationSetting.last - end - - def self.expire - Rails.cache.delete(CACHE_KEY) - rescue - # Gracefully handle when Redis is not available. For example, - # omnibus may fail here during gitlab:assets:compile. - end - - def self.cached - value = Rails.cache.read(CACHE_KEY) - ensure_cache_setup if value.present? - value - end - - def self.ensure_cache_setup - # This is a workaround for a Rails bug that causes attribute methods not - # to be loaded when read from cache: https://github.com/rails/rails/issues/27348 - ApplicationSetting.define_attribute_methods + reset_memoized_terms end + after_commit :expire_performance_bar_allowed_user_ids_cache, if: -> { previous_changes.key?('performance_bar_allowed_group_id') } def self.defaults { @@ -331,7 +293,8 @@ class ApplicationSetting < ActiveRecord::Base gitaly_timeout_fast: 10, gitaly_timeout_medium: 30, gitaly_timeout_default: 55, - allow_local_requests_from_hooks_and_services: false + allow_local_requests_from_hooks_and_services: false, + mirror_available: true } end @@ -359,6 +322,11 @@ class ApplicationSetting < ActiveRecord::Base ::Gitlab::Database.cached_column_exists?(:application_settings, :sidekiq_throttling_enabled) end + def disabled_oauth_sign_in_sources=(sources) + sources = (sources || []).map(&:to_s) & Devise.omniauth_providers.map(&:to_s) + super(sources) + end + def domain_whitelist_raw self.domain_whitelist&.join("\n") end @@ -389,17 +357,6 @@ class ApplicationSetting < ActiveRecord::Base Array(read_attribute(:repository_storages)) end - # DEPRECATED - # repository_storage is still required in the API. Remove in 9.0 - # Still used in API v3 - def repository_storage - repository_storages.first - end - - def repository_storage=(value) - self.repository_storages = [value] - end - def default_project_visibility=(level) super(Gitlab::VisibilityLevel.level_value(level)) end @@ -416,31 +373,6 @@ class ApplicationSetting < ActiveRecord::Base super(levels.map { |level| Gitlab::VisibilityLevel.level_value(level) }) end - def performance_bar_allowed_group_id=(group_full_path) - group_full_path = nil if group_full_path.blank? - - if group_full_path.nil? - if group_full_path != performance_bar_allowed_group_id - super(group_full_path) - Gitlab::PerformanceBar.expire_allowed_user_ids_cache - end - - return - end - - group = Group.find_by_full_path(group_full_path) - - if group - if group.id != performance_bar_allowed_group_id - super(group.id) - Gitlab::PerformanceBar.expire_allowed_user_ids_cache - end - else - super(nil) - Gitlab::PerformanceBar.expire_allowed_user_ids_cache - end - end - def performance_bar_allowed_group Group.find_by_id(performance_bar_allowed_group_id) end @@ -450,15 +382,6 @@ class ApplicationSetting < ActiveRecord::Base performance_bar_allowed_group_id.present? end - # - If `enable` is true, we early return since the actual attribute that holds - # the enabling/disabling is `performance_bar_allowed_group_id` - # - If `enable` is false, we set `performance_bar_allowed_group_id` to `nil` - def performance_bar_enabled=(enable) - return if Gitlab::Utils.to_boolean(enable) - - self.performance_bar_allowed_group_id = nil - end - # Choose one of the available repository storage options. Currently all have # equal weighting. def pick_repository_storage @@ -507,6 +430,16 @@ class ApplicationSetting < ActiveRecord::Base password_authentication_enabled_for_web? || password_authentication_enabled_for_git? end + delegate :terms, to: :latest_terms, allow_nil: true + def latest_terms + @latest_terms ||= Term.latest + end + + def reset_memoized_terms + @latest_terms = nil + latest_terms + end + private def ensure_uuid! @@ -520,4 +453,14 @@ class ApplicationSetting < ActiveRecord::Base errors.add(:repository_storages, "can't include: #{invalid.join(", ")}") unless invalid.empty? end + + def terms_exist + return unless enforce_terms? + + errors.add(:terms, "You need to set terms to be enforced") unless terms.present? + end + + def expire_performance_bar_allowed_user_ids_cache + Gitlab::PerformanceBar.expire_allowed_user_ids_cache + end end diff --git a/app/models/application_setting/term.rb b/app/models/application_setting/term.rb new file mode 100644 index 00000000000..3b1dfe7e4ef --- /dev/null +++ b/app/models/application_setting/term.rb @@ -0,0 +1,19 @@ +class ApplicationSetting + class Term < ActiveRecord::Base + include CacheMarkdownField + has_many :term_agreements + + validates :terms, presence: true + + cache_markdown_field :terms + + def self.latest + order(:id).last + end + + def accepted_by_user?(user) + user.accepted_term_id == id || + term_agreements.accepted.where(user: user).exists? + end + end +end diff --git a/app/models/badge.rb b/app/models/badge.rb index f7e10c2ebfc..265c5d872d4 100644 --- a/app/models/badge.rb +++ b/app/models/badge.rb @@ -18,7 +18,7 @@ class Badge < ActiveRecord::Base scope :order_created_at_asc, -> { reorder(created_at: :asc) } - validates :link_url, :image_url, url_placeholder: { protocols: %w(http https), placeholder_regex: PLACEHOLDERS_REGEX } + validates :link_url, :image_url, url: { protocols: %w(http https) } validates :type, presence: true def rendered_link_url(project = nil) diff --git a/app/models/broadcast_message.rb b/app/models/broadcast_message.rb index 0b561203914..4aa236555cb 100644 --- a/app/models/broadcast_message.rb +++ b/app/models/broadcast_message.rb @@ -19,7 +19,7 @@ class BroadcastMessage < ActiveRecord::Base after_commit :flush_redis_cache def self.current - messages = Rails.cache.fetch(CACHE_KEY) { current_and_future_messages.to_a } + messages = Rails.cache.fetch(CACHE_KEY, expires_in: cache_expires_in) { current_and_future_messages.to_a } return messages if messages.empty? @@ -36,6 +36,10 @@ class BroadcastMessage < ActiveRecord::Base where('ends_at > :now', now: Time.zone.now).order_id_asc end + def self.cache_expires_in + nil + end + def active? started? && !ended? end diff --git a/app/models/ci/artifact_blob.rb b/app/models/ci/artifact_blob.rb index ec56cc53aea..760f01f225b 100644 --- a/app/models/ci/artifact_blob.rb +++ b/app/models/ci/artifact_blob.rb @@ -36,16 +36,15 @@ module Ci def external_url(project, job) return unless external_link?(job) - full_path_parts = project.full_path_components - top_level_group = full_path_parts.shift + url_project_path = project.full_path.partition('/').last artifact_path = [ - '-', *full_path_parts, '-', + '-', url_project_path, '-', 'jobs', job.id, 'artifacts', path ].join('/') - "#{pages_config.protocol}://#{top_level_group}.#{pages_config.host}/#{artifact_path}" + "#{project.pages_group_url}/#{artifact_path}" end def external_link?(job) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 1e066b69c6e..41446946a5e 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -3,8 +3,10 @@ module Ci prepend ArtifactMigratable include TokenAuthenticatable include AfterCommitQueue + include ObjectStorage::BackgroundMove include Presentable include Importable + include Gitlab::Utils::StrongMemoize MissingDependenciesError = Class.new(StandardError) @@ -17,18 +19,26 @@ module Ci has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment' has_many :trace_sections, class_name: 'Ci::BuildTraceSection' + has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id - has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id - # The "environment" field for builds is a String, and is the unexpanded name + has_one :metadata, class_name: 'Ci::BuildMetadata' + delegate :timeout, to: :metadata, prefix: true, allow_nil: true + delegate :gitlab_deploy_token, to: :project + + ## + # The "environment" field for builds is a String, and is the unexpanded name! + # def persisted_environment - @persisted_environment ||= Environment.find_by( - name: expanded_environment_name, - project: project - ) + return unless has_environment? + + strong_memoize(:persisted_environment) do + Environment.find_by(name: expanded_environment_name, project: project) + end end serialize :options # rubocop:disable Cop/ActiveRecordSerialize @@ -45,11 +55,18 @@ module Ci where('(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)', '', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').archive) end + + scope :without_archived_trace, ->() do + where('NOT EXISTS (?)', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').trace) + end + + scope :with_artifacts_stored_locally, -> { with_artifacts_archive.where(artifacts_file_store: [nil, LegacyArtifactUploader::Store::LOCAL]) } scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) } scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) } scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) } scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) } scope :ref_protected, -> { where(protected: true) } + scope :with_live_trace, -> { where('EXISTS (?)', Ci::BuildTraceChunk.where('ci_builds.id = ci_build_trace_chunks.build_id').select(1)) } scope :matches_tag_ids, -> (tag_ids) do matcher = ::ActsAsTaggableOn::Tagging @@ -81,12 +98,13 @@ module Ci before_save :ensure_token before_destroy { unscoped_project } + before_create :ensure_metadata after_create unless: :importing? do |build| run_after_commit { BuildHooksWorker.perform_async(build.id) } end - after_commit :update_project_statistics_after_save, on: [:create, :update] - after_commit :update_project_statistics, on: :destroy + after_save :update_project_statistics_after_save, if: :artifacts_size_changed? + after_destroy :update_project_statistics_after_destroy, unless: :project_destroyed? class << self # This is needed for url_for to work, @@ -132,6 +150,7 @@ module Ci after_transition any => [:success] do |build| build.run_after_commit do BuildSuccessWorker.perform_async(id) + PagesWorker.perform_async(:deploy, id) if build.pages_generator? end end @@ -151,6 +170,14 @@ module Ci before_transition any => [:running] do |build| build.validates_dependencies! unless Feature.enabled?('ci_disable_validates_dependencies') end + + after_transition pending: :running do |build| + build.ensure_metadata.update_timeout_state + end + end + + def ensure_metadata + metadata || build_metadata(project: project) end def detailed_status(current_user) @@ -163,8 +190,13 @@ module Ci pipeline.manual_actions.where.not(name: name) end + def pages_generator? + Gitlab.config.pages.enabled && + self.name == 'pages' + end + def playable? - action? && (manual? || complete?) + action? && (manual? || retryable?) end def action? @@ -198,7 +230,11 @@ module Ci end def expanded_environment_name - ExpandVariables.expand(environment, simple_variables) if environment + return unless has_environment? + + strong_memoize(:expanded_environment_name) do + ExpandVariables.expand(environment, simple_variables) + end end def has_environment? @@ -229,10 +265,6 @@ module Ci latest_builds.where('stage_idx < ?', stage_idx) end - def timeout - project.build_timeout - end - def triggered_by?(current_user) user == current_user end @@ -248,31 +280,52 @@ module Ci Gitlab::Utils.slugify(ref.to_s) end - # Variables whose value does not depend on environment - def simple_variables - variables(environment: nil) - end - - # All variables, including those dependent on environment, which could - # contain unexpanded variables. - def variables(environment: persisted_environment) - collection = Gitlab::Ci::Variables::Collection.new.tap do |variables| + ## + # Variables in the environment name scope. + # + def scoped_variables(environment: expanded_environment_name) + Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.concat(predefined_variables) variables.concat(project.predefined_variables) variables.concat(pipeline.predefined_variables) variables.concat(runner.predefined_variables) if runner - variables.concat(project.deployment_variables(environment: environment)) if has_environment? + variables.concat(project.deployment_variables(environment: environment)) if environment variables.concat(yaml_variables) variables.concat(user_variables) - variables.concat(project.group.secret_variables_for(ref, project)) if project.group - variables.concat(secret_variables(environment: environment)) + variables.concat(secret_group_variables) + variables.concat(secret_project_variables(environment: environment)) variables.concat(trigger_request.user_variables) if trigger_request variables.concat(pipeline.variables) variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline.pipeline_schedule - variables.concat(persisted_environment_variables) if environment end + end - collection.to_runner_variables + ## + # Variables that do not depend on the environment name. + # + def simple_variables + strong_memoize(:simple_variables) do + scoped_variables(environment: nil).to_runner_variables + end + end + + ## + # All variables, including persisted environment variables. + # + def variables + Gitlab::Ci::Variables::Collection.new + .concat(persisted_variables) + .concat(scoped_variables) + .concat(persisted_environment_variables) + .to_runner_variables + end + + ## + # Regular Ruby hash of scoped variables, without duplicates that are + # possible to be present in an array of hashes returned from `variables`. + # + def scoped_variables_hash + scoped_variables.to_hash end def features @@ -361,17 +414,21 @@ module Ci build_data = Gitlab::DataBuilder::Build.build(self) project.execute_hooks(build_data.dup, :job_hooks) project.execute_services(build_data.dup, :job_hooks) - PagesService.new(build_data).execute - project.running_or_pending_build_count(force: true) + end + + def browsable_artifacts? + artifacts_metadata? end def artifacts_metadata_entry(path, **options) - metadata = Gitlab::Ci::Build::Artifacts::Metadata.new( - artifacts_metadata.path, - path, - **options) + artifacts_metadata.use_file do |metadata_path| + metadata = Gitlab::Ci::Build::Artifacts::Metadata.new( + metadata_path, + path, + **options) - metadata.to_entry + metadata.to_entry + end end def erase_artifacts! @@ -434,7 +491,7 @@ module Ci def user_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| - return variables if user.blank? + break variables if user.blank? variables.append(key: 'GITLAB_USER_ID', value: user.id.to_s) variables.append(key: 'GITLAB_USER_EMAIL', value: user.email) @@ -443,9 +500,14 @@ module Ci end end - def secret_variables(environment: persisted_environment) + def secret_group_variables + return [] unless project.group + + project.group.secret_variables_for(ref, project) + end + + def secret_project_variables(environment: persisted_environment) project.secret_variables_for(ref: ref, environment: environment) - .map(&:to_runner_variable) end def steps @@ -542,24 +604,37 @@ module Ci CI_REGISTRY_USER = 'gitlab-ci-token'.freeze + def persisted_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + break variables unless persisted? + + variables + .concat(pipeline.persisted_variables) + .append(key: 'CI_JOB_ID', value: id.to_s) + .append(key: 'CI_JOB_URL', value: Gitlab::Routing.url_helpers.project_job_url(project, self)) + .append(key: 'CI_JOB_TOKEN', value: token, public: false) + .append(key: 'CI_BUILD_ID', value: id.to_s) + .append(key: 'CI_BUILD_TOKEN', value: token, public: false) + .append(key: 'CI_REGISTRY_USER', value: CI_REGISTRY_USER) + .append(key: 'CI_REGISTRY_PASSWORD', value: token, public: false) + .append(key: 'CI_REPOSITORY_URL', value: repo_url, public: false) + .concat(deploy_token_variables) + end + end + def predefined_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'CI', value: 'true') variables.append(key: 'GITLAB_CI', value: 'true') - variables.append(key: 'GITLAB_FEATURES', value: project.namespace.features.join(',')) + variables.append(key: 'GITLAB_FEATURES', value: project.licensed_features.join(',')) variables.append(key: 'CI_SERVER_NAME', value: 'GitLab') variables.append(key: 'CI_SERVER_VERSION', value: Gitlab::VERSION) - variables.append(key: 'CI_SERVER_REVISION', value: Gitlab::REVISION) - variables.append(key: 'CI_JOB_ID', value: id.to_s) + variables.append(key: 'CI_SERVER_REVISION', value: Gitlab.revision) variables.append(key: 'CI_JOB_NAME', value: name) variables.append(key: 'CI_JOB_STAGE', value: stage) - variables.append(key: 'CI_JOB_TOKEN', value: token, public: false) variables.append(key: 'CI_COMMIT_SHA', value: sha) variables.append(key: 'CI_COMMIT_REF_NAME', value: ref) variables.append(key: 'CI_COMMIT_REF_SLUG', value: ref_slug) - variables.append(key: 'CI_REGISTRY_USER', value: CI_REGISTRY_USER) - variables.append(key: 'CI_REGISTRY_PASSWORD', value: token, public: false) - variables.append(key: 'CI_REPOSITORY_URL', value: repo_url, public: false) variables.append(key: "CI_COMMIT_TAG", value: ref) if tag? variables.append(key: "CI_PIPELINE_TRIGGERED", value: 'true') if trigger_request variables.append(key: "CI_JOB_MANUAL", value: 'true') if action? @@ -567,9 +642,23 @@ module Ci end end + def legacy_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.append(key: 'CI_BUILD_REF', value: sha) + variables.append(key: 'CI_BUILD_BEFORE_SHA', value: before_sha) + variables.append(key: 'CI_BUILD_REF_NAME', value: ref) + variables.append(key: 'CI_BUILD_REF_SLUG', value: ref_slug) + variables.append(key: 'CI_BUILD_NAME', value: name) + variables.append(key: 'CI_BUILD_STAGE', value: stage) + variables.append(key: "CI_BUILD_TAG", value: ref) if tag? + variables.append(key: "CI_BUILD_TRIGGERED", value: 'true') if trigger_request + variables.append(key: "CI_BUILD_MANUAL", value: 'true') if action? + end + end + def persisted_environment_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| - return variables unless persisted_environment + break variables unless persisted? && persisted_environment.present? variables.concat(persisted_environment.predefined_variables) @@ -580,19 +669,12 @@ module Ci end end - def legacy_variables + def deploy_token_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| - variables.append(key: 'CI_BUILD_ID', value: id.to_s) - variables.append(key: 'CI_BUILD_TOKEN', value: token, public: false) - variables.append(key: 'CI_BUILD_REF', value: sha) - variables.append(key: 'CI_BUILD_BEFORE_SHA', value: before_sha) - variables.append(key: 'CI_BUILD_REF_NAME', value: ref) - variables.append(key: 'CI_BUILD_REF_SLUG', value: ref_slug) - variables.append(key: 'CI_BUILD_NAME', value: name) - variables.append(key: 'CI_BUILD_STAGE', value: stage) - variables.append(key: "CI_BUILD_TAG", value: ref) if tag? - variables.append(key: "CI_BUILD_TRIGGERED", value: 'true') if trigger_request - variables.append(key: "CI_BUILD_MANUAL", value: 'true') if action? + break variables unless gitlab_deploy_token + + variables.append(key: 'CI_DEPLOY_USER', value: gitlab_deploy_token.username) + variables.append(key: 'CI_DEPLOY_PASSWORD', value: gitlab_deploy_token.token, public: false) end end @@ -606,16 +688,20 @@ module Ci pipeline.config_processor.build_attributes(name) end - def update_project_statistics - return unless project + def update_project_statistics_after_save + update_project_statistics(read_attribute(:artifacts_size).to_i - artifacts_size_was.to_i) + end - ProjectCacheWorker.perform_async(project_id, [], [:build_artifacts_size]) + def update_project_statistics_after_destroy + update_project_statistics(-artifacts_size) end - def update_project_statistics_after_save - if previous_changes.include?('artifacts_size') - update_project_statistics - end + def update_project_statistics(difference) + ProjectStatistics.increment_statistic(project_id, :build_artifacts_size, difference) + end + + def project_destroyed? + project.pending_delete? end end end diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb new file mode 100644 index 00000000000..96762f8845c --- /dev/null +++ b/app/models/ci/build_metadata.rb @@ -0,0 +1,35 @@ +module Ci + # The purpose of this class is to store Build related data that can be disposed. + # Data that should be persisted forever, should be stored with Ci::Build model. + class BuildMetadata < ActiveRecord::Base + extend Gitlab::Ci::Model + include Presentable + include ChronicDurationAttribute + + self.table_name = 'ci_builds_metadata' + + belongs_to :build, class_name: 'Ci::Build' + belongs_to :project + + validates :build, presence: true + validates :project, presence: true + + chronic_duration_attr_reader :timeout_human_readable, :timeout + + enum timeout_source: { + unknown_timeout_source: 1, + project_timeout_source: 2, + runner_timeout_source: 3 + } + + def update_timeout_state + return unless build.runner.present? + + project_timeout = project&.build_timeout + timeout = [project_timeout, build.runner.maximum_timeout].compact.min + timeout_source = timeout < project_timeout ? :runner_timeout_source : :project_timeout_source + + update(timeout: timeout, timeout_source: timeout_source) + end + end +end diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb new file mode 100644 index 00000000000..4856f10846c --- /dev/null +++ b/app/models/ci/build_trace_chunk.rb @@ -0,0 +1,180 @@ +module Ci + class BuildTraceChunk < ActiveRecord::Base + include FastDestroyAll + extend Gitlab::Ci::Model + + belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id + + default_value_for :data_store, :redis + + WriteError = Class.new(StandardError) + + CHUNK_SIZE = 128.kilobytes + CHUNK_REDIS_TTL = 1.week + WRITE_LOCK_RETRY = 10 + WRITE_LOCK_SLEEP = 0.01.seconds + WRITE_LOCK_TTL = 1.minute + + enum data_store: { + redis: 1, + db: 2 + } + + class << self + def redis_data_key(build_id, chunk_index) + "gitlab:ci:trace:#{build_id}:chunks:#{chunk_index}" + end + + def redis_data_keys + redis.pluck(:build_id, :chunk_index).map do |data| + redis_data_key(data.first, data.second) + end + end + + def redis_delete_data(keys) + return if keys.empty? + + Gitlab::Redis::SharedState.with do |redis| + redis.del(keys) + end + end + + ## + # FastDestroyAll concerns + def begin_fast_destroy + redis_data_keys + end + + ## + # FastDestroyAll concerns + def finalize_fast_destroy(keys) + redis_delete_data(keys) + end + end + + ## + # Data is memoized for optimizing #size and #end_offset + def data + @data ||= get_data.to_s + end + + def truncate(offset = 0) + raise ArgumentError, 'Offset is out of range' if offset > size || offset < 0 + return if offset == size # Skip the following process as it doesn't affect anything + + self.append("", offset) + end + + def append(new_data, offset) + raise ArgumentError, 'Offset is out of range' if offset > size || offset < 0 + raise ArgumentError, 'Chunk size overflow' if CHUNK_SIZE < (offset + new_data.bytesize) + + set_data(data.byteslice(0, offset) + new_data) + end + + def size + data&.bytesize.to_i + end + + def start_offset + chunk_index * CHUNK_SIZE + end + + def end_offset + start_offset + size + end + + def range + (start_offset...end_offset) + end + + def use_database! + in_lock do + break if db? + break unless size > 0 + + self.update!(raw_data: data, data_store: :db) + self.class.redis_delete_data([redis_data_key]) + end + end + + private + + def get_data + if redis? + redis_data + elsif db? + raw_data + else + raise 'Unsupported data store' + end&.force_encoding(Encoding::BINARY) # Redis/Database return UTF-8 string as default + end + + def set_data(value) + raise ArgumentError, 'too much data' if value.bytesize > CHUNK_SIZE + + in_lock do + if redis? + redis_set_data(value) + elsif db? + self.raw_data = value + else + raise 'Unsupported data store' + end + + @data = value + + save! if changed? + end + + schedule_to_db if full? + end + + def schedule_to_db + return if db? + + Ci::BuildTraceChunkFlushWorker.perform_async(id) + end + + def full? + size == CHUNK_SIZE + end + + def redis_data + Gitlab::Redis::SharedState.with do |redis| + redis.get(redis_data_key) + end + end + + def redis_set_data(data) + Gitlab::Redis::SharedState.with do |redis| + redis.set(redis_data_key, data, ex: CHUNK_REDIS_TTL) + end + end + + def redis_data_key + self.class.redis_data_key(build_id, chunk_index) + end + + def in_lock + write_lock_key = "trace_write:#{build_id}:chunks:#{chunk_index}" + + lease = Gitlab::ExclusiveLease.new(write_lock_key, timeout: WRITE_LOCK_TTL) + retry_count = 0 + + until uuid = lease.try_obtain + # Keep trying until we obtain the lease. To prevent hammering Redis too + # much we'll wait for a bit between retries. + sleep(WRITE_LOCK_SLEEP) + break if WRITE_LOCK_RETRY < (retry_count += 1) + end + + raise WriteError, 'Failed to obtain write lock' unless uuid + + self.reload if self.persisted? + return yield + ensure + Gitlab::ExclusiveLease.cancel(write_lock_key, uuid) + end + end +end diff --git a/app/models/ci/group.rb b/app/models/ci/group.rb index 87898b086c6..9c1046e8715 100644 --- a/app/models/ci/group.rb +++ b/app/models/ci/group.rb @@ -31,6 +31,14 @@ module Ci end end + def self.fabricate(stage) + stage.statuses.ordered.latest + .sort_by(&:sortable_name).group_by(&:group_name) + .map do |group_name, grouped_statuses| + self.new(stage, name: group_name, jobs: grouped_statuses) + end + end + private def commit_statuses diff --git a/app/models/ci/group_variable.rb b/app/models/ci/group_variable.rb index 1dd0e050ba9..44cb583e1bd 100644 --- a/app/models/ci/group_variable.rb +++ b/app/models/ci/group_variable.rb @@ -4,7 +4,9 @@ module Ci include HasVariable include Presentable - belongs_to :group + belongs_to :group, class_name: "::Group" + + alias_attribute :secret_value, :value validates :key, uniqueness: { scope: :group_id, diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 0a599f72bc7..3b952391b7e 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -1,15 +1,23 @@ module Ci class JobArtifact < ActiveRecord::Base + include AfterCommitQueue + include ObjectStorage::BackgroundMove extend Gitlab::Ci::Model belongs_to :project belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id + mount_uploader :file, JobArtifactUploader + before_save :set_size, if: :file_changed? + after_save :update_project_statistics_after_save, if: :size_changed? + after_destroy :update_project_statistics_after_destroy, unless: :project_destroyed? - mount_uploader :file, JobArtifactUploader + after_save :update_file_store, if: :file_changed? - delegate :open, :exists?, to: :file + scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) } + + delegate :exists?, :open, to: :file enum file_type: { archive: 1, @@ -17,12 +25,18 @@ module Ci trace: 3 } + def update_file_store + # The file.object_store is set during `uploader.store!` + # which happens after object is inserted/updated + self.update_column(:file_store, file.object_store) + end + def self.artifacts_size_for(project) self.where(project: project).sum(:size) end - def set_size - self.size = file.size + def local_store? + [nil, ::JobArtifactUploader::Store::LOCAL].include?(self.file_store) end def expire_in @@ -35,5 +49,28 @@ module Ci ChronicDuration.parse(value)&.seconds&.from_now end end + + private + + def set_size + self.size = file.size + end + + def update_project_statistics_after_save + update_project_statistics(size.to_i - size_was.to_i) + end + + def update_project_statistics_after_destroy + update_project_statistics(-self.size) + end + + def update_project_statistics(difference) + ProjectStatistics.increment_statistic(project_id, :build_artifacts_size, difference) + end + + def project_destroyed? + # Use job.project to avoid extra DB query for project + job.project.pending_delete? + end end end diff --git a/app/models/ci/legacy_stage.rb b/app/models/ci/legacy_stage.rb index 9b536af672b..ce691875e42 100644 --- a/app/models/ci/legacy_stage.rb +++ b/app/models/ci/legacy_stage.rb @@ -16,11 +16,7 @@ module Ci end def groups - @groups ||= statuses.ordered.latest - .sort_by(&:sortable_name).group_by(&:group_name) - .map do |group_name, grouped_statuses| - Ci::Group.new(self, name: group_name, jobs: grouped_statuses) - end + @groups ||= Ci::Group.fabricate(self) end def to_param diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 44f9bdf111e..e5caa3ffa41 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -6,13 +6,20 @@ module Ci include AfterCommitQueue include Presentable include Gitlab::OptimisticLocking + include Gitlab::Utils::StrongMemoize + include AtomicInternalId + include EnumWithNil belongs_to :project, inverse_of: :pipelines belongs_to :user belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' - has_many :stages + has_internal_id :iid, scope: :project, presence: false, init: ->(s) do + s&.project&.pipelines&.maximum(:iid) || s&.project&.pipelines&.count + end + + has_many :stages, -> { order(position: :asc) }, inverse_of: :pipeline has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent @@ -31,18 +38,24 @@ module Ci has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id' has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id' + accepts_nested_attributes_for :variables, reject_if: :persisted? + delegate :id, to: :project, prefix: true delegate :full_path, to: :project, prefix: true - validates :source, exclusion: { in: %w(unknown), unless: :importing? }, on: :create validates :sha, presence: { unless: :importing? } validates :ref, presence: { unless: :importing? } validates :status, presence: { unless: :importing? } validate :valid_commit_sha, unless: :importing? + # Replace validator below with + # `validates :source, presence: { unless: :importing? }, on: :create` + # when removing Gitlab.rails5? code. + validate :valid_source, unless: :importing?, on: :create + after_create :keep_around_commits, unless: :importing? - enum source: { + enum_with_nil source: { unknown: nil, push: 1, web: 2, @@ -52,7 +65,7 @@ module Ci external: 6 } - enum config_source: { + enum_with_nil config_source: { unknown_source: nil, repository_source: 1, auto_devops_source: 2 @@ -242,6 +255,20 @@ module Ci stage unless stage.statuses_count.zero? end + ## + # TODO We do not completely switch to persisted stages because of + # race conditions with setting statuses gitlab-ce#23257. + # + def ordered_stages + return legacy_stages unless complete? + + if Feature.enabled?('ci_pipeline_persisted_stages') + stages + else + legacy_stages + end + end + def legacy_stages # TODO, this needs refactoring, see gitlab-ce#26481. @@ -268,19 +295,39 @@ module Ci end def git_author_name - commit.try(:author_name) + strong_memoize(:git_author_name) do + commit.try(:author_name) + end end def git_author_email - commit.try(:author_email) + strong_memoize(:git_author_email) do + commit.try(:author_email) + end end def git_commit_message - commit.try(:message) + strong_memoize(:git_commit_message) do + commit.try(:message) + end end def git_commit_title - commit.try(:title) + strong_memoize(:git_commit_title) do + commit.try(:title) + end + end + + def git_commit_full_title + strong_memoize(:git_commit_full_title) do + commit.try(:full_title) + end + end + + def git_commit_description + strong_memoize(:git_commit_description) do + commit.try(:description) + end end def short_sha @@ -361,23 +408,36 @@ module Ci def stage_seeds return [] unless config_processor - @stage_seeds ||= config_processor.stage_seeds(self) + strong_memoize(:stage_seeds) do + seeds = config_processor.stages_attributes.map do |attributes| + Gitlab::Ci::Pipeline::Seed::Stage.new(self, attributes) + end + + seeds.select(&:included?) + end end def seeds_size - @seeds_size ||= stage_seeds.sum(&:size) + stage_seeds.sum(&:size) end def has_kubernetes_active? project.deployment_platform&.active? end - def has_stage_seeds? - stage_seeds.any? + def has_warnings? + number_of_warnings.positive? end - def has_warnings? - builds.latest.failed_but_allowed.any? + def number_of_warnings + BatchLoader.for(id).batch(default_value: 0) do |pipeline_ids, loader| + ::Ci::Build.where(commit_id: pipeline_ids) + .latest + .failed_but_allowed + .group(:commit_id) + .count + .each { |id, amount| loader.call(id, amount) } + end end def set_config_source @@ -388,6 +448,9 @@ module Ci end end + ## + # TODO, setting yaml_errors should be moved to the pipeline creation chain. + # def config_processor return unless ci_yaml_file return @config_processor if defined?(@config_processor) @@ -460,7 +523,8 @@ module Ci def update_status retry_optimistic_lock(self) do - case latest_builds_status + case latest_builds_status.to_s + when 'created' then nil when 'pending' then enqueue when 'running' then run when 'success' then succeed @@ -468,15 +532,38 @@ module Ci when 'canceled' then cancel when 'skipped' then skip when 'manual' then block + else + raise HasStatus::UnknownStatusError, + "Unknown status `#{latest_builds_status}`" end end end + def protected_ref? + strong_memoize(:protected_ref) { project.protected_for?(ref) } + end + + def legacy_trigger + strong_memoize(:legacy_trigger) { trigger_requests.first } + end + + def persisted_variables + Gitlab::Ci::Variables::Collection.new.tap do |variables| + break variables unless persisted? + + variables.append(key: 'CI_PIPELINE_ID', value: id.to_s) + variables.append(key: 'CI_PIPELINE_URL', value: Gitlab::Routing.url_helpers.project_pipeline_url(project, self)) + end + end + def predefined_variables Gitlab::Ci::Variables::Collection.new - .append(key: 'CI_PIPELINE_ID', value: id.to_s) + .append(key: 'CI_PIPELINE_IID', value: iid.to_s) .append(key: 'CI_CONFIG_PATH', value: ci_yaml_file_path) .append(key: 'CI_PIPELINE_SOURCE', value: source.to_s) + .append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message.to_s) + .append(key: 'CI_COMMIT_TITLE', value: git_commit_full_title.to_s) + .append(key: 'CI_COMMIT_DESCRIPTION', value: git_commit_description.to_s) end def queued_duration @@ -551,5 +638,11 @@ module Ci project.repository.keep_around(self.sha) project.repository.keep_around(self.before_sha) end + + def valid_source + if source.nil? || source == "unknown" + errors.add(:source, "invalid source") + end + end end end diff --git a/app/models/ci/pipeline_schedule_variable.rb b/app/models/ci/pipeline_schedule_variable.rb index af989fb14b4..03df4e3e638 100644 --- a/app/models/ci/pipeline_schedule_variable.rb +++ b/app/models/ci/pipeline_schedule_variable.rb @@ -5,6 +5,8 @@ module Ci belongs_to :pipeline_schedule + alias_attribute :secret_value, :value + validates :key, uniqueness: { scope: :pipeline_schedule_id } end end diff --git a/app/models/ci/pipeline_variable.rb b/app/models/ci/pipeline_variable.rb index de5aae17a15..38e14ffbc0c 100644 --- a/app/models/ci/pipeline_variable.rb +++ b/app/models/ci/pipeline_variable.rb @@ -5,6 +5,8 @@ module Ci belongs_to :pipeline + alias_attribute :secret_value, :value + validates :key, uniqueness: { scope: :pipeline_id } end end diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 7173f88f1c7..8c9aacca8de 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -3,42 +3,67 @@ module Ci extend Gitlab::Ci::Model include Gitlab::SQL::Pattern include RedisCacheable + include ChronicDurationAttribute RUNNER_QUEUE_EXPIRY_TIME = 60.minutes ONLINE_CONTACT_TIMEOUT = 1.hour UPDATE_DB_RUNNER_INFO_EVERY = 40.minutes AVAILABLE_SCOPES = %w[specific shared active paused online].freeze - FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level].freeze + FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level maximum_timeout_human_readable].freeze has_many :builds - has_many :runner_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :runner_projects, inverse_of: :runner, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :runner_projects + has_many :runner_namespaces, inverse_of: :runner + has_many :groups, through: :runner_namespaces has_one :last_build, ->() { order('id DESC') }, class_name: 'Ci::Build' before_validation :set_default_values - scope :specific, ->() { where(is_shared: false) } - scope :shared, ->() { where(is_shared: true) } - scope :active, ->() { where(active: true) } - scope :paused, ->() { where(active: false) } - scope :online, ->() { where('contacted_at > ?', contact_time_deadline) } - scope :ordered, ->() { order(id: :desc) } + scope :specific, -> { where(is_shared: false) } + scope :shared, -> { where(is_shared: true) } + scope :active, -> { where(active: true) } + scope :paused, -> { where(active: false) } + scope :online, -> { where('contacted_at > ?', contact_time_deadline) } + scope :ordered, -> { order(id: :desc) } - scope :owned_or_shared, ->(project_id) do - joins('LEFT JOIN ci_runner_projects ON ci_runner_projects.runner_id = ci_runners.id') - .where("ci_runner_projects.project_id = :project_id OR ci_runners.is_shared = true", project_id: project_id) + scope :belonging_to_project, -> (project_id) { + joins(:runner_projects).where(ci_runner_projects: { project_id: project_id }) + } + + scope :belonging_to_parent_group_of_project, -> (project_id) { + project_groups = ::Group.joins(:projects).where(projects: { id: project_id }) + hierarchy_groups = Gitlab::GroupHierarchy.new(project_groups).base_and_ancestors + + joins(:groups).where(namespaces: { id: hierarchy_groups }) + } + + scope :owned_or_shared, -> (project_id) do + union = Gitlab::SQL::Union.new( + [belonging_to_project(project_id), belonging_to_parent_group_of_project(project_id), shared], + remove_duplicates: false + ) + from("(#{union.to_sql}) ci_runners") end scope :assignable_for, ->(project) do # FIXME: That `to_sql` is needed to workaround a weird Rails bug. # Without that, placeholders would miss one and couldn't match. where(locked: false) - .where.not("id IN (#{project.runners.select(:id).to_sql})").specific + .where.not("ci_runners.id IN (#{project.runners.select(:id).to_sql})") + .project_type end validate :tag_constraints validates :access_level, presence: true + validates :runner_type, presence: true + + validate :no_projects, unless: :project_type? + validate :no_groups, unless: :group_type? + validate :any_project, if: :project_type? + validate :exactly_one_group, if: :group_type? + validate :validate_is_shared acts_as_taggable @@ -49,7 +74,19 @@ module Ci ref_protected: 1 } - cached_attr_reader :version, :revision, :platform, :architecture, :contacted_at, :ip_address + enum runner_type: { + instance_type: 1, + group_type: 2, + project_type: 3 + } + + cached_attr_reader :version, :revision, :platform, :architecture, :ip_address, :contacted_at + + chronic_duration_attr :maximum_timeout_human_readable, :maximum_timeout + + validates :maximum_timeout, allow_nil: true, + numericality: { greater_than_or_equal_to: 600, + message: 'needs to be at least 10 minutes' } # Searches for runners matching the given query. # @@ -76,9 +113,22 @@ module Ci end def assign_to(project, current_user = nil) - self.is_shared = false if shared? - self.save - project.runner_projects.create(runner_id: self.id) + if shared? + self.is_shared = false if shared? + self.runner_type = :project_type + elsif group_type? + raise ArgumentError, 'Transitioning a group runner to a project runner is not supported' + end + + begin + transaction do + self.projects << project + self.save! + end + rescue ActiveRecord::RecordInvalid => e + self.errors.add(:assign_to, e.message) + false + end end def display_name @@ -113,6 +163,14 @@ module Ci !shared? end + def assigned_to_group? + runner_namespaces.any? + end + + def assigned_to_project? + runner_projects.any? + end + def can_pick?(build) return false if self.ref_protected? && !build.protected? @@ -161,9 +219,13 @@ module Ci cache_attributes(values) - if persist_cached_data? - self.assign_attributes(values) - self.save if self.changed? + # We save data without validation, it will always change due to `contacted_at` + self.update_columns(values) if persist_cached_data? + end + + def pick_build!(build) + if can_pick?(build) + tick_runner_queue end end @@ -198,7 +260,37 @@ module Ci end def assignable_for?(project_id) - is_shared? || projects.exists?(id: project_id) + self.class.owned_or_shared(project_id).where(id: self.id).any? + end + + def no_projects + if projects.any? + errors.add(:runner, 'cannot have projects assigned') + end + end + + def no_groups + if groups.any? + errors.add(:runner, 'cannot have groups assigned') + end + end + + def any_project + unless projects.any? + errors.add(:runner, 'needs to be assigned to at least one project') + end + end + + def exactly_one_group + unless groups.one? + errors.add(:runner, 'needs to be assigned to exactly one group') + end + end + + def validate_is_shared + unless is_shared? == instance_type? + errors.add(:is_shared, 'is not equal to instance_type?') + end end def accepting_tags?(build) diff --git a/app/models/ci/runner_namespace.rb b/app/models/ci/runner_namespace.rb new file mode 100644 index 00000000000..29508fdd326 --- /dev/null +++ b/app/models/ci/runner_namespace.rb @@ -0,0 +1,11 @@ +module Ci + class RunnerNamespace < ActiveRecord::Base + extend Gitlab::Ci::Model + + belongs_to :runner, inverse_of: :runner_namespaces, validate: true + belongs_to :namespace, inverse_of: :runner_namespaces, class_name: '::Namespace' + belongs_to :group, class_name: '::Group', foreign_key: :namespace_id + + validates :runner_id, uniqueness: { scope: :namespace_id } + end +end diff --git a/app/models/ci/runner_project.rb b/app/models/ci/runner_project.rb index 505d178ba8e..52437047300 100644 --- a/app/models/ci/runner_project.rb +++ b/app/models/ci/runner_project.rb @@ -2,8 +2,8 @@ module Ci class RunnerProject < ActiveRecord::Base extend Gitlab::Ci::Model - belongs_to :runner - belongs_to :project + belongs_to :runner, inverse_of: :runner_projects + belongs_to :project, inverse_of: :runner_projects validates :runner_id, uniqueness: { scope: :project_id } end diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 75b8ea2a371..ea07f37e6c1 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -13,14 +13,27 @@ module Ci has_many :statuses, class_name: 'CommitStatus', foreign_key: :stage_id has_many :builds, foreign_key: :stage_id - validates :project, presence: true, unless: :importing? - validates :pipeline, presence: true, unless: :importing? - validates :name, presence: true, unless: :importing? + with_options unless: :importing? do + validates :project, presence: true + validates :pipeline, presence: true + validates :name, presence: true + validates :position, presence: true + end - after_initialize do |stage| + after_initialize do self.status = DEFAULT_STATUS if self.status.nil? end + before_validation unless: :importing? do + next if position.present? + + self.position = statuses.select(:stage_idx) + .where('stage_idx IS NOT NULL') + .group(:stage_idx) + .order('COUNT(*) DESC') + .first&.stage_idx.to_i + end + state_machine :status, initial: :created do event :enqueue do transition created: :pending @@ -55,16 +68,44 @@ module Ci def update_status retry_optimistic_lock(self) do case statuses.latest.status + when 'created' then nil when 'pending' then enqueue when 'running' then run when 'success' then succeed when 'failed' then drop when 'canceled' then cancel when 'manual' then block - when 'skipped' then skip - else skip + when 'skipped', nil then skip + else + raise HasStatus::UnknownStatusError, + "Unknown status `#{statuses.latest.status}`" end end end + + def groups + @groups ||= Ci::Group.fabricate(self) + end + + def has_warnings? + number_of_warnings.positive? + end + + def number_of_warnings + BatchLoader.for(id).batch(default_value: 0) do |stage_ids, loader| + ::Ci::Build.where(stage_id: stage_ids) + .latest + .failed_but_allowed + .group(:stage_id) + .count + .each { |id, amount| loader.call(id, amount) } + end + end + + def detailed_status(current_user) + Gitlab::Ci::Status::Stage::Factory + .new(self, current_user) + .fabricate! + end end end diff --git a/app/models/ci/variable.rb b/app/models/ci/variable.rb index 7c71291de84..452cb910bca 100644 --- a/app/models/ci/variable.rb +++ b/app/models/ci/variable.rb @@ -6,6 +6,8 @@ module Ci belongs_to :project + alias_attribute :secret_value, :value + validates :key, uniqueness: { scope: [:project_id, :environment_scope], message: "(%{value}) has already been taken" diff --git a/app/models/clusters/applications/jupyter.rb b/app/models/clusters/applications/jupyter.rb new file mode 100644 index 00000000000..975d434e1a4 --- /dev/null +++ b/app/models/clusters/applications/jupyter.rb @@ -0,0 +1,92 @@ +module Clusters + module Applications + class Jupyter < ActiveRecord::Base + VERSION = '0.0.1'.freeze + + self.table_name = 'clusters_applications_jupyter' + + include ::Clusters::Concerns::ApplicationCore + include ::Clusters::Concerns::ApplicationStatus + include ::Clusters::Concerns::ApplicationData + + belongs_to :oauth_application, class_name: 'Doorkeeper::Application' + + default_value_for :version, VERSION + + def set_initial_status + return unless not_installable? + + if cluster&.application_ingress_installed? && cluster.application_ingress.external_ip + self.status = 'installable' + end + end + + def chart + "#{name}/jupyterhub" + end + + def repository + 'https://jupyterhub.github.io/helm-chart/' + end + + def values + content_values.to_yaml + end + + def install_command + Gitlab::Kubernetes::Helm::InstallCommand.new( + name, + chart: chart, + values: values, + repository: repository + ) + end + + def callback_url + "http://#{hostname}/hub/oauth_callback" + end + + private + + def specification + { + "ingress" => { + "hosts" => [hostname] + }, + "hub" => { + "extraEnv" => { + "GITLAB_HOST" => gitlab_url + }, + "cookieSecret" => cookie_secret + }, + "proxy" => { + "secretToken" => secret_token + }, + "auth" => { + "gitlab" => { + "clientId" => oauth_application.uid, + "clientSecret" => oauth_application.secret, + "callbackUrl" => callback_url + } + } + } + end + + def gitlab_url + Gitlab.config.gitlab.url + end + + def content_values + YAML.load_file(chart_values_file).deep_merge!(specification) + end + + def secret_token + @secret_token ||= SecureRandom.hex(32) + end + + def cookie_secret + @cookie_secret ||= SecureRandom.hex(32) + end + end + end +end diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb index 7b25d8c4089..48137c2ed68 100644 --- a/app/models/clusters/applications/prometheus.rb +++ b/app/models/clusters/applications/prometheus.rb @@ -3,7 +3,7 @@ module Clusters class Prometheus < ActiveRecord::Base include PrometheusAdapter - VERSION = "2.0.0".freeze + VERSION = '6.7.3'.freeze self.table_name = 'clusters_applications_prometheus' @@ -37,6 +37,7 @@ module Clusters Gitlab::Kubernetes::Helm::InstallCommand.new( name, chart: chart, + version: version, values: values ) end @@ -49,6 +50,11 @@ module Clusters # ensures headers containing auth data are appended to original k8s client options options = kube_client.rest_client.options.merge(headers: kube_client.headers) RestClient::Resource.new(proxy_url, options) + rescue Kubeclient::HttpError + # If users have mistakenly set parameters or removed the depended clusters, + # `proxy_url` could raise an exception because gitlab can not communicate with the cluster. + # Since `PrometheusAdapter#can_query?` is eargely loaded on environement pages in gitlab, + # we need to silence the exceptions end private diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index 16efe90fa27..e6f795f3e0b 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -43,12 +43,21 @@ module Clusters def create_and_assign_runner transaction do - project.runners.create!(name: 'kubernetes-cluster', tag_list: %w(kubernetes cluster)).tap do |runner| + Ci::Runner.create!(runner_create_params).tap do |runner| update!(runner_id: runner.id) end end end + def runner_create_params + { + name: 'kubernetes-cluster', + runner_type: :project_type, + tag_list: %w(kubernetes cluster), + projects: [project] + } + end + def gitlab_url Gitlab::Routing.url_helpers.root_url(only_path: false) end diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 49eb069016a..b426b1bf8a1 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -8,8 +8,10 @@ module Clusters Applications::Helm.application_name => Applications::Helm, Applications::Ingress.application_name => Applications::Ingress, Applications::Prometheus.application_name => Applications::Prometheus, - Applications::Runner.application_name => Applications::Runner + Applications::Runner.application_name => Applications::Runner, + Applications::Jupyter.application_name => Applications::Jupyter }.freeze + DEFAULT_ENVIRONMENT = '*'.freeze belongs_to :user @@ -25,6 +27,7 @@ module Clusters has_one :application_ingress, class_name: 'Clusters::Applications::Ingress' has_one :application_prometheus, class_name: 'Clusters::Applications::Prometheus' has_one :application_runner, class_name: 'Clusters::Applications::Runner' + has_one :application_jupyter, class_name: 'Clusters::Applications::Jupyter' accepts_nested_attributes_for :provider_gcp, update_only: true accepts_nested_attributes_for :platform_kubernetes, update_only: true @@ -38,6 +41,7 @@ module Clusters delegate :active?, to: :platform_kubernetes, prefix: true, allow_nil: true delegate :installed?, to: :application_helm, prefix: true, allow_nil: true + delegate :installed?, to: :application_ingress, prefix: true, allow_nil: true enum platform_type: { kubernetes: 1 @@ -50,6 +54,11 @@ module Clusters scope :enabled, -> { where(enabled: true) } scope :disabled, -> { where(enabled: false) } + scope :user_provided, -> { where(provider_type: ::Clusters::Cluster.provider_types[:user]) } + scope :gcp_provided, -> { where(provider_type: ::Clusters::Cluster.provider_types[:gcp]) } + scope :gcp_installed, -> { gcp_provided.includes(:provider_gcp).where(cluster_providers_gcp: { status: ::Clusters::Providers::Gcp.state_machines[:status].states[:created].value }) } + + scope :default_environment, -> { where(environment_scope: DEFAULT_ENVIRONMENT) } def status_name if provider @@ -68,7 +77,8 @@ module Clusters application_helm || build_application_helm, application_ingress || build_application_ingress, application_prometheus || build_application_prometheus, - application_runner || build_application_runner + application_runner || build_application_runner, + application_jupyter || build_application_jupyter ] end diff --git a/app/models/clusters/concerns/application_status.rb b/app/models/clusters/concerns/application_status.rb index 7b7c8eac773..8f3eb75bfa9 100644 --- a/app/models/clusters/concerns/application_status.rb +++ b/app/models/clusters/concerns/application_status.rb @@ -4,6 +4,8 @@ module Clusters extend ActiveSupport::Concern included do + scope :installed, -> { where(status: self.state_machines[:status].states[:installed].value) } + state_machine :status, initial: :not_installable do state :not_installable, value: -2 state :errored, value: -1 diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index ba6552f238f..36631d57ad1 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -11,12 +11,12 @@ module Clusters attr_encrypted :password, mode: :per_attribute_iv, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base_truncated, algorithm: 'aes-256-cbc' attr_encrypted :token, mode: :per_attribute_iv, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base_truncated, algorithm: 'aes-256-cbc' before_validation :enforce_namespace_to_lower_case diff --git a/app/models/clusters/providers/gcp.rb b/app/models/clusters/providers/gcp.rb index 7fac32466ab..4db1bb35c12 100644 --- a/app/models/clusters/providers/gcp.rb +++ b/app/models/clusters/providers/gcp.rb @@ -11,7 +11,7 @@ module Clusters attr_encrypted :access_token, mode: :per_attribute_iv, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base_truncated, algorithm: 'aes-256-cbc' validates :gcp_project_id, diff --git a/app/models/commit.rb b/app/models/commit.rb index cceae5efb72..56d4c86774e 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -30,9 +30,12 @@ class Commit MIN_SHA_LENGTH = Gitlab::Git::Commit::MIN_SHA_LENGTH COMMIT_SHA_PATTERN = /\h{#{MIN_SHA_LENGTH},40}/.freeze + # Used by GFM to match and present link extensions on node texts and hrefs. + LINK_EXTENSION_PATTERN = /(patch)/.freeze def banzai_render_context(field) - context = { pipeline: :single_line, project: self.project } + pipeline = field == :description ? :commit_description : :single_line + context = { pipeline: pipeline, project: self.project } context[:author] = self.author if self.author context @@ -102,6 +105,10 @@ class Commit end end end + + def parent_class + ::Project + end end attr_accessor :raw @@ -142,7 +149,8 @@ class Commit end def self.link_reference_pattern - @link_reference_pattern ||= super("commit", /(?<commit>#{COMMIT_SHA_PATTERN})/) + @link_reference_pattern ||= + super("commit", /(?<commit>#{COMMIT_SHA_PATTERN})?(\.(?<extension>#{LINK_EXTENSION_PATTERN}))?/) end def to_reference(from = nil, full: false) @@ -175,7 +183,7 @@ class Commit if safe_message.blank? no_commit_message else - safe_message.split("\n", 2).first + safe_message.split(/[\r\n]/, 2).first end end @@ -216,8 +224,34 @@ class Commit Gitlab::ClosingIssueExtractor.new(project, current_user).closed_by_message(safe_message) end + def lazy_author + BatchLoader.for(author_email.downcase).batch do |emails, loader| + # A Hash that maps user Emails to the corresponding User objects. The + # Emails at this point are the _primary_ Emails of the Users. + users_for_emails = User + .by_any_email(emails) + .each_with_object({}) { |user, hash| hash[user.email] = user } + + users_for_ids = users_for_emails + .values + .each_with_object({}) { |user, hash| hash[user.id] = user } + + # Some commits may have used an alternative Email address. In this case we + # need to query the "emails" table to map those addresses to User objects. + Email + .where(email: emails - users_for_emails.keys) + .pluck(:email, :user_id) + .each { |(email, id)| users_for_emails[email] = users_for_ids[id] } + + users_for_emails.each { |email, user| loader.call(email, user) } + end + end + def author - User.find_by_any_email(author_email.downcase) + # We use __sync so that we get the actual objects back (including an actual + # nil), instead of a wrapper, as returning a wrapped nil breaks a lot of + # code. + lazy_author.__sync end request_cache(:author) { author_email.downcase } @@ -244,7 +278,7 @@ class Commit end def notes_with_associations - notes.includes(:author) + notes.includes(:author, :award_emoji) end def merge_requests @@ -416,6 +450,12 @@ class Commit # no-op but needs to be defined since #persisted? is defined end + def touch_later + # No-op. + # This method is called by ActiveRecord. + # We don't want to do anything for `Commit` model, so this is empty. + end + WIP_REGEX = /\A\s*(((?i)(\[WIP\]|WIP:|WIP)\s|WIP$))|(fixup!|squash!)\s/.freeze def work_in_progress? diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 9fb5b7efec6..97516079b66 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -2,6 +2,8 @@ class CommitStatus < ActiveRecord::Base include HasStatus include Importable include AfterCommitQueue + include Presentable + include EnumWithNil self.table_name = 'ci_builds' @@ -38,7 +40,7 @@ class CommitStatus < ActiveRecord::Base scope :retried_ordered, -> { retried.ordered.includes(project: :namespace) } scope :after_stage, -> (index) { where('stage_idx > ?', index) } - enum failure_reason: { + enum_with_nil failure_reason: { unknown_failure: nil, script_failure: 1, api_failure: 2, @@ -87,7 +89,7 @@ class CommitStatus < ActiveRecord::Base transition [:created, :pending, :running, :manual] => :canceled end - before_transition created: [:pending, :running] do |commit_status| + before_transition [:created, :skipped, :manual] => :pending do |commit_status| commit_status.queued_at = Time.now end @@ -141,7 +143,7 @@ class CommitStatus < ActiveRecord::Base end def group_name - name.to_s.gsub(%r{\d+[\.\s:/\\]+\d+\s*}, '').strip + name.to_s.gsub(%r{\d+[\s:/\\]+\d+\s*}, '').strip end def failed_but_allowed? diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb index 4b66725a3e6..164c704260e 100644 --- a/app/models/concerns/atomic_internal_id.rb +++ b/app/models/concerns/atomic_internal_id.rb @@ -25,22 +25,23 @@ module AtomicInternalId extend ActiveSupport::Concern module ClassMethods - def has_internal_id(column, scope:, init:) # rubocop:disable Naming/PredicateName - before_validation(on: :create) do - if read_attribute(column).blank? - scope_attrs = { scope => association(scope).reader } + def has_internal_id(column, scope:, init:, presence: true) # rubocop:disable Naming/PredicateName + before_validation :"ensure_#{scope}_#{column}!", on: :create + validates column, presence: presence + + define_method("ensure_#{scope}_#{column}!") do + scope_value = association(scope).reader + + if read_attribute(column).blank? && scope_value + scope_attrs = { scope_value.class.table_name.singularize.to_sym => scope_value } usage = self.class.table_name.to_sym new_iid = InternalId.generate_next(self, scope_attrs, usage, init) write_attribute(column, new_iid) end - end - validates column, presence: true, numericality: true + read_attribute(column) + end end end - - def to_param - iid.to_s - end end diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb index d35e37935fb..095897b08e3 100644 --- a/app/models/concerns/avatarable.rb +++ b/app/models/concerns/avatarable.rb @@ -3,11 +3,15 @@ module Avatarable included do prepend ShadowMethods + include ObjectStorage::BackgroundMove + include Gitlab::Utils::StrongMemoize validate :avatar_type, if: ->(user) { user.avatar.present? && user.avatar_changed? } validates :avatar, file_size: { maximum: 200.kilobytes.to_i } mount_uploader :avatar, AvatarUploader + + after_initialize :add_avatar_to_batch end module ShadowMethods @@ -17,11 +21,22 @@ module Avatarable avatar_path(only_path: args.fetch(:only_path, true)) || super end + + def retrieve_upload(identifier, paths) + upload = retrieve_upload_from_batch(identifier) + + # This fallback is needed when deleting an upload, because we may have + # already been removed from the DB. We have to check an explicit `#nil?` + # because it's a BatchLoader instance. + upload = super if upload.nil? + + upload + end end def avatar_type unless self.avatar.image? - self.errors.add :avatar, "only images allowed" + errors.add :avatar, "file format is not supported. Please try one of the following supported formats: #{AvatarUploader::IMAGE_EXT.join(', ')}" end end @@ -30,12 +45,13 @@ module Avatarable asset_host = ActionController::Base.asset_host use_asset_host = asset_host.present? + use_authentication = respond_to?(:public?) && !public? # Avatars for private and internal groups and projects require authentication to be viewed, # which means they can only be served by Rails, on the regular GitLab host. # If an asset host is configured, we need to return the fully qualified URL # instead of only the avatar path, so that Rails doesn't prefix it with the asset host. - if use_asset_host && respond_to?(:public?) && !public? + if use_asset_host && use_authentication use_asset_host = false only_path = false end @@ -48,6 +64,39 @@ module Avatarable url_base << gitlab_config.relative_url_root end - url_base + avatar.url + url_base + avatar.local_url + end + + # Path that is persisted in the tracking Upload model. Used to fetch the + # upload from the model. + def upload_paths(identifier) + avatar_mounter.blank_uploader.store_dirs.map { |store, path| File.join(path, identifier) } + end + + private + + def retrieve_upload_from_batch(identifier) + BatchLoader.for(identifier: identifier, model: self).batch(key: self.class) do |upload_params, loader, args| + model_class = args[:key] + paths = upload_params.flat_map do |params| + params[:model].upload_paths(params[:identifier]) + end + + Upload.where(uploader: AvatarUploader, path: paths).find_each do |upload| + model = model_class.instantiate('id' => upload.model_id) + + loader.call({ model: model, identifier: File.basename(upload.path) }, upload) + end + end + end + + def add_avatar_to_batch + return unless avatar_mounter + + avatar_mounter.read_identifiers.each { |identifier| retrieve_upload_from_batch(identifier) } + end + + def avatar_mounter + strong_memoize(:avatar_mounter) { _mounter(:avatar) } end end diff --git a/app/models/concerns/awardable.rb b/app/models/concerns/awardable.rb index d8394415362..fce37e7f78e 100644 --- a/app/models/concerns/awardable.rb +++ b/app/models/concerns/awardable.rb @@ -79,11 +79,7 @@ module Awardable end def user_can_award?(current_user, name) - if user_authored?(current_user) - !awardable_votes?(normalize_name(name)) - else - true - end + awardable_by_user?(current_user, name) && Ability.allowed?(current_user, :award_emoji, self) end def user_authored?(current_user) @@ -119,4 +115,12 @@ module Awardable def normalize_name(name) Gitlab::Emoji.normalize_emoji_name(name) end + + def awardable_by_user?(current_user, name) + if user_authored?(current_user) + !awardable_votes?(normalize_name(name)) + else + true + end + end end diff --git a/app/models/concerns/batch_destroy_dependent_associations.rb b/app/models/concerns/batch_destroy_dependent_associations.rb new file mode 100644 index 00000000000..353ee2e73d0 --- /dev/null +++ b/app/models/concerns/batch_destroy_dependent_associations.rb @@ -0,0 +1,28 @@ +# Provides a way to work around Rails issue where dependent objects are all +# loaded into memory before destroyed: https://github.com/rails/rails/issues/22510. +# +# This concern allows an ActiveRecord module to destroy all its dependent +# associations in batches. The idea is borrowed from https://github.com/thisismydesign/batch_dependent_associations. +# +# The differences here with that gem: +# +# 1. We allow excluding certain associations. +# 2. We don't need to support delete_all since we can use the EachBatch concern. +module BatchDestroyDependentAssociations + extend ActiveSupport::Concern + + DEPENDENT_ASSOCIATIONS_BATCH_SIZE = 1000 + + def dependent_associations_to_destroy + self.class.reflect_on_all_associations(:has_many).select { |assoc| assoc.options[:dependent] == :destroy } + end + + def destroy_dependent_associations_in_batches(exclude: []) + dependent_associations_to_destroy.each do |association| + next if exclude.include?(association.name) + + # rubocop:disable GitlabSecurity/PublicSend + public_send(association.name).find_each(batch_size: DEPENDENT_ASSOCIATIONS_BATCH_SIZE, &:destroy) + end + end +end diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index 4ae5dd8c677..9f6358cecbe 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -11,7 +11,9 @@ module CacheMarkdownField extend ActiveSupport::Concern # Increment this number every time the renderer changes its output - CACHE_VERSION = 3 + CACHE_REDCARPET_VERSION = 3 + CACHE_COMMONMARK_VERSION_START = 10 + CACHE_COMMONMARK_VERSION = 11 # changes to these attributes cause the cache to be invalidates INVALIDATED_BY = %w[author project].freeze @@ -49,12 +51,14 @@ module CacheMarkdownField # Always include a project key, or Banzai complains project = self.project if self.respond_to?(:project) - group = self.group if self.respond_to?(:group) + group = self.group if self.respond_to?(:group) context = cached_markdown_fields[field].merge(project: project, group: group) # Banzai is less strict about authors, so don't always have an author key context[:author] = self.author if self.respond_to?(:author) + context[:markdown_engine] = markdown_engine + context end @@ -69,7 +73,7 @@ module CacheMarkdownField Banzai::Renderer.cacheless_render_field(self, markdown_field, options) ] end.to_h - updates['cached_markdown_version'] = CacheMarkdownField::CACHE_VERSION + updates['cached_markdown_version'] = latest_cached_markdown_version updates.each {|html_field, data| write_attribute(html_field, data) } end @@ -90,7 +94,7 @@ module CacheMarkdownField markdown_changed = attribute_changed?(markdown_field) || false html_changed = attribute_changed?(html_field) || false - CacheMarkdownField::CACHE_VERSION == cached_markdown_version && + latest_cached_markdown_version == cached_markdown_version && (html_changed || markdown_changed == html_changed) end @@ -109,6 +113,24 @@ module CacheMarkdownField __send__(cached_markdown_fields.html_field(markdown_field)) # rubocop:disable GitlabSecurity/PublicSend end + def latest_cached_markdown_version + return CacheMarkdownField::CACHE_COMMONMARK_VERSION unless cached_markdown_version + + if cached_markdown_version < CacheMarkdownField::CACHE_COMMONMARK_VERSION_START + CacheMarkdownField::CACHE_REDCARPET_VERSION + else + CacheMarkdownField::CACHE_COMMONMARK_VERSION + end + end + + def markdown_engine + if latest_cached_markdown_version < CacheMarkdownField::CACHE_COMMONMARK_VERSION_START + :redcarpet + else + :common_mark + end + end + included do cattr_reader :cached_markdown_fields do FieldData.new diff --git a/app/models/concerns/cacheable_attributes.rb b/app/models/concerns/cacheable_attributes.rb new file mode 100644 index 00000000000..d58d7165969 --- /dev/null +++ b/app/models/concerns/cacheable_attributes.rb @@ -0,0 +1,74 @@ +module CacheableAttributes + extend ActiveSupport::Concern + + included do + after_commit { self.class.expire } + end + + class_methods do + def cache_key + "#{name}:#{Gitlab::VERSION}:#{Gitlab.migrations_hash}:#{Rails.version}".freeze + end + + # Can be overriden + def current_without_cache + last + end + + # Can be overriden + def defaults + {} + end + + def build_from_defaults(attributes = {}) + new(defaults.merge(attributes)) + end + + def cached + if RequestStore.active? + RequestStore[:"#{name}_cached_attributes"] ||= retrieve_from_cache + else + retrieve_from_cache + end + end + + def retrieve_from_cache + record = Rails.cache.read(cache_key) + ensure_cache_setup if record.present? + + record + end + + def current + cached_record = cached + return cached_record if cached_record.present? + + current_without_cache.tap { |current_record| current_record&.cache! } + rescue => e + if Rails.env.production? + Rails.logger.warn("Cached record for #{name} couldn't be loaded, falling back to uncached record: #{e}") + else + raise e + end + # Fall back to an uncached value if there are any problems (e.g. Redis down) + current_without_cache + end + + def expire + Rails.cache.delete(cache_key) + rescue + # Gracefully handle when Redis is not available. For example, + # omnibus may fail here during gitlab:assets:compile. + end + + def ensure_cache_setup + # This is a workaround for a Rails bug that causes attribute methods not + # to be loaded when read from cache: https://github.com/rails/rails/issues/27348 + define_attribute_methods + end + end + + def cache! + Rails.cache.write(self.class.cache_key, self) + end +end diff --git a/app/models/concerns/chronic_duration_attribute.rb b/app/models/concerns/chronic_duration_attribute.rb new file mode 100644 index 00000000000..593a9b3d71d --- /dev/null +++ b/app/models/concerns/chronic_duration_attribute.rb @@ -0,0 +1,39 @@ +module ChronicDurationAttribute + extend ActiveSupport::Concern + + class_methods do + def chronic_duration_attr_reader(virtual_attribute, source_attribute) + define_method(virtual_attribute) do + chronic_duration_attributes[virtual_attribute] || output_chronic_duration_attribute(source_attribute) + end + end + + def chronic_duration_attr_writer(virtual_attribute, source_attribute, parameters = {}) + chronic_duration_attr_reader(virtual_attribute, source_attribute) + + define_method("#{virtual_attribute}=") do |value| + chronic_duration_attributes[virtual_attribute] = value.presence || parameters[:default].presence.to_s + + begin + new_value = value.present? ? ChronicDuration.parse(value).to_i : parameters[:default].presence + assign_attributes(source_attribute => new_value) + rescue ChronicDuration::DurationParseError + # ignore error as it will be caught by validation + end + end + + validates virtual_attribute, allow_nil: true, duration: true + end + + alias_method :chronic_duration_attr, :chronic_duration_attr_writer + end + + def chronic_duration_attributes + @chronic_duration_attributes ||= {} + end + + def output_chronic_duration_attribute(source_attribute) + value = attributes[source_attribute.to_s] + ChronicDuration.output(value, format: :short) if value + end +end diff --git a/app/models/concerns/deployment_platform.rb b/app/models/concerns/deployment_platform.rb index faa94204e33..52851b3d0b2 100644 --- a/app/models/concerns/deployment_platform.rb +++ b/app/models/concerns/deployment_platform.rb @@ -1,16 +1,24 @@ module DeploymentPlatform - # EE would override this and utilize the extra argument + # EE would override this and utilize environment argument + # rubocop:disable Gitlab/ModuleWithInstanceVariables def deployment_platform(environment: nil) - @deployment_platform ||= - find_cluster_platform_kubernetes || - find_kubernetes_service_integration || - build_cluster_and_deployment_platform + @deployment_platform ||= {} + + @deployment_platform[environment] ||= find_deployment_platform(environment) end private - def find_cluster_platform_kubernetes - clusters.find_by(enabled: true)&.platform_kubernetes + def find_deployment_platform(environment) + find_cluster_platform_kubernetes(environment: environment) || + find_kubernetes_service_integration || + build_cluster_and_deployment_platform + end + + # EE would override this and utilize environment argument + def find_cluster_platform_kubernetes(environment: nil) + clusters.enabled.default_environment + .last&.platform_kubernetes end def find_kubernetes_service_integration diff --git a/app/models/concerns/diff_file.rb b/app/models/concerns/diff_file.rb new file mode 100644 index 00000000000..72332072012 --- /dev/null +++ b/app/models/concerns/diff_file.rb @@ -0,0 +1,9 @@ +module DiffFile + extend ActiveSupport::Concern + + def to_hash + keys = Gitlab::Git::Diff::SERIALIZE_KEYS - [:diff] + + as_json(only: keys).merge(diff: diff).with_indifferent_access + end +end diff --git a/app/models/concerns/enum_with_nil.rb b/app/models/concerns/enum_with_nil.rb new file mode 100644 index 00000000000..6b37903da20 --- /dev/null +++ b/app/models/concerns/enum_with_nil.rb @@ -0,0 +1,33 @@ +module EnumWithNil + extend ActiveSupport::Concern + + included do + def self.enum_with_nil(definitions) + # use original `enum` to auto-define all methods + enum(definitions) + + # override auto-defined methods only for the + # key which uses nil value + definitions.each do |name, values| + next unless key_with_nil = values.key(nil) + + # E.g. for enum_with_nil failure_reason: { unknown_failure: nil } + # this overrides auto-generated method `unknown_failure?` + define_method("#{key_with_nil}?") do + Gitlab.rails5? ? self[name].nil? : super() + end + + # E.g. for enum_with_nil failure_reason: { unknown_failure: nil } + # this overrides auto-generated method `failure_reason` + define_method(name) do + orig = super() + + return orig unless Gitlab.rails5? + return orig unless orig.nil? + + self.class.public_send(name.to_s.pluralize).key(nil) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + end +end diff --git a/app/models/concerns/fast_destroy_all.rb b/app/models/concerns/fast_destroy_all.rb new file mode 100644 index 00000000000..7ea042c6742 --- /dev/null +++ b/app/models/concerns/fast_destroy_all.rb @@ -0,0 +1,91 @@ +## +# This module is for replacing `dependent: :destroy` and `before_destroy` hooks. +# +# In general, `destroy_all` is inefficient because it calls each callback with `DELETE` queries i.e. O(n), whereas, +# `delete_all` is efficient as it deletes all rows with a single `DELETE` query. +# +# It's better to use `delete_all` as our best practice, however, +# if external data (e.g. ObjectStorage, FileStorage or Redis) are assosiated with database records, +# it is difficult to accomplish it. +# +# This module defines a format to use `delete_all` and delete associated external data. +# Here is an exmaple +# +# Situation +# - `Project` has many `Ci::BuildTraceChunk` through `Ci::Build` +# - `Ci::BuildTraceChunk` stores associated data in Redis, so it relies on `dependent: :destroy` and `before_destroy` for the deletion +# +# How to use +# - Define `use_fast_destroy :build_trace_chunks` in `Project` model. +# - Define `begin_fast_destroy` and `finalize_fast_destroy(params)` in `Ci::BuildTraceChunk` model. +# - Use `fast_destroy_all` instead of `destroy` and `destroy_all` +# - Remove `dependent: :destroy` and `before_destroy` as it's no longer need +# +# Expectation +# - When a project is `destroy`ed, the associated trace_chunks will be deleted by `delete_all`, +# and the associated data will be removed, too. +# - When `fast_destroy_all` is called, it also performns as same. +module FastDestroyAll + extend ActiveSupport::Concern + + ForbiddenActionError = Class.new(StandardError) + + included do + before_destroy do + raise ForbiddenActionError, '`destroy` and `destroy_all` are forbbiden. Please use `fast_destroy_all`' + end + end + + class_methods do + ## + # This method delete rows and associated external data efficiently + # + # This method can replace `destroy` and `destroy_all` without having `after_destroy` hook + def fast_destroy_all + params = begin_fast_destroy + + delete_all + + finalize_fast_destroy(params) + end + + ## + # This method returns identifiers to delete associated external data (e.g. file paths, redis keys) + # + # This method must be defined in fast destroyable model + def begin_fast_destroy + raise NotImplementedError + end + + ## + # This method deletes associated external data with the identifiers returned by `begin_fast_destroy` + # + # This method must be defined in fast destroyable model + def finalize_fast_destroy(params) + raise NotImplementedError + end + end + + module Helpers + extend ActiveSupport::Concern + + class_methods do + ## + # This method is to be defined on models which have fast destroyable models as children, + # and let us avoid to use `dependent: :destroy` hook + def use_fast_destroy(relation) + before_destroy(prepend: true) do + perform_fast_destroy(public_send(relation)) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + + def perform_fast_destroy(subject) + params = subject.begin_fast_destroy + + run_after_commit do + subject.finalize_fast_destroy(params) + end + end + end +end diff --git a/app/models/concerns/group_descendant.rb b/app/models/concerns/group_descendant.rb index 01957da0bf3..261ace57a17 100644 --- a/app/models/concerns/group_descendant.rb +++ b/app/models/concerns/group_descendant.rb @@ -37,7 +37,20 @@ module GroupDescendant parent ||= preloaded.detect { |possible_parent| possible_parent.is_a?(Group) && possible_parent.id == child.parent_id } if parent.nil? && !child.parent_id.nil? - raise ArgumentError.new('parent was not preloaded') + parent = child.parent + + exception = ArgumentError.new <<~MSG + parent: [GroupDescendant: #{parent.inspect}] was not preloaded for [#{child.inspect}]") + This error is not user facing, but causes a +1 query. + MSG + extras = { + parent: parent, + child: child, + preloaded: preloaded.map(&:full_path) + } + issue_url = 'https://gitlab.com/gitlab-org/gitlab-ce/issues/40785' + + Gitlab::Sentry.track_exception(exception, issue_url: issue_url, extra: extras) end if parent.nil? && hierarchy_top.present? diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb index 7c3ed96bc28..72c236a0fc7 100644 --- a/app/models/concerns/has_status.rb +++ b/app/models/concerns/has_status.rb @@ -11,6 +11,8 @@ module HasStatus STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3, failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze + UnknownStatusError = Class.new(StandardError) + class_methods do def status_sql scope_relevant = respond_to?(:exclude_ignored) ? exclude_ignored : all diff --git a/app/models/concerns/has_variable.rb b/app/models/concerns/has_variable.rb index 8a241e4374a..c8e20c0ab81 100644 --- a/app/models/concerns/has_variable.rb +++ b/app/models/concerns/has_variable.rb @@ -13,7 +13,7 @@ module HasVariable attr_encrypted :value, mode: :per_attribute_iv_and_salt, insecure_mode: true, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base, algorithm: 'aes-256-cbc' def key=(new_key) diff --git a/app/models/concerns/iid_routes.rb b/app/models/concerns/iid_routes.rb new file mode 100644 index 00000000000..246748cf52c --- /dev/null +++ b/app/models/concerns/iid_routes.rb @@ -0,0 +1,9 @@ +module IidRoutes + ## + # This automagically enforces all related routes to use `iid` instead of `id` + # If you want to use `iid` for some routes and `id` for other routes, this module should not to be included, + # instead you should define `iid` or `id` explictly at each route generators. e.g. pipeline_path(project.id, pipeline.iid) + def to_param + iid.to_s + end +end diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 5a566f3ac02..b93c1145f82 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -97,8 +97,6 @@ module Issuable strip_attributes :title - after_save :ensure_metrics, unless: :imported? - # We want to use optimistic lock for cases when only title or description are involved # http://api.rubyonrails.org/classes/ActiveRecord/Locking/Optimistic.html def locking_enabled? @@ -109,6 +107,10 @@ module Issuable false end + def etag_caching_enabled? + false + end + def has_multiple_assignees? assignees.count > 1 end @@ -137,7 +139,7 @@ module Issuable fuzzy_search(query, [:title, :description]) end - def sort(method, excluded_labels: []) + def sort_by_attribute(method, excluded_labels: []) sorted = case method.to_s when 'downvotes_desc' then order_downvotes_desc diff --git a/app/models/concerns/milestoneish.rb b/app/models/concerns/milestoneish.rb index caf8afa97f9..967fd9c5eea 100644 --- a/app/models/concerns/milestoneish.rb +++ b/app/models/concerns/milestoneish.rb @@ -45,11 +45,11 @@ module Milestoneish end def sorted_issues(user) - issues_visible_to_user(user).preload_associations.sort('label_priority') + issues_visible_to_user(user).preload_associations.sort_by_attribute('label_priority') end def sorted_merge_requests - merge_requests.sort('label_priority') + merge_requests.sort_by_attribute('label_priority') end def upcoming? @@ -102,14 +102,14 @@ module Milestoneish Gitlab::TimeTrackingFormatter.output(total_issue_time_estimate) end - private - def count_issues_by_state(user) memoize_per_user(user, :count_issues_by_state) do issues_visible_to_user(user).reorder(nil).group(:state).count end end + private + def memoize_per_user(user, method_name) memoized_users[method_name][user&.id] ||= yield end diff --git a/app/models/concerns/nonatomic_internal_id.rb b/app/models/concerns/nonatomic_internal_id.rb deleted file mode 100644 index 9d0c9b8512f..00000000000 --- a/app/models/concerns/nonatomic_internal_id.rb +++ /dev/null @@ -1,22 +0,0 @@ -module NonatomicInternalId - extend ActiveSupport::Concern - - included do - validate :set_iid, on: :create - validates :iid, presence: true, numericality: true - end - - def set_iid - if iid.blank? - parent = project || group - records = parent.public_send(self.class.name.tableize) # rubocop:disable GitlabSecurity/PublicSend - max_iid = records.maximum(:iid) - - self.iid = max_iid.to_i + 1 - end - end - - def to_param - iid.to_s - end -end diff --git a/app/models/concerns/participable.rb b/app/models/concerns/participable.rb index e48bc0be410..01b1ef9f82c 100644 --- a/app/models/concerns/participable.rb +++ b/app/models/concerns/participable.rb @@ -98,6 +98,10 @@ module Participable participants.merge(ext.users) + filter_by_ability(participants) + end + + def filter_by_ability(participants) case self when PersonalSnippet Ability.users_that_can_read_personal_snippet(participants.to_a, self) diff --git a/app/models/concerns/presentable.rb b/app/models/concerns/presentable.rb index 7b33b837004..bc4fbd19a02 100644 --- a/app/models/concerns/presentable.rb +++ b/app/models/concerns/presentable.rb @@ -1,4 +1,12 @@ module Presentable + extend ActiveSupport::Concern + + class_methods do + def present(attributes) + all.map { |klass_object| klass_object.present(attributes) } + end + end + def present(**attributes) Gitlab::View::Presenter::Factory .new(self, attributes) diff --git a/app/models/concerns/project_features_compatibility.rb b/app/models/concerns/project_features_compatibility.rb index b3fec99c816..1f7d78a2efe 100644 --- a/app/models/concerns/project_features_compatibility.rb +++ b/app/models/concerns/project_features_compatibility.rb @@ -1,4 +1,4 @@ -# Makes api V3 compatible with old project features permissions methods +# Makes api V4 compatible with old project features permissions methods # # After migrating issues_enabled merge_requests_enabled builds_enabled snippets_enabled and wiki_enabled # fields to a new table "project_features", support for the old fields is still needed in the API. diff --git a/app/models/concerns/protected_ref.rb b/app/models/concerns/protected_ref.rb index 454374121f3..94eef4ff7cd 100644 --- a/app/models/concerns/protected_ref.rb +++ b/app/models/concerns/protected_ref.rb @@ -31,7 +31,7 @@ module ProtectedRef end end - def protected_ref_accessible_to?(ref, user, action:, protected_refs: nil) + def protected_ref_accessible_to?(ref, user, project:, action:, protected_refs: nil) access_levels_for_ref(ref, action: action, protected_refs: protected_refs).any? do |access_level| access_level.check_access(user) end diff --git a/app/models/concerns/protected_ref_access.rb b/app/models/concerns/protected_ref_access.rb index bfda5b1678b..e3a7f2d5498 100644 --- a/app/models/concerns/protected_ref_access.rb +++ b/app/models/concerns/protected_ref_access.rb @@ -8,8 +8,8 @@ module ProtectedRefAccess ].freeze HUMAN_ACCESS_LEVELS = { - Gitlab::Access::MASTER => "Masters".freeze, - Gitlab::Access::DEVELOPER => "Developers + Masters".freeze, + Gitlab::Access::MASTER => "Maintainers".freeze, + Gitlab::Access::DEVELOPER => "Developers + Maintainers".freeze, Gitlab::Access::NO_ACCESS => "No one".freeze }.freeze diff --git a/app/models/concerns/reactive_caching.rb b/app/models/concerns/reactive_caching.rb index 2589215ad19..be0a5b49012 100644 --- a/app/models/concerns/reactive_caching.rb +++ b/app/models/concerns/reactive_caching.rb @@ -60,22 +60,26 @@ module ReactiveCaching end def with_reactive_cache(*args, &blk) - within_reactive_cache_lifetime(*args) do + bootstrap = !within_reactive_cache_lifetime?(*args) + Rails.cache.write(alive_reactive_cache_key(*args), true, expires_in: self.class.reactive_cache_lifetime) + + if bootstrap + ReactiveCachingWorker.perform_async(self.class, id, *args) + nil + else data = Rails.cache.read(full_reactive_cache_key(*args)) yield data if data.present? end - ensure - Rails.cache.write(alive_reactive_cache_key(*args), true, expires_in: self.class.reactive_cache_lifetime) - ReactiveCachingWorker.perform_async(self.class, id, *args) end def clear_reactive_cache!(*args) Rails.cache.delete(full_reactive_cache_key(*args)) + Rails.cache.delete(alive_reactive_cache_key(*args)) end def exclusively_update_reactive_cache!(*args) locking_reactive_cache(*args) do - within_reactive_cache_lifetime(*args) do + if within_reactive_cache_lifetime?(*args) enqueuing_update(*args) do value = calculate_reactive_cache(*args) Rails.cache.write(full_reactive_cache_key(*args), value) @@ -105,8 +109,8 @@ module ReactiveCaching Gitlab::ExclusiveLease.cancel(full_reactive_cache_key(*args), uuid) end - def within_reactive_cache_lifetime(*args) - yield if Rails.cache.read(alive_reactive_cache_key(*args)) + def within_reactive_cache_lifetime?(*args) + !!Rails.cache.read(alive_reactive_cache_key(*args)) end def enqueuing_update(*args) diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb index b889f4202dc..3bdc1330d23 100644 --- a/app/models/concerns/redis_cacheable.rb +++ b/app/models/concerns/redis_cacheable.rb @@ -7,7 +7,11 @@ module RedisCacheable class_methods do def cached_attr_reader(*attributes) attributes.each do |attribute| - define_method("#{attribute}") do + define_method(attribute) do + unless self.has_attribute?(attribute) + raise ArgumentError, "`cached_attr_reader` requires the #{self.class.name}\##{attribute} attribute to have a database column" + end + cached_attribute(attribute) || read_attribute(attribute) end end @@ -15,13 +19,16 @@ module RedisCacheable end def cached_attribute(attribute) - (cached_attributes || {})[attribute] + cached_value = (cached_attributes || {})[attribute] + cast_value_from_cache(attribute, cached_value) if cached_value end def cache_attributes(values) Gitlab::Redis::SharedState.with do |redis| redis.set(cache_attribute_key, values.to_json, ex: CACHED_ATTRIBUTES_EXPIRY_TIME) end + + clear_memoization(:cached_attributes) end private @@ -38,4 +45,12 @@ module RedisCacheable end end end + + def cast_value_from_cache(attribute, value) + if Gitlab.rails5? + self.class.type_for_attribute(attribute.to_s).cast(value) + else + self.class.column_for_attribute(attribute).type_cast_from_database(value) + end + end end diff --git a/app/models/concerns/resolvable_note.rb b/app/models/concerns/resolvable_note.rb index 668c5a079e3..4a0f8b92b3a 100644 --- a/app/models/concerns/resolvable_note.rb +++ b/app/models/concerns/resolvable_note.rb @@ -32,7 +32,7 @@ module ResolvableNote # Keep this method in sync with the `potentially_resolvable` scope def potentially_resolvable? - RESOLVABLE_TYPES.include?(self.class.name) && noteable.supports_resolvable_notes? + RESOLVABLE_TYPES.include?(self.class.name) && noteable&.supports_resolvable_notes? end # Keep this method in sync with the `resolvable` scope diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index dfd7d94450b..0176a12a131 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -4,7 +4,9 @@ module Routable extend ActiveSupport::Concern included do - has_one :route, as: :source, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + # Remove `inverse_of: source` when upgraded to rails 5.2 + # See https://github.com/rails/rails/pull/28808 + has_one :route, as: :source, autosave: true, dependent: :destroy, inverse_of: :source # rubocop:disable Cop/ActiveRecordDependent has_many :redirect_routes, as: :source, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent validates :route, presence: true @@ -102,7 +104,7 @@ module Routable # the route. Caching this per request ensures that even if we have multiple instances, # we will not have to duplicate work, avoiding N+1 queries in some cases. def full_path - return uncached_full_path unless RequestStore.active? + return uncached_full_path unless RequestStore.active? && persisted? RequestStore[full_path_key] ||= uncached_full_path end @@ -124,6 +126,11 @@ module Routable end end + # Group would override this to check from association + def owned_by?(user) + owner == user + end + private def set_path_errors diff --git a/app/models/concerns/sha_attribute.rb b/app/models/concerns/sha_attribute.rb index 703a72c355c..3796737427a 100644 --- a/app/models/concerns/sha_attribute.rb +++ b/app/models/concerns/sha_attribute.rb @@ -4,18 +4,34 @@ module ShaAttribute module ClassMethods def sha_attribute(name) return if ENV['STATIC_VERIFICATION'] - return unless table_exists? + + validate_binary_column_exists!(name) unless Rails.env.production? + + attribute(name, Gitlab::Database::ShaAttribute.new) + end + + # This only gets executed in non-production environments as an additional check to ensure + # the column is the correct type. In production it should behave like any other attribute. + # See https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/5502 for more discussion + def validate_binary_column_exists!(name) + unless table_exists? + warn "WARNING: sha_attribute #{name.inspect} is invalid since the table doesn't exist - you may need to run database migrations" + return + end column = columns.find { |c| c.name == name.to_s } - # In case the table doesn't exist we won't be able to find the column, - # thus we will only check the type if the column is present. - if column && column.type != :binary - raise ArgumentError, - "sha_attribute #{name.inspect} is invalid since the column type is not :binary" + unless column + warn "WARNING: sha_attribute #{name.inspect} is invalid since the column doesn't exist - you may need to run database migrations" + return end - attribute(name, Gitlab::Database::ShaAttribute.new) + unless column.type == :binary + raise ArgumentError.new("sha_attribute #{name.inspect} is invalid since the column type is not :binary") + end + rescue => error + Gitlab::AppLogger.error "ShaAttribute initialization: #{error.message}" + raise end end end diff --git a/app/models/concerns/sortable.rb b/app/models/concerns/sortable.rb index cefa5c13c5f..cb76ae971d4 100644 --- a/app/models/concerns/sortable.rb +++ b/app/models/concerns/sortable.rb @@ -12,8 +12,8 @@ module Sortable scope :order_created_asc, -> { reorder(created_at: :asc) } scope :order_updated_desc, -> { reorder(updated_at: :desc) } scope :order_updated_asc, -> { reorder(updated_at: :asc) } - scope :order_name_asc, -> { reorder(name: :asc) } - scope :order_name_desc, -> { reorder(name: :desc) } + scope :order_name_asc, -> { reorder(Arel::Nodes::Ascending.new(arel_table[:name].lower)) } + scope :order_name_desc, -> { reorder(Arel::Nodes::Descending.new(arel_table[:name].lower)) } end module ClassMethods diff --git a/app/models/concerns/storage/legacy_namespace.rb b/app/models/concerns/storage/legacy_namespace.rb index f05e606995d..f66bdd529f1 100644 --- a/app/models/concerns/storage/legacy_namespace.rb +++ b/app/models/concerns/storage/legacy_namespace.rb @@ -45,25 +45,25 @@ module Storage # Hooks - # Save the storage paths before the projects are destroyed to use them on after destroy + # Save the storages before the projects are destroyed to use them on after destroy def prepare_for_destroy - old_repository_storage_paths + old_repository_storages end private def move_repositories - # Move the namespace directory in all storage paths used by member projects - repository_storage_paths.each do |repository_storage_path| + # Move the namespace directory in all storages used by member projects + repository_storages.each do |repository_storage| # Ensure old directory exists before moving it - gitlab_shell.add_namespace(repository_storage_path, full_path_was) + gitlab_shell.add_namespace(repository_storage, full_path_was) # Ensure new directory exists before moving it (if there's a parent) - gitlab_shell.add_namespace(repository_storage_path, parent.full_path) if parent + gitlab_shell.add_namespace(repository_storage, parent.full_path) if parent - unless gitlab_shell.mv_namespace(repository_storage_path, full_path_was, full_path) + unless gitlab_shell.mv_namespace(repository_storage, full_path_was, full_path) - Rails.logger.error "Exception moving path #{repository_storage_path} from #{full_path_was} to #{full_path}" + Rails.logger.error "Exception moving path #{repository_storage} from #{full_path_was} to #{full_path}" # if we cannot move namespace directory we should rollback # db changes in order to prevent out of sync between db and fs @@ -72,33 +72,33 @@ module Storage end end - def old_repository_storage_paths - @old_repository_storage_paths ||= repository_storage_paths + def old_repository_storages + @old_repository_storage_paths ||= repository_storages end - def repository_storage_paths + def repository_storages # We need to get the storage paths for all the projects, even the ones that are # pending delete. Unscoping also get rids of the default order, which causes # problems with SELECT DISTINCT. Project.unscoped do - all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path) + all_projects.select('distinct(repository_storage)').to_a.map(&:repository_storage) end end def rm_dir # Remove the namespace directory in all storages paths used by member projects - old_repository_storage_paths.each do |repository_storage_path| + old_repository_storages.each do |repository_storage| # Move namespace directory into trash. # We will remove it later async new_path = "#{full_path}+#{id}+deleted" - if gitlab_shell.mv_namespace(repository_storage_path, full_path, new_path) + if gitlab_shell.mv_namespace(repository_storage, full_path, new_path) Gitlab::AppLogger.info %Q(Namespace directory "#{full_path}" moved to "#{new_path}") # Remove namespace directroy async with delay so # GitLab has time to remove all projects first run_after_commit do - GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path) + GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage, new_path) end end end diff --git a/app/models/concerns/time_trackable.rb b/app/models/concerns/time_trackable.rb index 5911b56c34c..0fc321c52bc 100644 --- a/app/models/concerns/time_trackable.rb +++ b/app/models/concerns/time_trackable.rb @@ -51,6 +51,10 @@ module TimeTrackable Gitlab::TimeTrackingFormatter.output(time_estimate) end + def time_estimate=(val) + val.is_a?(Integer) ? super([val, Gitlab::Database::MAX_INT_VALUE].min) : super(val) + end + private def reset_spent_time diff --git a/app/models/concerns/uniquify.rb b/app/models/concerns/uniquify.rb index a7fe5951b6e..549a76da20e 100644 --- a/app/models/concerns/uniquify.rb +++ b/app/models/concerns/uniquify.rb @@ -1,13 +1,21 @@ +# Uniquify +# +# Return a version of the given 'base' string that is unique +# by appending a counter to it. Uniqueness is determined by +# repeated calls to the passed block. +# +# You can pass an initial value for the counter, if not given +# counting starts from 1. +# +# If `base` is a function/proc, we expect that calling it with a +# candidate counter returns a string to test/return. class Uniquify - # Return a version of the given 'base' string that is unique - # by appending a counter to it. Uniqueness is determined by - # repeated calls to the passed block. - # - # If `base` is a function/proc, we expect that calling it with a - # candidate counter returns a string to test/return. + def initialize(counter = nil) + @counter = counter + end + def string(base) @base = base - @counter = nil increment_counter! while yield(base_string) base_string diff --git a/app/models/concerns/with_uploads.rb b/app/models/concerns/with_uploads.rb new file mode 100644 index 00000000000..4245d083a49 --- /dev/null +++ b/app/models/concerns/with_uploads.rb @@ -0,0 +1,43 @@ +# Mounted uploaders are destroyed by carrierwave's after_commit +# hook. This hook fetches upload location (local vs remote) from +# Upload model. So it's neccessary to make sure that during that +# after_commit hook model's associated uploads are not deleted yet. +# IOW we can not use dependent: :destroy : +# has_many :uploads, as: :model, dependent: :destroy +# +# And because not-mounted uploads require presence of upload's +# object model when destroying them (FileUploader's `build_upload` method +# references `model` on delete), we can not use after_commit hook for these +# uploads. +# +# Instead FileUploads are destroyed in before_destroy hook and remaining uploads +# are destroyed by the carrierwave's after_commit hook. + +module WithUploads + extend ActiveSupport::Concern + + # Currently there is no simple way how to select only not-mounted + # uploads, it should be all FileUploaders so we select them by + # `uploader` class + FILE_UPLOADERS = %w(PersonalFileUploader NamespaceFileUploader FileUploader).freeze + + included do + has_many :uploads, as: :model + + before_destroy :destroy_file_uploads + end + + # mounted uploads are deleted in carrierwave's after_commit hook, + # but FileUploaders which are not mounted must be deleted explicitly and + # it can not be done in after_commit because FileUploader requires loads + # associated model on destroy (which is already deleted in after_commit) + def destroy_file_uploads + self.uploads.where(uploader: FILE_UPLOADERS).find_each do |upload| + upload.destroy + end + end + + def retrieve_upload(_identifier, paths) + uploads.find_by(path: paths) + end +end diff --git a/app/models/deploy_key.rb b/app/models/deploy_key.rb index c2e0a5fa126..89a74b7dcb1 100644 --- a/app/models/deploy_key.rb +++ b/app/models/deploy_key.rb @@ -27,6 +27,10 @@ class DeployKey < Key self.private? end + def user + super || User.ghost + end + def has_access_to?(project) deploy_keys_project_for(project).present? end diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb new file mode 100644 index 00000000000..5082dc45368 --- /dev/null +++ b/app/models/deploy_token.rb @@ -0,0 +1,66 @@ +class DeployToken < ActiveRecord::Base + include Expirable + include TokenAuthenticatable + add_authentication_token_field :token + + AVAILABLE_SCOPES = %i(read_repository read_registry).freeze + GITLAB_DEPLOY_TOKEN_NAME = 'gitlab-deploy-token'.freeze + + default_value_for(:expires_at) { Forever.date } + + has_many :project_deploy_tokens, inverse_of: :deploy_token + has_many :projects, through: :project_deploy_tokens + + validate :ensure_at_least_one_scope + before_save :ensure_token + + accepts_nested_attributes_for :project_deploy_tokens + + scope :active, -> { where("revoked = false AND expires_at >= NOW()") } + + def self.gitlab_deploy_token + active.find_by(name: GITLAB_DEPLOY_TOKEN_NAME) + end + + def revoke! + update!(revoked: true) + end + + def active? + !revoked + end + + def scopes + AVAILABLE_SCOPES.select { |token_scope| read_attribute(token_scope) } + end + + def username + "gitlab+deploy-token-#{id}" + end + + def has_access_to?(requested_project) + active? && project == requested_project + end + + # This is temporal. Currently we limit DeployToken + # to a single project, later we're going to extend + # that to be for multiple projects and namespaces. + def project + projects.first + end + + def expires_at + expires_at = read_attribute(:expires_at) + expires_at != Forever.date ? expires_at : nil + end + + def expires_at=(value) + write_attribute(:expires_at, value.presence || Forever.date) + end + + private + + def ensure_at_least_one_scope + errors.add(:base, "Scopes can't be blank") unless read_repository || read_registry + end +end diff --git a/app/models/deployment.rb b/app/models/deployment.rb index e18ea8bfea4..ac86e9e8de0 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -1,11 +1,14 @@ class Deployment < ActiveRecord::Base - include NonatomicInternalId + include AtomicInternalId + include IidRoutes belongs_to :project, required: true belongs_to :environment, required: true belongs_to :user belongs_to :deployable, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations + has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.deployments&.maximum(:iid) } + validates :sha, presence: true validates :ref, presence: true diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb index 15122cbc693..d752d5bcdee 100644 --- a/app/models/diff_note.rb +++ b/app/models/diff_note.rb @@ -3,6 +3,7 @@ # A note of this type can be resolvable. class DiffNote < Note include NoteOnDiff + include Gitlab::Utils::StrongMemoize NOTEABLE_TYPES = %w(MergeRequest Commit).freeze @@ -12,7 +13,6 @@ class DiffNote < Note validates :original_position, presence: true validates :position, presence: true - validates :diff_line, presence: true, if: :on_text? validates :line_code, presence: true, line_code: true, if: :on_text? validates :noteable_type, inclusion: { in: NOTEABLE_TYPES } validate :positions_complete @@ -23,6 +23,7 @@ class DiffNote < Note before_validation :update_position, on: :create, if: :on_text? before_validation :set_line_code, if: :on_text? after_save :keep_around_commits + after_commit :create_diff_file, on: :create def discussion_class(*) DiffDiscussion @@ -53,8 +54,25 @@ class DiffNote < Note position.position_type == "image" end + def create_diff_file + return unless should_create_diff_file? + + diff_file = fetch_diff_file + diff_line = diff_file.line_for_position(self.original_position) + + creation_params = diff_file.diff.to_hash + .except(:too_large) + .merge(diff: diff_file.diff_hunk(diff_line)) + + create_note_diff_file(creation_params) + end + def diff_file - @diff_file ||= self.original_position.diff_file(self.project.repository) + strong_memoize(:diff_file) do + enqueue_diff_file_creation_job if should_create_diff_file? + + fetch_diff_file + end end def diff_line @@ -85,6 +103,38 @@ class DiffNote < Note private + def enqueue_diff_file_creation_job + # Avoid enqueuing multiple file creation jobs at once for a note (i.e. + # parallel calls to `DiffNote#diff_file`). + lease = Gitlab::ExclusiveLease.new("note_diff_file_creation:#{id}", timeout: 1.hour.to_i) + return unless lease.try_obtain + + CreateNoteDiffFileWorker.perform_async(id) + end + + def should_create_diff_file? + on_text? && note_diff_file.nil? && self == discussion.first_note + end + + def fetch_diff_file + if note_diff_file + diff = Gitlab::Git::Diff.new(note_diff_file.to_hash) + Gitlab::Diff::File.new(diff, + repository: project.repository, + diff_refs: original_position.diff_refs) + elsif created_at_diff?(noteable.diff_refs) + # We're able to use the already persisted diffs (Postgres) if we're + # presenting a "current version" of the MR discussion diff. + # So no need to make an extra Gitaly diff request for it. + # As an extra benefit, the returned `diff_file` already + # has `highlighted_diff_lines` data set from Redis on + # `Diff::FileCollection::MergeRequestDiff`. + noteable.diffs(paths: original_position.paths, expanded: true).diff_files.first + else + original_position.diff_file(self.project.repository) + end + end + def supported? for_commit? || self.noteable.has_complete_diff_refs? end diff --git a/app/models/discussion.rb b/app/models/discussion.rb index 92482a1a875..35a0ef00856 100644 --- a/app/models/discussion.rb +++ b/app/models/discussion.rb @@ -17,6 +17,10 @@ class Discussion to: :first_note + def project_id + project&.id + end + def self.build(notes, context_noteable = nil) notes.first.discussion_class(context_noteable).new(notes, context_noteable) end diff --git a/app/models/environment.rb b/app/models/environment.rb index 9517723d9d9..8d523dae324 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -32,7 +32,7 @@ class Environment < ActiveRecord::Base validates :external_url, length: { maximum: 255 }, allow_nil: true, - addressable_url: true + url: true delegate :stop_action, :manual_actions, to: :last_deployment, allow_nil: true @@ -224,7 +224,7 @@ class Environment < ActiveRecord::Base end def deployment_platform - project.deployment_platform(environment: self) + project.deployment_platform(environment: self.name) end private diff --git a/app/models/event.rb b/app/models/event.rb index 17a198d52c7..ac0b1c7b27c 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -40,6 +40,7 @@ class Event < ActiveRecord::Base ).freeze RESET_PROJECT_ACTIVITY_INTERVAL = 1.hour + REPOSITORY_UPDATED_AT_INTERVAL = 5.minutes delegate :name, :email, :public_email, :username, to: :author, prefix: true, allow_nil: true delegate :title, to: :issue, prefix: true, allow_nil: true @@ -52,12 +53,12 @@ class Event < ActiveRecord::Base belongs_to :target, -> { # If the association for "target" defines an "author" association we want to # eager-load this so Banzai & friends don't end up performing N+1 queries to - # get the authors of notes, issues, etc. - if reflections['events'].active_record.reflect_on_association(:author) - includes(:author) - else - self + # get the authors of notes, issues, etc. (likewise for "noteable"). + incs = %i(author noteable).select do |a| + reflections['events'].active_record.reflect_on_association(a) end + + incs.reduce(self) { |obj, a| obj.includes(a) } }, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations has_one :push_event_payload @@ -110,7 +111,10 @@ class Event < ActiveRecord::Base end end + # Remove this method when removing Gitlab.rails5? code. def subclass_from_attributes(attrs) + return super if Gitlab.rails5? + # Without this Rails will keep calling this method on the returned class, # resulting in an infinite loop. return unless self == Event @@ -388,6 +392,7 @@ class Event < ActiveRecord::Base def set_last_repository_updated_at Project.unscoped.where(id: project_id) + .where("last_repository_updated_at < ? OR last_repository_updated_at IS NULL", REPOSITORY_UPDATED_AT_INTERVAL.ago) .update_all(last_repository_updated_at: created_at) end diff --git a/app/models/generic_commit_status.rb b/app/models/generic_commit_status.rb index 532b8f4ad69..5ac8bde44cd 100644 --- a/app/models/generic_commit_status.rb +++ b/app/models/generic_commit_status.rb @@ -1,7 +1,7 @@ class GenericCommitStatus < CommitStatus before_validation :set_default_values - validates :target_url, addressable_url: true, + validates :target_url, url: true, length: { maximum: 255 }, allow_nil: true diff --git a/app/models/group.rb b/app/models/group.rb index f669b1a7009..9c171de7fc3 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -9,6 +9,9 @@ class Group < Namespace include SelectForProjectAuthorization include LoadedInGroupList include GroupDescendant + include TokenAuthenticatable + include WithUploads + include Gitlab::Utils::StrongMemoize has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent alias_method :members, :group_members @@ -24,13 +27,15 @@ class Group < Namespace has_many :milestones has_many :project_group_links, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :shared_projects, through: :project_group_links, source: :project + + # Overridden on another method + # Left here just to be dependent: :destroy has_many :notification_settings, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent + has_many :labels, class_name: 'GroupLabel' has_many :variables, class_name: 'Ci::GroupVariable' has_many :custom_attributes, class_name: 'GroupCustomAttribute' - has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :boards has_many :badges, class_name: 'GroupBadge' @@ -43,6 +48,8 @@ class Group < Namespace validates :two_factor_grace_period, presence: true, numericality: { greater_than_or_equal_to: 0 } + add_authentication_token_field :runners_token + after_create :post_create_hook after_destroy :post_destroy_hook after_save :update_two_factor_requirement @@ -53,7 +60,7 @@ class Group < Namespace Gitlab::Database.postgresql? end - def sort(method) + def sort_by_attribute(method) if method == 'storage_size_desc' # storage_size is a virtual column so we need to # pass a string to avoid AR adding the table name @@ -86,6 +93,15 @@ class Group < Namespace end end + # Overrides notification_settings has_many association + # This allows to apply notification settings from parent groups + # to child groups and projects. + def notification_settings + source_type = self.class.base_class.name + + NotificationSetting.where(source_type: source_type, source_id: self_and_ancestors_ids) + end + def to_reference(_from = nil, full: nil) "#{self.class.reference_prefix}#{full_path}" end @@ -125,6 +141,10 @@ class Group < Namespace self[:lfs_enabled] end + def owned_by?(user) + owners.include?(user) + end + def add_users(users, access_level, current_user: nil, expires_at: nil) GroupMember.add_users( self, @@ -135,13 +155,14 @@ class Group < Namespace ) end - def add_user(user, access_level, current_user: nil, expires_at: nil) + def add_user(user, access_level, current_user: nil, expires_at: nil, ldap: false) GroupMember.add_user( self, user, access_level, current_user: current_user, - expires_at: expires_at + expires_at: expires_at, + ldap: ldap ) end @@ -189,10 +210,8 @@ class Group < Namespace owners.include?(user) && owners.size == 1 end - def avatar_type - unless self.avatar.image? - self.errors.add :avatar, "only images allowed" - end + def ldap_synced? + false end def post_create_hook @@ -220,6 +239,12 @@ class Group < Namespace members_with_parents.pluck(:user_id) end + def self_and_ancestors_ids + strong_memoize(:self_and_ancestors_ids) do + self_and_ancestors.pluck(:id) + end + end + def members_with_parents # Avoids an unnecessary SELECT when the group has no parents source_ids = @@ -240,6 +265,13 @@ class Group < Namespace .where(source_id: self_and_descendants.reorder(nil).select(:id)) end + # Returns all members that are part of the group, it's subgroups, and ancestor groups + def direct_and_indirect_members + GroupMember + .active_without_invites_and_requests + .where(source_id: self_and_hierarchy.reorder(nil).select(:id)) + end + def users_with_parents User .where(id: members_with_parents.select(:user_id)) @@ -252,6 +284,30 @@ class Group < Namespace .reorder(nil) end + # Returns all users that are members of the group because: + # 1. They belong to the group + # 2. They belong to a project that belongs to the group + # 3. They belong to a sub-group or project in such sub-group + # 4. They belong to an ancestor group + def direct_and_indirect_users + union = Gitlab::SQL::Union.new([ + User + .where(id: direct_and_indirect_members.select(:user_id)) + .reorder(nil), + project_users_with_descendants + ]) + + User.from("(#{union.to_sql}) #{User.table_name}") + end + + # Returns all users that are members of projects + # belonging to the current group or sub-groups + def project_users_with_descendants + User + .joins(projects: :group) + .where(namespaces: { id: self_and_descendants.select(:id) }) + end + def max_member_access_for_user(user) return GroupMember::OWNER if user.admin? @@ -292,6 +348,17 @@ class Group < Namespace false end + def refresh_project_authorizations + refresh_members_authorized_projects(blocking: false) + end + + # each existing group needs to have a `runners_token`. + # we do this on read since migrating all existing groups is not a feasible + # solution. + def runners_token + ensure_runners_token! + end + private def update_two_factor_requirement diff --git a/app/models/hooks/project_hook.rb b/app/models/hooks/project_hook.rb index b6dd39b860b..ec072882cc9 100644 --- a/app/models/hooks/project_hook.rb +++ b/app/models/hooks/project_hook.rb @@ -7,6 +7,7 @@ class ProjectHook < WebHook :issue_hooks, :confidential_issue_hooks, :note_hooks, + :confidential_note_hooks, :merge_request_hooks, :job_hooks, :pipeline_hooks, diff --git a/app/models/hooks/system_hook.rb b/app/models/hooks/system_hook.rb index 0528266e5b3..6bef00f26ea 100644 --- a/app/models/hooks/system_hook.rb +++ b/app/models/hooks/system_hook.rb @@ -11,4 +11,9 @@ class SystemHook < WebHook default_value_for :push_events, false default_value_for :repository_update_events, true default_value_for :merge_requests_events, false + + # Allow urls pointing localhost and the local network + def allow_local_requests? + true + end end diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb index 27729deeac9..e353abdda9c 100644 --- a/app/models/hooks/web_hook.rb +++ b/app/models/hooks/web_hook.rb @@ -3,7 +3,9 @@ class WebHook < ActiveRecord::Base has_many :web_hook_logs, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - validates :url, presence: true, url: true + validates :url, presence: true, public_url: { allow_localhost: lambda(&:allow_local_requests?), + allow_local_network: lambda(&:allow_local_requests?) } + validates :token, format: { without: /\n/ } def execute(data, hook_name) @@ -13,4 +15,9 @@ class WebHook < ActiveRecord::Base def async_execute(data, hook_name) WebHookService.new(self, data, hook_name).async_execute end + + # Allow urls pointing localhost and the local network + def allow_local_requests? + false + end end diff --git a/app/models/identity.rb b/app/models/identity.rb index 1011b9f1109..3fd0c5e751d 100644 --- a/app/models/identity.rb +++ b/app/models/identity.rb @@ -1,12 +1,16 @@ class Identity < ActiveRecord::Base + def self.uniqueness_scope + :provider + end + include Sortable include CaseSensitivity belongs_to :user validates :provider, presence: true - validates :extern_uid, allow_blank: true, uniqueness: { scope: :provider, case_sensitive: false } - validates :user_id, uniqueness: { scope: :provider } + validates :extern_uid, allow_blank: true, uniqueness: { scope: uniqueness_scope, case_sensitive: false } + validates :user_id, uniqueness: { scope: uniqueness_scope } before_save :ensure_normalized_extern_uid, if: :extern_uid_changed? after_destroy :clear_user_synced_attributes, if: :user_synced_attributes_metadata_from_provider? diff --git a/app/models/internal_id.rb b/app/models/internal_id.rb index cbec735c2dd..f50f28deffe 100644 --- a/app/models/internal_id.rb +++ b/app/models/internal_id.rb @@ -12,8 +12,9 @@ # * (Optionally) add columns to `internal_ids` if needed for scope. class InternalId < ActiveRecord::Base belongs_to :project + belongs_to :namespace - enum usage: { issues: 0 } + enum usage: { issues: 0, merge_requests: 1, deployments: 2, milestones: 3, epics: 4, ci_pipelines: 5 } validates :usage, presence: true diff --git a/app/models/issue.rb b/app/models/issue.rb index 7bfc45c1f43..d3df2da14e2 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -2,6 +2,7 @@ require 'carrierwave/orm/activerecord' class Issue < ActiveRecord::Base include AtomicInternalId + include IidRoutes include Issuable include Noteable include Referable @@ -14,15 +15,17 @@ class Issue < ActiveRecord::Base ignore_column :assignee_id, :branch_name, :deleted_at - DueDateStruct = Struct.new(:title, :name).freeze - NoDueDate = DueDateStruct.new('No Due Date', '0').freeze - AnyDueDate = DueDateStruct.new('Any Due Date', '').freeze - Overdue = DueDateStruct.new('Overdue', 'overdue').freeze - DueThisWeek = DueDateStruct.new('Due This Week', 'week').freeze - DueThisMonth = DueDateStruct.new('Due This Month', 'month').freeze + DueDateStruct = Struct.new(:title, :name).freeze + NoDueDate = DueDateStruct.new('No Due Date', '0').freeze + AnyDueDate = DueDateStruct.new('Any Due Date', '').freeze + Overdue = DueDateStruct.new('Overdue', 'overdue').freeze + DueThisWeek = DueDateStruct.new('Due This Week', 'week').freeze + DueThisMonth = DueDateStruct.new('Due This Month', 'month').freeze + DueNextMonthAndPreviousTwoWeeks = DueDateStruct.new('Due Next Month And Previous Two Weeks', 'next_month_and_previous_two_weeks').freeze belongs_to :project belongs_to :moved_to, class_name: 'Issue' + belongs_to :closed_by, class_name: 'User' has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.issues&.maximum(:iid) } @@ -45,18 +48,22 @@ class Issue < ActiveRecord::Base scope :unassigned, -> { where('NOT EXISTS (SELECT TRUE FROM issue_assignees WHERE issue_id = issues.id)') } scope :assigned_to, ->(u) { where('EXISTS (SELECT TRUE FROM issue_assignees WHERE user_id = ? AND issue_id = issues.id)', u.id)} + scope :with_due_date, -> { where('due_date IS NOT NULL') } scope :without_due_date, -> { where(due_date: nil) } scope :due_before, ->(date) { where('issues.due_date < ?', date) } scope :due_between, ->(from_date, to_date) { where('issues.due_date >= ?', from_date).where('issues.due_date <= ?', to_date) } + scope :due_tomorrow, -> { where(due_date: Date.tomorrow) } scope :order_due_date_asc, -> { reorder('issues.due_date IS NULL, issues.due_date ASC') } scope :order_due_date_desc, -> { reorder('issues.due_date IS NULL, issues.due_date DESC') } + scope :order_closest_future_date, -> { reorder('CASE WHEN due_date >= CURRENT_DATE THEN 0 ELSE 1 END ASC, ABS(CURRENT_DATE - due_date) ASC') } scope :preload_associations, -> { preload(:labels, project: :namespace) } scope :public_only, -> { where(confidential: false) } after_save :expire_etag_cache + after_save :ensure_metrics, unless: :imported? attr_spammable :title, spam_title: true attr_spammable :description, spam_description: true @@ -78,6 +85,11 @@ class Issue < ActiveRecord::Base before_transition any => :closed do |issue| issue.closed_at = Time.zone.now end + + before_transition closed: :opened do |issue| + issue.closed_at = nil + issue.closed_by = nil + end end class << self @@ -110,8 +122,9 @@ class Issue < ActiveRecord::Base 'project_id' end - def self.sort(method, excluded_labels: []) + def self.sort_by_attribute(method, excluded_labels: []) case method.to_s + when 'closest_future_date' then order_closest_future_date when 'due_date' then order_due_date_asc when 'due_date_asc' then order_due_date_asc when 'due_date_desc' then order_due_date_desc @@ -187,6 +200,15 @@ class Issue < ActiveRecord::Base branches_with_iid - branches_with_merge_request end + def suggested_branch_name + return to_branch_name unless project.repository.branch_exists?(to_branch_name) + + start_counting_from = 2 + Uniquify.new(start_counting_from).string(-> (counter) { "#{to_branch_name}-#{counter}" }) do |suggested_branch_name| + project.repository.branch_exists?(suggested_branch_name) + end + end + # Returns boolean if a related branch exists for the current issue # ignores merge requests branchs def has_related_branch? @@ -241,11 +263,8 @@ class Issue < ActiveRecord::Base end end - def can_be_worked_on?(current_user) - !self.closed? && - !self.project.forked? && - self.related_branches(current_user).empty? && - self.closed_by_merge_requests(current_user).empty? + def can_be_worked_on? + !self.closed? && !self.project.forked? end # Returns `true` if the current issue can be viewed by either a logged in User @@ -266,11 +285,17 @@ class Issue < ActiveRecord::Base def as_json(options = {}) super(options).tap do |json| - if options.key?(:sidebar_endpoints) && project + if options.key?(:issue_endpoints) && project url_helper = Gitlab::Routing.url_helpers - json.merge!(issue_sidebar_endpoint: url_helper.project_issue_path(project, self, format: :json, serializer: 'sidebar'), - toggle_subscription_endpoint: url_helper.toggle_subscription_project_issue_path(project, self)) + issue_reference = options[:include_full_project_path] ? to_reference(full: true) : to_reference + + json.merge!( + reference_path: issue_reference, + real_path: url_helper.project_issue_path(project, self), + issue_sidebar_endpoint: url_helper.project_issue_path(project, self, format: :json, serializer: 'sidebar'), + toggle_subscription_endpoint: url_helper.toggle_subscription_project_issue_path(project, self) + ) end if options.key?(:labels) @@ -283,6 +308,10 @@ class Issue < ActiveRecord::Base end end + def etag_caching_enabled? + true + end + def discussions_rendered_on_frontend? true end diff --git a/app/models/label.rb b/app/models/label.rb index de7f1d56c64..7bbcaa121ca 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -85,11 +85,16 @@ class Label < ActiveRecord::Base (#{Project.reference_pattern})? #{Regexp.escape(reference_prefix)} (?: - (?<label_id>\d+(?!\S\w)\b) | # Integer-based label ID, or - (?<label_name> - [A-Za-z0-9_\-\?\.&]+ | # String-based single-word label title, or - ".+?" # String-based multi-word label surrounded in quotes - ) + (?<label_id>\d+(?!\S\w)\b) + | # Integer-based label ID, or + (?<label_name> + # String-based single-word label title, or + [A-Za-z0-9_\-\?\.&]+ + (?<!\.|\?) + | + # String-based multi-word label surrounded in quotes + ".+?" + ) ) }x end @@ -137,6 +142,10 @@ class Label < ActiveRecord::Base priority.try(:priority) end + def priority? + priorities.present? + end + def template? template end diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index b444812a4cf..84487031ee5 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -1,15 +1,32 @@ class LfsObject < ActiveRecord::Base + include AfterCommitQueue + include ObjectStorage::BackgroundMove + has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :lfs_objects_projects + scope :with_files_stored_locally, -> { where(file_store: [nil, LfsObjectUploader::Store::LOCAL]) } + validates :oid, presence: true, uniqueness: true mount_uploader :file, LfsObjectUploader + after_save :update_file_store, if: :file_changed? + + def update_file_store + # The file.object_store is set during `uploader.store!` + # which happens after object is inserted/updated + self.update_column(:file_store, file.object_store) + end + def project_allowed_access?(project) projects.exists?(project.lfs_storage_project.id) end + def local_store? + [nil, LfsObjectUploader::Store::LOCAL].include?(self.file_store) + end + def self.destroy_unreferenced joins("LEFT JOIN lfs_objects_projects ON lfs_objects_projects.lfs_object_id = #{table_name}.id") .where(lfs_objects_projects: { id: nil }) diff --git a/app/models/list.rb b/app/models/list.rb index 918275be142..4edcfa78835 100644 --- a/app/models/list.rb +++ b/app/models/list.rb @@ -2,17 +2,27 @@ class List < ActiveRecord::Base belongs_to :board belongs_to :label - enum list_type: { backlog: 0, label: 1, closed: 2 } + enum list_type: { backlog: 0, label: 1, closed: 2, assignee: 3 } validates :board, :list_type, presence: true validates :label, :position, presence: true, if: :label? validates :label_id, uniqueness: { scope: :board_id }, if: :label? - validates :position, numericality: { only_integer: true, greater_than_or_equal_to: 0 }, if: :label? + validates :position, numericality: { only_integer: true, greater_than_or_equal_to: 0 }, if: :movable? before_destroy :can_be_destroyed - scope :destroyable, -> { where(list_type: list_types[:label]) } - scope :movable, -> { where(list_type: list_types[:label]) } + scope :destroyable, -> { where(list_type: list_types.slice(*destroyable_types).values) } + scope :movable, -> { where(list_type: list_types.slice(*movable_types).values) } + + class << self + def destroyable_types + [:label] + end + + def movable_types + [:label] + end + end def destroyable? label? @@ -31,7 +41,8 @@ class List < ActiveRecord::Base if options.key?(:label) json[:label] = label.as_json( project: board.project, - only: [:id, :title, :description, :color] + only: [:id, :title, :description, :color], + methods: [:text_color] ) end end diff --git a/app/models/member.rb b/app/models/member.rb index e1a32148538..68572f2e33a 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -96,7 +96,18 @@ class Member < ActiveRecord::Base joins(:user).merge(User.search(query)) end - def sort(method) + def filter_by_2fa(value) + case value + when 'enabled' + left_join_users.merge(User.with_two_factor_indistinct) + when 'disabled' + left_join_users.merge(User.without_two_factor) + else + all + end + end + + def sort_by_attribute(method) case method.to_s when 'access_level_asc' then reorder(access_level: :asc) when 'access_level_desc' then reorder(access_level: :desc) diff --git a/app/models/members/group_member.rb b/app/models/members/group_member.rb index 661e668dbf9..5da739f9618 100644 --- a/app/models/members/group_member.rb +++ b/app/models/members/group_member.rb @@ -37,20 +37,20 @@ class GroupMember < Member private def send_invite - notification_service.invite_group_member(self, @raw_invite_token) + run_after_commit_or_now { notification_service.invite_group_member(self, @raw_invite_token) } super end def post_create_hook - notification_service.new_group_member(self) + run_after_commit_or_now { notification_service.new_group_member(self) } super end def post_update_hook if access_level_changed? - notification_service.update_group_member(self) + run_after_commit { notification_service.update_group_member(self) } end super diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb index 1c7ed4a96df..024106056b4 100644 --- a/app/models/members/project_member.rb +++ b/app/models/members/project_member.rb @@ -92,7 +92,7 @@ class ProjectMember < Member private def send_invite - notification_service.invite_project_member(self, @raw_invite_token) + run_after_commit_or_now { notification_service.invite_project_member(self, @raw_invite_token) } super end @@ -100,7 +100,7 @@ class ProjectMember < Member def post_create_hook unless owner? event_service.join_project(self.project, self.user) - notification_service.new_project_member(self) + run_after_commit_or_now { notification_service.new_project_member(self) } end super @@ -108,7 +108,7 @@ class ProjectMember < Member def post_update_hook if access_level_changed? - notification_service.update_project_member(self) + run_after_commit { notification_service.update_project_member(self) } end super diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 7e6d89ec9c7..6c96c8ca391 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -1,5 +1,6 @@ class MergeRequest < ActiveRecord::Base - include NonatomicInternalId + include AtomicInternalId + include IidRoutes include Issuable include Noteable include Referable @@ -18,6 +19,8 @@ class MergeRequest < ActiveRecord::Base belongs_to :source_project, class_name: "Project" belongs_to :merge_user, class_name: "User" + has_internal_id :iid, scope: :target_project, init: ->(s) { s&.target_project&.merge_requests&.maximum(:iid) } + has_many :merge_request_diffs has_one :merge_request_diff, @@ -56,6 +59,7 @@ class MergeRequest < ActiveRecord::Base after_create :ensure_merge_request_diff, unless: :importing? after_update :clear_memoized_shas after_update :reload_diff_if_branch_changed + after_save :ensure_metrics # When this attribute is true some MR validation is ignored # It allows us to close or modify broken merge requests @@ -102,24 +106,42 @@ class MergeRequest < ActiveRecord::Base state_machine :merge_status, initial: :unchecked do event :mark_as_unchecked do - transition [:can_be_merged, :cannot_be_merged] => :unchecked + transition [:can_be_merged, :unchecked] => :unchecked + transition [:cannot_be_merged, :cannot_be_merged_recheck] => :cannot_be_merged_recheck end event :mark_as_mergeable do - transition [:unchecked, :cannot_be_merged] => :can_be_merged + transition [:unchecked, :cannot_be_merged_recheck] => :can_be_merged end event :mark_as_unmergeable do - transition [:unchecked, :can_be_merged] => :cannot_be_merged + transition [:unchecked, :cannot_be_merged_recheck] => :cannot_be_merged end state :unchecked + state :cannot_be_merged_recheck state :can_be_merged state :cannot_be_merged around_transition do |merge_request, transition, block| Gitlab::Timeless.timeless(merge_request, &block) end + + after_transition unchecked: :cannot_be_merged do |merge_request, transition| + begin + if merge_request.notify_conflict? + NotificationService.new.merge_request_unmergeable(merge_request) + TodoService.new.merge_request_became_unmergeable(merge_request) + end + rescue Gitlab::Git::CommandError + # Checking mergeability can trigger exception, e.g. non-utf8 + # We ignore this type of errors. + end + end + + def check_state?(merge_status) + [:unchecked, :cannot_be_merged_recheck].include?(merge_status.to_sym) + end end validates :source_project, presence: true, unless: [:allow_broken, :importing?, :closed_without_fork?] @@ -321,10 +343,20 @@ class MergeRequest < ActiveRecord::Base # updates `merge_jid` with the MergeWorker#jid. # This helps tracking enqueued and ongoing merge jobs. def merge_async(user_id, params) - jid = MergeWorker.perform_async(id, user_id, params) + jid = MergeWorker.perform_async(id, user_id, params.to_h) update_column(:merge_jid, jid) end + def merge_participants + participants = [author] + + if merge_when_pipeline_succeeds? && !participants.include?(merge_user) + participants << merge_user + end + + participants + end + def first_commit merge_request_diff ? merge_request_diff.first_commit : compare_commits.first end @@ -344,6 +376,10 @@ class MergeRequest < ActiveRecord::Base end end + def non_latest_diffs + merge_request_diffs.where.not(id: merge_request_diff.id) + end + def diff_size # Calling `merge_request_diff.diffs.real_size` will also perform # highlighting, which we don't need here. @@ -536,18 +572,25 @@ class MergeRequest < ActiveRecord::Base merge_request_diff(true) end + def viewable_diffs + @viewable_diffs ||= merge_request_diffs.viewable.to_a + end + def merge_request_diff_for(diff_refs_or_sha) - @merge_request_diffs_by_diff_refs_or_sha ||= Hash.new do |h, diff_refs_or_sha| - diffs = merge_request_diffs.viewable - h[diff_refs_or_sha] = - if diff_refs_or_sha.is_a?(Gitlab::Diff::DiffRefs) - diffs.find_by_diff_refs(diff_refs_or_sha) - else - diffs.find_by(head_commit_sha: diff_refs_or_sha) - end - end + matcher = + if diff_refs_or_sha.is_a?(Gitlab::Diff::DiffRefs) + { + 'start_commit_sha' => diff_refs_or_sha.start_sha, + 'head_commit_sha' => diff_refs_or_sha.head_sha, + 'base_commit_sha' => diff_refs_or_sha.base_sha + } + else + { 'head_commit_sha' => diff_refs_or_sha } + end - @merge_request_diffs_by_diff_refs_or_sha[diff_refs_or_sha] + viewable_diffs.find do |diff| + diff.attributes.slice(*matcher.keys) == matcher + end end def version_params_for(diff_refs) @@ -578,22 +621,11 @@ class MergeRequest < ActiveRecord::Base def reload_diff(current_user = nil) return unless open? - old_diff_refs = self.diff_refs - new_diff = create_merge_request_diff - - MergeRequests::MergeRequestDiffCacheService.new.execute(self, new_diff) - - new_diff_refs = self.diff_refs - - update_diff_discussion_positions( - old_diff_refs: old_diff_refs, - new_diff_refs: new_diff_refs, - current_user: current_user - ) + MergeRequests::ReloadDiffsService.new(self, current_user).execute end def check_if_can_be_merged - return unless unchecked? && Gitlab::Database.read_write? + return unless self.class.state_machines[:merge_status].check_state?(merge_status) && Gitlab::Database.read_write? can_be_merged = !broken? && project.repository.can_be_merged?(diff_head_sha, target_branch) @@ -674,6 +706,10 @@ class MergeRequest < ActiveRecord::Base should_remove_source_branch? || force_remove_source_branch? end + def notify_conflict? + (opened? || locked?) && !project.repository.can_be_merged?(diff_head_sha, target_branch) + end + def related_notes # Fetch comments only from last 100 commits commits_for_notes_limit = 100 @@ -998,6 +1034,10 @@ class MergeRequest < ActiveRecord::Base @merge_commit ||= project.commit(merge_commit_sha) if merge_commit_sha end + def short_merge_commit_sha + Commit.truncate_sha(merge_commit_sha) if merge_commit_sha + end + def can_be_reverted?(current_user) return false unless merge_commit @@ -1079,6 +1119,10 @@ class MergeRequest < ActiveRecord::Base true end + def discussions_rendered_on_frontend? + true + end + def update_project_counter_caches Projects::OpenMergeRequestsCountService.new(target_project).refresh_cache end @@ -1089,21 +1133,31 @@ class MergeRequest < ActiveRecord::Base project.merge_requests.merged.where(author_id: author_id).empty? end - def allow_maintainer_to_push - maintainer_push_possible? && super + # TODO: remove once production database rename completes + alias_attribute :allow_collaboration, :allow_maintainer_to_push + + def allow_collaboration + collaborative_push_possible? && allow_maintainer_to_push end - alias_method :allow_maintainer_to_push?, :allow_maintainer_to_push + alias_method :allow_collaboration?, :allow_collaboration - def maintainer_push_possible? + def collaborative_push_possible? source_project.present? && for_fork? && target_project.visibility_level > Gitlab::VisibilityLevel::PRIVATE && source_project.visibility_level > Gitlab::VisibilityLevel::PRIVATE && !ProtectedBranch.protected?(source_project, source_branch) end - def can_allow_maintainer_to_push?(user) - maintainer_push_possible? && + def can_allow_collaboration?(user) + collaborative_push_possible? && Ability.allowed?(user, :push_code, source_project) end + + def squash_in_progress? + # The source project can be deleted + return false unless source_project + + source_project.repository.squash_in_progress?(id) + end end diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index c1c27ccf3e5..3d72c447b4b 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -3,6 +3,7 @@ class MergeRequestDiff < ActiveRecord::Base include Importable include ManualInverseAssociation include IgnorableColumn + include EachBatch # Don't display more than 100 commits at once COMMITS_SAFE_SIZE = 100 @@ -17,8 +18,14 @@ class MergeRequestDiff < ActiveRecord::Base has_many :merge_request_diff_commits, -> { order(:merge_request_diff_id, :relative_order) } state_machine :state, initial: :empty do + event :clean do + transition any => :without_files + end + state :collected state :overflow + # Diff files have been deleted by the system + state :without_files # Deprecated states: these are no longer used but these values may still occur # in the database. state :timeout @@ -27,6 +34,7 @@ class MergeRequestDiff < ActiveRecord::Base state :overflow_diff_lines_limit end + scope :with_files, -> { without_states(:without_files, :empty) } scope :viewable, -> { without_state(:empty) } scope :by_commit_sha, ->(sha) do joins(:merge_request_diff_commits).where(merge_request_diff_commits: { sha: sha }).reorder(nil) @@ -42,6 +50,10 @@ class MergeRequestDiff < ActiveRecord::Base find_by(start_commit_sha: diff_refs.start_sha, head_commit_sha: diff_refs.head_sha, base_commit_sha: diff_refs.base_sha) end + def viewable? + collected? || without_files? || overflow? + end + # Collect information about commits and diff from repository # and save it to the database as serialized data def save_git_content @@ -170,6 +182,21 @@ class MergeRequestDiff < ActiveRecord::Base end def diffs(diff_options = nil) + if without_files? && comparison = diff_refs.compare_in(project) + # It should fetch the repository when diffs are cleaned by the system. + # We don't keep these for storage overload purposes. + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/37639 + comparison.diffs(diff_options) + else + diffs_collection(diff_options) + end + end + + # Should always return the DB persisted diffs collection + # (e.g. Gitlab::Diff::FileCollection::MergeRequestDiff. + # It's useful when trying to invalidate old caches through + # FileCollection::MergeRequestDiff#clear_cache! + def diffs_collection(diff_options = nil) Gitlab::Diff::FileCollection::MergeRequestDiff.new(self, diff_options: diff_options) end @@ -197,10 +224,6 @@ class MergeRequestDiff < ActiveRecord::Base CompareService.new(project, head_commit_sha).execute(project, sha, straight: true) end - def commits_count - super || merge_request_diff_commits.size - end - private def create_merge_request_diff_files(diffs) diff --git a/app/models/merge_request_diff_commit.rb b/app/models/merge_request_diff_commit.rb index b75387e236e..1c2e57bb01f 100644 --- a/app/models/merge_request_diff_commit.rb +++ b/app/models/merge_request_diff_commit.rb @@ -17,7 +17,7 @@ class MergeRequestDiffCommit < ActiveRecord::Base commit_hash.merge( merge_request_diff_id: merge_request_diff_id, relative_order: index, - sha: sha_attribute.type_cast_for_database(sha), + sha: sha_attribute.serialize(sha), # rubocop:disable Cop/ActiveRecordSerialize authored_date: Gitlab::Database.sanitize_timestamp(commit_hash[:authored_date]), committed_date: Gitlab::Database.sanitize_timestamp(commit_hash[:committed_date]) ) diff --git a/app/models/merge_request_diff_file.rb b/app/models/merge_request_diff_file.rb index 1199ff5af22..cd8ba6b904d 100644 --- a/app/models/merge_request_diff_file.rb +++ b/app/models/merge_request_diff_file.rb @@ -1,5 +1,6 @@ class MergeRequestDiffFile < ActiveRecord::Base include Gitlab::EncodingHelper + include DiffFile belongs_to :merge_request_diff @@ -12,10 +13,4 @@ class MergeRequestDiffFile < ActiveRecord::Base def diff binary? ? super.unpack('m0').first : super end - - def to_hash - keys = Gitlab::Git::Diff::SERIALIZE_KEYS - [:diff] - - as_json(only: keys).merge(diff: diff).with_indifferent_access - end end diff --git a/app/models/milestone.rb b/app/models/milestone.rb index e7d397f40f5..d05dcfd083a 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -8,7 +8,8 @@ class Milestone < ActiveRecord::Base Started = MilestoneStruct.new('Started', '#started', -3) include CacheMarkdownField - include NonatomicInternalId + include AtomicInternalId + include IidRoutes include Sortable include Referable include StripAttribute @@ -21,6 +22,9 @@ class Milestone < ActiveRecord::Base belongs_to :project belongs_to :group + has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.milestones&.maximum(:iid) } + has_internal_id :iid, scope: :group, init: ->(s) { s&.group&.milestones&.maximum(:iid) } + has_many :issues has_many :labels, -> { distinct.reorder('labels.title') }, through: :issues has_many :merge_requests @@ -34,8 +38,8 @@ class Milestone < ActiveRecord::Base scope :for_projects_and_groups, -> (project_ids, group_ids) do conditions = [] - conditions << arel_table[:project_id].in(project_ids) if project_ids.compact.any? - conditions << arel_table[:group_id].in(group_ids) if group_ids.compact.any? + conditions << arel_table[:project_id].in(project_ids) if project_ids&.compact&.any? + conditions << arel_table[:group_id].in(group_ids) if group_ids&.compact&.any? where(conditions.reduce(:or)) end @@ -138,7 +142,7 @@ class Milestone < ActiveRecord::Base User.joins(assigned_issues: :milestone).where("milestones.id = ?", id).uniq end - def self.sort(method) + def self.sort_by_attribute(method) case method.to_s when 'due_date_asc' reorder(Gitlab::Database.nulls_last_order('due_date', 'ASC')) diff --git a/app/models/namespace.rb b/app/models/namespace.rb index e350b675639..7034c633268 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -21,6 +21,9 @@ class Namespace < ActiveRecord::Base has_many :projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :project_statistics + has_many :runner_namespaces, inverse_of: :namespace, class_name: 'Ci::RunnerNamespace' + has_many :runners, through: :runner_namespaces, source: :runner, class_name: 'Ci::Runner' + # This should _not_ be `inverse_of: :namespace`, because that would also set # `user.namespace` when this user creates a group with themselves as `owner`. belongs_to :owner, class_name: "User" @@ -163,6 +166,13 @@ class Namespace < ActiveRecord::Base projects.with_shared_runners.any? end + # Returns all ancestors, self, and descendants of the current namespace. + def self_and_hierarchy + Gitlab::GroupHierarchy + .new(self.class.where(id: id)) + .all_groups + end + # Returns all the ancestors of the current namespaces. def ancestors return self.class.none unless parent_id @@ -218,6 +228,10 @@ class Namespace < ActiveRecord::Base parent.present? end + def root_ancestor + ancestors.reorder(nil).find_by(parent_id: nil) + end + def subgroup? has_parent? end @@ -248,8 +262,8 @@ class Namespace < ActiveRecord::Base all_projects.with_storage_feature(:repository).find_each(&:remove_exports) end - def features - [] + def refresh_project_authorizations + owner.refresh_authorized_projects end private diff --git a/app/models/note.rb b/app/models/note.rb index 787a80f0196..abc40d9016e 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -63,6 +63,7 @@ class Note < ActiveRecord::Base has_many :todos has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_one :system_note_metadata + has_one :note_diff_file, inverse_of: :diff_note, foreign_key: :diff_note_id delegate :gfm_reference, :local_reference, to: :noteable delegate :name, to: :project, prefix: true @@ -100,7 +101,8 @@ class Note < ActiveRecord::Base scope :inc_author_project, -> { includes(:project, :author) } scope :inc_author, -> { includes(:author) } scope :inc_relations_for_view, -> do - includes(:project, :author, :updated_by, :resolved_by, :award_emoji, :system_note_metadata) + includes(:project, :author, :updated_by, :resolved_by, :award_emoji, + :system_note_metadata, :note_diff_file) end scope :diff_notes, -> { where(type: %w(LegacyDiffNote DiffNote)) } @@ -268,6 +270,10 @@ class Note < ActiveRecord::Base self.special_role = Note::SpecialRole::FIRST_TIME_CONTRIBUTOR end + def confidential? + noteable.try(:confidential?) + end + def editable? !system? end @@ -378,13 +384,17 @@ class Note < ActiveRecord::Base def expire_etag_cache return unless noteable&.discussions_rendered_on_frontend? + return unless noteable&.etag_caching_enabled? + + Gitlab::EtagCaching::Store.new.touch(etag_key) + end - key = Gitlab::Routing.url_helpers.project_noteable_notes_path( + def etag_key + Gitlab::Routing.url_helpers.project_noteable_notes_path( project, target_type: noteable_type.underscore, target_id: noteable_id ) - Gitlab::EtagCaching::Store.new.touch(key) end def touch(*args) @@ -426,6 +436,10 @@ class Note < ActiveRecord::Base super.merge(noteable: noteable) end + def retrieve_upload(_identifier, paths) + Upload.find_by(model: self, path: paths) + end + private def keep_around_commit diff --git a/app/models/note_diff_file.rb b/app/models/note_diff_file.rb new file mode 100644 index 00000000000..e688018a6d9 --- /dev/null +++ b/app/models/note_diff_file.rb @@ -0,0 +1,7 @@ +class NoteDiffFile < ActiveRecord::Base + include DiffFile + + belongs_to :diff_note, inverse_of: :note_diff_file + + validates :diff_note, presence: true +end diff --git a/app/models/notification_recipient.rb b/app/models/notification_recipient.rb index e95655e19f8..1a03dd9df56 100644 --- a/app/models/notification_recipient.rb +++ b/app/models/notification_recipient.rb @@ -1,4 +1,6 @@ class NotificationRecipient + include Gitlab::Utils::StrongMemoize + attr_reader :user, :type, :reason def initialize(user, type, **opts) unless NotificationSetting.levels.key?(type) || type == :subscription @@ -48,7 +50,7 @@ class NotificationRecipient when :custom custom_enabled? || %i[participating mention].include?(@type) when :watch, :participating - !excluded_watcher_action? + !action_excluded? when :mention @type == :mention else @@ -64,7 +66,7 @@ class NotificationRecipient return false unless @target return false unless @target.respond_to?(:subscriptions) - subscription = @target.subscriptions.find_by_user_id(@user.id) + subscription = @target.subscriptions.find { |subscription| subscription.user_id == @user.id } subscription && !subscription.subscribed end @@ -83,26 +85,35 @@ class NotificationRecipient def has_access? DeclarativePolicy.subject_scope do - return false unless user.can?(:receive_notifications) - return true if @skip_read_ability + break false unless user.can?(:receive_notifications) + break true if @skip_read_ability - return false if @target && !user.can?(:read_cross_project) - return false if @project && !user.can?(:read_project, @project) + break false if @target && !user.can?(:read_cross_project) + break false if @project && !user.can?(:read_project, @project) - return true unless read_ability - return true unless DeclarativePolicy.has_policy?(@target) + break true unless read_ability + break true unless DeclarativePolicy.has_policy?(@target) user.can?(read_ability, @target) end end + def action_excluded? + excluded_watcher_action? || excluded_participating_action? + end + def excluded_watcher_action? - return false unless @custom_action - return false if notification_level == :custom + return false unless @custom_action && notification_level == :watch NotificationSetting::EXCLUDED_WATCHER_EVENTS.include?(@custom_action) end + def excluded_participating_action? + return false unless @custom_action && notification_level == :participating + + NotificationSetting::EXCLUDED_PARTICIPATING_EVENTS.include?(@custom_action) + end + private def read_ability @@ -133,10 +144,33 @@ class NotificationRecipient return project_setting unless project_setting.nil? || project_setting.global? - group_setting = @group && user.notification_settings_for(@group) + group_setting = closest_non_global_group_notification_settting - return group_setting unless group_setting.nil? || group_setting.global? + return group_setting unless group_setting.nil? user.global_notification_setting end + + # Returns the notificaton_setting of the lowest group in hierarchy with non global level + def closest_non_global_group_notification_settting + return unless @group + return if indexed_group_notification_settings.empty? + + notification_setting = nil + + @group.self_and_ancestors_ids.each do |id| + notification_setting = indexed_group_notification_settings[id] + break if notification_setting + end + + notification_setting + end + + def indexed_group_notification_settings + strong_memoize(:indexed_group_notification_settings) do + @group.notification_settings.where(user_id: user.id) + .where.not(level: NotificationSetting.levels[:global]) + .index_by(&:source_id) + end + end end diff --git a/app/models/notification_setting.rb b/app/models/notification_setting.rb index 245f8dddcf9..9195408551f 100644 --- a/app/models/notification_setting.rb +++ b/app/models/notification_setting.rb @@ -33,6 +33,7 @@ class NotificationSetting < ActiveRecord::Base :close_issue, :reassign_issue, :new_merge_request, + :push_to_merge_request, :reopen_merge_request, :close_merge_request, :reassign_merge_request, @@ -41,10 +42,15 @@ class NotificationSetting < ActiveRecord::Base :success_pipeline ].freeze - EXCLUDED_WATCHER_EVENTS = [ + EXCLUDED_PARTICIPATING_EVENTS = [ :success_pipeline ].freeze + EXCLUDED_WATCHER_EVENTS = [ + :push_to_merge_request, + :issue_due + ].push(*EXCLUDED_PARTICIPATING_EVENTS).freeze + def self.find_or_create_for(source) setting = find_or_initialize_by(source: source) diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index 588bd50ed77..bfea64c3759 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -6,8 +6,10 @@ class PagesDomain < ActiveRecord::Base validates :domain, hostname: { allow_numeric_hostname: true } validates :domain, uniqueness: { case_sensitive: false } - validates :certificate, certificate: true, allow_nil: true, allow_blank: true - validates :key, certificate_key: true, allow_nil: true, allow_blank: true + validates :certificate, presence: { message: 'must be present if HTTPS-only is enabled' }, if: ->(domain) { domain.project&.pages_https_only? } + validates :certificate, certificate: true, if: ->(domain) { domain.certificate.present? } + validates :key, presence: { message: 'must be present if HTTPS-only is enabled' }, if: ->(domain) { domain.project&.pages_https_only? } + validates :key, certificate_key: true, if: ->(domain) { domain.key.present? } validates :verification_code, presence: true, allow_blank: false validate :validate_pages_domain @@ -17,7 +19,7 @@ class PagesDomain < ActiveRecord::Base attr_encrypted :key, mode: :per_attribute_iv_and_salt, insecure_mode: true, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base, algorithm: 'aes-256-cbc' after_initialize :set_verification_code @@ -46,6 +48,10 @@ class PagesDomain < ActiveRecord::Base !Gitlab::CurrentSettings.pages_domain_verification_enabled? || enabled_until.present? end + def https? + certificate.present? + end + def to_param domain end diff --git a/app/models/personal_snippet.rb b/app/models/personal_snippet.rb index 82c1c4de3a0..355624fd552 100644 --- a/app/models/personal_snippet.rb +++ b/app/models/personal_snippet.rb @@ -1,2 +1,3 @@ class PersonalSnippet < Snippet + include WithUploads end diff --git a/app/models/project.rb b/app/models/project.rb index 250680e2a2c..d91d7dcfe9a 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -21,6 +21,11 @@ class Project < ActiveRecord::Base include Gitlab::SQL::Pattern include DeploymentPlatform include ::Gitlab::Utils::StrongMemoize + include ChronicDurationAttribute + include FastDestroyAll::Helpers + include WithUploads + include BatchDestroyDependentAssociations + extend Gitlab::Cache::RequestCache extend Gitlab::ConfigHelper @@ -63,15 +68,29 @@ class Project < ActiveRecord::Base default_value_for :only_allow_merge_if_all_discussions_are_resolved, false add_authentication_token_field :runners_token + + before_validation :mark_remote_mirrors_for_removal, if: -> { RemoteMirror.table_exists? } + before_save :ensure_runners_token after_save :update_project_statistics, if: :namespace_id_changed? + + after_save :create_import_state, if: ->(project) { project.import? && project.import_state.nil? } + after_create :create_project_feature, unless: :project_feature + + after_create :create_ci_cd_settings, + unless: :ci_cd_settings, + if: proc { ProjectCiCdSetting.available? } + after_create :set_last_activity_at after_create :set_last_repository_updated_at after_update :update_forks_visibility_level before_destroy :remove_private_deploy_keys + + use_fast_destroy :build_trace_chunks + after_destroy -> { run_after_commit { remove_pages } } after_destroy :remove_exports @@ -151,6 +170,8 @@ class Project < ActiveRecord::Base has_one :fork_network_member has_one :fork_network, through: :fork_network_member + has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project + # Merge Requests for target project should be removed with it has_many :merge_requests, foreign_key: 'target_project_id' has_many :source_of_merge_requests, foreign_key: 'source_project_id', class_name: 'MergeRequest' @@ -199,6 +220,7 @@ class Project < ActiveRecord::Base has_one :cluster_project, class_name: 'Clusters::Project' has_many :clusters, through: :cluster_project, class_name: 'Clusters::Cluster' + has_many :cluster_ingresses, through: :clusters, source: :application_ingress, class_name: 'Clusters::Applications::Ingress' # Container repositories need to remove data from the container registry, # which is not managed by the DB. Hence we're still using dependent: :destroy @@ -207,6 +229,7 @@ class Project < ActiveRecord::Base has_many :commit_statuses has_many :pipelines, class_name: 'Ci::Pipeline', inverse_of: :project + has_many :stages, class_name: 'Ci::Stage', inverse_of: :project # Ci::Build objects store data on the file system such as artifact files and # build traces. Currently there's no efficient way of removing this data in @@ -214,30 +237,39 @@ class Project < ActiveRecord::Base # still using `dependent: :destroy` here. has_many :builds, class_name: 'Ci::Build', inverse_of: :project, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :build_trace_section_names, class_name: 'Ci::BuildTraceSectionName' - has_many :runner_projects, class_name: 'Ci::RunnerProject' + has_many :build_trace_chunks, class_name: 'Ci::BuildTraceChunk', through: :builds, source: :trace_chunks + has_many :runner_projects, class_name: 'Ci::RunnerProject', inverse_of: :project has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' has_many :variables, class_name: 'Ci::Variable' has_many :triggers, class_name: 'Ci::Trigger' has_many :environments has_many :deployments has_many :pipeline_schedules, class_name: 'Ci::PipelineSchedule' - - has_many :active_runners, -> { active }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' + has_many :project_deploy_tokens + has_many :deploy_tokens, through: :project_deploy_tokens has_one :auto_devops, class_name: 'ProjectAutoDevops' has_many :custom_attributes, class_name: 'ProjectCustomAttribute' has_many :project_badges, class_name: 'ProjectBadge' + has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + + has_many :remote_mirrors, inverse_of: :project accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature, update_only: true accepts_nested_attributes_for :import_data accepts_nested_attributes_for :auto_devops, update_only: true + accepts_nested_attributes_for :remote_mirrors, + allow_destroy: true, + reject_if: ->(attrs) { attrs[:id].blank? && attrs[:url].blank? } + delegate :name, to: :owner, allow_nil: true, prefix: true delegate :members, to: :team, prefix: true delegate :add_user, :add_users, to: :team delegate :add_guest, :add_reporter, :add_developer, :add_master, :add_role, to: :team + delegate :group_runners_enabled, :group_runners_enabled=, :group_runners_enabled?, to: :ci_cd_settings # Validations validates :creator, presence: true, on: :create @@ -259,21 +291,22 @@ class Project < ActiveRecord::Base validates :namespace, presence: true validates :name, uniqueness: { scope: :namespace_id } - validates :import_url, addressable_url: true, if: :external_import? - validates :import_url, importable_url: true, if: [:external_import?, :import_url_changed?] + validates :import_url, url: { protocols: %w(http https ssh git), + allow_localhost: false, + enforce_user: true, + ports: VALID_IMPORT_PORTS }, if: [:external_import?, :import_url_changed?] validates :star_count, numericality: { greater_than_or_equal_to: 0 } validate :check_limit, on: :create validate :check_repository_path_availability, on: :update, if: ->(project) { project.renamed? } validate :visibility_level_allowed_by_group validate :visibility_level_allowed_as_fork validate :check_wiki_path_conflict + validate :validate_pages_https_only, if: -> { changes.has_key?(:pages_https_only) } validates :repository_storage, presence: true, inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } } validates :variables, variable_duplicates: { scope: :environment_scope } - has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - # Scopes scope :pending_delete, -> { where(pending_delete: true) } scope :without_deleted, -> { where(pending_delete: false) } @@ -321,9 +354,21 @@ class Project < ActiveRecord::Base scope :with_issues_enabled, -> { with_feature_enabled(:issues) } scope :with_issues_available_for_user, ->(current_user) { with_feature_available_for_user(:issues, current_user) } scope :with_merge_requests_enabled, -> { with_feature_enabled(:merge_requests) } + scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct } + + scope :with_group_runners_enabled, -> do + joins(:ci_cd_settings) + .where(project_ci_cd_settings: { group_runners_enabled: true }) + end enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 } + chronic_duration_attr :build_timeout_human_readable, :build_timeout, default: 3600 + + validates :build_timeout, allow_nil: true, + numericality: { greater_than_or_equal_to: 600, + message: 'needs to be at least 10 minutes' } + # Returns a collection of projects that is either public or visible to the # logged in user. def self.public_or_visible_to_user(user = nil) @@ -365,55 +410,9 @@ class Project < ActiveRecord::Base scope :abandoned, -> { where('projects.last_activity_at < ?', 6.months.ago) } scope :excluding_project, ->(project) { where.not(id: project) } - scope :import_started, -> { where(import_status: 'started') } - - state_machine :import_status, initial: :none do - event :import_schedule do - transition [:none, :finished, :failed] => :scheduled - end - - event :force_import_start do - transition [:none, :finished, :failed] => :started - end - event :import_start do - transition scheduled: :started - end - - event :import_finish do - transition started: :finished - end - - event :import_fail do - transition [:scheduled, :started] => :failed - end - - event :import_retry do - transition failed: :started - end - - state :scheduled - state :started - state :finished - state :failed - - after_transition [:none, :finished, :failed] => :scheduled do |project, _| - project.run_after_commit do - job_id = add_import_job - update(import_jid: job_id) if job_id - end - end - - after_transition started: :finished do |project, _| - project.reset_cache_and_import_attrs - - if Gitlab::ImportSources.importer_names.include?(project.import_type) && project.repo_exists? - project.run_after_commit do - Projects::AfterImportService.new(project).execute - end - end - end - end + scope :joins_import_state, -> { joins("LEFT JOIN project_mirror_data import_state ON import_state.project_id = projects.id") } + scope :import_started, -> { joins_import_state.where("import_state.status = 'started' OR projects.import_status = 'started'") } class << self # Searches for a list of projects based on the query given in `query`. @@ -435,7 +434,7 @@ class Project < ActiveRecord::Base Gitlab::VisibilityLevel.options end - def sort(method) + def sort_by_attribute(method) case method.to_s when 'storage_size_desc' # storage_size is a joined column so we need to @@ -502,10 +501,6 @@ class Project < ActiveRecord::Base repository.empty? end - def repository_storage_path - Gitlab.config.repositories.storages[repository_storage].try(:[], 'path') - end - def team @team ||= ProjectTeam.new(self) end @@ -565,9 +560,7 @@ class Project < ActiveRecord::Base def add_import_job job_id = if forked? - RepositoryForkWorker.perform_async(id, - forked_from_project.repository_storage_path, - forked_from_project.disk_path) + RepositoryForkWorker.perform_async(id) elsif gitlab_project_import? # Do not retry on Import/Export until https://gitlab.com/gitlab-org/gitlab-ce/issues/26189 is solved. RepositoryImportWorker.set(retry: false).perform_async(self.id) @@ -631,7 +624,7 @@ class Project < ActiveRecord::Base end def create_or_update_import_data(data: nil, credentials: nil) - return unless import_url.present? && valid_import_url? + return if data.nil? && credentials.nil? project_import_data = import_data || build_import_data if data @@ -649,10 +642,6 @@ class Project < ActiveRecord::Base external_import? || forked? || gitlab_project_import? || bare_repository_import? end - def no_import? - import_status == 'none' - end - def external_import? import_url.present? end @@ -665,6 +654,105 @@ class Project < ActiveRecord::Base import_started? || import_scheduled? end + def import_state_args + { + status: self[:import_status], + jid: self[:import_jid], + last_error: self[:import_error] + } + end + + def ensure_import_state(force: false) + return if !force && (self[:import_status] == 'none' || self[:import_status].nil?) + return unless import_state.nil? + + if persisted? + create_import_state(import_state_args) + + update_column(:import_status, 'none') + else + build_import_state(import_state_args) + + self[:import_status] = 'none' + end + end + + def human_import_status_name + ensure_import_state + + import_state.human_status_name + end + + def import_schedule + ensure_import_state(force: true) + + import_state.schedule + end + + def force_import_start + ensure_import_state(force: true) + + import_state.force_start + end + + def import_start + ensure_import_state(force: true) + + import_state.start + end + + def import_fail + ensure_import_state(force: true) + + import_state.fail_op + end + + def import_finish + ensure_import_state(force: true) + + import_state.finish + end + + def import_jid=(new_jid) + ensure_import_state(force: true) + + import_state.jid = new_jid + end + + def import_jid + ensure_import_state + + import_state&.jid + end + + def import_error=(new_error) + ensure_import_state(force: true) + + import_state.last_error = new_error + end + + def import_error + ensure_import_state + + import_state&.last_error + end + + def import_status=(new_status) + ensure_import_state(force: true) + + import_state.status = new_status + end + + def import_status + ensure_import_state + + import_state&.status || 'none' + end + + def no_import? + import_status == 'none' + end + def import_started? # import? does SQL work so only run it if it looks like there's an import running import_status == 'started' && import? @@ -698,6 +786,37 @@ class Project < ActiveRecord::Base import_type == 'gitea' end + def has_remote_mirror? + remote_mirror_available? && remote_mirrors.enabled.exists? + end + + def updating_remote_mirror? + remote_mirrors.enabled.started.exists? + end + + def update_remote_mirrors + return unless remote_mirror_available? + + remote_mirrors.enabled.each(&:sync) + end + + def mark_stuck_remote_mirrors_as_failed! + remote_mirrors.stuck.update_all( + update_status: :failed, + last_error: 'The remote mirror took to long to complete.', + last_update_at: Time.now + ) + end + + def mark_remote_mirrors_for_removal + remote_mirrors.each(&:mark_for_delete_if_blank_url) + end + + def remote_mirror_available? + remote_mirror_available_overridden || + ::Gitlab::CurrentSettings.mirror_available + end + def check_limit unless creator.can_create_project? || namespace.kind == 'group' projects_limit = creator.projects_limit @@ -737,6 +856,26 @@ class Project < ActiveRecord::Base end end + def pages_https_only + return false unless Gitlab.config.pages.external_https + + super + end + + def pages_https_only? + return false unless Gitlab.config.pages.external_https + + super + end + + def validate_pages_https_only + return unless pages_https_only? + + unless pages_domains.all?(&:https?) + errors.add(:pages_https_only, "cannot be enabled unless all domains have TLS certificates") + end + end + def to_param if persisted? && errors.include?(:path) path_was @@ -766,6 +905,13 @@ class Project < ActiveRecord::Base Gitlab::Routing.url_helpers.project_url(self) end + def readme_url + readme = repository.readme + if readme + Gitlab::Routing.url_helpers.project_blob_url(self, File.join(default_branch, readme.path)) + end + end + def new_issuable_address(author, address_type) return unless Gitlab::IncomingEmail.supports_issue_creation? && author @@ -869,7 +1015,7 @@ class Project < ActiveRecord::Base available_services_names = Service.available_services_names - exceptions - available_services_names.map do |service_name| + available_services = available_services_names.map do |service_name| service = find_service(services, service_name) if service @@ -886,6 +1032,14 @@ class Project < ActiveRecord::Base end end end + + available_services.reject do |service| + disabled_services.include?(service.to_param) + end + end + + def disabled_services + [] end def find_or_initialize_service(name) @@ -1013,13 +1167,6 @@ class Project < ActiveRecord::Base "#{web_url}.git" end - def user_can_push_to_empty_repo?(user) - return false unless empty_repo? - return false unless Ability.allowed?(user, :push_code, self) - - !ProtectedBranch.default_branch_protected? || team.max_member_access(user.id) > Gitlab::Access::DEVELOPER - end - def forked? return true if fork_network && fork_network.root_project != self @@ -1047,6 +1194,16 @@ class Project < ActiveRecord::Base end end + # This will return all `lfs_objects` that are accessible to the project. + # So this might be `self.lfs_objects` if the project is not part of a fork + # network, or it is the base of the fork network. + # + # TODO: refactor this to get the correct lfs objects when implementing + # https://gitlab.com/gitlab-org/gitlab-ce/issues/39769 + def all_lfs_objects + lfs_storage_project.lfs_objects + end + def personal? !group end @@ -1068,7 +1225,7 @@ class Project < ActiveRecord::Base # Check if repository already exists on disk def check_repository_path_availability return true if skip_disk_validation - return false unless repository_storage_path + return false unless repository_storage expires_full_path_cache # we need to clear cache to validate renames correctly @@ -1268,28 +1425,31 @@ class Project < ActiveRecord::Base @shared_runners ||= shared_runners_available? ? Ci::Runner.shared : Ci::Runner.none end - def active_shared_runners - @active_shared_runners ||= shared_runners.active + def group_runners + @group_runners ||= group_runners_enabled? ? Ci::Runner.belonging_to_parent_group_of_project(self.id) : Ci::Runner.none end - def any_runners?(&block) - active_runners.any?(&block) || active_shared_runners.any?(&block) + def all_runners + union = Gitlab::SQL::Union.new([runners, group_runners, shared_runners]) + Ci::Runner.from("(#{union.to_sql}) ci_runners") end - def valid_runners_token?(token) - self.runners_token && ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.runners_token) + def active_runners + strong_memoize(:active_runners) do + all_runners.active + end end - def build_timeout_in_minutes - build_timeout / 60 + def any_runners?(&block) + active_runners.any?(&block) end - def build_timeout_in_minutes=(value) - self.build_timeout = value.to_i * 60 + def valid_runners_token?(token) + self.runners_token && ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.runners_token) end - def open_issues_count - Projects::OpenIssuesCountService.new(self).count + def open_issues_count(current_user = nil) + Projects::OpenIssuesCountService.new(self, current_user).count end def open_merge_requests_count @@ -1325,20 +1485,19 @@ class Project < ActiveRecord::Base Dir.exist?(public_pages_path) end - def pages_url - subdomain, _, url_path = full_path.partition('/') - - # The hostname always needs to be in downcased - # All web servers convert hostname to lowercase - host = "#{subdomain}.#{Settings.pages.host}".downcase - + def pages_group_url # The host in URL always needs to be downcased - url = Gitlab.config.pages.url.sub(%r{^https?://}) do |prefix| - "#{prefix}#{subdomain}." + Gitlab.config.pages.url.sub(%r{^https?://}) do |prefix| + "#{prefix}#{pages_subdomain}." end.downcase + end + + def pages_url + url = pages_group_url + url_path = full_path.partition('/').last # If the project path is the same as host, we serve it as group page - return url if host == url_path + return url if url == "#{Settings.pages.protocol}://#{url_path}" "#{url}/#{url_path}" end @@ -1445,7 +1604,9 @@ class Project < ActiveRecord::Base end def rename_repo_notify! - send_move_instructions(full_path_was) + # When we import a project overwriting the original project, there + # is a move operation. In that case we don't want to send the instructions. + send_move_instructions(full_path_was) unless import_started? expires_full_path_cache self.old_path_with_namespace = full_path_was @@ -1456,10 +1617,12 @@ class Project < ActiveRecord::Base def after_import repository.after_import + wiki.repository.after_import import_finish remove_import_jid update_project_counter_caches after_create_default_branch + refresh_markdown_cache! end def update_project_counter_caches @@ -1498,13 +1661,8 @@ class Project < ActiveRecord::Base return unless import_jid Gitlab::SidekiqStatus.unset(import_jid) - update_column(:import_jid, nil) - end - def running_or_pending_build_count(force: false) - Rails.cache.fetch(['projects', id, 'running_or_pending_build_count'], force: force) do - builds.running_or_pending.count(:all) - end + import_state.update_column(:jid, nil) end # Lazy loading of the `pipeline_status` attribute @@ -1517,15 +1675,16 @@ class Project < ActiveRecord::Base sanitized_message = Gitlab::UrlSanitizer.sanitize(error_message) import_fail - update_column(:import_error, sanitized_message) + + import_state.update_column(:last_error, sanitized_message) rescue ActiveRecord::ActiveRecordError => e Rails.logger.error("Error setting import status to failed: #{e.message}. Original error: #{sanitized_message}") ensure @errors = original_errors end - def add_export_job(current_user:, params: {}) - job_id = ProjectExportWorker.perform_async(current_user.id, self.id, params) + def add_export_job(current_user:, after_export_strategy: nil, params: {}) + job_id = ProjectExportWorker.perform_async(current_user.id, self.id, after_export_strategy, params) if job_id Rails.logger.info "Export job started for project ID #{self.id} with job ID #{job_id}" @@ -1551,6 +1710,8 @@ class Project < ActiveRecord::Base def export_status if export_in_progress? :started + elsif after_export_in_progress? + :after_export_action elsif export_project_path :finished else @@ -1562,12 +1723,22 @@ class Project < ActiveRecord::Base import_export_shared.active_export_count > 0 end + def after_export_in_progress? + import_export_shared.after_export_in_progress? + end + def remove_exports return nil unless export_path.present? FileUtils.rm_rf(export_path) end + def remove_exported_project_file + return unless export_project_path.present? + + FileUtils.rm_f(export_project_path) + end + def full_path_slug Gitlab::Utils.slugify(full_path.to_s) end @@ -1593,7 +1764,7 @@ class Project < ActiveRecord::Base def container_registry_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| - return variables unless Gitlab.config.registry.enabled + break variables unless Gitlab.config.registry.enabled variables.append(key: 'CI_REGISTRY', value: Gitlab.config.registry.host_port) @@ -1813,24 +1984,45 @@ class Project < ActiveRecord::Base .limit(1) .select(1) source_of_merge_requests.opened - .where(allow_maintainer_to_push: true) + .where(allow_collaboration: true) .where('EXISTS (?)', developer_access_exists) end - def branch_allows_maintainer_push?(user, branch_name) + def branch_allows_collaboration?(user, branch_name) return false unless user cache_key = "user:#{user.id}:#{branch_name}:branch_allows_push" - memoized_results = strong_memoize(:branch_allows_maintainer_push) do + memoized_results = strong_memoize(:branch_allows_collaboration) do Hash.new do |result, cache_key| - result[cache_key] = fetch_branch_allows_maintainer_push?(user, branch_name) + result[cache_key] = fetch_branch_allows_collaboration?(user, branch_name) end end memoized_results[cache_key] end + def licensed_features + [] + end + + def toggle_ci_cd_settings!(settings_attribute) + ci_cd_settings.toggle!(settings_attribute) + end + + def gitlab_deploy_token + @gitlab_deploy_token ||= deploy_tokens.gitlab_deploy_token + end + + def any_lfs_file_locks? + lfs_file_locks.any? + end + request_cache(:any_lfs_file_locks?) { self.id } + + def auto_cancel_pending_pipelines? + auto_cancel_pending_pipelines == 'enabled' + end + private def storage @@ -1859,14 +2051,14 @@ class Project < ActiveRecord::Base def check_repository_absence! return if skip_disk_validation - if repository_storage_path.blank? || repository_with_same_path_already_exists? + if repository_storage.blank? || repository_with_same_path_already_exists? errors.add(:base, 'There is already a repository with that name on disk') throw :abort end end def repository_with_same_path_already_exists? - gitlab_shell.exists?(repository_storage_path, "#{disk_path}.git") + gitlab_shell.exists?(repository_storage, "#{disk_path}.git") end # set last_activity_at to the same as created_at @@ -1954,17 +2146,22 @@ class Project < ActiveRecord::Base raise ex end - def fetch_branch_allows_maintainer_push?(user, branch_name) + def fetch_branch_allows_collaboration?(user, branch_name) check_access = -> do - merge_request = source_of_merge_requests.opened - .where(allow_maintainer_to_push: true) - .find_by(source_branch: branch_name) + next false if empty_repo? + + merge_requests = source_of_merge_requests.opened + .where(allow_collaboration: true) - merge_request&.can_be_merged_by?(user) + if branch_name + merge_requests.find_by(source_branch: branch_name)&.can_be_merged_by?(user) + else + merge_requests.any? { |merge_request| merge_request.can_be_merged_by?(user) } + end end if RequestStore.active? - RequestStore.fetch("project-#{id}:branch-#{branch_name}:user-#{user.id}:branch_allows_maintainer_push") do + RequestStore.fetch("project-#{id}:branch-#{branch_name}:user-#{user.id}:branch_allows_collaboration") do check_access.call end else diff --git a/app/models/project_auto_devops.rb b/app/models/project_auto_devops.rb index ed6c1eddbc1..faa831b1949 100644 --- a/app/models/project_auto_devops.rb +++ b/app/models/project_auto_devops.rb @@ -1,11 +1,18 @@ class ProjectAutoDevops < ActiveRecord::Base belongs_to :project + enum deploy_strategy: { + continuous: 0, + manual: 1 + } + scope :enabled, -> { where(enabled: true) } scope :disabled, -> { where(enabled: false) } validates :domain, allow_blank: true, hostname: { allow_numeric_hostname: true } + after_save :create_gitlab_deploy_token, if: :needs_to_create_deploy_token? + def instance_domain Gitlab::CurrentSettings.auto_devops_domain end @@ -20,6 +27,30 @@ class ProjectAutoDevops < ActiveRecord::Base variables.append(key: 'AUTO_DEVOPS_DOMAIN', value: domain.presence || instance_domain) end + + if manual? + variables.append(key: 'STAGING_ENABLED', value: '1') + variables.append(key: 'INCREMENTAL_ROLLOUT_ENABLED', value: '1') + end end end + + private + + def create_gitlab_deploy_token + project.deploy_tokens.create!( + name: DeployToken::GITLAB_DEPLOY_TOKEN_NAME, + read_registry: true + ) + end + + def needs_to_create_deploy_token? + auto_devops_enabled? && + !project.public? && + !project.deploy_tokens.find_by(name: DeployToken::GITLAB_DEPLOY_TOKEN_NAME).present? + end + + def auto_devops_enabled? + Gitlab::CurrentSettings.auto_devops_enabled? || enabled? + end end diff --git a/app/models/project_ci_cd_setting.rb b/app/models/project_ci_cd_setting.rb new file mode 100644 index 00000000000..588cced5781 --- /dev/null +++ b/app/models/project_ci_cd_setting.rb @@ -0,0 +1,16 @@ +class ProjectCiCdSetting < ActiveRecord::Base + belongs_to :project, inverse_of: :ci_cd_settings + + # The version of the schema that first introduced this model/table. + MINIMUM_SCHEMA_VERSION = 20180403035759 + + def self.available? + @available ||= + ActiveRecord::Migrator.current_version >= MINIMUM_SCHEMA_VERSION + end + + def self.reset_column_information + @available = nil + super + end +end diff --git a/app/models/project_deploy_token.rb b/app/models/project_deploy_token.rb new file mode 100644 index 00000000000..ab4482f0c0b --- /dev/null +++ b/app/models/project_deploy_token.rb @@ -0,0 +1,8 @@ +class ProjectDeployToken < ActiveRecord::Base + belongs_to :project + belongs_to :deploy_token, inverse_of: :project_deploy_tokens + + validates :deploy_token, presence: true + validates :project, presence: true + validates :deploy_token_id, uniqueness: { scope: [:project_id] } +end diff --git a/app/models/project_import_data.rb b/app/models/project_import_data.rb index 6da6632f4f2..1d7089ccfc7 100644 --- a/app/models/project_import_data.rb +++ b/app/models/project_import_data.rb @@ -3,7 +3,7 @@ require 'carrierwave/orm/activerecord' class ProjectImportData < ActiveRecord::Base belongs_to :project, inverse_of: :import_data attr_encrypted :credentials, - key: Gitlab::Application.secrets.db_key_base, + key: Settings.attr_encrypted_db_key_base, marshal: true, encode: true, mode: :per_attribute_iv_and_salt, diff --git a/app/models/project_import_state.rb b/app/models/project_import_state.rb new file mode 100644 index 00000000000..1605317ae14 --- /dev/null +++ b/app/models/project_import_state.rb @@ -0,0 +1,55 @@ +class ProjectImportState < ActiveRecord::Base + include AfterCommitQueue + + self.table_name = "project_mirror_data" + + belongs_to :project, inverse_of: :import_state + + validates :project, presence: true + + state_machine :status, initial: :none do + event :schedule do + transition [:none, :finished, :failed] => :scheduled + end + + event :force_start do + transition [:none, :finished, :failed] => :started + end + + event :start do + transition scheduled: :started + end + + event :finish do + transition started: :finished + end + + event :fail_op do + transition [:scheduled, :started] => :failed + end + + state :scheduled + state :started + state :finished + state :failed + + after_transition [:none, :finished, :failed] => :scheduled do |state, _| + state.run_after_commit do + job_id = project.add_import_job + update(jid: job_id) if job_id + end + end + + after_transition started: :finished do |state, _| + project = state.project + + project.reset_cache_and_import_attrs + + if Gitlab::ImportSources.importer_names.include?(project.import_type) && project.repo_exists? + state.run_after_commit do + Projects::AfterImportService.new(project).execute + end + end + end + end +end diff --git a/app/models/project_services/bamboo_service.rb b/app/models/project_services/bamboo_service.rb index 54e4b3278db..7f4c47a6d14 100644 --- a/app/models/project_services/bamboo_service.rb +++ b/app/models/project_services/bamboo_service.rb @@ -3,7 +3,7 @@ class BambooService < CiService prop_accessor :bamboo_url, :build_key, :username, :password - validates :bamboo_url, presence: true, url: true, if: :activated? + validates :bamboo_url, presence: true, public_url: true, if: :activated? validates :build_key, presence: true, if: :activated? validates :username, presence: true, diff --git a/app/models/project_services/bugzilla_service.rb b/app/models/project_services/bugzilla_service.rb index 046e2809f45..e4e3a80976b 100644 --- a/app/models/project_services/bugzilla_service.rb +++ b/app/models/project_services/bugzilla_service.rb @@ -1,5 +1,5 @@ class BugzillaService < IssueTrackerService - validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated? + validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url diff --git a/app/models/project_services/buildkite_service.rb b/app/models/project_services/buildkite_service.rb index d2aaff8817a..35884c4560c 100644 --- a/app/models/project_services/buildkite_service.rb +++ b/app/models/project_services/buildkite_service.rb @@ -8,7 +8,7 @@ class BuildkiteService < CiService prop_accessor :project_url, :token boolean_accessor :enable_ssl_verification - validates :project_url, presence: true, url: true, if: :activated? + validates :project_url, presence: true, public_url: true, if: :activated? validates :token, presence: true, if: :activated? after_save :compose_service_hook, if: :activated? diff --git a/app/models/project_services/chat_message/base_message.rb b/app/models/project_services/chat_message/base_message.rb index 22a65b5145e..f710fa85b5d 100644 --- a/app/models/project_services/chat_message/base_message.rb +++ b/app/models/project_services/chat_message/base_message.rb @@ -26,13 +26,18 @@ module ChatMessage end end - def pretext + def summary return message if markdown format(message) end + def pretext + summary + end + def fallback + format(message) end def attachments diff --git a/app/models/project_services/chat_message/pipeline_message.rb b/app/models/project_services/chat_message/pipeline_message.rb index 2135122278a..96fd23aede3 100644 --- a/app/models/project_services/chat_message/pipeline_message.rb +++ b/app/models/project_services/chat_message/pipeline_message.rb @@ -23,10 +23,6 @@ module ChatMessage '' end - def fallback - format(message) - end - def attachments return message if markdown diff --git a/app/models/project_services/chat_notification_service.rb b/app/models/project_services/chat_notification_service.rb index dab0ea1a681..a60b4c7fd0d 100644 --- a/app/models/project_services/chat_notification_service.rb +++ b/app/models/project_services/chat_notification_service.rb @@ -8,7 +8,7 @@ class ChatNotificationService < Service prop_accessor :webhook, :username, :channel boolean_accessor :notify_only_broken_pipelines, :notify_only_default_branch - validates :webhook, presence: true, url: true, if: :activated? + validates :webhook, presence: true, public_url: true, if: :activated? def initialize_properties # Custom serialized properties initialization @@ -21,8 +21,16 @@ class ChatNotificationService < Service end end + def confidential_issue_channel + properties['confidential_issue_channel'].presence || properties['issue_channel'] + end + + def confidential_note_channel + properties['confidential_note_channel'].presence || properties['note_channel'] + end + def self.supported_events - %w[push issue confidential_issue merge_request note tag_push + %w[push issue confidential_issue merge_request note confidential_note tag_push pipeline wiki_page] end @@ -55,7 +63,9 @@ class ChatNotificationService < Service return false unless message - channel_name = get_channel_field(object_kind).presence || channel + event_type = data[:event_type] || object_kind + + channel_name = get_channel_field(event_type).presence || channel opts = {} opts[:channel] = channel_name if channel_name @@ -145,6 +155,7 @@ class ChatNotificationService < Service end def notify_for_ref?(data) + return true if data[:object_kind] == 'tag_push' return true if data.dig(:object_attributes, :tag) return true unless notify_only_default_branch? diff --git a/app/models/project_services/custom_issue_tracker_service.rb b/app/models/project_services/custom_issue_tracker_service.rb index b9e3e982b64..456c7f5cee2 100644 --- a/app/models/project_services/custom_issue_tracker_service.rb +++ b/app/models/project_services/custom_issue_tracker_service.rb @@ -1,5 +1,5 @@ class CustomIssueTrackerService < IssueTrackerService - validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated? + validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url diff --git a/app/models/project_services/drone_ci_service.rb b/app/models/project_services/drone_ci_service.rb index 71b10fc6bc1..ab4e46da89f 100644 --- a/app/models/project_services/drone_ci_service.rb +++ b/app/models/project_services/drone_ci_service.rb @@ -4,7 +4,7 @@ class DroneCiService < CiService prop_accessor :drone_url, :token boolean_accessor :enable_ssl_verification - validates :drone_url, presence: true, url: true, if: :activated? + validates :drone_url, presence: true, public_url: true, if: :activated? validates :token, presence: true, if: :activated? after_save :compose_service_hook, if: :activated? @@ -115,6 +115,6 @@ class DroneCiService < CiService def merge_request_valid?(data) data[:object_attributes][:state] == 'opened' && - data[:object_attributes][:merge_status] == 'unchecked' + MergeRequest.state_machines[:merge_status].check_state?(data[:object_attributes][:merge_status]) end end diff --git a/app/models/project_services/external_wiki_service.rb b/app/models/project_services/external_wiki_service.rb index 1553f169827..a4b1ef09e93 100644 --- a/app/models/project_services/external_wiki_service.rb +++ b/app/models/project_services/external_wiki_service.rb @@ -1,7 +1,7 @@ class ExternalWikiService < Service prop_accessor :external_wiki_url - validates :external_wiki_url, presence: true, url: true, if: :activated? + validates :external_wiki_url, presence: true, public_url: true, if: :activated? def title 'External Wiki' diff --git a/app/models/project_services/flowdock_service.rb b/app/models/project_services/flowdock_service.rb index 4d23a17a545..da01ac1b7cf 100644 --- a/app/models/project_services/flowdock_service.rb +++ b/app/models/project_services/flowdock_service.rb @@ -1,5 +1,51 @@ require "flowdock-git-hook" +# Flow dock depends on Grit to compute the number of commits between two given +# commits. To make this depend on Gitaly, a monkey patch is applied +module Flowdock + class Git + # pass down a Repository all the way down + def repo + @options[:repo] + end + + def config + {} + end + + def messages + Git::Builder.new(repo: repo, + ref: @ref, + before: @from, + after: @to, + commit_url: @commit_url, + branch_url: @branch_url, + diff_url: @diff_url, + repo_url: @repo_url, + repo_name: @repo_name, + permanent_refs: @permanent_refs, + tags: tags + ).to_hashes + end + + class Builder + def commits + @repo.commits_between(@before, @after).map do |commit| + { + url: @opts[:commit_url] ? @opts[:commit_url] % [commit.sha] : nil, + id: commit.sha, + message: commit.message, + author: { + name: commit.author_name, + email: commit.author_email + } + } + end + end + end + end +end + class FlowdockService < Service prop_accessor :token validates :token, presence: true, if: :activated? @@ -34,7 +80,7 @@ class FlowdockService < Service data[:before], data[:after], token: token, - repo: project.repository.path_to_repo, + repo: project.repository, repo_url: "#{Gitlab.config.gitlab.url}/#{project.full_path}", commit_url: "#{Gitlab.config.gitlab.url}/#{project.full_path}/commit/%s", diff_url: "#{Gitlab.config.gitlab.url}/#{project.full_path}/compare/%s...%s" diff --git a/app/models/project_services/gemnasium_service.rb b/app/models/project_services/gemnasium_service.rb index 017a9b2df6e..8a6b0ed1a5f 100644 --- a/app/models/project_services/gemnasium_service.rb +++ b/app/models/project_services/gemnasium_service.rb @@ -3,6 +3,7 @@ require "gemnasium/gitlab_service" class GemnasiumService < Service prop_accessor :token, :api_key validates :token, :api_key, presence: true, if: :activated? + validate :deprecation_validation def title 'Gemnasium' @@ -27,16 +28,33 @@ class GemnasiumService < Service %w(push) end + def deprecated? + true + end + + def deprecation_message + "Gemnasium has been acquired by GitLab in January 2018. Since May 15, 2018, the service provided by Gemnasium is no longer available." + end + + def deprecation_validation + errors[:base] << deprecation_message + end + def execute(data) return unless supported_events.include?(data[:object_kind]) + # Gitaly: this class will be removed https://gitlab.com/gitlab-org/gitlab-ee/issues/6010 + repo_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + project.repository.path_to_repo + end + Gemnasium::GitlabService.execute( ref: data[:ref], before: data[:before], after: data[:after], token: token, api_key: api_key, - repo: project.repository.path_to_repo + repo: repo_path ) end end diff --git a/app/models/project_services/gitlab_issue_tracker_service.rb b/app/models/project_services/gitlab_issue_tracker_service.rb index 88c428b4aae..16e32a4139e 100644 --- a/app/models/project_services/gitlab_issue_tracker_service.rb +++ b/app/models/project_services/gitlab_issue_tracker_service.rb @@ -1,7 +1,7 @@ class GitlabIssueTrackerService < IssueTrackerService include Gitlab::Routing - validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated? + validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url diff --git a/app/models/project_services/hipchat_service.rb b/app/models/project_services/hipchat_service.rb index f31c3f02af2..dce878e485f 100644 --- a/app/models/project_services/hipchat_service.rb +++ b/app/models/project_services/hipchat_service.rb @@ -46,7 +46,7 @@ class HipchatService < Service end def self.supported_events - %w(push issue confidential_issue merge_request note tag_push pipeline) + %w(push issue confidential_issue merge_request note confidential_note tag_push pipeline) end def execute(data) diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index ed4bbfb6cfc..412d62388f0 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -3,8 +3,8 @@ class JiraService < IssueTrackerService include ApplicationHelper include ActionView::Helpers::AssetUrlHelper - validates :url, url: true, presence: true, if: :activated? - validates :api_url, url: true, allow_blank: true + validates :url, public_url: true, presence: true, if: :activated? + validates :api_url, public_url: true, allow_blank: true validates :username, presence: true, if: :activated? validates :password, presence: true, if: :activated? @@ -265,7 +265,7 @@ class JiraService < IssueTrackerService title: title, status: status, icon: { - title: 'GitLab', url16x16: asset_url('favicon.ico', host: gitlab_config.url) + title: 'GitLab', url16x16: asset_url(Gitlab::Favicon.main, host: gitlab_config.url) } } } diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb index 20fed432e55..ddd4026019b 100644 --- a/app/models/project_services/kubernetes_service.rb +++ b/app/models/project_services/kubernetes_service.rb @@ -24,7 +24,7 @@ class KubernetesService < DeploymentService prop_accessor :ca_pem with_options presence: true, if: :activated? do - validates :api_url, url: true + validates :api_url, public_url: true validates :token end diff --git a/app/models/project_services/microsoft_teams_service.rb b/app/models/project_services/microsoft_teams_service.rb index 2facff53e26..99500caec0e 100644 --- a/app/models/project_services/microsoft_teams_service.rb +++ b/app/models/project_services/microsoft_teams_service.rb @@ -44,7 +44,7 @@ class MicrosoftTeamsService < ChatNotificationService def notify(message, opts) MicrosoftTeams::Notifier.new(webhook).ping( title: message.project_name, - pretext: message.pretext, + summary: message.summary, activity: message.activity, attachments: message.attachments ) diff --git a/app/models/project_services/mock_ci_service.rb b/app/models/project_services/mock_ci_service.rb index 2221459c90b..b89dc07a73e 100644 --- a/app/models/project_services/mock_ci_service.rb +++ b/app/models/project_services/mock_ci_service.rb @@ -3,7 +3,7 @@ class MockCiService < CiService ALLOWED_STATES = %w[failed canceled running pending success success_with_warnings skipped not_found].freeze prop_accessor :mock_service_url - validates :mock_service_url, presence: true, url: true, if: :activated? + validates :mock_service_url, presence: true, public_url: true, if: :activated? def title 'MockCI' diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb index dcaeb65dc32..df4254e0523 100644 --- a/app/models/project_services/prometheus_service.rb +++ b/app/models/project_services/prometheus_service.rb @@ -6,7 +6,7 @@ class PrometheusService < MonitoringService boolean_accessor :manual_configuration with_options presence: true, if: :manual_configuration? do - validates :api_url, url: true + validates :api_url, public_url: true end before_save :synchronize_service_state diff --git a/app/models/project_services/redmine_service.rb b/app/models/project_services/redmine_service.rb index 6acf611eba5..3721093a6d1 100644 --- a/app/models/project_services/redmine_service.rb +++ b/app/models/project_services/redmine_service.rb @@ -1,5 +1,5 @@ class RedmineService < IssueTrackerService - validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated? + validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url diff --git a/app/models/project_services/teamcity_service.rb b/app/models/project_services/teamcity_service.rb index 145313b8e71..802678147cf 100644 --- a/app/models/project_services/teamcity_service.rb +++ b/app/models/project_services/teamcity_service.rb @@ -3,7 +3,7 @@ class TeamcityService < CiService prop_accessor :teamcity_url, :build_type, :username, :password - validates :teamcity_url, presence: true, url: true, if: :activated? + validates :teamcity_url, presence: true, public_url: true, if: :activated? validates :build_type, presence: true, if: :activated? validates :username, presence: true, diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb index 87a4350f022..5d4e3c34b39 100644 --- a/app/models/project_statistics.rb +++ b/app/models/project_statistics.rb @@ -4,15 +4,15 @@ class ProjectStatistics < ActiveRecord::Base before_save :update_storage_size - STORAGE_COLUMNS = [:repository_size, :lfs_objects_size, :build_artifacts_size].freeze - STATISTICS_COLUMNS = [:commit_count] + STORAGE_COLUMNS + COLUMNS_TO_REFRESH = [:repository_size, :lfs_objects_size, :commit_count].freeze + INCREMENTABLE_COLUMNS = [:build_artifacts_size].freeze def total_repository_size repository_size + lfs_objects_size end def refresh!(only: nil) - STATISTICS_COLUMNS.each do |column, generator| + COLUMNS_TO_REFRESH.each do |column, generator| if only.blank? || only.include?(column) public_send("update_#{column}") # rubocop:disable GitlabSecurity/PublicSend end @@ -34,13 +34,15 @@ class ProjectStatistics < ActiveRecord::Base self.lfs_objects_size = project.lfs_objects.sum(:size) end - def update_build_artifacts_size - self.build_artifacts_size = - project.builds.sum(:artifacts_size) + - Ci::JobArtifact.artifacts_size_for(self.project) + def update_storage_size + self.storage_size = repository_size + lfs_objects_size + build_artifacts_size end - def update_storage_size - self.storage_size = STORAGE_COLUMNS.sum(&method(:read_attribute)) + def self.increment_statistic(project_id, key, amount) + raise ArgumentError, "Cannot increment attribute: #{key}" unless key.in?(INCREMENTABLE_COLUMNS) + return if amount == 0 + + where(project_id: project_id) + .update_all(["#{key} = COALESCE(#{key}, 0) + (?)", amount]) end end diff --git a/app/models/project_team.rb b/app/models/project_team.rb index 33280eda0b9..9a38806baab 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -24,7 +24,7 @@ class ProjectTeam end def add_role(user, role, current_user: nil) - send(:"add_#{role}", user, current_user: current_user) # rubocop:disable GitlabSecurity/PublicSend + public_send(:"add_#{role}", user, current_user: current_user) # rubocop:disable GitlabSecurity/PublicSend end def find_member(user_id) diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index e70b7dc2325..05b07804ea8 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -24,7 +24,7 @@ class ProjectWiki end delegate :empty?, to: :pages - delegate :repository_storage_path, :hashed_storage?, to: :project + delegate :repository_storage, :hashed_storage?, to: :project def path @project.path + '.wiki' @@ -147,10 +147,6 @@ class ProjectWiki [title, title_array.join("/")] end - def search_files(query) - repository.search_files_by_content(query, default_branch) - end - def repository @repository ||= Repository.new(full_path, @project, disk_path: disk_path, is_wiki: true) end @@ -186,7 +182,11 @@ class ProjectWiki def commit_details(action, message = nil, title = nil) commit_message = message || default_message(action, title) - Gitlab::Git::Wiki::CommitDetails.new(@user.name, @user.email, commit_message) + Gitlab::Git::Wiki::CommitDetails.new(@user.id, + @user.username, + @user.name, + @user.email, + commit_message) end def default_message(action, title) diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index 609780c5587..dff99cfca35 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -4,6 +4,15 @@ class ProtectedBranch < ActiveRecord::Base protected_ref_access_levels :merge, :push + def self.protected_ref_accessible_to?(ref, user, project:, action:, protected_refs: nil) + # Maintainers, owners and admins are allowed to create the default branch + if default_branch_protected? && project.empty_repo? + return true if user.admin? || project.team.max_member_access(user.id) > Gitlab::Access::DEVELOPER + end + + super + end + # Check if branch name is marked as protected in the system def self.protected?(project, ref_name) return true if project.empty_repo? && default_branch_protected? diff --git a/app/models/redirect_route.rb b/app/models/redirect_route.rb index 20532527346..31de204d824 100644 --- a/app/models/redirect_route.rb +++ b/app/models/redirect_route.rb @@ -17,32 +17,4 @@ class RedirectRoute < ActiveRecord::Base where(wheres, path, "#{sanitize_sql_like(path)}/%") end - - scope :permanent, -> do - if column_permanent_exists? - where(permanent: true) - else - none - end - end - - scope :temporary, -> do - if column_permanent_exists? - where(permanent: [false, nil]) - else - all - end - end - - default_value_for :permanent, false - - def permanent=(value) - if self.class.column_permanent_exists? - super - end - end - - def self.column_permanent_exists? - ActiveRecord::Base.connection.column_exists?(:redirect_routes, :permanent) - end end diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb new file mode 100644 index 00000000000..c4b5dd2dc96 --- /dev/null +++ b/app/models/remote_mirror.rb @@ -0,0 +1,218 @@ +class RemoteMirror < ActiveRecord::Base + include AfterCommitQueue + + PROTECTED_BACKOFF_DELAY = 1.minute + UNPROTECTED_BACKOFF_DELAY = 5.minutes + + attr_encrypted :credentials, + key: Settings.attr_encrypted_db_key_base, + marshal: true, + encode: true, + mode: :per_attribute_iv_and_salt, + insecure_mode: true, + algorithm: 'aes-256-cbc' + + default_value_for :only_protected_branches, true + + belongs_to :project, inverse_of: :remote_mirrors + + validates :url, presence: true, url: { protocols: %w(ssh git http https), allow_blank: true, enforce_user: true } + + before_save :set_new_remote_name, if: :mirror_url_changed? + + after_save :set_override_remote_mirror_available, unless: -> { Gitlab::CurrentSettings.current_application_settings.mirror_available } + after_save :refresh_remote, if: :mirror_url_changed? + after_update :reset_fields, if: :mirror_url_changed? + + after_commit :remove_remote, on: :destroy + + scope :enabled, -> { where(enabled: true) } + scope :started, -> { with_update_status(:started) } + scope :stuck, -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.day.ago, 1.day.ago) } + + state_machine :update_status, initial: :none do + event :update_start do + transition [:none, :finished, :failed] => :started + end + + event :update_finish do + transition started: :finished + end + + event :update_fail do + transition started: :failed + end + + state :started + state :finished + state :failed + + after_transition any => :started do |remote_mirror, _| + Gitlab::Metrics.add_event(:remote_mirrors_running, path: remote_mirror.project.full_path) + + remote_mirror.update(last_update_started_at: Time.now) + end + + after_transition started: :finished do |remote_mirror, _| + Gitlab::Metrics.add_event(:remote_mirrors_finished, path: remote_mirror.project.full_path) + + timestamp = Time.now + remote_mirror.update_attributes!( + last_update_at: timestamp, last_successful_update_at: timestamp, last_error: nil + ) + end + + after_transition started: :failed do |remote_mirror, _| + Gitlab::Metrics.add_event(:remote_mirrors_failed, path: remote_mirror.project.full_path) + + remote_mirror.update(last_update_at: Time.now) + end + end + + def remote_name + super || fallback_remote_name + end + + def update_failed? + update_status == 'failed' + end + + def update_in_progress? + update_status == 'started' + end + + def update_repository(options) + raw.update(options) + end + + def sync? + enabled? + end + + def sync + return unless sync? + + if recently_scheduled? + RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now) + else + RepositoryUpdateRemoteMirrorWorker.perform_async(self.id, Time.now) + end + end + + def enabled + return false unless project && super + return false unless project.remote_mirror_available? + return false unless project.repository_exists? + return false if project.pending_delete? + + true + end + alias_method :enabled?, :enabled + + def updated_since?(timestamp) + last_update_started_at && last_update_started_at > timestamp && !update_failed? + end + + def mark_for_delete_if_blank_url + mark_for_destruction if url.blank? + end + + def mark_as_failed(error_message) + update_fail + update_column(:last_error, Gitlab::UrlSanitizer.sanitize(error_message)) + end + + def url=(value) + super(value) && return unless Gitlab::UrlSanitizer.valid?(value) + + mirror_url = Gitlab::UrlSanitizer.new(value) + self.credentials = mirror_url.credentials + + super(mirror_url.sanitized_url) + end + + def url + if super + Gitlab::UrlSanitizer.new(super, credentials: credentials).full_url + end + rescue + super + end + + def safe_url + return if url.nil? + + result = URI.parse(url) + result.password = '*****' if result.password + result.user = '*****' if result.user && result.user != "git" # tokens or other data may be saved as user + result.to_s + end + + private + + def raw + @raw ||= Gitlab::Git::RemoteMirror.new(project.repository.raw, remote_name) + end + + def fallback_remote_name + return unless id + + "remote_mirror_#{id}" + end + + def recently_scheduled? + return false unless self.last_update_started_at + + self.last_update_started_at >= Time.now - backoff_delay + end + + def backoff_delay + if self.only_protected_branches + PROTECTED_BACKOFF_DELAY + else + UNPROTECTED_BACKOFF_DELAY + end + end + + def reset_fields + update_columns( + last_error: nil, + last_update_at: nil, + last_successful_update_at: nil, + update_status: 'finished' + ) + end + + def set_override_remote_mirror_available + enabled = read_attribute(:enabled) + + project.update(remote_mirror_available_overridden: enabled) + end + + def set_new_remote_name + self.remote_name = "remote_mirror_#{SecureRandom.hex}" + end + + def refresh_remote + return unless project + + # Before adding a new remote we have to delete the data from + # the previous remote name + prev_remote_name = remote_name_was || fallback_remote_name + run_after_commit do + project.repository.async_remove_remote(prev_remote_name) + end + + project.repository.add_remote(remote_name, url) + end + + def remove_remote + return unless project # could be pending to delete so don't need to touch the git repository + + project.repository.async_remove_remote(remote_name) + end + + def mirror_url_changed? + url_changed? || encrypted_credentials_changed? + end +end diff --git a/app/models/repository.rb b/app/models/repository.rb index 42f1ac43e29..3056c20516a 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -21,7 +21,7 @@ class Repository attr_accessor :full_path, :disk_path, :project, :is_wiki delegate :ref_name_for_sha, to: :raw_repository - delegate :bundle_to_disk, :create_from_bundle, to: :raw_repository + delegate :bundle_to_disk, to: :raw_repository CreateTreeError = Class.new(StandardError) @@ -37,7 +37,7 @@ class Repository changelog license_blob license_key gitignore koding_yml gitlab_ci_yml branch_names tag_names branch_count tag_count avatar exists? root_ref has_visible_content? - issue_template_names merge_request_template_names).freeze + issue_template_names merge_request_template_names xcode_project?).freeze # Methods that use cache_method but only memoize the value MEMOIZED_CACHED_METHODS = %i(license).freeze @@ -55,7 +55,8 @@ class Repository gitlab_ci: :gitlab_ci_yml, avatar: :avatar, issue_template: :issue_template_names, - merge_request_template: :merge_request_template_names + merge_request_template: :merge_request_template_names, + xcode_config: :xcode_project? }.freeze def initialize(full_path, project, disk_path: nil, is_wiki: false) @@ -84,19 +85,20 @@ class Repository # Return absolute path to repository def path_to_repo - @path_to_repo ||= File.expand_path( - File.join(repository_storage_path, disk_path + '.git') - ) + @path_to_repo ||= + begin + storage = Gitlab.config.repositories.storages[@project.repository_storage] + + File.expand_path( + File.join(storage.legacy_disk_path, disk_path + '.git') + ) + end end def inspect "#<#{self.class.name}:#{@disk_path}>" end - def create_hooks - Gitlab::Git::Repository.create_hooks(path_to_repo, Gitlab.config.gitlab_shell.hooks_path) - end - def commit(ref = 'HEAD') return nil unless exists? return ref if ref.is_a?(::Commit) @@ -152,7 +154,10 @@ class Repository # Returns a list of commits that are not present in any reference def new_commits(newrev) - refs = ::Gitlab::Git::RevList.new(raw, newrev: newrev).new_refs + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/1233 + refs = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + ::Gitlab::Git::RevList.new(raw, newrev: newrev).new_refs + end refs.map { |sha| commit(sha.strip) } end @@ -253,13 +258,13 @@ class Repository end def diverging_commit_counts(branch) - root_ref_hash = raw_repository.commit(root_ref).id + @root_ref_hash ||= raw_repository.commit(root_ref).id cache.fetch(:"diverging_commit_counts_#{branch.name}") do # Rugged seems to throw a `ReferenceError` when given branch_names rather # than SHA-1 hashes number_commits_behind, number_commits_ahead = raw_repository.count_commits_between( - root_ref_hash, + @root_ref_hash, branch.dereferenced_target.sha, left_right: true, max_count: MAX_DIVERGING_COUNT) @@ -268,6 +273,16 @@ class Repository end end + def archive_metadata(ref, storage_path, format = "tar.gz", append_sha:) + raw_repository.archive_metadata( + ref, + storage_path, + project.path, + format, + append_sha: append_sha + ) + end + def expire_tags_cache expire_method_caches(%i(tag_names tag_count)) @tags = nil @@ -335,6 +350,7 @@ class Repository return unless empty? expire_method_caches(%i(has_visible_content?)) + raw_repository.expire_has_local_branches_cache end def lookup_cache @@ -592,6 +608,11 @@ class Repository end cache_method :gitlab_ci_yml + def xcode_project? + file_on_head(:xcode_config, :tree).present? + end + cache_method :xcode_project? + def head_commit @head_commit ||= commit(self.root_ref) end @@ -829,7 +850,7 @@ class Repository @root_ref_sha ||= commit(root_ref).sha end - delegate :merged_branch_names, :can_be_merged?, to: :raw_repository + delegate :merged_branch_names, to: :raw_repository def merge_base(first_commit_id, second_commit_id) first_commit_id = commit(first_commit_id).try(:id) || first_commit_id @@ -852,13 +873,27 @@ class Repository add_remote(remote_name, url, mirror_refmap: refmap) fetch_remote(remote_name, forced: forced, prune: prune) ensure - remove_remote(remote_name) if tmp_remote_name + async_remove_remote(remote_name) if tmp_remote_name end def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false, prune: true) gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) end + def async_remove_remote(remote_name) + return unless remote_name + + job_id = RepositoryRemoveRemoteWorker.perform_async(project.id, remote_name) + + if job_id + Rails.logger.info("Remove remote job scheduled for #{project.id} with remote name: #{remote_name} job ID #{job_id}.") + else + Rails.logger.info("Remove remote job failed to create for #{project.id} with remote name #{remote_name}.") + end + + job_id + end + def fetch_source_branch!(source_repository, source_branch, local_ref) raw_repository.fetch_source_branch!(source_repository.raw_repository, source_branch, local_ref) end @@ -898,11 +933,21 @@ class Repository end end - def file_on_head(type) - if head = tree(:head) - head.blobs.find do |blob| - Gitlab::FileDetector.type_of(blob.path) == type + def file_on_head(type, object_type = :blob) + return unless head = tree(:head) + + objects = + case object_type + when :blob + head.blobs + when :tree + head.trees + else + raise ArgumentError, "Object type #{object_type} is not supported" end + + objects.find do |object| + Gitlab::FileDetector.type_of(object.path) == type end end @@ -914,12 +959,12 @@ class Repository blob_data_at(sha, path) end - def fetch_ref(source_repository, source_ref:, target_ref:) - raw_repository.fetch_ref(source_repository.raw_repository, source_ref: source_ref, target_ref: target_ref) + def lfsconfig_for(sha) + blob_data_at(sha, '.lfsconfig') end - def repository_storage_path - @project.repository_storage_path + def fetch_ref(source_repository, source_ref:, target_ref:) + raw_repository.fetch_ref(source_repository.raw_repository, source_ref: source_ref, target_ref: target_ref) end def rebase(user, merge_request) @@ -929,6 +974,22 @@ class Repository remote_branch: merge_request.target_branch) end + def blob_data_at(sha, path) + blob = blob_at(sha, path) + return unless blob + + blob.load_all_data! + blob.data + end + + def squash(user, merge_request) + raw.squash(user, merge_request.id, branch: merge_request.target_branch, + start_sha: merge_request.diff_start_sha, + end_sha: merge_request.diff_head_sha, + author: merge_request.author, + message: merge_request.title) + end + private # TODO Generice finder, later split this on finders by Ref or Oid @@ -943,14 +1004,6 @@ class Repository ::Commit.new(commit, @project) if commit end - def blob_data_at(sha, path) - blob = blob_at(sha, path) - return unless blob - - blob.load_all_data! - blob.data - end - def cache @cache ||= Gitlab::RepositoryCache.new(self) end diff --git a/app/models/route.rb b/app/models/route.rb index 07d96c21cf1..2d609920051 100644 --- a/app/models/route.rb +++ b/app/models/route.rb @@ -10,8 +10,6 @@ class Route < ActiveRecord::Base presence: true, uniqueness: { case_sensitive: false } - validate :ensure_permanent_paths, if: :path_changed? - before_validation :delete_conflicting_orphaned_routes after_create :delete_conflicting_redirects after_update :delete_conflicting_redirects, if: :path_changed? @@ -45,7 +43,7 @@ class Route < ActiveRecord::Base # We are not calling route.delete_conflicting_redirects here, in hopes # of avoiding deadlocks. The parent (self, in this method) already # called it, which deletes conflicts for all descendants. - route.create_redirect(old_path, permanent: permanent_redirect?) if attributes[:path] + route.create_redirect(old_path) if attributes[:path] end end end @@ -55,31 +53,17 @@ class Route < ActiveRecord::Base end def conflicting_redirects - RedirectRoute.temporary.matching_path_and_descendants(path) + RedirectRoute.matching_path_and_descendants(path) end - def create_redirect(path, permanent: false) - RedirectRoute.create(source: source, path: path, permanent: permanent) + def create_redirect(path) + RedirectRoute.create(source: source, path: path) end private def create_redirect_for_old_path - create_redirect(path_was, permanent: permanent_redirect?) if path_changed? - end - - def permanent_redirect? - source_type != "Project" - end - - def ensure_permanent_paths - return if path.nil? - - errors.add(:path, "has been taken before") if conflicting_redirect_exists? - end - - def conflicting_redirect_exists? - RedirectRoute.permanent.matching_path_and_descendants(path).exists? + create_redirect(path_was) if path_changed? end def delete_conflicting_orphaned_routes diff --git a/app/models/sent_notification.rb b/app/models/sent_notification.rb index 6e311806be1..3da7c301d28 100644 --- a/app/models/sent_notification.rb +++ b/app/models/sent_notification.rb @@ -5,14 +5,14 @@ class SentNotification < ActiveRecord::Base belongs_to :noteable, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations belongs_to :recipient, class_name: "User" - validates :project, :recipient, presence: true + validates :recipient, presence: true validates :reply_key, presence: true, uniqueness: true validates :noteable_id, presence: true, unless: :for_commit? validates :commit_id, presence: true, if: :for_commit? validates :in_reply_to_discussion_id, format: { with: /\A\h{40}\z/, allow_nil: true } validate :note_valid - after_save :keep_around_commit + after_save :keep_around_commit, if: :for_commit? class << self def reply_key diff --git a/app/models/service.rb b/app/models/service.rb index 1dcb79157a2..1d259bcfec7 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -14,6 +14,7 @@ class Service < ActiveRecord::Base default_value_for :merge_requests_events, true default_value_for :tag_push_events, true default_value_for :note_events, true + default_value_for :confidential_note_events, true default_value_for :job_events, true default_value_for :pipeline_events, true default_value_for :wiki_page_events, true @@ -42,6 +43,7 @@ class Service < ActiveRecord::Base scope :confidential_issue_hooks, -> { where(confidential_issues_events: true, active: true) } scope :merge_request_hooks, -> { where(merge_requests_events: true, active: true) } scope :note_hooks, -> { where(note_events: true, active: true) } + scope :confidential_note_hooks, -> { where(confidential_note_events: true, active: true) } scope :job_hooks, -> { where(job_events: true, active: true) } scope :pipeline_hooks, -> { where(pipeline_events: true, active: true) } scope :wiki_page_hooks, -> { where(wiki_page_events: true, active: true) } @@ -168,8 +170,10 @@ class Service < ActiveRecord::Base def self.prop_accessor(*args) args.each do |arg| class_eval %{ - def #{arg} - properties['#{arg}'] + unless method_defined?(arg) + def #{arg} + properties['#{arg}'] + end end def #{arg}=(value) @@ -202,7 +206,12 @@ class Service < ActiveRecord::Base args.each do |arg| class_eval %{ def #{arg}? - ActiveRecord::ConnectionAdapters::Column::TRUE_VALUES.include?(#{arg}) + # '!!' is used because nil or empty string is converted to nil + if Gitlab.rails5? + !!ActiveRecord::Type::Boolean.new.cast(#{arg}) + else + !!ActiveRecord::Type::Boolean.new.type_cast_from_database(#{arg}) + end end } end @@ -245,7 +254,6 @@ class Service < ActiveRecord::Base emails_on_push external_wiki flowdock - gemnasium hipchat irker jira @@ -273,6 +281,7 @@ class Service < ActiveRecord::Base def self.build_from_template(project_id, template) service = template.dup + service.active = false unless service.valid? service.template = false service.project_id = project_id service diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb index fae1b64961a..26b4b78ac64 100644 --- a/app/models/storage/hashed_project.rb +++ b/app/models/storage/hashed_project.rb @@ -1,7 +1,7 @@ module Storage class HashedProject attr_accessor :project - delegate :gitlab_shell, :repository_storage_path, to: :project + delegate :gitlab_shell, :repository_storage, to: :project ROOT_PATH_PREFIX = '@hashed'.freeze @@ -24,7 +24,7 @@ module Storage end def ensure_storage_path_exists - gitlab_shell.add_namespace(repository_storage_path, base_dir) + gitlab_shell.add_namespace(repository_storage, base_dir) end def rename_repo diff --git a/app/models/storage/legacy_project.rb b/app/models/storage/legacy_project.rb index 9d9e5e1d352..27cb388c702 100644 --- a/app/models/storage/legacy_project.rb +++ b/app/models/storage/legacy_project.rb @@ -1,7 +1,7 @@ module Storage class LegacyProject attr_accessor :project - delegate :namespace, :gitlab_shell, :repository_storage_path, to: :project + delegate :namespace, :gitlab_shell, :repository_storage, to: :project def initialize(project) @project = project @@ -24,18 +24,18 @@ module Storage def ensure_storage_path_exists return unless namespace - gitlab_shell.add_namespace(repository_storage_path, base_dir) + gitlab_shell.add_namespace(repository_storage, base_dir) end def rename_repo new_full_path = project.build_full_path - if gitlab_shell.mv_repository(repository_storage_path, project.full_path_was, new_full_path) + if gitlab_shell.mv_repository(repository_storage, project.full_path_was, new_full_path) # If repository moved successfully we need to send update instructions to users. # However we cannot allow rollback since we moved repository # So we basically we mute exceptions in next actions begin - gitlab_shell.mv_repository(repository_storage_path, "#{project.full_path_was}.wiki", "#{new_full_path}.wiki") + gitlab_shell.mv_repository(repository_storage, "#{project.full_path_was}.wiki", "#{new_full_path}.wiki") return true rescue => e Rails.logger.error "Exception renaming #{project.full_path_was} -> #{new_full_path}: #{e}" diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb index 29035480371..1c2161accc4 100644 --- a/app/models/system_note_metadata.rb +++ b/app/models/system_note_metadata.rb @@ -17,7 +17,11 @@ class SystemNoteMetadata < ActiveRecord::Base ].freeze validates :note, presence: true - validates :action, inclusion: ICON_TYPES, allow_nil: true + validates :action, inclusion: { in: :icon_types }, allow_nil: true belongs_to :note + + def icon_types + ICON_TYPES + end end diff --git a/app/models/term_agreement.rb b/app/models/term_agreement.rb new file mode 100644 index 00000000000..c317bd0c90b --- /dev/null +++ b/app/models/term_agreement.rb @@ -0,0 +1,8 @@ +class TermAgreement < ActiveRecord::Base + belongs_to :term, class_name: 'ApplicationSetting::Term' + belongs_to :user + + scope :accepted, -> { where(accepted: true) } + + validates :user, :term, presence: true +end diff --git a/app/models/timelog.rb b/app/models/timelog.rb index e166cf69703..659146f43e4 100644 --- a/app/models/timelog.rb +++ b/app/models/timelog.rb @@ -2,8 +2,8 @@ class Timelog < ActiveRecord::Base validates :time_spent, :user, presence: true validate :issuable_id_is_present - belongs_to :issue - belongs_to :merge_request + belongs_to :issue, touch: true + belongs_to :merge_request, touch: true belongs_to :user def issuable @@ -19,4 +19,9 @@ class Timelog < ActiveRecord::Base errors.add(:base, 'Issue or Merge Request ID is required') end end + + # Rails5 defaults to :touch_later, overwrite for normal touch + def belongs_to_touch_method + :touch + end end diff --git a/app/models/todo.rb b/app/models/todo.rb index 8afacd188e0..a2ab405fdbe 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -50,7 +50,7 @@ class Todo < ActiveRecord::Base # Priority sorting isn't displayed in the dropdown, because we don't show # milestones, but still show something if the user has a URL with that # selected. - def sort(method) + def sort_by_attribute(method) sorted = case method.to_s when 'priority', 'label_priority' then order_by_labels_priority diff --git a/app/models/upload.rb b/app/models/upload.rb index 99ad37dc892..cf71a7b76fc 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -9,6 +9,8 @@ class Upload < ActiveRecord::Base validates :model, presence: true validates :uploader, presence: true + scope :with_files_stored_locally, -> { where(store: [nil, ObjectStorage::Store::LOCAL]) } + before_save :calculate_checksum!, if: :foreground_checksummable? after_commit :schedule_checksum, if: :checksummable? @@ -21,6 +23,7 @@ class Upload < ActiveRecord::Base end def absolute_path + raise ObjectStorage::RemoteStoreError, "Remote object has no absolute path." unless local? return path unless relative_path? uploader_class.absolute_path(self) @@ -30,11 +33,11 @@ class Upload < ActiveRecord::Base self.checksum = nil return unless checksummable? - self.checksum = self.class.hexdigest(absolute_path) + self.checksum = Digest::SHA256.file(absolute_path).hexdigest end - def build_uploader - uploader_class.new(model, mount_point, **uploader_context).tap do |uploader| + def build_uploader(mounted_as = nil) + uploader_class.new(model, mounted_as || mount_point).tap do |uploader| uploader.upload = self uploader.retrieve_from_store!(identifier) end @@ -51,6 +54,12 @@ class Upload < ActiveRecord::Base }.compact end + def local? + return true if store.nil? + + store == ObjectStorage::Store::LOCAL + end + private def delete_file! @@ -61,10 +70,6 @@ class Upload < ActiveRecord::Base checksum.nil? && local? && exist? end - def local? - true - end - def foreground_checksummable? checksummable? && size <= CHECKSUM_THRESHOLD end diff --git a/app/models/user.rb b/app/models/user.rb index b8c55205ab8..8e0dc91b2a7 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -17,6 +17,7 @@ class User < ActiveRecord::Base include IgnorableColumn include BulkMemberAccessLoad include BlocksJsonSerialization + include WithUploads DEFAULT_NOTIFICATION_LEVEL = :participating @@ -25,7 +26,7 @@ class User < ActiveRecord::Base ignore_column :authentication_token add_authentication_token_field :incoming_email_token - add_authentication_token_field :rss_token + add_authentication_token_field :feed_token default_value_for :admin, false default_value_for(:external) { Gitlab::CurrentSettings.user_default_external } @@ -82,11 +83,8 @@ class User < ActiveRecord::Base has_one :namespace, -> { where(type: nil) }, dependent: :destroy, foreign_key: :owner_id, inverse_of: :owner, autosave: true # rubocop:disable Cop/ActiveRecordDependent # Profile - has_many :keys, -> do - type = Key.arel_table[:type] - where(type.not_eq('DeployKey').or(type.eq(nil))) - end, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :deploy_keys, -> { where(type: 'DeployKey') }, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :keys, -> { where(type: ['Key', nil]) }, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :deploy_keys, -> { where(type: 'DeployKey') }, dependent: :nullify # rubocop:disable Cop/ActiveRecordDependent has_many :gpg_keys has_many :emails, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -100,8 +98,8 @@ class User < ActiveRecord::Base has_many :members has_many :group_members, -> { where(requested_at: nil) }, source: 'GroupMember' has_many :groups, through: :group_members - has_many :owned_groups, -> { where members: { access_level: Gitlab::Access::OWNER } }, through: :group_members, source: :group - has_many :masters_groups, -> { where members: { access_level: Gitlab::Access::MASTER } }, through: :group_members, source: :group + has_many :owned_groups, -> { where(members: { access_level: Gitlab::Access::OWNER }) }, through: :group_members, source: :group + has_many :masters_groups, -> { where(members: { access_level: Gitlab::Access::MASTER }) }, through: :group_members, source: :group # Projects has_many :groups_projects, through: :groups, source: :projects @@ -111,7 +109,7 @@ class User < ActiveRecord::Base has_many :created_projects, foreign_key: :creator_id, class_name: 'Project' has_many :users_star_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :starred_projects, through: :users_star_projects, source: :project - has_many :project_authorizations + has_many :project_authorizations, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_many :authorized_projects, through: :project_authorizations, source: :project has_many :user_interacted_projects @@ -140,7 +138,8 @@ class User < ActiveRecord::Base has_many :custom_attributes, class_name: 'UserCustomAttribute' has_many :callouts, class_name: 'UserCallout' - has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :term_agreements + belongs_to :accepted_term, class_name: 'ApplicationSetting::Term' # # Validations @@ -166,19 +165,34 @@ class User < ActiveRecord::Base validate :signup_domain_valid?, on: :create, if: ->(user) { !user.created_by_id } before_validation :sanitize_attrs - before_validation :set_notification_email, if: :email_changed? + before_validation :set_notification_email, if: :new_record? before_validation :set_public_email, if: :public_email_changed? + before_save :set_public_email, if: :public_email_changed? # in case validation is skipped before_save :ensure_incoming_email_token before_save :ensure_user_rights_and_limits, if: ->(user) { user.new_record? || user.external_changed? } before_save :skip_reconfirmation!, if: ->(user) { user.email_changed? && user.read_only_attribute?(:email) } before_save :check_for_verified_email, if: ->(user) { user.email_changed? && !user.new_record? } before_validation :ensure_namespace_correct + before_save :ensure_namespace_correct # in case validation is skipped after_validation :set_username_errors after_update :username_changed_hook, if: :username_changed? after_destroy :post_destroy_hook after_destroy :remove_key_cache - after_commit :update_emails_with_primary_email, on: :update, if: -> { previous_changes.key?('email') } - after_commit :update_invalid_gpg_signatures, on: :update, if: -> { previous_changes.key?('email') } + after_commit(on: :update) do + if previous_changes.key?('email') + # Grab previous_email here since previous_changes changes after + # #update_emails_with_primary_email and #update_notification_email are called + previous_email = previous_changes[:email][0] + + update_emails_with_primary_email(previous_email) + update_invalid_gpg_signatures + + if previous_email == notification_email + self.notification_email = email + save + end + end + end after_initialize :set_projects_limit @@ -187,7 +201,7 @@ class User < ActiveRecord::Base # User's Dashboard preference # Note: When adding an option, it MUST go on the end of the array. - enum dashboard: [:projects, :stars, :project_activity, :starred_project_activity, :groups, :todos] + enum dashboard: [:projects, :stars, :project_activity, :starred_project_activity, :groups, :todos, :issues, :merge_requests] # User's Project preference # Note: When adding an option, it MUST go on the end of the array. @@ -235,14 +249,18 @@ class User < ActiveRecord::Base scope :order_recent_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'DESC')) } scope :order_oldest_sign_in, -> { reorder(Gitlab::Database.nulls_last_order('current_sign_in_at', 'ASC')) } - def self.with_two_factor + def self.with_two_factor_indistinct joins("LEFT OUTER JOIN u2f_registrations AS u2f ON u2f.user_id = users.id") - .where("u2f.id IS NOT NULL OR otp_required_for_login = ?", true).distinct(arel_table[:id]) + .where("u2f.id IS NOT NULL OR users.otp_required_for_login = ?", true) + end + + def self.with_two_factor + with_two_factor_indistinct.distinct(arel_table[:id]) end def self.without_two_factor joins("LEFT OUTER JOIN u2f_registrations AS u2f ON u2f.user_id = users.id") - .where("u2f.id IS NULL AND otp_required_for_login = ?", false) + .where("u2f.id IS NULL AND users.otp_required_for_login = ?", false) end # @@ -259,7 +277,7 @@ class User < ActiveRecord::Base end end - def sort(method) + def sort_by_attribute(method) order_method = method || 'id_desc' case order_method.to_s @@ -411,7 +429,6 @@ class User < ActiveRecord::Base unique_internal(where(ghost: true), 'ghost', email) do |u| u.bio = 'This is a "Ghost User", created to hold all issues authored by users that have since been deleted. This user cannot be removed.' u.name = 'Ghost User' - u.notification_email = email end end end @@ -541,8 +558,7 @@ class User < ActiveRecord::Base # hash and `_was` variables getting munged. # By using an `after_commit` instead of `after_update`, we avoid the recursive callback # scenario, though it then requires us to use the `previous_changes` hash - def update_emails_with_primary_email - previous_email = previous_changes[:email][0] # grab this before the DestroyService is called + def update_emails_with_primary_email(previous_email) primary_email_record = emails.find_by(email: email) Emails::DestroyService.new(self, user: self).execute(primary_email_record) if primary_email_record @@ -623,9 +639,7 @@ class User < ActiveRecord::Base end def owned_projects - @owned_projects ||= - Project.where('namespace_id IN (?) OR namespace_id = ?', - owned_groups.select(:id), namespace.id).joins(:namespace) + @owned_projects ||= Project.from("(#{owned_projects_union.to_sql}) AS projects") end # Returns projects which user can admin issues on (for example to move an issue to that project). @@ -703,10 +717,6 @@ class User < ActiveRecord::Base projects_limit - personal_projects_count end - def personal_projects_count - @personal_projects_count ||= personal_projects.count - end - def recent_push(project = nil) service = Users::LastPushEventService.new(self) @@ -773,13 +783,13 @@ class User < ActiveRecord::Base end def set_notification_email - if notification_email.blank? || !all_emails.include?(notification_email) + if notification_email.blank? || all_emails.exclude?(notification_email) self.notification_email = email end end def set_public_email - if public_email.blank? || !all_emails.include?(public_email) + if public_email.blank? || all_emails.exclude?(public_email) self.public_email = '' end end @@ -861,6 +871,16 @@ class User < ActiveRecord::Base confirmed? && !temp_oauth_email? end + def accept_pending_invitations! + pending_invitations.select do |member| + member.accept_invite!(self) + end + end + + def pending_invitations + Member.where(invite_email: verified_emails).invite + end + def all_emails all_emails = [] all_emails << email unless temp_oauth_email? @@ -917,7 +937,7 @@ class User < ActiveRecord::Base def delete_async(deleted_by:, params: {}) block if params[:hard_delete] - DeleteUserWorker.perform_async(deleted_by.id, id, params) + DeleteUserWorker.perform_async(deleted_by.id, id, params.to_h) end def notification_service @@ -954,10 +974,13 @@ class User < ActiveRecord::Base end def manageable_groups - union = Gitlab::SQL::Union.new([owned_groups.select(:id), - masters_groups.select(:id)]) - arel_union = Arel::Nodes::SqlLiteral.new(union.to_sql) - owned_and_master_groups = Group.where(Group.arel_table[:id].in(arel_union)) + union_sql = Gitlab::SQL::Union.new([owned_groups.select(:id), masters_groups.select(:id)]).to_sql + + # Update this line to not use raw SQL when migrated to Rails 5.2. + # Either ActiveRecord or Arel constructions are fine. + # This was replaced with the raw SQL construction because of bugs in the arel gem. + # Bugs were fixed in arel 9.0.0 (Rails 5.2). + owned_and_master_groups = Group.where("namespaces.id IN (#{union_sql})") # rubocop:disable GitlabSecurity/SqlInjection Gitlab::GroupHierarchy.new(owned_and_master_groups).base_and_descendants end @@ -997,18 +1020,28 @@ class User < ActiveRecord::Base !solo_owned_groups.present? end - def ci_authorized_runners - @ci_authorized_runners ||= begin - runner_ids = Ci::RunnerProject - .where("ci_runner_projects.project_id IN (#{ci_projects_union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection + def ci_owned_runners + @ci_owned_runners ||= begin + project_runner_ids = Ci::RunnerProject + .where(project: authorized_projects(Gitlab::Access::MASTER)) .select(:runner_id) - Ci::Runner.specific.where(id: runner_ids) + + group_runner_ids = Ci::RunnerNamespace + .where(namespace_id: owned_or_masters_groups.select(:id)) + .select(:runner_id) + + union = Gitlab::SQL::Union.new([project_runner_ids, group_runner_ids]) + + Ci::Runner.specific.where("ci_runners.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection end end def notification_settings_for(source) if notification_settings.loaded? - notification_settings.find { |notification| notification.source == source } + notification_settings.find do |notification| + notification.source_type == source.class.base_class.name && + notification.source_id == source.id + end else notification_settings.find_or_initialize_by(source: source) end @@ -1049,9 +1082,10 @@ class User < ActiveRecord::Base end end - def update_cache_counts - assigned_open_merge_requests_count(force: true) - assigned_open_issues_count(force: true) + def personal_projects_count(force: false) + Rails.cache.fetch(['users', id, 'personal_projects_count'], force: force, expires_in: 24.hours, raw: true) do + personal_projects.count + end.to_i end def update_todos_count_cache @@ -1064,6 +1098,7 @@ class User < ActiveRecord::Base invalidate_merge_request_cache_counts invalidate_todos_done_count invalidate_todos_pending_count + invalidate_personal_projects_count end def invalidate_issue_cache_counts @@ -1082,6 +1117,10 @@ class User < ActiveRecord::Base Rails.cache.delete(['users', id, 'todos_pending_count']) end + def invalidate_personal_projects_count + Rails.cache.delete(['users', id, 'personal_projects_count']) + end + # This is copied from Devise::Models::Lockable#valid_for_authentication?, as our auth # flow means we don't call that automatically (and can't conveniently do so). # @@ -1089,8 +1128,11 @@ class User < ActiveRecord::Base # <https://github.com/plataformatec/devise/blob/v4.0.0/lib/devise/models/lockable.rb#L92> # def increment_failed_attempts! + return if ::Gitlab::Database.read_only? + self.failed_attempts ||= 0 self.failed_attempts += 1 + if attempts_exceeded? lock_access! unless access_locked? else @@ -1128,11 +1170,11 @@ class User < ActiveRecord::Base save end - # each existing user needs to have an `rss_token`. + # each existing user needs to have an `feed_token`. # we do this on read since migrating all existing users is not a feasible # solution. - def rss_token - ensure_rss_token! + def feed_token + ensure_feed_token! end def sync_attribute?(attribute) @@ -1185,6 +1227,20 @@ class User < ActiveRecord::Base max_member_access_for_group_ids([group_id])[group_id] end + def terms_accepted? + accepted_term_id.present? + end + + def required_terms_not_accepted? + Gitlab::CurrentSettings.current_application_settings.enforce_terms? && + !terms_accepted? + end + + def owned_or_masters_groups + union = Gitlab::SQL::Union.new([owned_groups, masters_groups]) + Group.from("(#{union.to_sql}) namespaces") + end + protected # override, from Devise::Validatable @@ -1196,13 +1252,13 @@ class User < ActiveRecord::Base private - def ci_projects_union - scope = { access_level: [Gitlab::Access::MASTER, Gitlab::Access::OWNER] } - groups = groups_projects.where(members: scope) - other = projects.where(members: scope) - - Gitlab::SQL::Union.new([personal_projects.select(:id), groups.select(:id), - other.select(:id)]) + def owned_projects_union + Gitlab::SQL::Union.new([ + Project.where(namespace: namespace), + Project.joins(:project_authorizations) + .where("projects.namespace_id <> ?", namespace.id) + .where(project_authorizations: { user_id: id, access_level: Gitlab::Access::OWNER }) + ], remove_duplicates: false) end # Added according to https://github.com/plataformatec/devise/blob/7df57d5081f9884849ca15e4fde179ef164a575f/README.md#activejob-integration diff --git a/app/models/user_callout.rb b/app/models/user_callout.rb index e4b69382626..9d461c6750a 100644 --- a/app/models/user_callout.rb +++ b/app/models/user_callout.rb @@ -2,7 +2,8 @@ class UserCallout < ActiveRecord::Base belongs_to :user enum feature_name: { - gke_cluster_integration: 1 + gke_cluster_integration: 1, + gcp_signup_offer: 2 } validates :user, presence: true diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index 0f5536415f7..cde79b95062 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -265,6 +265,15 @@ class WikiPage title.present? && self.class.unhyphenize(@page.url_path) != title end + # Updates the current @attributes hash by merging a hash of params + def update_attributes(attrs) + attrs[:title] = process_title(attrs[:title]) if attrs[:title].present? + + attrs.slice!(:content, :format, :message, :title) + + @attributes.merge!(attrs) + end + private # Process and format the title based on the user input. @@ -290,15 +299,6 @@ class WikiPage File.join(components) end - # Updates the current @attributes hash by merging a hash of params - def update_attributes(attrs) - attrs[:title] = process_title(attrs[:title]) if attrs[:title].present? - - attrs.slice!(:content, :format, :message, :title) - - @attributes.merge!(attrs) - end - def set_attributes attributes[:slug] = @page.url_path attributes[:title] = @page.title |