diff options
author | Lin Jen-Shin <godfat@godfat.org> | 2018-07-13 12:34:27 +0300 |
---|---|---|
committer | Lin Jen-Shin <godfat@godfat.org> | 2018-07-13 12:34:27 +0300 |
commit | d9f26586b4ec00c00fb470e55af3dc5c1f4ce33a (patch) | |
tree | 1a16bb35e6b103042ed405dcb37227c4dc44244f /app/models | |
parent | 99c80156ae6058f70fde77baabd5f007f0246e6d (diff) | |
parent | a3c8525dc9a69d0e1048c114fd27317dcffc4221 (diff) |
Merge remote-tracking branch 'upstream/master' into 14995-custom_wiki_sidebar
* upstream/master: (467 commits)
Update docs board features tier
Upgrade grape-path-helpers to 1.0.6
Remove healthchecks from prometheus endpoint
Fix find_branch call sites
Ensure Encoding.default_external is set to UTF-8 when running QA scenarios
i18n: externalize strings from 'app/views/admin/groups'
Backport mr widget changes from EE
Allow to toggle notifications for issues due soon
Vuex test helper improvements
whitespace
Make more ref RPC's mandatory
Resolve "Improve performance of MR Changes tab: reduce event listeners on scroll event"
Remove old service architecture from Vue docs
Adding spec to test basic forking functionalities
Fix performance problem of accessing tag list for projects api endpoints
typo
Add sleep to QA test before installing tiller
Include Vue files that are not covered by tests in test coverage
Remove Repository#path memoization
Resolve "do not set updated_at when creating note"
...
Diffstat (limited to 'app/models')
34 files changed, 428 insertions, 198 deletions
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index bddeb8b0352..f770b219422 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -294,6 +294,7 @@ class ApplicationSetting < ActiveRecord::Base gitaly_timeout_medium: 30, gitaly_timeout_default: 55, allow_local_requests_from_hooks_and_services: false, + hide_third_party_offers: false, mirror_available: true } end diff --git a/app/models/board.rb b/app/models/board.rb index 3cede6fc99a..bb6bb753daf 100644 --- a/app/models/board.rb +++ b/app/models/board.rb @@ -26,4 +26,8 @@ class Board < ActiveRecord::Base def closed_list lists.merge(List.closed).take end + + def scoped? + false + end end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 41446946a5e..d8ddb4bc667 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -27,7 +27,13 @@ module Ci has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :metadata, class_name: 'Ci::BuildMetadata' + has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build + + accepts_nested_attributes_for :runner_session + delegate :timeout, to: :metadata, prefix: true, allow_nil: true + delegate :url, to: :runner_session, prefix: true, allow_nil: true + delegate :terminal_specification, to: :runner_session, allow_nil: true delegate :gitlab_deploy_token, to: :project ## @@ -174,6 +180,10 @@ module Ci after_transition pending: :running do |build| build.ensure_metadata.update_timeout_state end + + after_transition running: any do |build| + Ci::BuildRunnerSession.where(build: build).delete_all + end end def ensure_metadata @@ -361,7 +371,7 @@ module Ci def update_coverage coverage = trace.extract_coverage(coverage_regex) - update_attributes(coverage: coverage) if coverage.present? + update(coverage: coverage) if coverage.present? end def parse_trace_sections! @@ -376,6 +386,10 @@ module Ci trace.exist? end + def has_old_trace? + old_trace.present? + end + def trace=(data) raise NotImplementedError end @@ -385,6 +399,8 @@ module Ci end def erase_old_trace! + return unless has_old_trace? + update_column(:trace, nil) end @@ -421,9 +437,9 @@ module Ci end def artifacts_metadata_entry(path, **options) - artifacts_metadata.use_file do |metadata_path| + artifacts_metadata.open do |metadata_stream| metadata = Gitlab::Ci::Build::Artifacts::Metadata.new( - metadata_path, + metadata_stream, path, **options) @@ -584,6 +600,10 @@ module Ci super(options).merge(when: read_attribute(:when)) end + def has_terminal? + running? && runner_session_url.present? + end + private def update_artifacts_size diff --git a/app/models/ci/build_runner_session.rb b/app/models/ci/build_runner_session.rb new file mode 100644 index 00000000000..6f3be31d8e1 --- /dev/null +++ b/app/models/ci/build_runner_session.rb @@ -0,0 +1,25 @@ +module Ci + # The purpose of this class is to store Build related runner session. + # Data will be removed after transitioning from running to any state. + class BuildRunnerSession < ActiveRecord::Base + extend Gitlab::Ci::Model + + self.table_name = 'ci_builds_runner_session' + + belongs_to :build, class_name: 'Ci::Build', inverse_of: :runner_session + + validates :build, presence: true + validates :url, url: { protocols: %w(https) } + + def terminal_specification + return {} unless url.present? + + { + subprotocols: ['terminal.gitlab.com'].freeze, + url: "#{url}/exec".sub("https://", "wss://"), + headers: { Authorization: authorization.presence }.compact, + ca_pem: certificate.presence + } + end + end +end diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 4856f10846c..b442de34061 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -1,54 +1,58 @@ module Ci class BuildTraceChunk < ActiveRecord::Base include FastDestroyAll + include ::Gitlab::ExclusiveLeaseHelpers extend Gitlab::Ci::Model belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id default_value_for :data_store, :redis - WriteError = Class.new(StandardError) - CHUNK_SIZE = 128.kilobytes - CHUNK_REDIS_TTL = 1.week WRITE_LOCK_RETRY = 10 WRITE_LOCK_SLEEP = 0.01.seconds WRITE_LOCK_TTL = 1.minute + # Note: The ordering of this enum is related to the precedence of persist store. + # The bottom item takes the higest precedence, and the top item takes the lowest precedence. enum data_store: { redis: 1, - db: 2 + database: 2, + fog: 3 } class << self - def redis_data_key(build_id, chunk_index) - "gitlab:ci:trace:#{build_id}:chunks:#{chunk_index}" + def all_stores + @all_stores ||= self.data_stores.keys end - def redis_data_keys - redis.pluck(:build_id, :chunk_index).map do |data| - redis_data_key(data.first, data.second) - end + def persistable_store + # get first available store from the back of the list + all_stores.reverse.find { |store| get_store_class(store).available? } end - def redis_delete_data(keys) - return if keys.empty? - - Gitlab::Redis::SharedState.with do |redis| - redis.del(keys) - end + def get_store_class(store) + @stores ||= {} + @stores[store] ||= "Ci::BuildTraceChunks::#{store.capitalize}".constantize.new end ## # FastDestroyAll concerns def begin_fast_destroy - redis_data_keys + all_stores.each_with_object({}) do |store, result| + relation = public_send(store) # rubocop:disable GitlabSecurity/PublicSend + keys = get_store_class(store).keys(relation) + + result[store] = keys if keys.present? + end end ## # FastDestroyAll concerns def finalize_fast_destroy(keys) - redis_delete_data(keys) + keys.each do |store, value| + get_store_class(store).delete_keys(value) + end end end @@ -66,10 +70,15 @@ module Ci end def append(new_data, offset) + raise ArgumentError, 'New data is missing' unless new_data raise ArgumentError, 'Offset is out of range' if offset > size || offset < 0 raise ArgumentError, 'Chunk size overflow' if CHUNK_SIZE < (offset + new_data.bytesize) - set_data(data.byteslice(0, offset) + new_data) + in_lock(*lock_params) do # Write opetation is atomic + unsafe_set_data!(data.byteslice(0, offset) + new_data) + end + + schedule_to_persist if full? end def size @@ -88,93 +97,63 @@ module Ci (start_offset...end_offset) end - def use_database! - in_lock do - break if db? - break unless size > 0 - - self.update!(raw_data: data, data_store: :db) - self.class.redis_delete_data([redis_data_key]) + def persist_data! + in_lock(*lock_params) do # Write opetation is atomic + unsafe_persist_to!(self.class.persistable_store) end end private - def get_data - if redis? - redis_data - elsif db? - raw_data - else - raise 'Unsupported data store' - end&.force_encoding(Encoding::BINARY) # Redis/Database return UTF-8 string as default - end - - def set_data(value) - raise ArgumentError, 'too much data' if value.bytesize > CHUNK_SIZE - - in_lock do - if redis? - redis_set_data(value) - elsif db? - self.raw_data = value - else - raise 'Unsupported data store' - end + def unsafe_persist_to!(new_store) + return if data_store == new_store.to_s + raise ArgumentError, 'Can not persist empty data' unless size > 0 - @data = value + old_store_class = self.class.get_store_class(data_store) - save! if changed? + get_data.tap do |the_data| + self.raw_data = nil + self.data_store = new_store + unsafe_set_data!(the_data) end - schedule_to_db if full? - end - - def schedule_to_db - return if db? - - Ci::BuildTraceChunkFlushWorker.perform_async(id) + old_store_class.delete_data(self) end - def full? - size == CHUNK_SIZE + def get_data + self.class.get_store_class(data_store).data(self)&.force_encoding(Encoding::BINARY) # Redis/Database return UTF-8 string as default + rescue Excon::Error::NotFound + # If the data store is :fog and the file does not exist in the object storage, this method returns nil. end - def redis_data - Gitlab::Redis::SharedState.with do |redis| - redis.get(redis_data_key) - end - end + def unsafe_set_data!(value) + raise ArgumentError, 'New data size exceeds chunk size' if value.bytesize > CHUNK_SIZE - def redis_set_data(data) - Gitlab::Redis::SharedState.with do |redis| - redis.set(redis_data_key, data, ex: CHUNK_REDIS_TTL) - end - end + self.class.get_store_class(data_store).set_data(self, value) + @data = value - def redis_data_key - self.class.redis_data_key(build_id, chunk_index) + save! if changed? end - def in_lock - write_lock_key = "trace_write:#{build_id}:chunks:#{chunk_index}" + def schedule_to_persist + return if data_persisted? - lease = Gitlab::ExclusiveLease.new(write_lock_key, timeout: WRITE_LOCK_TTL) - retry_count = 0 + Ci::BuildTraceChunkFlushWorker.perform_async(id) + end - until uuid = lease.try_obtain - # Keep trying until we obtain the lease. To prevent hammering Redis too - # much we'll wait for a bit between retries. - sleep(WRITE_LOCK_SLEEP) - break if WRITE_LOCK_RETRY < (retry_count += 1) - end + def data_persisted? + !redis? + end - raise WriteError, 'Failed to obtain write lock' unless uuid + def full? + size == CHUNK_SIZE + end - self.reload if self.persisted? - return yield - ensure - Gitlab::ExclusiveLease.cancel(write_lock_key, uuid) + def lock_params + ["trace_write:#{build_id}:chunks:#{chunk_index}", + { ttl: WRITE_LOCK_TTL, + retries: WRITE_LOCK_RETRY, + sleep_sec: WRITE_LOCK_SLEEP }] end end end diff --git a/app/models/ci/build_trace_chunks/database.rb b/app/models/ci/build_trace_chunks/database.rb new file mode 100644 index 00000000000..3666d77c790 --- /dev/null +++ b/app/models/ci/build_trace_chunks/database.rb @@ -0,0 +1,29 @@ +module Ci + module BuildTraceChunks + class Database + def available? + true + end + + def keys(relation) + [] + end + + def delete_keys(keys) + # no-op + end + + def data(model) + model.raw_data + end + + def set_data(model, data) + model.raw_data = data + end + + def delete_data(model) + model.update_columns(raw_data: nil) unless model.raw_data.nil? + end + end + end +end diff --git a/app/models/ci/build_trace_chunks/fog.rb b/app/models/ci/build_trace_chunks/fog.rb new file mode 100644 index 00000000000..7506c40a39d --- /dev/null +++ b/app/models/ci/build_trace_chunks/fog.rb @@ -0,0 +1,59 @@ +module Ci + module BuildTraceChunks + class Fog + def available? + object_store.enabled + end + + def data(model) + connection.get_object(bucket_name, key(model))[:body] + end + + def set_data(model, data) + connection.put_object(bucket_name, key(model), data) + end + + def delete_data(model) + delete_keys([[model.build_id, model.chunk_index]]) + end + + def keys(relation) + return [] unless available? + + relation.pluck(:build_id, :chunk_index) + end + + def delete_keys(keys) + keys.each do |key| + connection.delete_object(bucket_name, key_raw(*key)) + end + end + + private + + def key(model) + key_raw(model.build_id, model.chunk_index) + end + + def key_raw(build_id, chunk_index) + "tmp/builds/#{build_id.to_i}/chunks/#{chunk_index.to_i}.log" + end + + def bucket_name + return unless available? + + object_store.remote_directory + end + + def connection + return unless available? + + @connection ||= ::Fog::Storage.new(object_store.connection.to_hash.deep_symbolize_keys) + end + + def object_store + Gitlab.config.artifacts.object_store + end + end + end +end diff --git a/app/models/ci/build_trace_chunks/redis.rb b/app/models/ci/build_trace_chunks/redis.rb new file mode 100644 index 00000000000..fdb6065e2a0 --- /dev/null +++ b/app/models/ci/build_trace_chunks/redis.rb @@ -0,0 +1,51 @@ +module Ci + module BuildTraceChunks + class Redis + CHUNK_REDIS_TTL = 1.week + + def available? + true + end + + def data(model) + Gitlab::Redis::SharedState.with do |redis| + redis.get(key(model)) + end + end + + def set_data(model, data) + Gitlab::Redis::SharedState.with do |redis| + redis.set(key(model), data, ex: CHUNK_REDIS_TTL) + end + end + + def delete_data(model) + delete_keys([[model.build_id, model.chunk_index]]) + end + + def keys(relation) + relation.pluck(:build_id, :chunk_index) + end + + def delete_keys(keys) + return if keys.empty? + + keys = keys.map { |key| key_raw(*key) } + + Gitlab::Redis::SharedState.with do |redis| + redis.del(keys) + end + end + + private + + def key(model) + key_raw(model.build_id, model.chunk_index) + end + + def key_raw(build_id, chunk_index) + "gitlab:ci:trace:#{build_id.to_i}:chunks:#{chunk_index.to_i}" + end + end + end +end diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 8c9aacca8de..bcd0c206bca 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -2,6 +2,7 @@ module Ci class Runner < ActiveRecord::Base extend Gitlab::Ci::Model include Gitlab::SQL::Pattern + include IgnorableColumn include RedisCacheable include ChronicDurationAttribute @@ -11,6 +12,8 @@ module Ci AVAILABLE_SCOPES = %w[specific shared active paused online].freeze FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level maximum_timeout_human_readable].freeze + ignore_column :is_shared + has_many :builds has_many :runner_projects, inverse_of: :runner, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :runner_projects @@ -21,13 +24,16 @@ module Ci before_validation :set_default_values - scope :specific, -> { where(is_shared: false) } - scope :shared, -> { where(is_shared: true) } scope :active, -> { where(active: true) } scope :paused, -> { where(active: false) } scope :online, -> { where('contacted_at > ?', contact_time_deadline) } scope :ordered, -> { order(id: :desc) } + # BACKWARD COMPATIBILITY: There are needed to maintain compatibility with `AVAILABLE_SCOPES` used by `lib/api/runners.rb` + scope :deprecated_shared, -> { instance_type } + # this should get replaced with `project_type.or(group_type)` once using Rails5 + scope :deprecated_specific, -> { where(runner_type: [runner_types[:project_type], runner_types[:group_type]]) } + scope :belonging_to_project, -> (project_id) { joins(:runner_projects).where(ci_runner_projects: { project_id: project_id }) } @@ -39,9 +45,9 @@ module Ci joins(:groups).where(namespaces: { id: hierarchy_groups }) } - scope :owned_or_shared, -> (project_id) do + scope :owned_or_instance_wide, -> (project_id) do union = Gitlab::SQL::Union.new( - [belonging_to_project(project_id), belonging_to_parent_group_of_project(project_id), shared], + [belonging_to_project(project_id), belonging_to_parent_group_of_project(project_id), instance_type], remove_duplicates: false ) from("(#{union.to_sql}) ci_runners") @@ -63,7 +69,6 @@ module Ci validate :no_groups, unless: :group_type? validate :any_project, if: :project_type? validate :exactly_one_group, if: :group_type? - validate :validate_is_shared acts_as_taggable @@ -113,8 +118,7 @@ module Ci end def assign_to(project, current_user = nil) - if shared? - self.is_shared = false if shared? + if instance_type? self.runner_type = :project_type elsif group_type? raise ArgumentError, 'Transitioning a group runner to a project runner is not supported' @@ -137,10 +141,6 @@ module Ci description end - def shared? - is_shared - end - def online? contacted_at && contacted_at > self.class.contact_time_deadline end @@ -159,10 +159,6 @@ module Ci runner_projects.count == 1 end - def specific? - !shared? - end - def assigned_to_group? runner_namespaces.any? end @@ -260,7 +256,7 @@ module Ci end def assignable_for?(project_id) - self.class.owned_or_shared(project_id).where(id: self.id).any? + self.class.owned_or_instance_wide(project_id).where(id: self.id).any? end def no_projects @@ -287,12 +283,6 @@ module Ci end end - def validate_is_shared - unless is_shared? == instance_type? - errors.add(:is_shared, 'is not equal to instance_type?') - end - end - def accepting_tags?(build) (run_untagged? || build.has_tags?) && (build.tag_list - tag_list).empty? end diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index 9f6358cecbe..b05bf909058 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -40,6 +40,18 @@ module CacheMarkdownField end end + class MarkdownEngine + def self.from_version(version = nil) + return :common_mark if version.nil? || version == 0 + + if version < CacheMarkdownField::CACHE_COMMONMARK_VERSION_START + :redcarpet + else + :common_mark + end + end + end + def skip_project_check? false end @@ -57,7 +69,7 @@ module CacheMarkdownField # Banzai is less strict about authors, so don't always have an author key context[:author] = self.author if self.respond_to?(:author) - context[:markdown_engine] = markdown_engine + context[:markdown_engine] = MarkdownEngine.from_version(latest_cached_markdown_version) context end @@ -123,14 +135,6 @@ module CacheMarkdownField end end - def markdown_engine - if latest_cached_markdown_version < CacheMarkdownField::CACHE_COMMONMARK_VERSION_START - :redcarpet - else - :common_mark - end - end - included do cattr_reader :cached_markdown_fields do FieldData.new diff --git a/app/models/concerns/cacheable_attributes.rb b/app/models/concerns/cacheable_attributes.rb index d58d7165969..606549b947f 100644 --- a/app/models/concerns/cacheable_attributes.rb +++ b/app/models/concerns/cacheable_attributes.rb @@ -7,7 +7,7 @@ module CacheableAttributes class_methods do def cache_key - "#{name}:#{Gitlab::VERSION}:#{Gitlab.migrations_hash}:#{Rails.version}".freeze + "#{name}:#{Gitlab::VERSION}:#{Rails.version}".freeze end # Can be overriden @@ -69,6 +69,6 @@ module CacheableAttributes end def cache! - Rails.cache.write(self.class.cache_key, self) + Rails.cache.write(self.class.cache_key, self, expires_in: 1.minute) end end diff --git a/app/models/concerns/group_descendant.rb b/app/models/concerns/group_descendant.rb index 261ace57a17..5e9a95c3282 100644 --- a/app/models/concerns/group_descendant.rb +++ b/app/models/concerns/group_descendant.rb @@ -44,8 +44,8 @@ module GroupDescendant This error is not user facing, but causes a +1 query. MSG extras = { - parent: parent, - child: child, + parent: parent.inspect, + child: child.inspect, preloaded: preloaded.map(&:full_path) } issue_url = 'https://gitlab.com/gitlab-org/gitlab-ce/issues/40785' diff --git a/app/models/concerns/protected_ref.rb b/app/models/concerns/protected_ref.rb index 94eef4ff7cd..dbe8d31de37 100644 --- a/app/models/concerns/protected_ref.rb +++ b/app/models/concerns/protected_ref.rb @@ -23,7 +23,7 @@ module ProtectedRef # If we don't `protected_branch` or `protected_tag` would be empty and # `project` cannot be delegated to it, which in turn would cause validations # to fail. - has_many :"#{type}_access_levels", inverse_of: self.model_name.singular # rubocop:disable Cop/ActiveRecordDependent + has_many :"#{type}_access_levels", inverse_of: self.model_name.singular validates :"#{type}_access_levels", length: { is: 1, message: "are restricted to a single instance per #{self.model_name.human}." } diff --git a/app/models/concerns/protected_ref_access.rb b/app/models/concerns/protected_ref_access.rb index e3a7f2d5498..71b0c3468b9 100644 --- a/app/models/concerns/protected_ref_access.rb +++ b/app/models/concerns/protected_ref_access.rb @@ -2,19 +2,20 @@ module ProtectedRefAccess extend ActiveSupport::Concern ALLOWED_ACCESS_LEVELS = [ - Gitlab::Access::MASTER, + Gitlab::Access::MAINTAINER, Gitlab::Access::DEVELOPER, Gitlab::Access::NO_ACCESS ].freeze HUMAN_ACCESS_LEVELS = { - Gitlab::Access::MASTER => "Maintainers".freeze, + Gitlab::Access::MAINTAINER => "Maintainers".freeze, Gitlab::Access::DEVELOPER => "Developers + Maintainers".freeze, Gitlab::Access::NO_ACCESS => "No one".freeze }.freeze included do - scope :master, -> { where(access_level: Gitlab::Access::MASTER) } + scope :master, -> { maintainer } # @deprecated + scope :maintainer, -> { where(access_level: Gitlab::Access::MAINTAINER) } scope :developer, -> { where(access_level: Gitlab::Access::DEVELOPER) } validates :access_level, presence: true, if: :role?, inclusion: { diff --git a/app/models/concerns/select_for_project_authorization.rb b/app/models/concerns/select_for_project_authorization.rb index 58194b0ea13..7af0fdbd618 100644 --- a/app/models/concerns/select_for_project_authorization.rb +++ b/app/models/concerns/select_for_project_authorization.rb @@ -6,8 +6,11 @@ module SelectForProjectAuthorization select("projects.id AS project_id, members.access_level") end - def select_as_master_for_project_authorization - select(["projects.id AS project_id", "#{Gitlab::Access::MASTER} AS access_level"]) + def select_as_maintainer_for_project_authorization + select(["projects.id AS project_id", "#{Gitlab::Access::MAINTAINER} AS access_level"]) end + + # @deprecated + alias_method :select_as_master_for_project_authorization, :select_as_maintainer_for_project_authorization end end diff --git a/app/models/group.rb b/app/models/group.rb index 9c171de7fc3..ddebaff50b0 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -178,10 +178,13 @@ class Group < Namespace add_user(user, :developer, current_user: current_user) end - def add_master(user, current_user = nil) - add_user(user, :master, current_user: current_user) + def add_maintainer(user, current_user = nil) + add_user(user, :maintainer, current_user: current_user) end + # @deprecated + alias_method :add_master, :add_maintainer + def add_owner(user, current_user = nil) add_user(user, :owner, current_user: current_user) end @@ -198,12 +201,15 @@ class Group < Namespace members_with_parents.owners.where(user_id: user).any? end - def has_master?(user) + def has_maintainer?(user) return false unless user - members_with_parents.masters.where(user_id: user).any? + members_with_parents.maintainers.where(user_id: user).any? end + # @deprecated + alias_method :has_master?, :has_maintainer? + # Check if user is a last owner of the group. # Parent owners are ignored for nested groups. def last_owner?(user) diff --git a/app/models/hooks/web_hook_log.rb b/app/models/hooks/web_hook_log.rb index e72c125fb69..59a1f2aed69 100644 --- a/app/models/hooks/web_hook_log.rb +++ b/app/models/hooks/web_hook_log.rb @@ -7,6 +7,11 @@ class WebHookLog < ActiveRecord::Base validates :web_hook, presence: true + def self.recent + where('created_at >= ?', 2.days.ago.beginning_of_day) + .order(created_at: :desc) + end + def success? response_status =~ /^2/ end diff --git a/app/models/import_export_upload.rb b/app/models/import_export_upload.rb new file mode 100644 index 00000000000..60d53d6c2c8 --- /dev/null +++ b/app/models/import_export_upload.rb @@ -0,0 +1,13 @@ +class ImportExportUpload < ActiveRecord::Base + include WithUploads + include ObjectStorage::BackgroundMove + + belongs_to :project + + mount_uploader :import_file, ImportExportUploader + mount_uploader :export_file, ImportExportUploader + + def retrieve_upload(_identifier, paths) + Upload.find_by(model: self, path: paths) + end +end diff --git a/app/models/member.rb b/app/models/member.rb index 68572f2e33a..00a13a279a9 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -69,9 +69,11 @@ class Member < ActiveRecord::Base scope :guests, -> { active.where(access_level: GUEST) } scope :reporters, -> { active.where(access_level: REPORTER) } scope :developers, -> { active.where(access_level: DEVELOPER) } - scope :masters, -> { active.where(access_level: MASTER) } + scope :maintainers, -> { active.where(access_level: MAINTAINER) } + scope :masters, -> { maintainers } # @deprecated scope :owners, -> { active.where(access_level: OWNER) } - scope :owners_and_masters, -> { active.where(access_level: [OWNER, MASTER]) } + scope :owners_and_maintainers, -> { active.where(access_level: [OWNER, MAINTAINER]) } + scope :owners_and_masters, -> { owners_and_maintainers } # @deprecated scope :order_name_asc, -> { left_join_users.reorder(Gitlab::Database.nulls_last_order('users.name', 'ASC')) } scope :order_name_desc, -> { left_join_users.reorder(Gitlab::Database.nulls_last_order('users.name', 'DESC')) } diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb index 024106056b4..4f27d0aeaf8 100644 --- a/app/models/members/project_member.rb +++ b/app/models/members/project_member.rb @@ -17,19 +17,19 @@ class ProjectMember < Member # Add users to projects with passed access option # # access can be an integer representing a access code - # or symbol like :master representing role + # or symbol like :maintainer representing role # # Ex. # add_users_to_projects( # project_ids, # user_ids, - # ProjectMember::MASTER + # ProjectMember::MAINTAINER # ) # # add_users_to_projects( # project_ids, # user_ids, - # :master + # :maintainer # ) # def add_users_to_projects(project_ids, users, access_level, current_user: nil, expires_at: nil) diff --git a/app/models/milestone.rb b/app/models/milestone.rb index d05dcfd083a..14cc12b38a5 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -131,9 +131,10 @@ class Milestone < ActiveRecord::Base rel.order(:project_id, :due_date).select('DISTINCT ON (project_id) id') else rel - .group(:project_id) + .group(:project_id, :due_date, :id) .having('due_date = MIN(due_date)') .pluck(:id, :project_id, :due_date) + .uniq(&:second) .map(&:first) end end diff --git a/app/models/network/commit.rb b/app/models/network/commit.rb index 22d48c9e661..d667948deae 100644 --- a/app/models/network/commit.rb +++ b/app/models/network/commit.rb @@ -11,8 +11,8 @@ module Network @parent_spaces = [] end - def method_missing(m, *args, &block) - @commit.__send__(m, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + def method_missing(msg, *args, &block) + @commit.__send__(msg, *args, &block) # rubocop:disable GitlabSecurity/PublicSend end def space diff --git a/app/models/notification_setting.rb b/app/models/notification_setting.rb index 9195408551f..1933c46ee44 100644 --- a/app/models/notification_setting.rb +++ b/app/models/notification_setting.rb @@ -32,6 +32,7 @@ class NotificationSetting < ActiveRecord::Base :reopen_issue, :close_issue, :reassign_issue, + :issue_due, :new_merge_request, :push_to_merge_request, :reopen_merge_request, diff --git a/app/models/project.rb b/app/models/project.rb index d91d7dcfe9a..1894de6ceed 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -171,6 +171,7 @@ class Project < ActiveRecord::Base has_one :fork_network, through: :fork_network_member has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project + has_one :import_export_upload, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent # Merge Requests for target project should be removed with it has_many :merge_requests, foreign_key: 'target_project_id' @@ -268,7 +269,8 @@ class Project < ActiveRecord::Base delegate :name, to: :owner, allow_nil: true, prefix: true delegate :members, to: :team, prefix: true delegate :add_user, :add_users, to: :team - delegate :add_guest, :add_reporter, :add_developer, :add_master, :add_role, to: :team + delegate :add_guest, :add_reporter, :add_developer, :add_maintainer, :add_role, to: :team + delegate :add_master, to: :team # @deprecated delegate :group_runners_enabled, :group_runners_enabled=, :group_runners_enabled?, to: :ci_cd_settings # Validations @@ -1422,7 +1424,7 @@ class Project < ActiveRecord::Base end def shared_runners - @shared_runners ||= shared_runners_available? ? Ci::Runner.shared : Ci::Runner.none + @shared_runners ||= shared_runners_available? ? Ci::Runner.instance_type : Ci::Runner.none end def group_runners @@ -1646,10 +1648,10 @@ class Project < ActiveRecord::Base params = { name: default_branch, push_access_levels_attributes: [{ - access_level: Gitlab::CurrentSettings.default_branch_protection == Gitlab::Access::PROTECTION_DEV_CAN_PUSH ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER + access_level: Gitlab::CurrentSettings.default_branch_protection == Gitlab::Access::PROTECTION_DEV_CAN_PUSH ? Gitlab::Access::DEVELOPER : Gitlab::Access::MAINTAINER }], merge_access_levels_attributes: [{ - access_level: Gitlab::CurrentSettings.default_branch_protection == Gitlab::Access::PROTECTION_DEV_CAN_MERGE ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER + access_level: Gitlab::CurrentSettings.default_branch_protection == Gitlab::Access::PROTECTION_DEV_CAN_MERGE ? Gitlab::Access::DEVELOPER : Gitlab::Access::MAINTAINER }] } @@ -1712,7 +1714,7 @@ class Project < ActiveRecord::Base :started elsif after_export_in_progress? :after_export_action - elsif export_project_path + elsif export_project_path || export_project_object_exists? :finished else :none @@ -1727,16 +1729,21 @@ class Project < ActiveRecord::Base import_export_shared.after_export_in_progress? end - def remove_exports - return nil unless export_path.present? - - FileUtils.rm_rf(export_path) + def remove_exports(path = export_path) + if path.present? + FileUtils.rm_rf(path) + elsif export_project_object_exists? + import_export_upload.remove_export_file! + import_export_upload.save + end end def remove_exported_project_file - return unless export_project_path.present? + remove_exports(export_project_path) + end - FileUtils.rm_f(export_project_path) + def export_project_object_exists? + Gitlab::ImportExport.object_storage? && import_export_upload&.export_file&.file end def full_path_slug @@ -1774,6 +1781,15 @@ class Project < ActiveRecord::Base end end + def default_environment + production_first = "(CASE WHEN name = 'production' THEN 0 ELSE 1 END), id ASC" + + environments + .with_state(:available) + .reorder(production_first) + .first + end + def secret_variables_for(ref:, environment: nil) # EE would use the environment if protected_for?(ref) diff --git a/app/models/project_group_link.rb b/app/models/project_group_link.rb index ac1e9ab2b0b..cf8fc41e870 100644 --- a/app/models/project_group_link.rb +++ b/app/models/project_group_link.rb @@ -4,7 +4,8 @@ class ProjectGroupLink < ActiveRecord::Base GUEST = 10 REPORTER = 20 DEVELOPER = 30 - MASTER = 40 + MAINTAINER = 40 + MASTER = MAINTAINER # @deprecated belongs_to :project belongs_to :group diff --git a/app/models/project_services/bamboo_service.rb b/app/models/project_services/bamboo_service.rb index 7f4c47a6d14..edc5c00d9c4 100644 --- a/app/models/project_services/bamboo_service.rb +++ b/app/models/project_services/bamboo_service.rb @@ -67,11 +67,11 @@ class BambooService < CiService def execute(data) return unless supported_events.include?(data[:object_kind]) - get_path("updateAndBuild.action?buildKey=#{build_key}") + get_path("updateAndBuild.action", { buildKey: build_key }) end def calculate_reactive_cache(sha, ref) - response = get_path("rest/api/latest/result?label=#{sha}") + response = get_path("rest/api/latest/result/byChangeset/#{sha}") { build_page: read_build_page(response), commit_status: read_commit_status(response) } end @@ -113,18 +113,20 @@ class BambooService < CiService URI.join("#{bamboo_url}/", path).to_s end - def get_path(path) + def get_path(path, query_params = {}) url = build_url(path) if username.blank? && password.blank? - Gitlab::HTTP.get(url, verify: false) + Gitlab::HTTP.get(url, verify: false, query: query_params) else - url << '&os_authType=basic' - Gitlab::HTTP.get(url, verify: false, - basic_auth: { - username: username, - password: password - }) + query_params[:os_authType] = 'basic' + Gitlab::HTTP.get(url, + verify: false, + query: query_params, + basic_auth: { + username: username, + password: password + }) end end end diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb index ddd4026019b..722642f6da7 100644 --- a/app/models/project_services/kubernetes_service.rb +++ b/app/models/project_services/kubernetes_service.rb @@ -240,7 +240,7 @@ class KubernetesService < DeploymentService end def deprecation_validation - return if active_changed?(from: true, to: false) + return if active_changed?(from: true, to: false) || (new_record? && !active?) if deprecated? errors[:base] << deprecation_message diff --git a/app/models/project_team.rb b/app/models/project_team.rb index 9a38806baab..c7d0f49d837 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -19,10 +19,13 @@ class ProjectTeam add_user(user, :developer, current_user: current_user) end - def add_master(user, current_user: nil) - add_user(user, :master, current_user: current_user) + def add_maintainer(user, current_user: nil) + add_user(user, :maintainer, current_user: current_user) end + # @deprecated + alias_method :add_master, :add_maintainer + def add_role(user, role, current_user: nil) public_send(:"add_#{role}", user, current_user: current_user) # rubocop:disable GitlabSecurity/PublicSend end @@ -81,10 +84,13 @@ class ProjectTeam @developers ||= fetch_members(Gitlab::Access::DEVELOPER) end - def masters - @masters ||= fetch_members(Gitlab::Access::MASTER) + def maintainers + @maintainers ||= fetch_members(Gitlab::Access::MAINTAINER) end + # @deprecated + alias_method :masters, :maintainers + def owners @owners ||= if group @@ -136,10 +142,13 @@ class ProjectTeam max_member_access(user.id) == Gitlab::Access::DEVELOPER end - def master?(user) - max_member_access(user.id) == Gitlab::Access::MASTER + def maintainer?(user) + max_member_access(user.id) == Gitlab::Access::MAINTAINER end + # @deprecated + alias_method :master?, :maintainer? + # Checks if `user` is authorized for this project, with at least the # `min_access_level` (if given). def member?(user, min_access_level = Gitlab::Access::GUEST) diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index 05b07804ea8..3aa56b3983f 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -23,7 +23,6 @@ class ProjectWiki @user = user end - delegate :empty?, to: :pages delegate :repository_storage, :hashed_storage?, to: :project def path @@ -77,6 +76,10 @@ class ProjectWiki !!find_page('home') end + def empty? + pages(limit: 1).empty? + end + # Returns an Array of Gitlab WikiPage instances or an # empty Array if this Wiki has no pages. def pages(limit: nil) @@ -114,7 +117,7 @@ class ProjectWiki update_project_activity rescue Gitlab::Git::Wiki::DuplicatePageError => e @error_message = "Duplicate page: #{e.message}" - return false + false end def update_page(page, content:, title: nil, format: :markdown, message: nil) diff --git a/app/models/remote_mirror.rb b/app/models/remote_mirror.rb index c4b5dd2dc96..976b501e297 100644 --- a/app/models/remote_mirror.rb +++ b/app/models/remote_mirror.rb @@ -57,7 +57,7 @@ class RemoteMirror < ActiveRecord::Base Gitlab::Metrics.add_event(:remote_mirrors_finished, path: remote_mirror.project.full_path) timestamp = Time.now - remote_mirror.update_attributes!( + remote_mirror.update!( last_update_at: timestamp, last_successful_update_at: timestamp, last_error: nil ) end diff --git a/app/models/repository.rb b/app/models/repository.rb index 5f9894f1168..a96c73e6ab7 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -83,7 +83,7 @@ class Repository @raw_repository&.cleanup end - # Return absolute path to repository + # Don't use this! It's going away. Use Gitaly to read or write from repos. def path_to_repo @path_to_repo ||= begin @@ -174,8 +174,8 @@ class Repository CommitCollection.new(project, commits, ref) end - def find_branch(name, fresh_repo: true) - raw_repository.find_branch(name, fresh_repo) + def find_branch(name) + raw_repository.find_branch(name) end def find_tag(name) @@ -250,7 +250,7 @@ class Repository # This will still fail if the file is corrupted (e.g. 0 bytes) raw_repository.write_ref(keep_around_ref_name(sha), sha, shell: false) rescue Gitlab::Git::CommandError => ex - Rails.logger.error "Unable to create keep-around reference for repository #{path}: #{ex}" + Rails.logger.error "Unable to create keep-around reference for repository #{disk_path}: #{ex}" end def kept_around?(sha) @@ -462,12 +462,12 @@ class Repository expire_branches_cache end - def method_missing(m, *args, &block) - if m == :lookup && !block_given? - lookup_cache[m] ||= {} - lookup_cache[m][args.join(":")] ||= raw_repository.__send__(m, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + def method_missing(msg, *args, &block) + if msg == :lookup && !block_given? + lookup_cache[msg] ||= {} + lookup_cache[msg][args.join(":")] ||= raw_repository.__send__(msg, *args, &block) # rubocop:disable GitlabSecurity/PublicSend else - raw_repository.__send__(m, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + raw_repository.__send__(msg, *args, &block) # rubocop:disable GitlabSecurity/PublicSend end end @@ -564,7 +564,7 @@ class Repository end def rendered_readme - MarkupHelper.markup_unsafe(readme.name, readme.data, project: project) if readme + MarkupHelper.markup_unsafe(readme.name, readme.data, project: project, markdown_engine: :redcarpet) if readme end cache_method :rendered_readme diff --git a/app/models/service.rb b/app/models/service.rb index 1d259bcfec7..ad835293b46 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -281,9 +281,9 @@ class Service < ActiveRecord::Base def self.build_from_template(project_id, template) service = template.dup - service.active = false unless service.valid? service.template = false service.project_id = project_id + service.active = false if service.active? && !service.valid? service end diff --git a/app/models/user.rb b/app/models/user.rb index 48629c58490..4987d01aac6 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -99,7 +99,8 @@ class User < ActiveRecord::Base has_many :group_members, -> { where(requested_at: nil) }, source: 'GroupMember' has_many :groups, through: :group_members has_many :owned_groups, -> { where(members: { access_level: Gitlab::Access::OWNER }) }, through: :group_members, source: :group - has_many :masters_groups, -> { where(members: { access_level: Gitlab::Access::MASTER }) }, through: :group_members, source: :group + has_many :maintainers_groups, -> { where(members: { access_level: Gitlab::Access::MAINTAINER }) }, through: :group_members, source: :group + alias_attribute :masters_groups, :maintainers_groups # Projects has_many :groups_projects, through: :groups, source: :projects @@ -496,7 +497,7 @@ class User < ActiveRecord::Base def disable_two_factor! transaction do - update_attributes( + update( otp_required_for_login: false, encrypted_otp_secret: nil, encrypted_otp_secret_iv: nil, @@ -728,7 +729,7 @@ class User < ActiveRecord::Base end def several_namespaces? - owned_groups.any? || masters_groups.any? + owned_groups.any? || maintainers_groups.any? end def namespace_id @@ -974,15 +975,15 @@ class User < ActiveRecord::Base end def manageable_groups - union_sql = Gitlab::SQL::Union.new([owned_groups.select(:id), masters_groups.select(:id)]).to_sql + union_sql = Gitlab::SQL::Union.new([owned_groups.select(:id), maintainers_groups.select(:id)]).to_sql # Update this line to not use raw SQL when migrated to Rails 5.2. # Either ActiveRecord or Arel constructions are fine. # This was replaced with the raw SQL construction because of bugs in the arel gem. # Bugs were fixed in arel 9.0.0 (Rails 5.2). - owned_and_master_groups = Group.where("namespaces.id IN (#{union_sql})") # rubocop:disable GitlabSecurity/SqlInjection + owned_and_maintainer_groups = Group.where("namespaces.id IN (#{union_sql})") # rubocop:disable GitlabSecurity/SqlInjection - Gitlab::GroupHierarchy.new(owned_and_master_groups).base_and_descendants + Gitlab::GroupHierarchy.new(owned_and_maintainer_groups).base_and_descendants end def namespaces @@ -1023,16 +1024,16 @@ class User < ActiveRecord::Base def ci_owned_runners @ci_owned_runners ||= begin project_runner_ids = Ci::RunnerProject - .where(project: authorized_projects(Gitlab::Access::MASTER)) + .where(project: authorized_projects(Gitlab::Access::MAINTAINER)) .select(:runner_id) group_runner_ids = Ci::RunnerNamespace - .where(namespace_id: owned_or_masters_groups.select(:id)) + .where(namespace_id: owned_or_maintainers_groups.select(:id)) .select(:runner_id) union = Gitlab::SQL::Union.new([project_runner_ids, group_runner_ids]) - Ci::Runner.specific.where("ci_runners.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection + Ci::Runner.where("ci_runners.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection end end @@ -1053,7 +1054,7 @@ class User < ActiveRecord::Base return @global_notification_setting if defined?(@global_notification_setting) @global_notification_setting = notification_settings.find_or_initialize_by(source: nil) - @global_notification_setting.update_attributes(level: NotificationSetting.levels[DEFAULT_NOTIFICATION_LEVEL]) unless @global_notification_setting.persisted? + @global_notification_setting.update(level: NotificationSetting.levels[DEFAULT_NOTIFICATION_LEVEL]) unless @global_notification_setting.persisted? @global_notification_setting end @@ -1236,11 +1237,14 @@ class User < ActiveRecord::Base !terms_accepted? end - def owned_or_masters_groups - union = Gitlab::SQL::Union.new([owned_groups, masters_groups]) + def owned_or_maintainers_groups + union = Gitlab::SQL::Union.new([owned_groups, maintainers_groups]) Group.from("(#{union.to_sql}) namespaces") end + # @deprecated + alias_method :owned_or_masters_groups, :owned_or_maintainers_groups + protected # override, from Devise::Validatable @@ -1333,8 +1337,8 @@ class User < ActiveRecord::Base end end - def self.unique_internal(scope, username, email_pattern, &b) - scope.first || create_unique_internal(scope, username, email_pattern, &b) + def self.unique_internal(scope, username, email_pattern, &block) + scope.first || create_unique_internal(scope, username, email_pattern, &block) end def self.create_unique_internal(scope, username, email_pattern, &creation_block) diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index cde79b95062..4b49edb01a5 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -1,3 +1,4 @@ +# rubocop:disable Rails/ActiveRecordAliases class WikiPage PageChangedError = Class.new(StandardError) PageRenameError = Class.new(StandardError) |