diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-09-02 15:10:35 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-09-02 15:10:35 +0300 |
commit | 4fa04f789e6fed5f0dfeafe718eeb7f56a5086e9 (patch) | |
tree | 5ef2d1d8232d3bd359ec79bf95c9a35ce650ae0b /lib | |
parent | 4b9ace6c1fead1b44f173eaee0cfaa58f46a258a (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r-- | lib/gitlab/cross_project_access.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/data_builder/push.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/database/partitioning/partition_monitoring.rb | 34 | ||||
-rw-r--r-- | lib/gitlab/highlight.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/search/recent_issues.rb | 60 | ||||
-rw-r--r-- | lib/gitlab/sql/except.rb | 22 | ||||
-rw-r--r-- | lib/gitlab/sql/intersect.rb | 23 | ||||
-rw-r--r-- | lib/gitlab/sql/set_operator.rb | 53 | ||||
-rw-r--r-- | lib/gitlab/sql/union.rb | 27 | ||||
-rw-r--r-- | lib/gitlab/usage_data.rb | 17 | ||||
-rw-r--r-- | lib/gitlab/usage_data_counters/track_unique_events.rb | 7 | ||||
-rw-r--r-- | lib/tasks/gitlab/lfs/migrate.rake | 10 |
12 files changed, 220 insertions, 41 deletions
diff --git a/lib/gitlab/cross_project_access.rb b/lib/gitlab/cross_project_access.rb index 4ddc7e02d1b..93baf1e596c 100644 --- a/lib/gitlab/cross_project_access.rb +++ b/lib/gitlab/cross_project_access.rb @@ -18,7 +18,7 @@ module Gitlab end def add_check( - klass, + klass, actions: {}, positive_condition: nil, negative_condition: nil, diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index af363705bed..f941c57a6dd 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -86,7 +86,7 @@ module Gitlab # # rubocop:disable Metrics/ParameterLists def build( - project:, user:, ref:, oldrev: nil, newrev: nil, + project:, user:, ref:, oldrev: nil, newrev: nil, commits: [], commits_count: nil, message: nil, push_options: {}, with_changed_files: true) diff --git a/lib/gitlab/database/partitioning/partition_monitoring.rb b/lib/gitlab/database/partitioning/partition_monitoring.rb new file mode 100644 index 00000000000..9ec9ae684a5 --- /dev/null +++ b/lib/gitlab/database/partitioning/partition_monitoring.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class PartitionMonitoring + attr_reader :models + + def initialize(models = PartitionCreator.models) + @models = models + end + + def report_metrics + models.each do |model| + strategy = model.partitioning_strategy + + gauge_present.set({ table: model.table_name }, strategy.current_partitions.size) + gauge_missing.set({ table: model.table_name }, strategy.missing_partitions.size) + end + end + + private + + def gauge_present + @gauge_present ||= Gitlab::Metrics.gauge(:db_partitions_present, 'Number of database partitions present') + end + + def gauge_missing + @gauge_missing ||= Gitlab::Metrics.gauge(:db_partitions_missing, 'Number of database partitions currently expected, but not present') + end + end + end + end +end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 63cf80a10ed..40dee0142b9 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -31,8 +31,8 @@ module Gitlab def lexer @lexer ||= custom_language || begin Rouge::Lexer.guess(filename: @blob_name, source: @blob_content).new - rescue Rouge::Guesser::Ambiguous => e - e.alternatives.min_by(&:tag) + rescue Rouge::Guesser::Ambiguous => e + e.alternatives.min_by(&:tag) end end diff --git a/lib/gitlab/search/recent_issues.rb b/lib/gitlab/search/recent_issues.rb new file mode 100644 index 00000000000..755ca35bd66 --- /dev/null +++ b/lib/gitlab/search/recent_issues.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module Search + class RecentIssues + ITEMS_LIMIT = 100 + EXPIRES_AFTER = 7.days + + def initialize(user:, items_limit: ITEMS_LIMIT, expires_after: EXPIRES_AFTER) + @user = user + @items_limit = items_limit + @expires_after = expires_after + end + + def log_view(issue) + return unless recent_items_enabled? + + with_redis do |redis| + redis.zadd(key, Time.now.to_f, issue.id) + redis.expire(key, @expires_after) + + # There is a race condition here where we could end up removing an + # item from 2 places concurrently but this is fine since worst case + # scenario we remove an extra item from the end of the list. + if redis.zcard(key) > @items_limit + redis.zremrangebyrank(key, 0, 0) # Remove least recent + end + end + end + + def search(term) + return Issue.none unless recent_items_enabled? + + ids = with_redis do |redis| + redis.zrevrange(key, 0, @items_limit - 1) + end.map(&:to_i) + + IssuesFinder.new(@user, search: term, in: 'title').execute.reorder(nil).id_in_ordered(ids) # rubocop: disable CodeReuse/ActiveRecord + end + + private + + def with_redis(&blk) + Gitlab::Redis::SharedState.with(&blk) # rubocop: disable CodeReuse/ActiveRecord + end + + def key + "recent_items:#{type.name.downcase}:#{@user.id}" + end + + def type + Issue + end + + def recent_items_enabled? + Feature.enabled?(:recent_items_search, @user) + end + end + end +end diff --git a/lib/gitlab/sql/except.rb b/lib/gitlab/sql/except.rb new file mode 100644 index 00000000000..82cbfa8d4ab --- /dev/null +++ b/lib/gitlab/sql/except.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module SQL + # Class for building SQL EXCEPT statements. + # + # ORDER BYs are dropped from the relations as the final sort order is not + # guaranteed any way. + # + # Example usage: + # + # except = Gitlab::SQL::Except.new([user.projects, user.personal_projects]) + # sql = except.to_sql + # + # Project.where("id IN (#{sql})") + class Except < SetOperator + def self.operator_keyword + 'EXCEPT' + end + end + end +end diff --git a/lib/gitlab/sql/intersect.rb b/lib/gitlab/sql/intersect.rb new file mode 100644 index 00000000000..c661db3d4c5 --- /dev/null +++ b/lib/gitlab/sql/intersect.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module SQL + # Class for building SQL INTERSECT statements. + # + # ORDER BYs are dropped from the relations as the final sort order is not + # guaranteed any way. + # + # Example usage: + # + # hierarchies = [group1.self_and_hierarchy, group2.self_and_hierarchy] + # intersect = Gitlab::SQL::Intersect.new(hierarchies) + # sql = intersect.to_sql + # + # Project.where("id IN (#{sql})") + class Intersect < SetOperator + def self.operator_keyword + 'INTERSECT' + end + end + end +end diff --git a/lib/gitlab/sql/set_operator.rb b/lib/gitlab/sql/set_operator.rb new file mode 100644 index 00000000000..d58a1415493 --- /dev/null +++ b/lib/gitlab/sql/set_operator.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module SQL + # Class for building SQL set operator statements (UNION, INTERSECT, and + # EXCEPT). + # + # ORDER BYs are dropped from the relations as the final sort order is not + # guaranteed any way. + # + # Example usage: + # + # union = Gitlab::SQL::Union.new([user.personal_projects, user.projects]) + # sql = union.to_sql + # + # Project.where("id IN (#{sql})") + class SetOperator + def initialize(relations, remove_duplicates: true) + @relations = relations + @remove_duplicates = remove_duplicates + end + + def self.operator_keyword + raise NotImplementedError + end + + def to_sql + # Some relations may include placeholders for prepared statements, these + # aren't incremented properly when joining relations together this way. + # By using "unprepared_statements" we remove the usage of placeholders + # (thus fixing this problem), at a slight performance cost. + fragments = ActiveRecord::Base.connection.unprepared_statement do + relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?) + end + + if fragments.any? + "(" + fragments.join(")\n#{operator_keyword_fragment}\n(") + ")" + else + 'NULL' + end + end + + # UNION [ALL] | INTERSECT [ALL] | EXCEPT [ALL] + def operator_keyword_fragment + remove_duplicates ? self.class.operator_keyword : "#{self.class.operator_keyword} ALL" + end + + private + + attr_reader :relations, :remove_duplicates + end + end +end diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb index b15f2ca385a..7fb3487a5e5 100644 --- a/lib/gitlab/sql/union.rb +++ b/lib/gitlab/sql/union.rb @@ -13,30 +13,9 @@ module Gitlab # sql = union.to_sql # # Project.where("id IN (#{sql})") - class Union - def initialize(relations, remove_duplicates: true) - @relations = relations - @remove_duplicates = remove_duplicates - end - - def to_sql - # Some relations may include placeholders for prepared statements, these - # aren't incremented properly when joining relations together this way. - # By using "unprepared_statements" we remove the usage of placeholders - # (thus fixing this problem), at a slight performance cost. - fragments = ActiveRecord::Base.connection.unprepared_statement do - @relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?) - end - - if fragments.any? - "(" + fragments.join(")\n#{union_keyword}\n(") + ")" - else - 'NULL' - end - end - - def union_keyword - @remove_duplicates ? 'UNION' : 'UNION ALL' + class Union < SetOperator + def self.operator_keyword + 'UNION' end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 504ffe0a2ad..1dd6eab1743 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -426,16 +426,17 @@ module Gitlab {} # augmented in EE end - # rubocop: disable CodeReuse/ActiveRecord def merge_requests_users(time_period) - distinct_count( - Event.where(target_type: Event::TARGET_TYPES[:merge_request].to_s).where(time_period), - :author_id, - start: user_minimum_id, - finish: user_maximum_id - ) + counter = Gitlab::UsageDataCounters::TrackUniqueEvents + + redis_usage_data do + counter.count_unique_events( + event_action: Gitlab::UsageDataCounters::TrackUniqueEvents::MERGE_REQUEST_ACTION, + date_from: time_period[:created_at].first, + date_to: time_period[:created_at].last + ) + end end - # rubocop: enable CodeReuse/ActiveRecord def installation_type if Rails.env.production? diff --git a/lib/gitlab/usage_data_counters/track_unique_events.rb b/lib/gitlab/usage_data_counters/track_unique_events.rb index f2a217e980e..71a4574eea9 100644 --- a/lib/gitlab/usage_data_counters/track_unique_events.rb +++ b/lib/gitlab/usage_data_counters/track_unique_events.rb @@ -6,6 +6,7 @@ module Gitlab WIKI_ACTION = :wiki_action DESIGN_ACTION = :design_action PUSH_ACTION = :project_action + MERGE_REQUEST_ACTION = :merge_request_action ACTION_TRANSFORMATIONS = HashWithIndifferentAccess.new({ wiki: { @@ -20,6 +21,12 @@ module Gitlab }, project: { pushed: PUSH_ACTION + }, + merge_request: { + closed: MERGE_REQUEST_ACTION, + merged: MERGE_REQUEST_ACTION, + created: MERGE_REQUEST_ACTION, + commented: MERGE_REQUEST_ACTION } }).freeze diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake index 470a12c39cd..3d4c847a0f0 100644 --- a/lib/tasks/gitlab/lfs/migrate.rake +++ b/lib/tasks/gitlab/lfs/migrate.rake @@ -9,12 +9,12 @@ namespace :gitlab do LfsObject.with_files_stored_locally .find_each(batch_size: 10) do |lfs_object| - lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE) + lfs_object.file.migrate!(LfsObjectUploader::Store::REMOTE) - logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage") - rescue => e - logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}") - end + logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to object storage") + rescue => e + logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}") + end end task migrate_to_local: :environment do |