diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-10-05 16:54:15 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-10-05 16:54:15 +0300 |
commit | be834a25982746ffd85252ff502df42bb88cb9d5 (patch) | |
tree | b4d6a8ba0931e12fac08f05abea33a3b8ec2c8a2 /lib/gitlab | |
parent | ee925a3597f27e92f83a50937a64068109675b3d (diff) |
Add latest changes from gitlab-org/gitlab@13-5-stable-eev13.5.0-rc32
Diffstat (limited to 'lib/gitlab')
113 files changed, 1780 insertions, 830 deletions
diff --git a/lib/gitlab/alert_management/alert_params.rb b/lib/gitlab/alert_management/alert_params.rb deleted file mode 100644 index 3bb839c1114..00000000000 --- a/lib/gitlab/alert_management/alert_params.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module AlertManagement - class AlertParams - MONITORING_TOOLS = { - prometheus: 'Prometheus' - }.freeze - - def self.from_generic_alert(project:, payload:) - parsed_payload = Gitlab::Alerting::NotificationPayloadParser.call(payload, project).with_indifferent_access - annotations = parsed_payload[:annotations] - - { - project_id: project.id, - title: annotations[:title], - description: annotations[:description], - monitoring_tool: annotations[:monitoring_tool], - service: annotations[:service], - hosts: Array(annotations[:hosts]), - payload: payload, - started_at: parsed_payload['startsAt'], - ended_at: parsed_payload['endsAt'], - severity: annotations[:severity], - fingerprint: annotations[:fingerprint], - environment: annotations[:environment] - } - end - - def self.from_prometheus_alert(project:, parsed_alert:) - { - project_id: project.id, - title: parsed_alert.title, - description: parsed_alert.description, - monitoring_tool: MONITORING_TOOLS[:prometheus], - payload: parsed_alert.payload, - started_at: parsed_alert.starts_at, - ended_at: parsed_alert.ends_at, - fingerprint: parsed_alert.gitlab_fingerprint, - environment: parsed_alert.environment, - prometheus_alert: parsed_alert.gitlab_alert - } - end - end - end -end diff --git a/lib/gitlab/alert_management/alert_status_counts.rb b/lib/gitlab/alert_management/alert_status_counts.rb index 382026236e0..e88436d479b 100644 --- a/lib/gitlab/alert_management/alert_status_counts.rb +++ b/lib/gitlab/alert_management/alert_status_counts.rb @@ -30,7 +30,7 @@ module Gitlab end def all - counts.values.sum # rubocop:disable CodeReuse/ActiveRecord + counts.values.sum end private diff --git a/lib/gitlab/alert_management/payload/generic.rb b/lib/gitlab/alert_management/payload/generic.rb index 7efdfac75dc..e8e85155bef 100644 --- a/lib/gitlab/alert_management/payload/generic.rb +++ b/lib/gitlab/alert_management/payload/generic.rb @@ -8,6 +8,8 @@ module Gitlab DEFAULT_TITLE = 'New: Incident' DEFAULT_SEVERITY = 'critical' + attribute :description, paths: 'description' + attribute :ends_at, paths: 'end_time', type: :time attribute :environment_name, paths: 'gitlab_environment_name' attribute :hosts, paths: 'hosts' attribute :monitoring_tool, paths: 'monitoring_tool' @@ -23,3 +25,5 @@ module Gitlab end end end + +Gitlab::AlertManagement::Payload::Generic.prepend_if_ee('EE::Gitlab::AlertManagement::Payload::Generic') diff --git a/lib/gitlab/alerting/alert.rb b/lib/gitlab/alerting/alert.rb deleted file mode 100644 index 94b81b7d290..00000000000 --- a/lib/gitlab/alerting/alert.rb +++ /dev/null @@ -1,215 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Alerting - class Alert - include ActiveModel::Model - include Gitlab::Utils::StrongMemoize - include Presentable - - attr_accessor :project, :payload, :am_alert - - def self.for_alert_management_alert(project:, alert:) - params = if alert.prometheus? - alert.payload - else - Gitlab::Alerting::NotificationPayloadParser.call(alert.payload.to_h, alert.project) - end - - self.new(project: project, payload: params, am_alert: alert) - end - - def gitlab_alert - strong_memoize(:gitlab_alert) do - parse_gitlab_alert_from_payload - end - end - - def metric_id - strong_memoize(:metric_id) do - payload&.dig('labels', 'gitlab_alert_id') - end - end - - def gitlab_prometheus_alert_id - strong_memoize(:gitlab_prometheus_alert_id) do - payload&.dig('labels', 'gitlab_prometheus_alert_id') - end - end - - def title - strong_memoize(:title) do - gitlab_alert&.title || parse_title_from_payload - end - end - - def description - strong_memoize(:description) do - parse_description_from_payload - end - end - - def environment - strong_memoize(:environment) do - gitlab_alert&.environment || parse_environment_from_payload - end - end - - def annotations - strong_memoize(:annotations) do - parse_annotations_from_payload || [] - end - end - - def starts_at - strong_memoize(:starts_at) do - parse_datetime_from_payload('startsAt') - end - end - - def starts_at_raw - strong_memoize(:starts_at_raw) do - payload&.dig('startsAt') - end - end - - def ends_at - strong_memoize(:ends_at) do - parse_datetime_from_payload('endsAt') - end - end - - def full_query - strong_memoize(:full_query) do - gitlab_alert&.full_query || parse_expr_from_payload - end - end - - def y_label - strong_memoize(:y_label) do - parse_y_label_from_payload || title - end - end - - def alert_markdown - strong_memoize(:alert_markdown) do - parse_alert_markdown_from_payload - end - end - - def status - strong_memoize(:status) do - payload&.dig('status') - end - end - - def firing? - status == 'firing' - end - - def resolved? - status == 'resolved' - end - - def gitlab_managed? - metric_id.present? - end - - def gitlab_fingerprint - Gitlab::AlertManagement::Fingerprint.generate(plain_gitlab_fingerprint) - end - - def valid? - payload.respond_to?(:dig) && project && title && starts_at - end - - def present - super(presenter_class: Projects::Prometheus::AlertPresenter) - end - - private - - def plain_gitlab_fingerprint - if gitlab_managed? - [metric_id, starts_at_raw].join('/') - else # self managed - [starts_at_raw, title, full_query].join('/') - end - end - - def parse_environment_from_payload - environment_name = payload&.dig('labels', 'gitlab_environment_name') - - return unless environment_name - - EnvironmentsFinder.new(project, nil, { name: environment_name }) - .find - &.first - end - - def parse_gitlab_alert_from_payload - alerts_found = matching_gitlab_alerts - - return if alerts_found.blank? || alerts_found.size > 1 - - alerts_found.first - end - - def matching_gitlab_alerts - return unless metric_id || gitlab_prometheus_alert_id - - Projects::Prometheus::AlertsFinder - .new(project: project, metric: metric_id, id: gitlab_prometheus_alert_id) - .execute - end - - def parse_title_from_payload - payload&.dig('annotations', 'title') || - payload&.dig('annotations', 'summary') || - payload&.dig('labels', 'alertname') - end - - def parse_description_from_payload - payload&.dig('annotations', 'description') - end - - def parse_annotations_from_payload - payload&.dig('annotations')&.map do |label, value| - Alerting::AlertAnnotation.new(label: label, value: value) - end - end - - def parse_datetime_from_payload(field) - value = payload&.dig(field) - return unless value - - # value is a rfc3339 timestamp - # Timestamps from Prometheus and Alertmanager are UTC RFC3339 timestamps like: '2018-03-12T09:06:00Z' (Z represents 0 offset or UTC) - # .utc sets the datetime zone to `UTC` - Time.rfc3339(value).utc - rescue ArgumentError - end - - # Parses `g0.expr` from `generatorURL`. - # - # Example: http://localhost:9090/graph?g0.expr=vector%281%29&g0.tab=1 - def parse_expr_from_payload - url = payload&.dig('generatorURL') - return unless url - - uri = URI(url) - - Rack::Utils.parse_query(uri.query).fetch('g0.expr') - rescue URI::InvalidURIError, KeyError - end - - def parse_alert_markdown_from_payload - payload&.dig('annotations', 'gitlab_incident_markdown') - end - - def parse_y_label_from_payload - payload&.dig('annotations', 'gitlab_y_label') - end - end - end -end diff --git a/lib/gitlab/alerting/alert_annotation.rb b/lib/gitlab/alerting/alert_annotation.rb deleted file mode 100644 index a4b3a97b08c..00000000000 --- a/lib/gitlab/alerting/alert_annotation.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Alerting - class AlertAnnotation - include ActiveModel::Model - - attr_accessor :label, :value - end - end -end diff --git a/lib/gitlab/alerting/notification_payload_parser.rb b/lib/gitlab/alerting/notification_payload_parser.rb deleted file mode 100644 index 348f851f551..00000000000 --- a/lib/gitlab/alerting/notification_payload_parser.rb +++ /dev/null @@ -1,107 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Alerting - class NotificationPayloadParser - BadPayloadError = Class.new(StandardError) - - DEFAULT_TITLE = 'New: Incident' - DEFAULT_SEVERITY = 'critical' - - def initialize(payload, project) - @payload = payload.to_h.with_indifferent_access - @project = project - end - - def self.call(payload, project) - new(payload, project).call - end - - def call - { - 'annotations' => annotations, - 'startsAt' => starts_at, - 'endsAt' => ends_at - }.compact - end - - private - - attr_reader :payload, :project - - def title - payload[:title].presence || DEFAULT_TITLE - end - - def severity - payload[:severity].presence || DEFAULT_SEVERITY - end - - def fingerprint - Gitlab::AlertManagement::Fingerprint.generate(payload[:fingerprint]) - end - - def annotations - primary_params - .reverse_merge(flatten_secondary_params) - .transform_values(&:presence) - .compact - end - - def primary_params - { - 'title' => title, - 'description' => payload[:description], - 'monitoring_tool' => payload[:monitoring_tool], - 'service' => payload[:service], - 'hosts' => hosts.presence, - 'severity' => severity, - 'fingerprint' => fingerprint, - 'environment' => environment - } - end - - def hosts - Array(payload[:hosts]).reject(&:blank?) - end - - def current_time - Time.current.change(usec: 0).rfc3339 - end - - def starts_at - Time.parse(payload[:start_time].to_s).rfc3339 - rescue ArgumentError - current_time - end - - def ends_at - Time.parse(payload[:end_time].to_s).rfc3339 - rescue ArgumentError - nil - end - - def environment - environment_name = payload[:gitlab_environment_name] - - return unless environment_name - - EnvironmentsFinder.new(project, nil, { name: environment_name }) - .find - &.first - end - - def secondary_params - payload.except(:start_time, :end_time) - end - - def flatten_secondary_params - Gitlab::Utils::SafeInlineHash.merge_keys!(secondary_params) - rescue ArgumentError - raise BadPayloadError, 'The payload is too big' - end - end - end -end - -Gitlab::Alerting::NotificationPayloadParser.prepend_if_ee('EE::Gitlab::Alerting::NotificationPayloadParser') diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 609eef5e365..001c083c778 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -371,7 +371,7 @@ module Gitlab end def find_build_by_token(token) - ::Ci::Build.running.find_by_token(token) + ::Ci::AuthJobFinder.new(token: token).execute end def user_auth_attempt!(user, success:) diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index ccf52bae9a5..3d3f7212053 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -290,7 +290,7 @@ module Gitlab end def api_request? - current_request.path.starts_with?('/api/') + current_request.path.starts_with?(Gitlab::Utils.append_path(Gitlab.config.gitlab.relative_url_root, '/api/')) end def archive_request? diff --git a/lib/gitlab/background_migration/backfill_snippet_repositories.rb b/lib/gitlab/background_migration/backfill_snippet_repositories.rb index 21538000fec..8befade8c3a 100644 --- a/lib/gitlab/background_migration/backfill_snippet_repositories.rb +++ b/lib/gitlab/background_migration/backfill_snippet_repositories.rb @@ -109,7 +109,7 @@ module Gitlab end def create_commit(snippet) - snippet.snippet_repository.multi_files_action(commit_author(snippet), snippet_action(snippet), commit_attrs) + snippet.snippet_repository.multi_files_action(commit_author(snippet), snippet_action(snippet), **commit_attrs) end # If the user is not allowed to access git or update the snippet diff --git a/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb b/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb index ca64d13b118..bbe2164ae4e 100644 --- a/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb +++ b/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb @@ -13,8 +13,6 @@ module Gitlab end def perform(start_id, stop_id) - return if Feature.disabled?(:migrate_bio_to_user_details, default_enabled: true) - relation = User .select("id AS user_id", "substring(COALESCE(bio, '') from 1 for 255) AS bio") .where("(COALESCE(bio, '') IS DISTINCT FROM '')") diff --git a/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb b/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb new file mode 100644 index 00000000000..cd305adc7cd --- /dev/null +++ b/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class RemoveDuplicatedCsFindingsWithoutVulnerabilityId + def perform(start_id, stop_id) + end + end + end +end + +Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId') diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb new file mode 100644 index 00000000000..3def5eb3369 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Banzai + # isolated Banzai::ReferenceParser + module ReferenceParser + # Returns the reference parser class for the given type + # + # Example: + # + # Banzai::ReferenceParser['isolated_mentioned_group'] + # + # This would return the `::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser::IsolatedMentionedGroupParser` class. + def self.[](name) + const_get("::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser::#{name.to_s.camelize}Parser", false) + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb new file mode 100644 index 00000000000..d3d032ba433 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Banzai + module ReferenceParser + # isolated Banzai::ReferenceParser::MentionedGroupParser + class IsolatedMentionedGroupParser < ::Banzai::ReferenceParser::MentionedGroupParser + extend ::Gitlab::Utils::Override + + self.reference_type = :user + + override :references_relation + def references_relation + ::Gitlab::BackgroundMigration::UserMentions::Models::Group + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb new file mode 100644 index 00000000000..1d3a3af81a1 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Gitlab + # Extract possible GFM references from an arbitrary String for further processing. + class IsolatedReferenceExtractor < ::Gitlab::ReferenceExtractor + REFERABLES = %i(isolated_mentioned_group).freeze + + REFERABLES.each do |type| + define_method("#{type}s") do + @references[type] ||= isolated_references(type) + end + end + + def isolated_references(type) + context = ::Banzai::RenderContext.new(project, current_user) + processor = ::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser[type].new(context) + + refs = processor.process(html_documents) + refs[:visible] + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb index 69ba3f9132b..be9c0ad2b3a 100644 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb @@ -36,7 +36,8 @@ module Gitlab if extractor extractors[current_user] = extractor else - extractor = extractors[current_user] ||= ::Gitlab::ReferenceExtractor.new(project, current_user) + extractor = extractors[current_user] ||= + Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedReferenceExtractor.new(project, current_user) extractor.reset_memoized_values end @@ -71,7 +72,7 @@ module Gitlab mentioned_users_ids = array_to_sql(refs.mentioned_users.pluck(:id)) mentioned_projects_ids = array_to_sql(refs.mentioned_projects.pluck(:id)) - mentioned_groups_ids = array_to_sql(refs.mentioned_groups.pluck(:id)) + mentioned_groups_ids = array_to_sql(refs.isolated_mentioned_groups.pluck(:id)) return if mentioned_users_ids.blank? && mentioned_projects_ids.blank? && mentioned_groups_ids.blank? diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb new file mode 100644 index 00000000000..5cadfa45b5b --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + module Concerns + module Namespace + # extracted methods for recursive traversing of namespace hierarchy + module RecursiveTraversal + extend ActiveSupport::Concern + + def root_ancestor + return self if persisted? && parent_id.nil? + + strong_memoize(:root_ancestor) do + Gitlab::ObjectHierarchy + .new(self.class.where(id: id)) + .base_and_ancestors + .reorder(nil) + .find_by(parent_id: nil) + end + end + + # Returns all ancestors, self, and descendants of the current namespace. + def self_and_hierarchy + Gitlab::ObjectHierarchy + .new(self.class.where(id: id)) + .all_objects + end + + # Returns all the ancestors of the current namespaces. + def ancestors + return self.class.none unless parent_id + + Gitlab::ObjectHierarchy + .new(self.class.where(id: parent_id)) + .base_and_ancestors + end + + # returns all ancestors upto but excluding the given namespace + # when no namespace is given, all ancestors upto the top are returned + def ancestors_upto(top = nil, hierarchy_order: nil) + Gitlab::ObjectHierarchy.new(self.class.where(id: id)) + .ancestors(upto: top, hierarchy_order: hierarchy_order) + end + + def self_and_ancestors(hierarchy_order: nil) + return self.class.where(id: id) unless parent_id + + Gitlab::ObjectHierarchy + .new(self.class.where(id: id)) + .base_and_ancestors(hierarchy_order: hierarchy_order) + end + + # Returns all the descendants of the current namespace. + def descendants + Gitlab::ObjectHierarchy + .new(self.class.where(parent_id: id)) + .base_and_descendants + end + + def self_and_descendants + Gitlab::ObjectHierarchy + .new(self.class.where(id: id)) + .base_and_descendants + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/group.rb b/lib/gitlab/background_migration/user_mentions/models/group.rb new file mode 100644 index 00000000000..bc04172b9a2 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/group.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + # isolated Group model + class Group < ::Gitlab::BackgroundMigration::UserMentions::Models::Namespace + self.store_full_sti_class = false + has_one :saml_provider + + def self.declarative_policy_class + "GroupPolicy" + end + + def max_member_access_for_user(user) + return GroupMember::NO_ACCESS unless user + + return GroupMember::OWNER if user.admin? + + max_member_access = members_with_parents.where(user_id: user) + .reorder(access_level: :desc) + .first + &.access_level + + max_member_access || GroupMember::NO_ACCESS + end + + def members_with_parents + # Avoids an unnecessary SELECT when the group has no parents + source_ids = + if has_parent? + self_and_ancestors.reorder(nil).select(:id) + else + id + end + + group_hierarchy_members = GroupMember.active_without_invites_and_requests + .where(source_id: source_ids) + + GroupMember.from_union([group_hierarchy_members, + members_from_self_and_ancestor_group_shares]) + end + + # rubocop: disable Metrics/AbcSize + def members_from_self_and_ancestor_group_shares + group_group_link_table = GroupGroupLink.arel_table + group_member_table = GroupMember.arel_table + + source_ids = + if has_parent? + self_and_ancestors.reorder(nil).select(:id) + else + id + end + + group_group_links_query = GroupGroupLink.where(shared_group_id: source_ids) + cte = Gitlab::SQL::CTE.new(:group_group_links_cte, group_group_links_query) + cte_alias = cte.table.alias(GroupGroupLink.table_name) + + # Instead of members.access_level, we need to maximize that access_level at + # the respective group_group_links.group_access. + member_columns = GroupMember.attribute_names.map do |column_name| + if column_name == 'access_level' + smallest_value_arel([cte_alias[:group_access], group_member_table[:access_level]], + 'access_level') + else + group_member_table[column_name] + end + end + + GroupMember + .with(cte.to_arel) + .select(*member_columns) + .from([group_member_table, cte.alias_to(group_group_link_table)]) + .where(group_member_table[:requested_at].eq(nil)) + .where(group_member_table[:source_id].eq(group_group_link_table[:shared_with_group_id])) + .where(group_member_table[:source_type].eq('Namespace')) + end + # rubocop: enable Metrics/AbcSize + + def smallest_value_arel(args, column_alias) + Arel::Nodes::As.new( + Arel::Nodes::NamedFunction.new('LEAST', args), + Arel::Nodes::SqlLiteral.new(column_alias)) + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb new file mode 100644 index 00000000000..6d7b9a86e69 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/namespace.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + # isolated Namespace model + class Namespace < ApplicationRecord + include ::Gitlab::VisibilityLevel + include ::Gitlab::Utils::StrongMemoize + include Gitlab::BackgroundMigration::UserMentions::Models::Concerns::Namespace::RecursiveTraversal + + belongs_to :parent, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Namespace" + + def visibility_level_field + :visibility_level + end + + def has_parent? + parent_id.present? || parent.present? + end + + # Overridden in EE::Namespace + def feature_available?(_feature) + false + end + end + end + end + end +end + +Namespace.prepend_if_ee('::EE::Namespace') diff --git a/lib/gitlab/checks/matching_merge_request.rb b/lib/gitlab/checks/matching_merge_request.rb index 71361b12d07..db7af0088d0 100644 --- a/lib/gitlab/checks/matching_merge_request.rb +++ b/lib/gitlab/checks/matching_merge_request.rb @@ -20,3 +20,5 @@ module Gitlab end end end + +Gitlab::Checks::MatchingMergeRequest.prepend_if_ee('EE::Gitlab::Checks::MatchingMergeRequest') diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb index 0373a12ab69..6d152c052dc 100644 --- a/lib/gitlab/ci/ansi2json/converter.rb +++ b/lib/gitlab/ci/ansi2json/converter.rb @@ -104,23 +104,24 @@ module Gitlab action = scanner[1] timestamp = scanner[2] section = scanner[3] + options = parse_section_options(scanner[4]) section_name = sanitize_section_name(section) - if action == "start" - handle_section_start(scanner, section_name, timestamp) - elsif action == "end" + if action == 'start' + handle_section_start(scanner, section_name, timestamp, options) + elsif action == 'end' handle_section_end(scanner, section_name, timestamp) else raise 'unsupported action' end end - def handle_section_start(scanner, section, timestamp) + def handle_section_start(scanner, section, timestamp, options) # We make a new line for new section flush_current_line - @state.open_section(section, timestamp) + @state.open_section(section, timestamp, options) # we need to consume match after handling # the open of section, as we want the section @@ -157,6 +158,18 @@ module Gitlab def sanitize_section_name(section) section.to_s.downcase.gsub(/[^a-z0-9]/, '-') end + + def parse_section_options(raw_options) + return unless raw_options + + # We need to remove the square brackets and split + # by comma to get a list of the options + options = raw_options[1...-1].split ',' + + # Now split each option by equals to separate + # each in the format [key, value] + options.to_h { |option| option.split '=' } + end end end end diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb index 21aa1f84353..b1dee0e1ecc 100644 --- a/lib/gitlab/ci/ansi2json/line.rb +++ b/lib/gitlab/ci/ansi2json/line.rb @@ -32,7 +32,7 @@ module Gitlab end attr_reader :offset, :sections, :segments, :current_segment, - :section_header, :section_duration + :section_header, :section_duration, :section_options def initialize(offset:, style:, sections: []) @offset = offset @@ -68,6 +68,10 @@ module Gitlab @sections << section end + def set_section_options(options) + @section_options = options + end + def set_as_section_header @section_header = true end @@ -90,6 +94,7 @@ module Gitlab result[:section] = sections.last if sections.any? result[:section_header] = true if @section_header result[:section_duration] = @section_duration if @section_duration + result[:section_options] = @section_options if @section_options end end end diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb index 38d36e6950c..b2b6ce649ed 100644 --- a/lib/gitlab/ci/ansi2json/state.rb +++ b/lib/gitlab/ci/ansi2json/state.rb @@ -26,10 +26,11 @@ module Gitlab Base64.urlsafe_encode64(state.to_json) end - def open_section(section, timestamp) + def open_section(section, timestamp, options) @open_sections[section] = timestamp @current_line.add_section(section) + @current_line.set_section_options(options) @current_line.set_as_section_header end diff --git a/lib/gitlab/ci/config/entry/product/matrix.rb b/lib/gitlab/ci/config/entry/product/matrix.rb index 6af809d46c1..d4ee0978e1b 100644 --- a/lib/gitlab/ci/config/entry/product/matrix.rb +++ b/lib/gitlab/ci/config/entry/product/matrix.rb @@ -46,13 +46,11 @@ module Gitlab end end - # rubocop:disable CodeReuse/ActiveRecord def number_of_generated_jobs value.sum do |config| config.values.reduce(1) { |acc, values| acc * values.size } end end - # rubocop:enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb index ac4f70fb69e..2481989060e 100644 --- a/lib/gitlab/ci/config/entry/product/variables.rb +++ b/lib/gitlab/ci/config/entry/product/variables.rb @@ -14,7 +14,7 @@ module Gitlab validations do validates :config, variables: { array_values: true } validates :config, length: { - minimum: 2, + minimum: :minimum, too_short: 'requires at least %{count} items' } end @@ -28,6 +28,10 @@ module Gitlab .map { |key, value| [key.to_s, Array(value).map(&:to_s)] } .to_h end + + def minimum + ::Gitlab::Ci::Features.one_dimensional_matrix_enabled? ? 1 : 2 + end end end end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index e770187b124..e14d56af978 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -10,10 +10,6 @@ module Gitlab ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: true) end - def self.job_heartbeats_runner?(project) - ::Feature.enabled?(:ci_job_heartbeats_runner, project, default_enabled: true) - end - def self.instance_variables_ui_enabled? ::Feature.enabled?(:ci_instance_variables_ui, default_enabled: true) end @@ -35,10 +31,6 @@ module Gitlab ::Feature.enabled?(:ci_raise_job_rules_without_workflow_rules_warning, default_enabled: true) end - def self.bulk_insert_on_create?(project) - ::Feature.enabled?(:ci_bulk_insert_on_create, project, default_enabled: true) - end - # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project` # is a safe switch to disable the feature for a parituclar project when something went wrong, # therefore it's not supposed to be enabled by default. @@ -54,10 +46,6 @@ module Gitlab Feature.enabled?(:project_transactionless_destroy, project, default_enabled: false) end - def self.coverage_report_view?(project) - ::Feature.enabled?(:coverage_report_view, project, default_enabled: true) - end - def self.child_of_child_pipeline_enabled?(project) ::Feature.enabled?(:ci_child_of_child_pipeline, project, default_enabled: true) end @@ -72,7 +60,11 @@ module Gitlab end def self.new_artifact_file_reader_enabled?(project) - ::Feature.enabled?(:ci_new_artifact_file_reader, project, default_enabled: false) + ::Feature.enabled?(:ci_new_artifact_file_reader, project, default_enabled: true) + end + + def self.one_dimensional_matrix_enabled? + ::Feature.enabled?(:one_dimensional_matrix, default_enabled: false) end end end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index d1882059dd8..06096a33f27 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -16,7 +16,7 @@ module Gitlab ) do include Gitlab::Utils::StrongMemoize - def initialize(**params) + def initialize(params = {}) params.each do |key, value| self[key] = value end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 34649fe16f3..81ef3bb074d 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -8,7 +8,7 @@ module Gitlab include Chain::Helpers def perform! - BulkInsertableAssociations.with_bulk_insert(enabled: ::Gitlab::Ci::Features.bulk_insert_on_create?(project)) do + BulkInsertableAssociations.with_bulk_insert do pipeline.save! end rescue ActiveRecord::RecordInvalid => e diff --git a/lib/gitlab/ci/runner/backoff.rb b/lib/gitlab/ci/runner/backoff.rb new file mode 100644 index 00000000000..95d7719e9cb --- /dev/null +++ b/lib/gitlab/ci/runner/backoff.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Runner + ## + # Runner Backoff class is an implementation of an exponential backoff + # used when a runner communicates with GitLab. We typically use it when a + # runner retries sending a build status after we created a build pending + # state. + # + # Backoff is calculated based on the backoff slot which is always a power + # of 2: + # + # 0s - 3s duration -> 1 second backoff + # 4s - 7s duration -> 2 seconds backoff + # 8s - 15s duration -> 4 seconds backoff + # 16s - 31s duration -> 8 seconds backoff + # 32s - 63s duration -> 16 seconds backoff + # 64s - 127s duration -> 32 seconds backoff + # 127s - 256s+ duration -> 64 seconds backoff + # + # It means that first 15 requests made by a runner will need to respect + # following backoffs: + # + # 0s -> 1 second backoff (backoff started, slot 0, 2^0 backoff) + # 1s -> 1 second backoff + # 2s -> 1 second backoff + # 3s -> 1 seconds backoff + # (slot 1 - 2^1 backoff) + # 4s -> 2 seconds backoff + # 6s -> 2 seconds backoff + # (slot 2 - 2^2 backoff) + # 8s -> 4 seconds backoff + # 12s -> 4 seconds backoff + # (slot 3 - 2^3 backoff) + # 16s -> 8 seconds backoff + # 24s -> 8 seconds backoff + # (slot 4 - 2^4 backoff) + # 32s -> 16 seconds backoff + # 48s -> 16 seconds backoff + # (slot 5 - 2^5 backoff) + # 64s -> 32 seconds backoff + # 96s -> 32 seconds backoff + # (slot 6 - 2^6 backoff) + # 128s -> 64 seconds backoff + # + # There is a cap on the backoff - it will never exceed 64 seconds. + # + class Backoff + def initialize(started) + @started = started + + if duration < 0 + raise ArgumentError, 'backoff duration negative' + end + end + + def duration + (Time.current - @started).ceil + end + + def slot + return 0 if duration < 2 + + Math.log(duration, 2).floor - 1 + end + + def to_seconds + 2**[slot, 6].min + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 829fd7a722f..8b921305c11 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.0.0-beta.2" dependencies: [] review: @@ -91,7 +91,7 @@ canary: - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret - - auto-deploy deploy canary + - auto-deploy deploy canary 50 environment: name: production url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN @@ -114,7 +114,6 @@ canary: - auto-deploy create_secret - auto-deploy deploy - auto-deploy delete canary - - auto-deploy delete rollout - auto-deploy persist_environment_url environment: name: production @@ -163,9 +162,7 @@ production_manual: - auto-deploy ensure_namespace - auto-deploy initialize_tiller - auto-deploy create_secret - - auto-deploy deploy rollout $ROLLOUT_PERCENTAGE - - auto-deploy scale stable $((100-ROLLOUT_PERCENTAGE)) - - auto-deploy delete canary + - auto-deploy deploy canary $ROLLOUT_PERCENTAGE - auto-deploy persist_environment_url environment: name: production diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml index da474f8ac88..317e8bfab0e 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -10,6 +10,7 @@ .deploy_to_ecs: image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest' + dependencies: [] script: - ecs update-task-definition diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 348e5472cb4..6fd32b3f1a0 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -16,6 +16,7 @@ module Gitlab ArchiveError = Class.new(StandardError) AlreadyArchivedError = Class.new(StandardError) + LockedError = Class.new(StandardError) attr_reader :job @@ -130,6 +131,12 @@ module Gitlab end end + def lock(&block) + in_write_lock(&block) + rescue FailedToObtainLockError + raise LockedError, "build trace `#{job.id}` is locked" + end + private def read_stream diff --git a/lib/gitlab/ci/trace/checksum.rb b/lib/gitlab/ci/trace/checksum.rb new file mode 100644 index 00000000000..b01136a6d24 --- /dev/null +++ b/lib/gitlab/ci/trace/checksum.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Trace + ## + # Trace::Checksum class is responsible for calculating a CRC32 checksum + # of an entire build trace using partial build trace chunks stored in a + # database. + # + # CRC32 checksum can be easily calculated by combining partial checksums + # in a right order. + # + # Then we compare CRC32 checksum provided by a GitLab Runner and expect + # it to be the same as the CRC32 checksum derived from partial chunks. + # + class Checksum + include Gitlab::Utils::StrongMemoize + + attr_reader :build + + def initialize(build) + @build = build + end + + def valid? + return false unless state_crc32.present? + + state_crc32 == chunks_crc32 + end + + def state_crc32 + strong_memoize(:crc32) { build.pending_state&.crc32 } + end + + def chunks_crc32 + trace_chunks.reduce(0) do |crc32, chunk| + Zlib.crc32_combine(crc32, chunk.crc32, chunk_size(chunk)) + end + end + + def last_chunk + strong_memoize(:last_chunk) { trace_chunks.max } + end + + ## + # Trace chunks will be persisted in a database if an object store is + # not configured - in that case we do not want to load entire raw data + # of all the chunks into memory. + # + # We ignore `raw_data` attribute instead, and rely on internal build + # trace chunk database adapter to handle + # `ActiveModel::MissingAttributeError` exception. + # + # Alternative solution would be separating chunk data from chunk + # metadata on the database level too. + # + def trace_chunks + strong_memoize(:trace_chunks) do + build.trace_chunks.persisted + .select(::Ci::BuildTraceChunk.metadata_attributes) + end + end + + private + + def chunk_size(chunk) + if chunk == last_chunk + chunk.size + else + ::Ci::BuildTraceChunk::CHUNK_SIZE + end + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb index 82a7d5fb83c..51372871f39 100644 --- a/lib/gitlab/ci/trace/metrics.rb +++ b/lib/gitlab/ci/trace/metrics.rb @@ -7,7 +7,8 @@ module Gitlab extend Gitlab::Utils::StrongMemoize OPERATIONS = [:appended, :streamed, :chunked, :mutated, :overwrite, - :accepted, :finalized, :discarded, :conflict].freeze + :accepted, :finalized, :discarded, :conflict, :locked, + :invalid].freeze def increment_trace_operation(operation: :unknown) unless OPERATIONS.include?(operation) @@ -18,7 +19,7 @@ module Gitlab end def increment_trace_bytes(size) - self.class.trace_bytes.increment(by: size.to_i) + self.class.trace_bytes.increment({}, size.to_i) end def self.trace_operations diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index a072036daa8..84a9280e507 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -36,9 +36,9 @@ module Gitlab def self.fabricate(resource) case resource when Hash - self.new(resource.symbolize_keys) + self.new(**resource.symbolize_keys) when ::Ci::HasVariable - self.new(resource.to_runner_variable) + self.new(**resource.to_runner_variable) when self resource.dup else diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 68f61e52df7..6c771b220ad 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -95,6 +95,10 @@ module Gitlab }.compact }.compact end + def merged_yaml + @ci_config&.to_hash&.to_yaml + end + private def variables diff --git a/lib/gitlab/cleanup/orphan_lfs_file_references.rb b/lib/gitlab/cleanup/orphan_lfs_file_references.rb index 14eac474e27..a6638b2cbc8 100644 --- a/lib/gitlab/cleanup/orphan_lfs_file_references.rb +++ b/lib/gitlab/cleanup/orphan_lfs_file_references.rb @@ -19,6 +19,11 @@ module Gitlab def run! log_info("Looking for orphan LFS files for project #{project.name_with_namespace}") + if project.lfs_objects.empty? + log_info("Project #{project.name_with_namespace} is linked to 0 LFS objects. Nothing to do") + return + end + remove_orphan_references end diff --git a/lib/gitlab/code_navigation_path.rb b/lib/gitlab/code_navigation_path.rb index 909d0536b5f..7d36f2f12cf 100644 --- a/lib/gitlab/code_navigation_path.rb +++ b/lib/gitlab/code_navigation_path.rb @@ -13,7 +13,6 @@ module Gitlab end def full_json_path_for(path) - return unless Feature.enabled?(:code_navigation, project, default_enabled: true) return unless build raw_project_job_artifacts_path(project, build, path: "lsif/#{path}.json", file_type: :lsif) diff --git a/lib/gitlab/config/entry/factory.rb b/lib/gitlab/config/entry/factory.rb index 7c5ffaa7621..f76c98f7cbf 100644 --- a/lib/gitlab/config/entry/factory.rb +++ b/lib/gitlab/config/entry/factory.rb @@ -79,7 +79,7 @@ module Gitlab end def fabricate(entry_class, value = nil) - entry_class.new(value, @metadata) do |node| + entry_class.new(value, **@metadata) do |node| node.key = @attributes[:key] node.parent = @attributes[:parent] node.default = @attributes[:default] diff --git a/lib/gitlab/config/entry/simplifiable.rb b/lib/gitlab/config/entry/simplifiable.rb index 315f1947e2c..ee28891a174 100644 --- a/lib/gitlab/config/entry/simplifiable.rb +++ b/lib/gitlab/config/entry/simplifiable.rb @@ -19,7 +19,7 @@ module Gitlab entry = self.class.entry_class(strategy) - @subject = entry.new(config, metadata, &blk) + @subject = entry.new(config, **metadata, &blk) super(@subject) end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 53406af2c4e..047600af267 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -21,11 +21,13 @@ module Gitlab def resolve(user, commit_message, files) msg = commit_message || default_commit_message resolution = Gitlab::Git::Conflict::Resolution.new(user, files, msg) - args = { + + resolver.resolve_conflicts( + @source_repo, + resolution, source_branch: merge_request.source_branch, target_branch: merge_request.target_branch - } - resolver.resolve_conflicts(@source_repo, resolution, args) + ) ensure @merge_request.clear_memoized_shas end diff --git a/lib/gitlab/danger/commit_linter.rb b/lib/gitlab/danger/commit_linter.rb index 954934518d7..7b01db125a9 100644 --- a/lib/gitlab/danger/commit_linter.rb +++ b/lib/gitlab/danger/commit_linter.rb @@ -10,7 +10,7 @@ module Gitlab MAX_LINE_LENGTH = 72 MAX_CHANGED_FILES_IN_COMMIT = 3 MAX_CHANGED_LINES_IN_COMMIT = 30 - SHORT_REFERENCE_REGEX = %r{([\w\-\/]+)?(#|!|&|%)\d+\b}.freeze + SHORT_REFERENCE_REGEX = %r{([\w\-\/]+)?(?<!`)(#|!|&|%)\d+(?<!`)}.freeze DEFAULT_SUBJECT_DESCRIPTION = 'commit subject' WIP_PREFIX = 'WIP: ' PROBLEMS = { @@ -118,7 +118,7 @@ module Gitlab next unless line_too_long?(line) - url_size = line.scan(%r((https?://\S+))).sum { |(url)| url.length } # rubocop:disable CodeReuse/ActiveRecord + url_size = line.scan(%r((https?://\S+))).sum { |(url)| url.length } # If the line includes a URL, we'll allow it to exceed MAX_LINE_LENGTH characters, but # only if the line _without_ the URL does not exceed this limit. diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 3626ec5bf5b..d01455c9ec4 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -214,6 +214,12 @@ module Gitlab title.gsub(DRAFT_REGEX, '').gsub(/`/, '\\\`') end + def draft_mr? + return false unless gitlab_helper + + DRAFT_REGEX.match?(gitlab_helper.mr_json['title']) + end + def security_mr? return false unless gitlab_helper diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb index a6866868e6c..e67e4a45bfe 100644 --- a/lib/gitlab/danger/roulette.rb +++ b/lib/gitlab/danger/roulette.rb @@ -146,13 +146,19 @@ module Gitlab %i[reviewer traintainer maintainer].map do |role| spin_role_for_category(team, role, project, category) end + hungry_reviewers = reviewers.select { |member| member.hungry } + hungry_traintainers = traintainers.select { |member| member.hungry } # TODO: take CODEOWNERS into account? # https://gitlab.com/gitlab-org/gitlab/issues/26723 - # Make traintainers have triple the chance to be picked as a reviewer random = new_random(mr_source_branch) - reviewer = spin_for_person(reviewers + traintainers + traintainers, random: random, timezone_experiment: timezone_experiment) + + # Make hungry traintainers have 4x the chance to be picked as a reviewer + # Make traintainers have 3x the chance to be picked as a reviewer + # Make hungry reviewers have 2x the chance to be picked as a reviewer + weighted_reviewers = reviewers + hungry_reviewers + traintainers + traintainers + traintainers + hungry_traintainers + reviewer = spin_for_person(weighted_reviewers, random: random, timezone_experiment: timezone_experiment) maintainer = spin_for_person(maintainers, random: random, timezone_experiment: timezone_experiment) Spin.new(category, reviewer, maintainer, false, timezone_experiment) diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index ebd96be40d7..4481977db15 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -3,7 +3,7 @@ module Gitlab module Danger class Teammate - attr_reader :options, :username, :name, :role, :projects, :available, :tz_offset_hours + attr_reader :options, :username, :name, :role, :projects, :available, :hungry, :tz_offset_hours # The options data are produced by https://gitlab.com/gitlab-org/gitlab-roulette/-/blob/master/lib/team_member.rb def initialize(options = {}) @@ -14,6 +14,7 @@ module Gitlab @role = options['role'] @projects = options['projects'] @available = options['available'] + @hungry = options['hungry'] @tz_offset_hours = options['tz_offset_hours'] end @@ -31,10 +32,8 @@ module Gitlab projects&.has_key?(name) end - # Traintainers also count as reviewers def reviewer?(project, category, labels) - has_capability?(project, category, :reviewer, labels) || - traintainer?(project, category, labels) + has_capability?(project, category, :reviewer, labels) end def traintainer?(project, category, labels) diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index accc6330253..d9c4b1cb280 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -92,10 +92,6 @@ module Gitlab @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] end - def self.postgresql_9_or_less? - version.to_f < 10 - end - def self.postgresql_minimum_supported_version? version.to_f >= MINIMUM_POSTGRES_VERSION end @@ -127,28 +123,6 @@ module Gitlab # ignore - happens when Rake tasks yet have to create a database, e.g. for testing end - # map some of the function names that changed between PostgreSQL 9 and 10 - # https://wiki.postgresql.org/wiki/New_in_postgres_10 - def self.pg_wal_lsn_diff - Gitlab::Database.postgresql_9_or_less? ? 'pg_xlog_location_diff' : 'pg_wal_lsn_diff' - end - - def self.pg_current_wal_insert_lsn - Gitlab::Database.postgresql_9_or_less? ? 'pg_current_xlog_insert_location' : 'pg_current_wal_insert_lsn' - end - - def self.pg_last_wal_receive_lsn - Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_receive_location' : 'pg_last_wal_receive_lsn' - end - - def self.pg_last_wal_replay_lsn - Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_replay_location' : 'pg_last_wal_replay_lsn' - end - - def self.pg_last_xact_replay_timestamp - 'pg_last_xact_replay_timestamp' - end - def self.nulls_last_order(field, direction = 'ASC') Arel.sql("#{field} #{direction} NULLS LAST") end diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index 1762b81b7d8..0de67ed8cf0 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -8,15 +8,20 @@ # In order to not use a possible complex time consuming query when calculating min and max for batch_distinct_count # the start and finish can be sent specifically # +# Grouped relations can be used as well. However, the preferred batch count should be around 10K because group by count is more expensive. +# # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 # # Examples: # extend ::Gitlab::Database::BatchCount # batch_count(User.active) # batch_count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) +# batch_count(Namespace.group(:type)) # batch_distinct_count(::Project, :creator_id) # batch_distinct_count(::Project.with_active_services.service_desk_enabled.where(time_period), start: ::User.minimum(:id), finish: ::User.maximum(:id)) +# batch_distinct_count(Project.group(:visibility_level), :creator_id) # batch_sum(User, :sign_in_count) +# batch_sum(Issue.group(:state_id), :weight)) module Gitlab module Database module BatchCount @@ -77,12 +82,12 @@ module Gitlab raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0 return FALLBACK if unwanted_configuration?(finish, batch_size, start) - counter = 0 + results = nil batch_start = start while batch_start <= finish begin - counter += batch_fetch(batch_start, batch_start + batch_size, mode) + results = merge_results(results, batch_fetch(batch_start, batch_start + batch_size, mode)) batch_start += batch_size rescue ActiveRecord::QueryCanceled # retry with a safe batch size & warmer cache @@ -95,7 +100,17 @@ module Gitlab sleep(SLEEP_TIME_IN_SECONDS) end - counter + results + end + + def merge_results(results, object) + return object unless results + + if object.is_a?(Hash) + results.merge!(object) { |_, a, b| a + b } + else + results + object + end end def batch_fetch(start, finish, mode) @@ -118,11 +133,11 @@ module Gitlab end def actual_start(start) - start || @relation.minimum(@column) || 0 + start || @relation.unscope(:group, :having).minimum(@column) || 0 end def actual_finish(finish) - finish || @relation.maximum(@column) || 0 + finish || @relation.unscope(:group, :having).maximum(@column) || 0 end def check_mode!(mode) diff --git a/lib/gitlab/database/concurrent_reindex.rb b/lib/gitlab/database/concurrent_reindex.rb deleted file mode 100644 index 485ab35e55d..00000000000 --- a/lib/gitlab/database/concurrent_reindex.rb +++ /dev/null @@ -1,143 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - class ConcurrentReindex - include Gitlab::Utils::StrongMemoize - include MigrationHelpers - - ReindexError = Class.new(StandardError) - - PG_IDENTIFIER_LENGTH = 63 - TEMPORARY_INDEX_PREFIX = 'tmp_reindex_' - REPLACED_INDEX_PREFIX = 'old_reindex_' - - attr_reader :index_name, :logger - - def initialize(index_name, logger:) - @index_name = index_name - @logger = logger - end - - def execute - raise ReindexError, "index #{index_name} does not exist" unless index_exists? - - raise ReindexError, 'UNIQUE indexes are currently not supported' if index_unique? - - logger.debug("dropping dangling index from previous run: #{replacement_index_name}") - remove_replacement_index - - begin - create_replacement_index - - unless replacement_index_valid? - message = 'replacement index was created as INVALID' - logger.error("#{message}, cleaning up") - raise ReindexError, "failed to reindex #{index_name}: #{message}" - end - - swap_replacement_index - rescue Gitlab::Database::WithLockRetries::AttemptsExhaustedError => e - logger.error('failed to obtain the required database locks to swap the indexes, cleaning up') - raise ReindexError, e.message - rescue ActiveRecord::ActiveRecordError, PG::Error => e - logger.error("database error while attempting reindex of #{index_name}: #{e.message}") - raise ReindexError, e.message - ensure - logger.info("dropping unneeded replacement index: #{replacement_index_name}") - remove_replacement_index - end - end - - private - - def connection - @connection ||= ActiveRecord::Base.connection - end - - def replacement_index_name - @replacement_index_name ||= constrained_index_name(TEMPORARY_INDEX_PREFIX) - end - - def index - strong_memoize(:index) do - find_index(index_name) - end - end - - def index_exists? - !index.nil? - end - - def index_unique? - index.indisunique - end - - def constrained_index_name(prefix) - "#{prefix}#{index_name}".slice(0, PG_IDENTIFIER_LENGTH) - end - - def create_replacement_index - create_replacement_index_statement = index.indexdef - .sub(/CREATE INDEX/, 'CREATE INDEX CONCURRENTLY') - .sub(/#{index_name}/, replacement_index_name) - - logger.info("creating replacement index #{replacement_index_name}") - logger.debug("replacement index definition: #{create_replacement_index_statement}") - - disable_statement_timeout do - connection.execute(create_replacement_index_statement) - end - end - - def replacement_index_valid? - find_index(replacement_index_name).indisvalid - end - - def find_index(index_name) - record = connection.select_one(<<~SQL) - SELECT - pg_index.indisunique, - pg_index.indisvalid, - pg_indexes.indexdef - FROM pg_index - INNER JOIN pg_class ON pg_class.oid = pg_index.indexrelid - INNER JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid - INNER JOIN pg_indexes ON pg_class.relname = pg_indexes.indexname - WHERE pg_namespace.nspname = 'public' - AND pg_class.relname = #{connection.quote(index_name)} - SQL - - OpenStruct.new(record) if record - end - - def swap_replacement_index - replaced_index_name = constrained_index_name(REPLACED_INDEX_PREFIX) - - logger.info("swapping replacement index #{replacement_index_name} with #{index_name}") - - with_lock_retries do - rename_index(index_name, replaced_index_name) - rename_index(replacement_index_name, index_name) - rename_index(replaced_index_name, replacement_index_name) - end - end - - def rename_index(old_index_name, new_index_name) - connection.execute("ALTER INDEX #{old_index_name} RENAME TO #{new_index_name}") - end - - def remove_replacement_index - disable_statement_timeout do - connection.execute("DROP INDEX CONCURRENTLY IF EXISTS #{replacement_index_name}") - end - end - - def with_lock_retries(&block) - arguments = { klass: self.class, logger: logger } - - Gitlab::Database::WithLockRetries.new(arguments).run(raise_on_exhaustion: true, &block) - end - end - end -end diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb index e226ed7613a..89190320cf9 100644 --- a/lib/gitlab/database/count/reltuples_count_strategy.rb +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -74,8 +74,9 @@ module Gitlab def get_statistics(table_names, check_statistics: true) time = 6.hours.ago - query = PgClass.joins("LEFT JOIN pg_stat_user_tables USING (relname)") + query = PgClass.joins("LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relid = pg_class.oid") .where(relname: table_names) + .where('schemaname = current_schema()') .select('pg_class.relname AS table_name, reltuples::bigint AS estimate') if check_statistics diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 723f0f6a308..4e2e1eaf21c 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -176,7 +176,7 @@ module Gitlab name: name.presence || concurrent_foreign_key_name(source, column) } - if foreign_key_exists?(source, target, options) + if foreign_key_exists?(source, target, **options) warning_message = "Foreign key not created because it exists already " \ "(this may be due to an aborted migration or similar): " \ "source: #{source}, target: #{target}, column: #{options[:column]}, "\ @@ -330,13 +330,13 @@ module Gitlab # * +timing_configuration+ - [[ActiveSupport::Duration, ActiveSupport::Duration], ...] lock timeout for the block, sleep time before the next iteration, defaults to `Gitlab::Database::WithLockRetries::DEFAULT_TIMING_CONFIGURATION` # * +logger+ - [Gitlab::JsonLogger] # * +env+ - [Hash] custom environment hash, see the example with `DISABLE_LOCK_RETRIES` - def with_lock_retries(**args, &block) + def with_lock_retries(*args, **kwargs, &block) merged_args = { klass: self.class, logger: Gitlab::BackgroundMigration::Logger - }.merge(args) + }.merge(kwargs) - Gitlab::Database::WithLockRetries.new(merged_args).run(&block) + Gitlab::Database::WithLockRetries.new(**merged_args).run(&block) end def true_value @@ -882,7 +882,7 @@ module Gitlab # column. opclasses[new] = opclasses.delete(old) if opclasses[old] - options[:opclasses] = opclasses + options[:opclass] = opclasses end add_concurrent_index(table, new_columns, options) @@ -994,10 +994,10 @@ into similar problems in the future (e.g. when new tables are created). def postgres_exists_by_name?(table, name) index_sql = <<~SQL SELECT COUNT(*) - FROM pg_index - JOIN pg_class i ON (indexrelid=i.oid) - JOIN pg_class t ON (indrelid=t.oid) - WHERE i.relname = '#{name}' AND t.relname = '#{table}' + FROM pg_catalog.pg_indexes + WHERE schemaname = #{connection.quote(current_schema)} + AND tablename = #{connection.quote(table)} + AND indexname = #{connection.quote(name)} SQL connection.select_value(index_sql).to_i > 0 @@ -1053,11 +1053,15 @@ into similar problems in the future (e.g. when new tables are created). # the table name in addition to using the constraint_name check_sql = <<~SQL SELECT COUNT(*) - FROM pg_constraint - JOIN pg_class ON pg_constraint.conrelid = pg_class.oid - WHERE pg_constraint.contype = 'c' - AND pg_constraint.conname = '#{constraint_name}' - AND pg_class.relname = '#{table}' + FROM pg_catalog.pg_constraint con + INNER JOIN pg_catalog.pg_class rel + ON rel.oid = con.conrelid + INNER JOIN pg_catalog.pg_namespace nsp + ON nsp.oid = con.connamespace + WHERE con.contype = 'c' + AND con.conname = #{connection.quote(constraint_name)} + AND nsp.nspname = #{connection.quote(current_schema)} + AND rel.relname = #{connection.quote(table)} SQL connection.select_value(check_sql) > 0 @@ -1284,8 +1288,9 @@ into similar problems in the future (e.g. when new tables are created). check_sql = <<~SQL SELECT c.is_nullable FROM information_schema.columns c - WHERE c.table_name = '#{table}' - AND c.column_name = '#{column}' + WHERE c.table_schema = #{connection.quote(current_schema)} + AND c.table_name = #{connection.quote(table)} + AND c.column_name = #{connection.quote(column)} SQL connection.select_value(check_sql) == 'YES' diff --git a/lib/gitlab/database/partitioning/partition_creator.rb b/lib/gitlab/database/partitioning/partition_creator.rb index 4c1b13fe3b5..547e0b9b957 100644 --- a/lib/gitlab/database/partitioning/partition_creator.rb +++ b/lib/gitlab/database/partitioning/partition_creator.rb @@ -72,10 +72,10 @@ module Gitlab end def with_lock_retries(&block) - Gitlab::Database::WithLockRetries.new({ + Gitlab::Database::WithLockRetries.new( klass: self.class, logger: Gitlab::AppLogger - }).run(&block) + ).run(&block) end def connection diff --git a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb index f9ad1e60776..17a42d997e6 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb @@ -11,8 +11,6 @@ module Gitlab PAUSE_SECONDS = 0.25 def perform(start_id, stop_id, source_table, partitioned_table, source_column) - return unless Feature.enabled?(:backfill_partitioned_audit_events, default_enabled: true) - if transaction_open? raise "Aborting job to backfill partitioned #{source_table} table! Do not run this job in a transaction block!" end diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb new file mode 100644 index 00000000000..2a9f23f0098 --- /dev/null +++ b/lib/gitlab/database/postgres_index.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class PostgresIndex < ActiveRecord::Base + self.table_name = 'postgres_indexes' + self.primary_key = 'identifier' + + scope :by_identifier, ->(identifier) do + raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + + find(identifier) + end + + # A 'regular' index is a non-unique index, + # that does not serve an exclusion constraint and + # is defined on a table that is not partitioned. + scope :regular, -> { where(unique: false, partitioned: false, exclusion: false)} + + scope :random_few, ->(how_many) do + limit(how_many).order(Arel.sql('RANDOM()')) + end + + scope :not_match, ->(regex) { where("name !~ ?", regex)} + + def to_s + name + end + end + end +end diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb new file mode 100644 index 00000000000..baffe28d9ed --- /dev/null +++ b/lib/gitlab/database/reindexing.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Reindexing + def self.perform(index_selector) + Array.wrap(index_selector).each do |index| + ReindexAction.keep_track_of(index) do + ConcurrentReindex.new(index).perform + end + end + end + + def self.candidate_indexes + Gitlab::Database::PostgresIndex + .regular + .not_match("^#{ConcurrentReindex::TEMPORARY_INDEX_PREFIX}") + .not_match("^#{ConcurrentReindex::REPLACED_INDEX_PREFIX}") + end + end + end +end diff --git a/lib/gitlab/database/reindexing/concurrent_reindex.rb b/lib/gitlab/database/reindexing/concurrent_reindex.rb new file mode 100644 index 00000000000..89fab4a183c --- /dev/null +++ b/lib/gitlab/database/reindexing/concurrent_reindex.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Reindexing + class ConcurrentReindex + include Gitlab::Utils::StrongMemoize + include MigrationHelpers + + ReindexError = Class.new(StandardError) + + PG_IDENTIFIER_LENGTH = 63 + TEMPORARY_INDEX_PREFIX = 'tmp_reindex_' + REPLACED_INDEX_PREFIX = 'old_reindex_' + + attr_reader :index, :logger + + def initialize(index, logger: Gitlab::AppLogger) + @index = index + @logger = logger + end + + def perform + raise ReindexError, 'UNIQUE indexes are currently not supported' if index.unique? + raise ReindexError, 'partitioned indexes are currently not supported' if index.partitioned? + raise ReindexError, 'indexes serving an exclusion constraint are currently not supported' if index.exclusion? + raise ReindexError, 'index is a left-over temporary index from a previous reindexing run' if index.name.start_with?(TEMPORARY_INDEX_PREFIX, REPLACED_INDEX_PREFIX) + + logger.info "Starting reindex of #{index}" + + with_rebuilt_index do |replacement_index| + swap_index(replacement_index) + end + end + + private + + def with_rebuilt_index + if Gitlab::Database::PostgresIndex.find_by(schema: index.schema, name: replacement_index_name) + logger.debug("dropping dangling index from previous run (if it exists): #{replacement_index_name}") + remove_index(index.schema, replacement_index_name) + end + + create_replacement_index_statement = index.definition + .sub(/CREATE INDEX #{index.name}/, "CREATE INDEX CONCURRENTLY #{replacement_index_name}") + + logger.info("creating replacement index #{replacement_index_name}") + logger.debug("replacement index definition: #{create_replacement_index_statement}") + + disable_statement_timeout do + connection.execute(create_replacement_index_statement) + end + + replacement_index = Gitlab::Database::PostgresIndex.find_by(schema: index.schema, name: replacement_index_name) + + unless replacement_index.valid_index? + message = 'replacement index was created as INVALID' + logger.error("#{message}, cleaning up") + raise ReindexError, "failed to reindex #{index}: #{message}" + end + + yield replacement_index + ensure + begin + remove_index(index.schema, replacement_index_name) + rescue => e + logger.error(e) + end + end + + def swap_index(replacement_index) + logger.info("swapping replacement index #{replacement_index} with #{index}") + + with_lock_retries do + rename_index(index.schema, index.name, replaced_index_name) + rename_index(replacement_index.schema, replacement_index.name, index.name) + rename_index(index.schema, replaced_index_name, replacement_index.name) + end + end + + def rename_index(schema, old_index_name, new_index_name) + connection.execute(<<~SQL) + ALTER INDEX #{quote_table_name(schema)}.#{quote_table_name(old_index_name)} + RENAME TO #{quote_table_name(new_index_name)} + SQL + end + + def remove_index(schema, name) + logger.info("Removing index #{schema}.#{name}") + + disable_statement_timeout do + connection.execute(<<~SQL) + DROP INDEX CONCURRENTLY + IF EXISTS #{quote_table_name(schema)}.#{quote_table_name(name)} + SQL + end + end + + def replacement_index_name + @replacement_index_name ||= "#{TEMPORARY_INDEX_PREFIX}#{index.indexrelid}" + end + + def replaced_index_name + @replaced_index_name ||= "#{REPLACED_INDEX_PREFIX}#{index.indexrelid}" + end + + def with_lock_retries(&block) + arguments = { klass: self.class, logger: logger } + + Gitlab::Database::WithLockRetries.new(**arguments).run(raise_on_exhaustion: true, &block) + end + + delegate :execute, :quote_table_name, to: :connection + def connection + @connection ||= ActiveRecord::Base.connection + end + end + end + end +end diff --git a/lib/gitlab/database/reindexing/reindex_action.rb b/lib/gitlab/database/reindexing/reindex_action.rb new file mode 100644 index 00000000000..0928ef90e5d --- /dev/null +++ b/lib/gitlab/database/reindexing/reindex_action.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Reindexing + class ReindexAction < ActiveRecord::Base + self.table_name = 'postgres_reindex_actions' + + enum state: { started: 0, finished: 1, failed: 2 } + + def self.keep_track_of(index, &block) + action = create!( + index_identifier: index.identifier, + action_start: Time.zone.now, + ondisk_size_bytes_start: index.ondisk_size_bytes + ) + + yield + + action.state = :finished + rescue + action.state = :failed + raise + ensure + index.reload # rubocop:disable Cop/ActiveRecordAssociationReload + + action.action_end = Time.zone.now + action.ondisk_size_bytes_end = index.ondisk_size_bytes + + action.save! + end + end + end + end +end diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb index dda4d8eecdb..3d929c62933 100644 --- a/lib/gitlab/database/schema_helpers.rb +++ b/lib/gitlab/database/schema_helpers.rb @@ -32,11 +32,14 @@ module Gitlab def trigger_exists?(table_name, name) connection.select_value(<<~SQL) SELECT 1 - FROM pg_trigger - INNER JOIN pg_class - ON pg_trigger.tgrelid = pg_class.oid - WHERE pg_class.relname = '#{table_name}' - AND pg_trigger.tgname = '#{name}' + FROM pg_catalog.pg_trigger trgr + INNER JOIN pg_catalog.pg_class rel + ON trgr.tgrelid = rel.oid + INNER JOIN pg_catalog.pg_namespace nsp + ON nsp.oid = rel.relnamespace + WHERE nsp.nspname = #{connection.quote(current_schema)} + AND rel.relname = #{connection.quote(table_name)} + AND trgr.tgname = #{connection.quote(name)} SQL end @@ -68,10 +71,10 @@ module Gitlab end def with_lock_retries(&block) - Gitlab::Database::WithLockRetries.new({ + Gitlab::Database::WithLockRetries.new( klass: self.class, logger: Gitlab::BackgroundMigration::Logger - }).run(&block) + ).run(&block) end def assert_not_in_transaction_block(scope:) diff --git a/lib/gitlab/database/similarity_score.rb b/lib/gitlab/database/similarity_score.rb index 2633c29438a..ff78fd0218c 100644 --- a/lib/gitlab/database/similarity_score.rb +++ b/lib/gitlab/database/similarity_score.rb @@ -6,6 +6,11 @@ module Gitlab EMPTY_STRING = Arel.sql("''").freeze EXPRESSION_ON_INVALID_INPUT = Arel::Nodes::NamedFunction.new('CAST', [Arel.sql('0').as('integer')]).freeze DEFAULT_MULTIPLIER = 1 + DISPLAY_NAME = self.name.underscore.freeze + + # Adds a "magic" comment in the generated SQL expression in order to be able to tell if we're sorting by similarity. + # Example: /* gitlab/database/similarity_score */ SIMILARITY(COALESCE... + SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION = "/* #{DISPLAY_NAME} */ SIMILARITY".freeze # This method returns an Arel expression that can be used in an ActiveRecord query to order the resultset by similarity. # @@ -74,6 +79,10 @@ module Gitlab end end + def self.order_by_similarity?(arel_query) + arel_query.to_sql.include?(SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION) + end + # (SIMILARITY(COALESCE(column, ''), 'search_string') * CAST(multiplier AS numeric)) def self.rule_to_arel(search, rule) Arel::Nodes::Grouping.new( @@ -91,7 +100,7 @@ module Gitlab # SIMILARITY(COALESCE(column, ''), 'search_string') def self.similarity_function_call(search, column) - Arel::Nodes::NamedFunction.new('SIMILARITY', [column, Arel.sql(search)]) + Arel::Nodes::NamedFunction.new(SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION, [column, Arel.sql(search)]) end # CAST(multiplier AS numeric) diff --git a/lib/gitlab/database/with_lock_retries.rb b/lib/gitlab/database/with_lock_retries.rb index a9c86e4e267..3fb52d786ad 100644 --- a/lib/gitlab/database/with_lock_retries.rb +++ b/lib/gitlab/database/with_lock_retries.rb @@ -95,7 +95,7 @@ module Gitlab run_block_with_transaction rescue ActiveRecord::LockWaitTimeout if retry_with_lock_timeout? - disable_idle_in_transaction_timeout + disable_idle_in_transaction_timeout if ActiveRecord::Base.connection.transaction_open? wait_until_next_retry reset_db_settings @@ -149,7 +149,7 @@ module Gitlab log(message: "Couldn't acquire lock to perform the migration", current_iteration: current_iteration) log(message: "Executing the migration without lock timeout", current_iteration: current_iteration) - execute("SET LOCAL lock_timeout TO '0'") + disable_lock_timeout if ActiveRecord::Base.connection.transaction_open? run_block @@ -184,6 +184,10 @@ module Gitlab execute("SET LOCAL idle_in_transaction_session_timeout TO '0'") end + def disable_lock_timeout + execute("SET LOCAL lock_timeout TO '0'") + end + def reset_db_settings execute('RESET idle_in_transaction_session_timeout; RESET lock_timeout') end diff --git a/lib/gitlab/design_management/copy_design_collection_model_attributes.yml b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml new file mode 100644 index 00000000000..1d341e6520e --- /dev/null +++ b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml @@ -0,0 +1,43 @@ +# This file exists to lock the attributes of Design Management models +# that get copied in `DesignManagement::CopyDesignCollection::CopyService` +# to specific schemas. +# +# This allows us to perform sanity checks and alert when there are changes +# to the schema by running expectations against the lists in this file +# and the actual schema of the models in `copy_designs_service_spec.rb`. +# +# If you are here because you received a failed test in +# `copy_designs_service_spec.rb`, you need to decide how to handle the +# changes and whether the new attribute(s) should be included in the copy +# or ignored. + +# COPY. +# Add attributes that should be copied to the `{model}_attributes` lists: +design_attributes: + - filename + - relative_position + +version_attributes: + - author_id + - created_at + +action_attributes: # (None) + +# IGNORE. +# Add attributes that should not be copied to the `ignore_{model}_attributes` lists: +ignore_design_attributes: + - id + - issue_id + - project_id + +ignore_version_attributes: + - id + - issue_id + - sha + +ignore_action_attributes: + - id + - design_id + - event + - image_v432x230 + - version_id diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb index d54e1aad19a..341572f9c94 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb @@ -11,7 +11,7 @@ module Gitlab super(merge_request_diff, project: merge_request_diff.project, - diff_options: diff_options, + diff_options: merged_diff_options(diff_options), diff_refs: merge_request_diff.diff_refs, fallback_diff_refs: merge_request_diff.fallback_diff_refs) end @@ -64,6 +64,11 @@ module Gitlab diff_stats_cache.read || super end end + + def merged_diff_options(diff_options) + max_diff_options = ::Commit.max_diff_options(project: @merge_request_diff.project) + diff_options.present? ? diff_options.merge(max_diff_options) : max_diff_options + end end end end diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 0eb22e6b3cb..e873e9c17d5 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -20,10 +20,23 @@ module Gitlab # - Assigns DiffFile#highlighted_diff_lines for cached files # def decorate(diff_file) - if content = read_file(diff_file) - diff_file.highlighted_diff_lines = content.map do |line| - Gitlab::Diff::Line.safe_init_from_hash(line) - end + content = read_file(diff_file) + + return [] unless content + + if content.empty? && recache_due_to_size?(diff_file) + # If the file is missing from the cache and there's reason to believe + # it is uncached due to a size issue around changing the values for + # max patch size, manually populate the hash and then set the value. + # + new_cache_content = {} + new_cache_content[diff_file.file_path] = diff_file.highlighted_diff_lines.map(&:to_hash) + + write_to_redis_hash(new_cache_content) + + set_highlighted_diff_lines(diff_file, read_file(diff_file)) + else + set_highlighted_diff_lines(diff_file, content) end end @@ -58,6 +71,28 @@ module Gitlab private + def set_highlighted_diff_lines(diff_file, content) + diff_file.highlighted_diff_lines = content.map do |line| + Gitlab::Diff::Line.safe_init_from_hash(line) + end + end + + def recache_due_to_size?(diff_file) + diff_file_class = diff_file.diff.class + + current_patch_safe_limit_bytes = diff_file_class.patch_safe_limit_bytes + default_patch_safe_limit_bytes = diff_file_class.patch_safe_limit_bytes(diff_file_class::DEFAULT_MAX_PATCH_BYTES) + + # If the diff is >= than the default limit, but less than the current + # limit, it is likely uncached due to having hit the default limit, + # making it eligible for recalculating. + # + diff_file.diff.diff_bytesize.between?( + default_patch_safe_limit_bytes, + current_patch_safe_limit_bytes + ) + end + def cacheable_files strong_memoize(:cacheable_files) do diff_files.select { |file| cacheable?(file) && read_file(file).nil? } diff --git a/lib/gitlab/exclusive_lease_helpers.rb b/lib/gitlab/exclusive_lease_helpers.rb index 10762d83588..da5b0afad38 100644 --- a/lib/gitlab/exclusive_lease_helpers.rb +++ b/lib/gitlab/exclusive_lease_helpers.rb @@ -35,7 +35,7 @@ module Gitlab lease.obtain(1 + retries) - yield(lease.retried?) + yield(lease.retried?, lease) ensure lease&.cancel end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index dca60c93fb2..72d3da6f0f2 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -62,6 +62,12 @@ module Gitlab }, invite_email: { tracking_category: 'Growth::Acquisition::Experiment::InviteEmail' + }, + invitation_reminders: { + tracking_category: 'Growth::Acquisition::Experiment::InvitationReminders' + }, + group_only_trials: { + tracking_category: 'Growth::Conversion::Experiment::GroupOnlyTrials' } }.freeze @@ -91,10 +97,17 @@ module Gitlab } end + def push_frontend_experiment(experiment_key) + var_name = experiment_key.to_s.camelize(:lower) + enabled = experiment_enabled?(experiment_key) + + gon.push({ experiments: { var_name => enabled } }, true) + end + def experiment_enabled?(experiment_key) return false if dnt_enabled? - return true if Experimentation.enabled_for_user?(experiment_key, experimentation_subject_index) + return true if Experimentation.enabled_for_value?(experiment_key, experimentation_subject_index) return true if forced_enabled?(experiment_key) false @@ -102,7 +115,7 @@ module Gitlab def track_experiment_event(experiment_key, action, value = nil) track_experiment_event_for(experiment_key, action, value) do |tracking_data| - ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), tracking_data) + ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data) end end @@ -183,9 +196,14 @@ module Gitlab experiment.enabled? && experiment.enabled_for_environment? end - def enabled_for_user?(experiment_key, experimentation_subject_index) + def enabled_for_attribute?(experiment_key, attribute) + index = Digest::SHA1.hexdigest(attribute).hex % 100 + enabled_for_value?(experiment_key, index) + end + + def enabled_for_value?(experiment_key, experimentation_subject_index) enabled?(experiment_key) && - experiment(experiment_key).enabled_for_experimentation_subject?(experimentation_subject_index) + experiment(experiment_key).enabled_for_index?(experimentation_subject_index) end end @@ -200,10 +218,10 @@ module Gitlab environment end - def enabled_for_experimentation_subject?(experimentation_subject_index) - return false if experimentation_subject_index.blank? + def enabled_for_index?(index) + return false if index.blank? - experimentation_subject_index <= experiment_percentage + index <= experiment_percentage end private diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 09a49b6c1ca..78c47023c08 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -120,8 +120,8 @@ module Gitlab # default. # # Patches surpassing this limit should still be persisted in the database. - def patch_safe_limit_bytes - patch_hard_limit_bytes / 10 + def patch_safe_limit_bytes(limit = patch_hard_limit_bytes) + limit / 10 end # Returns the limit for a single diff file (patch). @@ -174,9 +174,13 @@ module Gitlab @line_count ||= Util.count_lines(@diff) end + def diff_bytesize + @diff_bytesize ||= @diff.bytesize + end + def too_large? if @too_large.nil? - @too_large = @diff.bytesize >= self.class.patch_hard_limit_bytes + @too_large = diff_bytesize >= self.class.patch_hard_limit_bytes else @too_large end @@ -194,7 +198,7 @@ module Gitlab def collapsed? return @collapsed if defined?(@collapsed) - @collapsed = !expanded && @diff.bytesize >= self.class.patch_safe_limit_bytes + @collapsed = !expanded && diff_bytesize >= self.class.patch_safe_limit_bytes end def collapse! diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index e6121d688ba..2fa88973bae 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -7,19 +7,23 @@ module Gitlab class DiffCollection include Enumerable - DEFAULT_LIMITS = { max_files: 100, max_lines: 5000 }.freeze - attr_reader :limits delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits + def self.default_limits + { max_files: 100, max_lines: 5000 } + end + def self.limits(options = {}) limits = {} - limits[:max_files] = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) - limits[:max_lines] = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) + defaults = default_limits + limits[:max_files] = options.fetch(:max_files, defaults[:max_files]) + limits[:max_lines] = options.fetch(:max_lines, defaults[:max_lines]) limits[:max_bytes] = limits[:max_files] * 5.kilobytes # Average 5 KB per file - limits[:safe_max_files] = [limits[:max_files], DEFAULT_LIMITS[:max_files]].min - limits[:safe_max_lines] = [limits[:max_lines], DEFAULT_LIMITS[:max_lines]].min + + limits[:safe_max_files] = [limits[:max_files], defaults[:max_files]].min + limits[:safe_max_lines] = [limits[:max_lines], defaults[:max_lines]].min limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file limits[:max_patch_bytes] = Gitlab::Git::Diff.patch_hard_limit_bytes diff --git a/lib/gitlab/git/diff_stats_collection.rb b/lib/gitlab/git/diff_stats_collection.rb index 7e49d79676e..e30ec836a49 100644 --- a/lib/gitlab/git/diff_stats_collection.rb +++ b/lib/gitlab/git/diff_stats_collection.rb @@ -22,8 +22,8 @@ module Gitlab @collection.map(&:path) end - def real_size - max_files = ::Commit.max_diff_options[:max_files] + def real_size(project: nil) + max_files = ::Commit.max_diff_options(project: project)[:max_files] if paths.size > max_files "#{max_files}+" else diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index b67b3a37440..d8c992155cb 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -504,7 +504,7 @@ module Gitlab changes_size = 0 changes_list.each do |change| - changes_size += repository.new_blobs(change[:newrev]).sum(&:size) # rubocop: disable CodeReuse/ActiveRecord + changes_size += repository.new_blobs(change[:newrev]).sum(&:size) check_size_against_limit(changes_size) end diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb index ae83e45f2b3..b4ccca9df07 100644 --- a/lib/gitlab/git_access_snippet.rb +++ b/lib/gitlab/git_access_snippet.rb @@ -60,13 +60,17 @@ module Gitlab def check_valid_actor! # TODO: Investigate if expanding actor/authentication types are needed. # https://gitlab.com/gitlab-org/gitlab/issues/202190 - if actor && !actor.is_a?(User) && !actor.instance_of?(Key) + if actor && !allowed_actor? raise ForbiddenError, ERROR_MESSAGES[:authentication_mechanism] end super end + def allowed_actor? + actor.is_a?(User) || actor.instance_of?(Key) + end + def project_snippet? snippet.is_a?(ProjectSnippet) end @@ -138,3 +142,5 @@ module Gitlab end end end + +Gitlab::GitAccessSnippet.prepend_if_ee('EE::Gitlab::GitAccessSnippet') diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 66517ecd743..d2ddc608d3a 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -45,10 +45,11 @@ module Gitlab # made globally available to the frontend push_frontend_feature_flag(:snippets_vue, default_enabled: true) push_frontend_feature_flag(:monaco_blobs, default_enabled: true) - push_frontend_feature_flag(:monaco_ci, default_enabled: false) + push_frontend_feature_flag(:monaco_ci, default_enabled: true) push_frontend_feature_flag(:snippets_edit_vue, default_enabled: true) push_frontend_feature_flag(:webperf_experiment, default_enabled: false) push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false) + push_frontend_feature_flag(:usage_data_api, default_enabled: false) # Startup CSS feature is a special one as it can be enabled by means of cookies and params gon.push({ features: { 'startupCss' => use_startup_css? } }, true) diff --git a/lib/gitlab/graphql/global_id_compatibility.rb b/lib/gitlab/graphql/global_id_compatibility.rb new file mode 100644 index 00000000000..a96e4c4b976 --- /dev/null +++ b/lib/gitlab/graphql/global_id_compatibility.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module GlobalIDCompatibility + # TODO: remove this module once the compatibility layer is no longer needed. + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/257883 + def coerce_global_id_arguments!(args) + global_id_arguments = self.class.arguments.values.select do |arg| + arg.type.is_a?(Class) && arg.type <= ::Types::GlobalIDType + end + + global_id_arguments.each do |arg| + k = arg.keyword + args[k] &&= arg.type.coerce_isolated_input(args[k]) + end + end + end + end +end diff --git a/lib/gitlab/graphql/markdown_field.rb b/lib/gitlab/graphql/markdown_field.rb index 7be6810f7ba..0b5bde8d8d9 100644 --- a/lib/gitlab/graphql/markdown_field.rb +++ b/lib/gitlab/graphql/markdown_field.rb @@ -12,13 +12,19 @@ module Gitlab end method_name = kwargs.delete(:method) || name.to_s.sub(/_html$/, '') - kwargs[:resolve] = Gitlab::Graphql::MarkdownField::Resolver.new(method_name.to_sym).proc + resolver_method = "#{name}_resolver".to_sym + kwargs[:resolver_method] = resolver_method kwargs[:description] ||= "The GitLab Flavored Markdown rendering of `#{method_name}`" # Adding complexity to rendered notes since that could cause queries. kwargs[:complexity] ||= 5 field name, GraphQL::STRING_TYPE, **kwargs + + define_method resolver_method do + # We need to `dup` the context so the MarkdownHelper doesn't modify it + ::MarkupHelper.markdown_field(object, method_name.to_sym, context.to_h.dup) + end end end end diff --git a/lib/gitlab/graphql/markdown_field/resolver.rb b/lib/gitlab/graphql/markdown_field/resolver.rb deleted file mode 100644 index 11a01b95ad1..00000000000 --- a/lib/gitlab/graphql/markdown_field/resolver.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module MarkdownField - class Resolver - attr_reader :method_name - - def initialize(method_name) - @method_name = method_name - end - - def proc - -> (object, _args, ctx) do - # We need to `dup` the context so the MarkdownHelper doesn't modify it - ::MarkupHelper.markdown_field(object, method_name, ctx.to_h.dup) - end - end - end - end - end -end diff --git a/lib/gitlab/graphql/pagination/keyset/order_info.rb b/lib/gitlab/graphql/pagination/keyset/order_info.rb index f54695ddb9a..577f59911f5 100644 --- a/lib/gitlab/graphql/pagination/keyset/order_info.rb +++ b/lib/gitlab/graphql/pagination/keyset/order_info.rb @@ -94,6 +94,8 @@ module Gitlab [order_value.expr.expressions[0].name.to_s, order_value.direction, order_value.expr] elsif ordering_by_similarity?(order_value) ['similarity', order_value.direction, order_value.expr] + elsif ordering_by_case?(order_value) + [order_value.expr.case.name.to_s, order_value.direction, order_value.expr] else [order_value.expr.name, order_value.direction, nil] end @@ -106,7 +108,12 @@ module Gitlab # determine if ordering using SIMILARITY scoring based on Gitlab::Database::SimilarityScore def ordering_by_similarity?(order_value) - order_value.to_sql.match?(/SIMILARITY\(.+\*/) + Gitlab::Database::SimilarityScore.order_by_similarity?(order_value) + end + + # determine if ordering using CASE + def ordering_by_case?(order_value) + order_value.expr.is_a?(Arel::Nodes::Case) end end end diff --git a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb index 6b6bb72eb31..1e568e9dcbc 100644 --- a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb +++ b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb @@ -6,6 +6,8 @@ module Gitlab class LoggerAnalyzer COMPLEXITY_ANALYZER = GraphQL::Analysis::QueryComplexity.new { |query, complexity_value| complexity_value } DEPTH_ANALYZER = GraphQL::Analysis::QueryDepth.new { |query, depth_value| depth_value } + FIELD_USAGE_ANALYZER = GraphQL::Analysis::FieldUsage.new { |query, used_fields, used_deprecated_fields| [used_fields, used_deprecated_fields] } + ALL_ANALYZERS = [COMPLEXITY_ANALYZER, DEPTH_ANALYZER, FIELD_USAGE_ANALYZER].freeze def analyze?(query) Feature.enabled?(:graphql_logging, default_enabled: true) @@ -29,12 +31,13 @@ module Gitlab def final_value(memo) return if memo.nil? - analyzers = [COMPLEXITY_ANALYZER, DEPTH_ANALYZER] - complexity, depth = GraphQL::Analysis.analyze_query(memo[:query], analyzers) + complexity, depth, field_usages = GraphQL::Analysis.analyze_query(memo[:query], ALL_ANALYZERS) memo[:depth] = depth memo[:complexity] = complexity memo[:duration_s] = duration(memo[:time_started]).round(1) + memo[:used_fields] = field_usages.first + memo[:used_deprecated_fields] = field_usages.second GraphqlLogger.info(memo.except!(:time_started, :query)) rescue => e diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index 0cc3de297ba..751dedf4323 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -38,3 +38,5 @@ module Gitlab end end end + +Gitlab::GroupSearchResults.prepend_if_ee('EE::Gitlab::GroupSearchResults') diff --git a/lib/gitlab/health_checks/unicorn_check.rb b/lib/gitlab/health_checks/unicorn_check.rb index cdc6d2a7519..f0c6fdab600 100644 --- a/lib/gitlab/health_checks/unicorn_check.rb +++ b/lib/gitlab/health_checks/unicorn_check.rb @@ -22,7 +22,7 @@ module Gitlab def check return unless http_servers - http_servers.sum(&:worker_processes) # rubocop: disable CodeReuse/ActiveRecord + http_servers.sum(&:worker_processes) end # Traversal of ObjectSpace is expensive, on fully loaded application diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index 05b69362976..00c6f570f4f 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -53,6 +53,7 @@ module Gitlab @importable = importable @imported_object_retries = 0 @relation_hash[importable_column_name] = @importable.id + @original_user = {} # Remove excluded keys from relation_hash # We don't do this in the parsed_relation_hash because of the 'transformed attributes' @@ -112,6 +113,7 @@ module Gitlab def update_user_references self.class::USER_REFERENCES.each do |reference| if @relation_hash[reference] + @original_user[reference] = @relation_hash[reference] @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]] end end @@ -243,28 +245,20 @@ module Gitlab # will be used. Otherwise, a note stating the original author name # is left. def set_note_author - old_author_id = @relation_hash['author_id'] + old_author_id = @original_user['author_id'] author = @relation_hash.delete('author') - update_note_for_missing_author(author['name']) unless has_author?(old_author_id) - end - - def has_author?(old_author_id) - admin_user? && @members_mapper.include?(old_author_id) + unless @members_mapper.include?(old_author_id) + @relation_hash['note'] = "%{note}\n\n %{missing_author_note}" % { + note: @relation_hash['note'].presence || '*Blank note*', + missing_author_note: missing_author_note(@relation_hash['updated_at'], author['name']) + } + end end def missing_author_note(updated_at, author_name) timestamp = updated_at.split('.').first - "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" - end - - def update_note_for_missing_author(author_name) - @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? - @relation_hash['note'] = "#{@relation_hash['note']}#{missing_author_note(@relation_hash['updated_at'], author_name)}" - end - - def admin_user? - @user.admin? + "*By #{author_name} on #{timestamp} (imported from GitLab)*" end def existing_object? diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 515fd98630c..4964b8b16f4 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -21,10 +21,10 @@ module Gitlab save_lfs_object(lfs_object) end - append_lfs_json_for_batch(batch) if write_lfs_json_enabled? + append_lfs_json_for_batch(batch) end - write_lfs_json if write_lfs_json_enabled? + write_lfs_json true rescue => e @@ -35,10 +35,6 @@ module Gitlab private - def write_lfs_json_enabled? - ::Feature.enabled?(:export_lfs_objects_projects, default_enabled: true) - end - def save_lfs_object(lfs_object) if lfs_object.local_store? copy_file_for_lfs_object(lfs_object) diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 31d1f7b48bd..6b37683ea68 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -34,8 +34,8 @@ module Gitlab @user.id end - def include?(old_author_id) - map.has_key?(old_author_id) && map[old_author_id] != default_user_id + def include?(old_user_id) + map.has_key?(old_user_id) end private @@ -63,6 +63,8 @@ module Gitlab end def add_team_member(member, existing_user = nil) + return true if existing_user && @importable.members.exists?(user_id: existing_user.id) + member['user'] = existing_user member_hash = member_hash(member) diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb index 4a85a313fd7..d1ac6a55fb7 100644 --- a/lib/gitlab/instrumentation/redis.rb +++ b/lib/gitlab/instrumentation/redis.rb @@ -37,7 +37,7 @@ module Gitlab %i[get_request_count query_time read_bytes write_bytes].each do |method| define_method method do - STORAGES.sum(&method) # rubocop:disable CodeReuse/ActiveRecord + STORAGES.sum(&method) end end end diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index e7a8cc6305a..2cede524cac 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -23,7 +23,15 @@ module Gitlab TIMEOUTS_METRIC = :gitlab_job_waiter_timeouts_total def self.notify(key, jid) - Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) } + Gitlab::Redis::SharedState.with do |redis| + # Use a Redis MULTI transaction to ensure we always set an expiry + redis.multi do |multi| + multi.lpush(key, jid) + # This TTL needs to be long enough to allow whichever Sidekiq job calls + # JobWaiter#wait to reach BLPOP. + multi.expire(key, 6.hours.to_i) + end + end end def self.key?(key) @@ -52,10 +60,6 @@ module Gitlab increment_counter(STARTED_METRIC) Gitlab::Redis::SharedState.with do |redis| - # Fallback key expiry: allow a long grace period to reduce the chance of - # a job pushing to an expired key and recreating it - redis.expire(key, [timeout * 2, 10.minutes.to_i].max) - while jobs_remaining > 0 # Redis will not take fractional seconds. Prefer waiting too long over # not waiting long enough @@ -75,9 +79,6 @@ module Gitlab @finished << jid @jobs_remaining -= 1 end - - # All jobs have finished, so expire the key immediately - redis.expire(key, 0) if jobs_remaining == 0 end finished diff --git a/lib/gitlab/lfs/client.rb b/lib/gitlab/lfs/client.rb index e4d600694c2..95217f86d01 100644 --- a/lib/gitlab/lfs/client.rb +++ b/lib/gitlab/lfs/client.rb @@ -6,6 +6,14 @@ module Gitlab # * https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md # * https://github.com/git-lfs/git-lfs/blob/master/docs/api/basic-transfers.md class Client + GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs+json' + GIT_LFS_USER_AGENT = "GitLab #{Gitlab::VERSION} LFS client" + DEFAULT_HEADERS = { + 'Accept' => GIT_LFS_CONTENT_TYPE, + 'Content-Type' => GIT_LFS_CONTENT_TYPE, + 'User-Agent' => GIT_LFS_USER_AGENT + }.freeze + attr_reader :base_url def initialize(base_url, credentials:) @@ -13,19 +21,19 @@ module Gitlab @credentials = credentials end - def batch(operation, objects) + def batch!(operation, objects) body = { operation: operation, transfers: ['basic'], # We don't know `ref`, so can't send it - objects: objects.map { |object| { oid: object.oid, size: object.size } } + objects: objects.as_json(only: [:oid, :size]) } rsp = Gitlab::HTTP.post( batch_url, basic_auth: basic_auth, body: body.to_json, - headers: { 'Content-Type' => 'application/vnd.git-lfs+json' } + headers: build_request_headers ) raise BatchSubmitError unless rsp.success? @@ -40,14 +48,15 @@ module Gitlab body end - def upload(object, upload_action, authenticated:) + def upload!(object, upload_action, authenticated:) file = object.file.open params = { body_stream: file, headers: { 'Content-Length' => object.size.to_s, - 'Content-Type' => 'application/octet-stream' + 'Content-Type' => 'application/octet-stream', + 'User-Agent' => GIT_LFS_USER_AGENT }.merge(upload_action['header'] || {}) } @@ -60,8 +69,25 @@ module Gitlab file&.close end + def verify!(object, verify_action, authenticated:) + params = { + body: object.to_json(only: [:oid, :size]), + headers: build_request_headers(verify_action['header']) + } + + params[:basic_auth] = basic_auth unless authenticated + + rsp = Gitlab::HTTP.post(verify_action['href'], params) + + raise ObjectVerifyError unless rsp.success? + end + private + def build_request_headers(extra_headers = nil) + DEFAULT_HEADERS.merge(extra_headers || {}) + end + attr_reader :credentials def batch_url @@ -96,6 +122,12 @@ module Gitlab "Failed to upload object" end end + + class ObjectVerifyError < StandardError + def message + "Failed to verify object" + end + end end end end diff --git a/lib/gitlab/manifest_import/metadata.rb b/lib/gitlab/manifest_import/metadata.rb new file mode 100644 index 00000000000..80dff075391 --- /dev/null +++ b/lib/gitlab/manifest_import/metadata.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module ManifestImport + class Metadata + EXPIRY_TIME = 1.week + + attr_reader :user, :fallback + + def initialize(user, fallback: {}) + @user = user + @fallback = fallback + end + + def save(repositories, group_id) + Gitlab::Redis::SharedState.with do |redis| + redis.multi do + redis.set(key_for('repositories'), Gitlab::Json.dump(repositories), ex: EXPIRY_TIME) + redis.set(key_for('group_id'), group_id, ex: EXPIRY_TIME) + end + end + end + + def repositories + redis_get('repositories').then do |repositories| + next unless repositories + + Gitlab::Json.parse(repositories).map(&:symbolize_keys) + end || fallback[:manifest_import_repositories] + end + + def group_id + redis_get('group_id')&.to_i || fallback[:manifest_import_group_id] + end + + private + + def key_for(field) + "manifest_import:metadata:user:#{user.id}:#{field}" + end + + def redis_get(field) + Gitlab::Redis::SharedState.with do |redis| + redis.get(key_for(field)) + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb index d1490d5d9b6..8a176be30a2 100644 --- a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb +++ b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb @@ -13,11 +13,12 @@ module Gitlab @dashboard_hash = dashboard_hash @project = project @dashboard_path = dashboard_path + @affected_environment_ids = [] end def execute import - rescue ActiveRecord::RecordInvalid, ::Gitlab::Metrics::Dashboard::Transformers::TransformerError + rescue ActiveRecord::RecordInvalid, Dashboard::Transformers::Errors::BaseError false end @@ -32,28 +33,51 @@ module Gitlab def import delete_stale_metrics create_or_update_metrics + update_prometheus_environments end # rubocop: disable CodeReuse/ActiveRecord def create_or_update_metrics # TODO: use upsert and worker for callbacks? + + affected_metric_ids = [] prometheus_metrics_attributes.each do |attributes| - prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:identifier, :project)) + prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:dashboard_path, :identifier, :project)) prometheus_metric.update!(attributes.slice(*ALLOWED_ATTRIBUTES)) + + affected_metric_ids << prometheus_metric.id end + + @affected_environment_ids += find_alerts(affected_metric_ids).get_environment_id end # rubocop: enable CodeReuse/ActiveRecord def delete_stale_metrics - identifiers = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] } + identifiers_from_yml = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] } stale_metrics = PrometheusMetric.for_project(project) .for_dashboard_path(dashboard_path) .for_group(Enums::PrometheusMetric.groups[:custom]) - .not_identifier(identifiers) + .not_identifier(identifiers_from_yml) + + return unless stale_metrics.exists? + + delete_stale_alerts(stale_metrics) + stale_metrics.each_batch { |batch| batch.delete_all } + end + + def delete_stale_alerts(stale_metrics) + stale_alerts = find_alerts(stale_metrics) + + affected_environment_ids = stale_alerts.get_environment_id + return unless affected_environment_ids.present? - # TODO: use destroy_all and worker for callbacks? - stale_metrics.each(&:destroy) + @affected_environment_ids += affected_environment_ids + stale_alerts.each_batch { |batch| batch.delete_all } + end + + def find_alerts(metrics) + Projects::Prometheus::AlertsFinder.new(project: project, metric: metrics).execute end def prometheus_metrics_attributes @@ -65,6 +89,19 @@ module Gitlab ).execute end end + + def update_prometheus_environments + affected_environments = ::Environment.for_id(@affected_environment_ids.flatten.uniq).for_project(project) + + return unless affected_environments.exists? + + affected_environments.each do |affected_environment| + ::Clusters::Applications::ScheduleUpdateService.new( + affected_environment.cluster_prometheus_adapter, + project + ).execute + end + end end end end diff --git a/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb new file mode 100644 index 00000000000..5ed4466f440 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + # Acts on metrics which have been ingested from source controlled dashboards + class CustomDashboardMetricsInserter < BaseStage + # For each metric in the dashboard config, attempts to + # find a corresponding database record. If found, includes + # the record's id in the dashboard config. + def transform! + database_metrics = ::PrometheusMetricsFinder.new(common: false, group: :custom, project: project).execute + + for_metrics do |metric| + metric_record = database_metrics.find { |m| m.identifier == metric[:id] } + metric[:metric_id] = metric_record.id if metric_record + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/url_validator.rb b/lib/gitlab/metrics/dashboard/stages/url_validator.rb index 9e2bb0d1a70..ad9d78133af 100644 --- a/lib/gitlab/metrics/dashboard/stages/url_validator.rb +++ b/lib/gitlab/metrics/dashboard/stages/url_validator.rb @@ -46,7 +46,7 @@ module Gitlab links&.each do |link| next unless link.is_a? Hash - Gitlab::UrlBlocker.validate!(link[:url], blocker_args) + Gitlab::UrlBlocker.validate!(link[:url], **blocker_args) rescue Gitlab::UrlBlocker::BlockedUrlError link[:url] = '' end diff --git a/lib/gitlab/metrics/dashboard/transformers/errors.rb b/lib/gitlab/metrics/dashboard/transformers/errors.rb index 4d94ab098ae..bc85dc4e131 100644 --- a/lib/gitlab/metrics/dashboard/transformers/errors.rb +++ b/lib/gitlab/metrics/dashboard/transformers/errors.rb @@ -4,10 +4,10 @@ module Gitlab module Metrics module Dashboard module Transformers - TransformerError = Class.new(StandardError) - module Errors - class MissingAttribute < TransformerError + BaseError = Class.new(StandardError) + + class MissingAttribute < BaseError def initialize(attribute_name) super("Missing attribute: '#{attribute_name}'") end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index 15db3999fa4..f8ba254c2a7 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -4,15 +4,13 @@ module Gitlab module Metrics class RequestsRackMiddleware HTTP_METHODS = { - "delete" => %w(200 202 204 303 400 401 403 404 410 422 500 503), - "get" => %w(200 204 301 302 303 304 307 400 401 403 404 410 412 422 429 500 503), - "head" => %w(200 204 301 302 303 304 400 401 403 404 410 429 500 503), + "delete" => %w(200 202 204 303 400 401 403 404 500 503), + "get" => %w(200 204 301 302 303 304 307 400 401 403 404 410 422 429 500 503), + "head" => %w(200 204 301 302 303 401 403 404 410 500), "options" => %w(200 404), - "patch" => %w(200 202 204 400 403 404 409 416 422 500), - "post" => %w(200 201 202 204 301 302 303 304 400 401 403 404 406 409 410 412 413 415 422 429 500 503), - "propfind" => %w(404), - "put" => %w(200 202 204 400 401 403 404 405 406 409 410 415 422 500), - "report" => %w(404) + "patch" => %w(200 202 204 400 403 404 409 416 500), + "post" => %w(200 201 202 204 301 302 303 304 400 401 403 404 406 409 410 412 422 429 500 503), + "put" => %w(200 202 204 400 401 403 404 405 406 409 410 422 500) }.freeze HEALTH_ENDPOINT = /^\/-\/(liveness|readiness|health|metrics)\/?$/.freeze @@ -48,10 +46,12 @@ module Gitlab def call(env) method = env['REQUEST_METHOD'].downcase + method = 'INVALID' unless HTTP_METHODS.key?(method) started = Time.now.to_f + health_endpoint = health_endpoint?(env['PATH_INFO']) begin - if health_endpoint?(env['PATH_INFO']) + if health_endpoint RequestsRackMiddleware.http_health_requests_total.increment(method: method) else RequestsRackMiddleware.http_request_total.increment(method: method) @@ -60,7 +60,10 @@ module Gitlab status, headers, body = @app.call(env) elapsed = Time.now.to_f - started - RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method, status: status.to_s }, elapsed) + + unless health_endpoint + RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method, status: status.to_s }, elapsed) + end [status, headers, body] rescue diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index 8c4d150adad..d7935d65e12 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -54,7 +54,7 @@ module Gitlab end def unicorn_workers_count - http_servers.sum(&:worker_processes) # rubocop: disable CodeReuse/ActiveRecord + http_servers.sum(&:worker_processes) end # Traversal of ObjectSpace is expensive, on fully loaded application diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index 8e6ac7610f2..e7e18b3bb82 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -137,6 +137,7 @@ module Gitlab # TODO this class is meant to replace Handler when the feature flag # upload_middleware_jwt_params_handler is removed + # See https://gitlab.com/gitlab-org/gitlab/-/issues/233895#roll-out-steps class HandlerForJWTParams < Handler def with_open_files @rewritten_fields.keys.each do |field| diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb index 8796dd4d7ec..46c74b8fe3c 100644 --- a/lib/gitlab/pagination/offset_pagination.rb +++ b/lib/gitlab/pagination/offset_pagination.rb @@ -10,9 +10,9 @@ module Gitlab @request_context = request_context end - def paginate(relation) + def paginate(relation, exclude_total_headers: false) paginate_with_limit_optimization(add_default_order(relation)).tap do |data| - add_pagination_headers(data) + add_pagination_headers(data, exclude_total_headers) end end @@ -27,7 +27,7 @@ module Gitlab end return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation) - return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit) + return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit, type: :ops) limited_total_count = pagination_data.total_count_with_limit if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT @@ -47,14 +47,14 @@ module Gitlab relation end - def add_pagination_headers(paginated_data) + def add_pagination_headers(paginated_data, exclude_total_headers) header 'X-Per-Page', paginated_data.limit_value.to_s header 'X-Page', paginated_data.current_page.to_s header 'X-Next-Page', paginated_data.next_page.to_s header 'X-Prev-Page', paginated_data.prev_page.to_s header 'Link', pagination_links(paginated_data) - return if data_without_counts?(paginated_data) + return if exclude_total_headers || data_without_counts?(paginated_data) header 'X-Total', paginated_data.total_count.to_s header 'X-Total-Pages', total_pages(paginated_data).to_s diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index e6e599e079d..fa3af269bbf 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -52,6 +52,7 @@ module Gitlab ProjectTemplate.new('gitbook', 'Pages/GitBook', _('Everything you need to create a GitLab Pages site using GitBook.'), 'https://gitlab.com/pages/gitbook', 'illustrations/logos/gitbook.svg'), ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo.'), 'https://gitlab.com/pages/hexo', 'illustrations/logos/hexo.svg'), ProjectTemplate.new('sse_middleman', 'Static Site Editor/Middleman', _('Middleman project with Static Site Editor support'), 'https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman'), + ProjectTemplate.new('gitpod_spring_petclinic', 'Gitpod/Spring Petclinic', _('A Gitpod configured Webapplication in Spring and Java'), 'https://gitlab.com/gitlab-org/project-templates/gitpod-spring-petclinic', 'illustrations/logos/gitpod.svg'), ProjectTemplate.new('nfhugo', 'Netlify/Hugo', _('A Hugo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhugo', 'illustrations/logos/netlify.svg'), ProjectTemplate.new('nfjekyll', 'Netlify/Jekyll', _('A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfjekyll', 'illustrations/logos/netlify.svg'), ProjectTemplate.new('nfplainhtml', 'Netlify/Plain HTML', _('A plain HTML site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfplain-html', 'illustrations/logos/netlify.svg'), diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 98db8ff761e..c8c949a9363 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -121,6 +121,20 @@ module Gitlab result[:message] end end + + desc _('Approve a merge request') + explanation _('Approve the current merge request.') + types MergeRequest + condition do + quick_action_target.persisted? && quick_action_target.can_be_approved_by?(current_user) + end + command :approve do + success = MergeRequests::ApprovalService.new(quick_action_target.project, current_user).execute(quick_action_target) + + next unless success + + @execution_message[:approve] = _('Approved the current merge request.') + end end def merge_orchestration_service diff --git a/lib/gitlab/redis/hll.rb b/lib/gitlab/redis/hll.rb index ff5754675e2..496018c88cb 100644 --- a/lib/gitlab/redis/hll.rb +++ b/lib/gitlab/redis/hll.rb @@ -7,11 +7,11 @@ module Gitlab KeyFormatError = Class.new(StandardError) def self.count(params) - self.new.count(params) + self.new.count(**params) end def self.add(params) - self.new.add(params) + self.new.add(**params) end def count(keys:) diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 8e23ac6aca5..6511b84e947 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -61,6 +61,39 @@ module Gitlab )\z}xi.freeze end + def debian_package_name_regex + # See official parser + # https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/parsehelp.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n122 + # @debian_package_name_regex ||= %r{\A[a-z0-9][-+\._a-z0-9]*\z}i.freeze + # But we prefer a more strict version from Lintian + # https://salsa.debian.org/lintian/lintian/-/blob/5080c0068ffc4a9ddee92022a91d0c2ff53e56d1/lib/Lintian/Util.pm#L116 + @debian_package_name_regex ||= %r{\A[a-z0-9][-+\.a-z0-9]+\z}.freeze + end + + def debian_version_regex + # See official parser: https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/parsehelp.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n205 + @debian_version_regex ||= %r{ + \A(?: + (?:([0-9]{1,9}):)? (?# epoch) + ([0-9][0-9a-z\.+~-]*) (?# version) + (?:(-[0-0a-z\.+~]+))? (?# revision) + )\z}xi.freeze + end + + def debian_architecture_regex + # See official parser: https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/arch.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n43 + # But we limit to lower case + @debian_architecture_regex ||= %r{\A[a-z0-9][-a-z0-9]*\z}.freeze + end + + def debian_distribution_regex + @debian_distribution_regex ||= %r{\A[a-z0-9][a-z0-9\.-]*\z}i.freeze + end + + def debian_component_regex + @debian_component_regex ||= %r{#{debian_distribution_regex}}.freeze + end + def unbounded_semver_regex # See the official regex: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string @@ -80,6 +113,11 @@ module Gitlab @semver_regex ||= Regexp.new("\\A#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options) end + def prefixed_semver_regex + # identical to semver_regex, except starting with 'v' + @prefixed_semver_regex ||= Regexp.new("\\Av#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options) + end + def go_package_regex # A Go package name looks like a URL but is not; it: # - Must not have a scheme, such as http:// or https:// @@ -103,6 +141,14 @@ module Gitlab def generic_package_version_regex /\A\d+\.\d+\.\d+\z/ end + + def generic_package_name_regex + maven_file_name_regex + end + + def generic_package_file_name_regex + generic_package_name_regex + end end extend self @@ -211,8 +257,27 @@ module Gitlab "Must start with a letter, and cannot end with '-'" end + # The section start, e.g. section_start:12345678:NAME + def logs_section_prefix_regex + /section_((?:start)|(?:end)):(\d+):([a-zA-Z0-9_.-]+)/ + end + + # The optional section options, e.g. [collapsed=true] + def logs_section_options_regex + /(\[(?:\w+=\w+)(?:, ?(?:\w+=\w+))*\])?/ + end + + # The region end, always: \r\e\[0K + def logs_section_suffix_regex + /\r\033\[0K/ + end + def build_trace_section_regex - @build_trace_section_regexp ||= /section_((?:start)|(?:end)):(\d+):([a-zA-Z0-9_.-]+)\r\033\[0K/.freeze + @build_trace_section_regexp ||= %r{ + #{logs_section_prefix_regex} + #{logs_section_options_regex} + #{logs_section_suffix_regex} + }x.freeze end def markdown_code_or_html_blocks diff --git a/lib/gitlab/relative_positioning/item_context.rb b/lib/gitlab/relative_positioning/item_context.rb index cd03a347355..8f5495ece5e 100644 --- a/lib/gitlab/relative_positioning/item_context.rb +++ b/lib/gitlab/relative_positioning/item_context.rb @@ -131,12 +131,12 @@ module Gitlab def shift_left move_sequence_before(true) - object.reset + object.reset_relative_position end def shift_right move_sequence_after(true) - object.reset + object.reset_relative_position end def create_space_left diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 06d8dca2f70..1e78464ddf8 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -7,7 +7,7 @@ module Gitlab DEFAULT_PAGE = 1 DEFAULT_PER_PAGE = 20 - attr_reader :current_user, :query, :filters + attr_reader :current_user, :query, :sort, :filters # Limit search results by passed projects # It allows us to search only for projects user has access to @@ -19,31 +19,23 @@ module Gitlab # query attr_reader :default_project_filter - def initialize(current_user, query, limit_projects = nil, default_project_filter: false, filters: {}) + def initialize(current_user, query, limit_projects = nil, sort: nil, default_project_filter: false, filters: {}) @current_user = current_user @query = query @limit_projects = limit_projects || Project.all @default_project_filter = default_project_filter + @sort = sort @filters = filters end def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, without_count: true, preload_method: nil) should_preload = preload_method.present? - collection = case scope - when 'projects' - projects - when 'issues' - issues - when 'merge_requests' - merge_requests - when 'milestones' - milestones - when 'users' - users - else - should_preload = false - Kaminari.paginate_array([]) - end + collection = collection_for(scope) + + if collection.nil? + should_preload = false + collection = Kaminari.paginate_array([]) + end collection = collection.public_send(preload_method) if should_preload # rubocop:disable GitlabSecurity/PublicSend collection = collection.page(page).per(per_page) @@ -118,6 +110,34 @@ module Gitlab private + def collection_for(scope) + case scope + when 'projects' + projects + when 'issues' + issues + when 'merge_requests' + merge_requests + when 'milestones' + milestones + when 'users' + users + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def apply_sort(scope) + case sort + when 'oldest' + scope.reorder('created_at ASC') + when 'newest' + scope.reorder('created_at DESC') + else + scope + end + end + # rubocop: enable CodeReuse/ActiveRecord + def projects limit_projects.search(query) end @@ -129,7 +149,7 @@ module Gitlab issues = issues.where(project_id: project_ids_relation) # rubocop: disable CodeReuse/ActiveRecord end - issues + apply_sort(issues) end # rubocop: disable CodeReuse/ActiveRecord @@ -149,7 +169,7 @@ module Gitlab merge_requests = merge_requests.in_projects(project_ids_relation) end - merge_requests + apply_sort(merge_requests) end def default_scope @@ -193,6 +213,10 @@ module Gitlab end params[:state] = filters[:state] if filters.key?(:state) + + if Feature.enabled?(:search_filter_by_confidential) && filters.key?(:confidential) && %w(yes no).include?(filters[:confidential]) + params[:confidential] = filters[:confidential] == 'yes' + end end end diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index e1a87a77f04..8793a672693 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -231,7 +231,7 @@ module Gitlab def rss_increase_by_jobs Gitlab::SidekiqDaemon::Monitor.instance.jobs_mutex.synchronize do - Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job| # rubocop:disable CodeReuse/ActiveRecord + Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job| rss_increase_by_job(job) end end diff --git a/lib/gitlab/static_site_editor/config/file_config.rb b/lib/gitlab/static_site_editor/config/file_config.rb index f647c85e1c8..315c603c1dd 100644 --- a/lib/gitlab/static_site_editor/config/file_config.rb +++ b/lib/gitlab/static_site_editor/config/file_config.rb @@ -3,11 +3,38 @@ module Gitlab module StaticSiteEditor module Config + # + # Base GitLab Static Site Editor Configuration facade + # class FileConfig - def data - { - static_site_generator: 'middleman' - } + ConfigError = Class.new(StandardError) + + def initialize(yaml) + content_hash = content_hash(yaml) + @global = Entry::Global.new(content_hash) + @global.compose! + rescue Gitlab::Config::Loader::FormatError => e + raise FileConfig::ConfigError, e.message + end + + def valid? + @global.valid? + end + + def errors + @global.errors + end + + def to_hash_with_defaults + # NOTE: The current approach of simply mapping all the descendents' keys and values ('config') + # into a flat hash may need to be enhanced as we add more complex, non-scalar entries. + @global.descendants.map { |descendant| [descendant.key, descendant.config] }.to_h + end + + private + + def content_hash(yaml) + Gitlab::Config::Loader::Yaml.new(yaml).load! end end end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/global.rb b/lib/gitlab/static_site_editor/config/file_config/entry/global.rb new file mode 100644 index 00000000000..c295ccf1d11 --- /dev/null +++ b/lib/gitlab/static_site_editor/config/file_config/entry/global.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module StaticSiteEditor + module Config + class FileConfig + module Entry + ## + # This class represents a global entry - root Entry for entire + # GitLab StaticSiteEditor Configuration file. + # + class Global < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[ + image_upload_path + mounts + static_site_generator + ].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + end + + entry :image_upload_path, Entry::ImageUploadPath, + description: 'Configuration of the Static Site Editor image upload path.' + entry :mounts, Entry::Mounts, + description: 'Configuration of the Static Site Editor mounts.' + entry :static_site_generator, Entry::StaticSiteGenerator, + description: 'Configuration of the Static Site Editor static site generator.' + end + end + end + end + end +end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/image_upload_path.rb b/lib/gitlab/static_site_editor/config/file_config/entry/image_upload_path.rb new file mode 100644 index 00000000000..6a2b9e10d33 --- /dev/null +++ b/lib/gitlab/static_site_editor/config/file_config/entry/image_upload_path.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module StaticSiteEditor + module Config + class FileConfig + module Entry + ## + # Entry that represents the path to which images will be uploaded + # + class ImageUploadPath < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, type: String + end + + def self.default + 'source/images' + end + end + end + end + end + end +end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/mount.rb b/lib/gitlab/static_site_editor/config/file_config/entry/mount.rb new file mode 100644 index 00000000000..b10956e17a5 --- /dev/null +++ b/lib/gitlab/static_site_editor/config/file_config/entry/mount.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module StaticSiteEditor + module Config + class FileConfig + module Entry + ## + # Entry that represents the mappings of mounted source directories to target paths + # + class Mount < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[source target].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + + validates :source, type: String, presence: true + validates :target, type: String, presence: true, allow_blank: true + end + + def self.default + # NOTE: This is the default for middleman projects. Ideally, this would be determined + # based on the defaults for whatever `static_site_generator` is configured. + { + source: 'source', + target: '' + } + end + end + end + end + end + end +end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/mounts.rb b/lib/gitlab/static_site_editor/config/file_config/entry/mounts.rb new file mode 100644 index 00000000000..10bd377e419 --- /dev/null +++ b/lib/gitlab/static_site_editor/config/file_config/entry/mounts.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module StaticSiteEditor + module Config + class FileConfig + module Entry + ## + # Entry that represents the mappings of mounted source directories to target paths + # + class Mounts < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable + + entry :mount, Entry::Mount, description: 'Configuration of a Static Site Editor mount.' + + validations do + validates :config, type: Array, presence: true + end + + def skip_config_hash_validation? + true + end + + def self.default + [Entry::Mount.default] + end + end + end + end + end + end +end diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/static_site_generator.rb b/lib/gitlab/static_site_editor/config/file_config/entry/static_site_generator.rb new file mode 100644 index 00000000000..593c0951f93 --- /dev/null +++ b/lib/gitlab/static_site_editor/config/file_config/entry/static_site_generator.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module StaticSiteEditor + module Config + class FileConfig + module Entry + ## + # Entry that represents the static site generator tool/framework. + # + class StaticSiteGenerator < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, type: String, inclusion: { in: %w[middleman], message: "should be 'middleman'" } + end + + def self.default + 'middleman' + end + end + end + end + end + end +end diff --git a/lib/gitlab/static_site_editor/config/generated_config.rb b/lib/gitlab/static_site_editor/config/generated_config.rb index f3dce74a32f..ff24ec69ab0 100644 --- a/lib/gitlab/static_site_editor/config/generated_config.rb +++ b/lib/gitlab/static_site_editor/config/generated_config.rb @@ -4,8 +4,6 @@ module Gitlab module StaticSiteEditor module Config class GeneratedConfig - SUPPORTED_EXTENSIONS = %w[.md].freeze - def initialize(repository, ref, path, return_url) @repository = repository @ref = ref @@ -23,7 +21,7 @@ module Gitlab project: project.path, namespace: project.namespace.full_path, return_url: sanitize_url(return_url), - is_supported_content: supported_content?.to_s, + is_supported_content: supported_content?, base_url: Gitlab::Routing.url_helpers.project_show_sse_path(project, full_path), merge_requests_illustration_path: merge_requests_illustration_path } @@ -35,8 +33,12 @@ module Gitlab delegate :project, to: :repository + def supported_extensions + %w[.md].freeze + end + def commit_id - repository.commit(ref)&.id if ref + repository.commit(ref)&.id end def supported_content? @@ -50,7 +52,7 @@ module Gitlab def extension_supported? return true if path.end_with?('.md.erb') && Feature.enabled?(:sse_erb_support, project) - SUPPORTED_EXTENSIONS.any? { |ext| path.end_with?(ext) } + supported_extensions.any? { |ext| path.end_with?(ext) } end def file_exists? diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index 72783a2d682..16e7b8a7eca 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -10,21 +10,21 @@ module Gitlab APPLICATION_DEFAULT = 1 # Struct class representing a single Theme - Theme = Struct.new(:id, :name, :css_class) + Theme = Struct.new(:id, :name, :css_class, :css_filename) # All available Themes THEMES = [ - Theme.new(1, 'Indigo', 'ui-indigo'), - Theme.new(6, 'Light Indigo', 'ui-light-indigo'), - Theme.new(4, 'Blue', 'ui-blue'), - Theme.new(7, 'Light Blue', 'ui-light-blue'), - Theme.new(5, 'Green', 'ui-green'), - Theme.new(8, 'Light Green', 'ui-light-green'), - Theme.new(9, 'Red', 'ui-red'), - Theme.new(10, 'Light Red', 'ui-light-red'), - Theme.new(2, 'Dark', 'ui-dark'), - Theme.new(3, 'Light', 'ui-light'), - Theme.new(11, 'Dark Mode (alpha)', 'gl-dark') + Theme.new(1, 'Indigo', 'ui-indigo', 'theme_indigo'), + Theme.new(6, 'Light Indigo', 'ui-light-indigo', 'theme_light_indigo'), + Theme.new(4, 'Blue', 'ui-blue', 'theme_blue'), + Theme.new(7, 'Light Blue', 'ui-light-blue', 'theme_light_blue'), + Theme.new(5, 'Green', 'ui-green', 'theme_green'), + Theme.new(8, 'Light Green', 'ui-light-green', 'theme_light_green'), + Theme.new(9, 'Red', 'ui-red', 'theme_red'), + Theme.new(10, 'Light Red', 'ui-light-red', 'theme_light_red'), + Theme.new(2, 'Dark', 'ui-dark', 'theme_dark'), + Theme.new(3, 'Light', 'ui-light', 'theme_light'), + Theme.new(11, 'Dark Mode (alpha)', 'gl-dark', nil) ].freeze # Convenience method to get a space-separated String of all the theme diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 89605ce5d07..fbfc7beed9a 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -138,8 +138,10 @@ module Gitlab pages_domains: count(PagesDomain), pool_repositories: count(PoolRepository), projects: count(Project), + projects_creating_incidents: distinct_count(Issue.incident, :project_id), projects_imported_from_github: count(Project.where(import_type: 'github')), projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)), + projects_with_tracing_enabled: count(ProjectTracingSetting), projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), projects_with_alerts_service_enabled: count(AlertsService.active), projects_with_prometheus_alerts: distinct_count(PrometheusAlert, :project_id), @@ -166,8 +168,7 @@ module Gitlab user_preferences_usage, ingress_modsecurity_usage, container_expiration_policies_usage, - service_desk_counts, - snowplow_event_counts + service_desk_counts ).tap do |data| data[:snippets] = data[:personal_snippets] + data[:project_snippets] end @@ -175,7 +176,7 @@ module Gitlab end # rubocop: enable Metrics/AbcSize - def snowplow_event_counts(time_period: {}) + def snowplow_event_counts(time_period) return {} unless report_snowplow_events? { @@ -242,7 +243,8 @@ module Gitlab signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? }, web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? }, ingress_modsecurity_enabled: Feature.enabled?(:ingress_modsecurity), - grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? } + grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? }, + gitpod_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gitpod_enabled? } } end @@ -444,8 +446,11 @@ module Gitlab # rubocop: enable UsageData/LargeTable # rubocop: enable CodeReuse/ActiveRecord + # augmented in EE def user_preferences_usage - {} # augmented in EE + { + user_preferences_user_gitpod_enabled: count(UserPreference.with_user.gitpod_enabled.merge(User.active)) + } end def merge_requests_users(time_period) @@ -469,7 +474,7 @@ module Gitlab end def last_28_days_time_period(column: :created_at) - { column => 28.days.ago..Time.current } + { column => 30.days.ago..2.days.ago } end # Source: https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv @@ -541,6 +546,7 @@ module Gitlab groups: distinct_count(::GroupMember.where(time_period), :user_id), users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id), omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' }, + user_auth_by_provider: distinct_count_user_auth_by_provider(time_period), projects_imported: { gitlab_project: projects_imported_count('gitlab_project', time_period), gitlab: projects_imported_count('gitlab', time_period), @@ -555,7 +561,8 @@ module Gitlab jira: distinct_count(::JiraImportState.where(time_period), :user_id), fogbugz: projects_imported_count('fogbugz', time_period), phabricator: projects_imported_count('phabricator', time_period) - } + }, + groups_imported: distinct_count(::GroupImportState.where(time_period), :user_id) } end # rubocop: enable CodeReuse/ActiveRecord @@ -567,7 +574,8 @@ module Gitlab clusters_applications_prometheus: cluster_applications_user_distinct_count(::Clusters::Applications::Prometheus, time_period), operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period), start: user_minimum_id, - finish: user_maximum_id) + finish: user_maximum_id), + projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id) } end # rubocop: enable CodeReuse/ActiveRecord @@ -696,10 +704,10 @@ module Gitlab counter = Gitlab::UsageDataCounters::EditorUniqueCounter { - action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(date_range) }, - action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(date_range) }, - action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(date_range) }, - action_monthly_active_users_ide_edit: redis_usage_data { counter.count_edit_using_editor(date_range) } + action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(**date_range) }, + action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(**date_range) }, + action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(**date_range) }, + action_monthly_active_users_ide_edit: redis_usage_data { counter.count_edit_using_editor(**date_range) } } end @@ -812,6 +820,7 @@ module Gitlab clear_memoization(:approval_merge_request_rule_maximum_id) clear_memoization(:project_minimum_id) clear_memoization(:project_maximum_id) + clear_memoization(:auth_providers) end # rubocop: disable CodeReuse/ActiveRecord @@ -843,6 +852,39 @@ module Gitlab def projects_imported_count(from, time_period) distinct_count(::Project.imported_from(from).where(time_period), :creator_id) # rubocop: disable CodeReuse/ActiveRecord end + + # rubocop:disable CodeReuse/ActiveRecord + def distinct_count_user_auth_by_provider(time_period) + counts = auth_providers_except_ldap.each_with_object({}) do |provider, hash| + hash[provider] = distinct_count( + ::AuthenticationEvent.success.for_provider(provider).where(time_period), :user_id) + end + + if any_ldap_auth_providers? + counts['ldap'] = distinct_count( + ::AuthenticationEvent.success.ldap.where(time_period), :user_id + ) + end + + counts + end + # rubocop:enable CodeReuse/ActiveRecord + + # rubocop:disable UsageData/LargeTable + def auth_providers + strong_memoize(:auth_providers) do + ::AuthenticationEvent.providers + end + end + # rubocop:enable UsageData/LargeTable + + def auth_providers_except_ldap + auth_providers.reject { |provider| provider.starts_with?('ldap') } + end + + def any_ldap_auth_providers? + auth_providers.any? { |provider| provider.starts_with?('ldap') } + end end end end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 53bf6daea4c..eb132ef0967 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -72,7 +72,8 @@ module Gitlab events_names = events_for_category(category) event_results = events_names.each_with_object({}) do |event, hash| - hash[event] = unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current) + hash["#{event}_weekly"] = unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current) + hash["#{event}_monthly"] = unique_events(event_names: event, start_date: 4.weeks.ago.to_date, end_date: Date.current) end if eligible_for_totals?(events_names) diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index fc1b5a59487..b1ce46a1ff5 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -3,14 +3,26 @@ module Gitlab module UsageDataCounters module IssueActivityUniqueCounter - ISSUE_TITLE_CHANGED = 'g_project_management_issue_title_changed' - ISSUE_DESCRIPTION_CHANGED = 'g_project_management_issue_description_changed' + ISSUE_CATEGORY = 'issues_edit' + ISSUE_ASSIGNEE_CHANGED = 'g_project_management_issue_assignee_changed' + ISSUE_CREATED = 'g_project_management_issue_created' + ISSUE_CLOSED = 'g_project_management_issue_closed' + ISSUE_DESCRIPTION_CHANGED = 'g_project_management_issue_description_changed' + ISSUE_ITERATION_CHANGED = 'g_project_management_issue_iteration_changed' + ISSUE_LABEL_CHANGED = 'g_project_management_issue_label_changed' ISSUE_MADE_CONFIDENTIAL = 'g_project_management_issue_made_confidential' ISSUE_MADE_VISIBLE = 'g_project_management_issue_made_visible' - ISSUE_CATEGORY = 'issues_edit' + ISSUE_MILESTONE_CHANGED = 'g_project_management_issue_milestone_changed' + ISSUE_REOPENED = 'g_project_management_issue_reopened' + ISSUE_TITLE_CHANGED = 'g_project_management_issue_title_changed' + ISSUE_WEIGHT_CHANGED = 'g_project_management_issue_weight_changed' class << self + def track_issue_created_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_CREATED, author, time) + end + def track_issue_title_changed_action(author:, time: Time.zone.now) track_unique_action(ISSUE_TITLE_CHANGED, author, time) end @@ -31,6 +43,30 @@ module Gitlab track_unique_action(ISSUE_MADE_VISIBLE, author, time) end + def track_issue_closed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_CLOSED, author, time) + end + + def track_issue_reopened_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_REOPENED, author, time) + end + + def track_issue_label_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_LABEL_CHANGED, author, time) + end + + def track_issue_milestone_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_MILESTONE_CHANGED, author, time) + end + + def track_issue_iteration_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_ITERATION_CHANGED, author, time) + end + + def track_issue_weight_changed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_WEIGHT_CHANGED, author, time) + end + private def track_unique_action(action, author, time) diff --git a/lib/gitlab/usage_data_counters/known_events.yml b/lib/gitlab/usage_data_counters/known_events.yml index 25e7f858bb1..9bbf804d87c 100644 --- a/lib/gitlab/usage_data_counters/known_events.yml +++ b/lib/gitlab/usage_data_counters/known_events.yml @@ -206,3 +206,31 @@ category: issues_edit redis_slot: project_management aggregation: daily +- name: g_project_management_issue_created + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_closed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_reopened + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_label_changed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_milestone_changed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_iteration_changed + category: issues_edit + redis_slot: project_management + aggregation: daily +- name: g_project_management_issue_weight_changed + category: issues_edit + redis_slot: project_management + aggregation: daily diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index bacd63ab282..c54e766230e 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -22,7 +22,7 @@ module Gitlab end def sum(relation, column, *rest) - relation.select(relation.all.table[column].sum).to_sql # rubocop:disable CodeReuse/ActiveRecord + relation.select(relation.all.table[column].sum).to_sql end private diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index ca6a36c9cea..5267733d220 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -39,9 +39,9 @@ module Gitlab FALLBACK = -1 - def count(relation, column = nil, batch: true, start: nil, finish: nil) + def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) if batch - Gitlab::Database::BatchCount.batch_count(relation, column, start: start, finish: finish) + Gitlab::Database::BatchCount.batch_count(relation, column, batch_size: batch_size, start: start, finish: finish) else relation.count end @@ -106,7 +106,6 @@ module Gitlab # @param values [Array|String] the values counted def track_usage_event(event_name, values) return unless Feature.enabled?(:"usage_data_#{event_name}", default_enabled: true) - return unless Gitlab::CurrentSettings.usage_ping_enabled? Gitlab::UsageDataCounters::HLLRedisCounter.track_event(values, event_name.to_s) end diff --git a/lib/gitlab/webpack/manifest.rb b/lib/gitlab/webpack/manifest.rb index d2c01bbd55e..6a8a7e24ebd 100644 --- a/lib/gitlab/webpack/manifest.rb +++ b/lib/gitlab/webpack/manifest.rb @@ -1,16 +1,33 @@ # frozen_string_literal: true -require 'webpack/rails/manifest' +require 'net/http' +require 'uri' module Gitlab module Webpack - class Manifest < ::Webpack::Rails::Manifest - # Raised if a supplied asset does not exist in the webpack manifest + class Manifest + # Raised if we can't read our webpack manifest for whatever reason + class ManifestLoadError < StandardError + def initialize(message, orig) + super "#{message} (original error #{orig})" + end + end + + # Raised if webpack couldn't build one of your entry points + class WebpackError < StandardError + def initialize(errors) + super "Error in webpack compile, details follow below:\n#{errors.join("\n\n")}" + end + end + + # Raised if a supplied entry point does not exist in the webpack manifest AssetMissingError = Class.new(StandardError) class << self + include Gitlab::Utils::StrongMemoize + def entrypoint_paths(source) - raise ::Webpack::Rails::Manifest::WebpackError, manifest["errors"] unless manifest_bundled? + raise WebpackError, manifest["errors"] unless manifest_bundled? dll_assets = manifest.fetch("dllAssets", []) entrypoint = manifest["entrypoints"][source] @@ -21,9 +38,86 @@ module Gitlab "/#{::Rails.configuration.webpack.public_path}/#{p}" end else + raise AssetMissingError, "Can't find asset '#{source}' in webpack manifest" + end + end + + def asset_paths(source) + raise WebpackError, manifest["errors"] unless manifest_bundled? + + paths = manifest["assetsByChunkName"][source] + if paths + # Can be either a string or an array of strings. + # Do not include source maps as they are not javascript + [paths].flatten.reject { |p| p =~ /.*\.map$/ }.map do |p| + "/#{::Rails.configuration.webpack.public_path}/#{p}" + end + else raise AssetMissingError, "Can't find entry point '#{source}' in webpack manifest" end end + + def clear_manifest! + clear_memoization(:manifest) + end + + private + + def manifest_bundled? + !manifest["errors"].any? { |error| error.include? "Module build failed" } + end + + def manifest + if ::Rails.configuration.webpack.dev_server.enabled + # Don't cache if we're in dev server mode, manifest may change ... + load_manifest + else + # ... otherwise cache at class level, as JSON loading/parsing can be expensive + strong_memoize(:manifest) { load_manifest } + end + end + + def load_manifest + data = if ::Rails.configuration.webpack.dev_server.enabled + load_dev_server_manifest + else + load_static_manifest + end + + Gitlab::Json.parse(data) + end + + def load_dev_server_manifest + host = ::Rails.configuration.webpack.dev_server.manifest_host + port = ::Rails.configuration.webpack.dev_server.manifest_port + uri = Addressable::URI.new(scheme: 'http', host: host, port: port, path: dev_server_path) + + # localhost could be blocked via Gitlab::HTTP + response = HTTParty.get(uri.to_s, verify: false) # rubocop:disable Gitlab/HTTParty + + return response.body if response.code == 200 + + raise "HTTP error #{response.code}" + rescue => e + raise ManifestLoadError.new("Could not load manifest from webpack-dev-server at #{uri} - is it running, and is stats-webpack-plugin loaded?", e) + end + + def load_static_manifest + File.read(static_manifest_path) + rescue => e + raise ManifestLoadError.new("Could not load compiled manifest from #{static_manifest_path} - have you run `rake webpack:compile`?", e) + end + + def static_manifest_path + ::Rails.root.join( + ::Rails.configuration.webpack.output_dir, + ::Rails.configuration.webpack.manifest_filename + ) + end + + def dev_server_path + "/#{::Rails.configuration.webpack.public_path}/#{::Rails.configuration.webpack.manifest_filename}" + end end end end |