diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/api/admin/migrations.rb | 62 | ||||
| -rw-r--r-- | lib/api/api.rb | 2 | ||||
| -rw-r--r-- | lib/api/entities/error_tracking.rb | 2 | ||||
| -rw-r--r-- | lib/api/error_tracking/collector.rb | 156 | ||||
| -rw-r--r-- | lib/generators/gitlab/analytics/internal_events_generator.rb | 1 | ||||
| -rw-r--r-- | lib/generators/gitlab/usage_metric/templates/instrumentation_class_spec.rb.template | 2 | ||||
| -rw-r--r-- | lib/gitlab/middleware/compressed_json.rb | 4 | ||||
| -rw-r--r-- | lib/gitlab/usage_data_counters/hll_redis_counter.rb | 80 | ||||
| -rw-r--r-- | lib/tasks/gitlab/packages/events.rake | 5 | ||||
| -rw-r--r-- | lib/tasks/gitlab/usage_data.rake | 9 |
10 files changed, 86 insertions, 237 deletions
diff --git a/lib/api/admin/migrations.rb b/lib/api/admin/migrations.rb new file mode 100644 index 00000000000..d4dbdbbb021 --- /dev/null +++ b/lib/api/admin/migrations.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module API + module Admin + class Migrations < ::API::Base + feature_category :database + urgency :low + + before do + authenticated_as_admin! + end + + namespace 'admin' do + resources 'migrations/:timestamp/mark' do + desc 'Mark the migration as successfully executed' do + success [ + { code: 201, message: '201 Created' } + ] + failure [ + { code: 401, message: '401 Unauthorized' }, + { code: 403, message: '403 Forbidden' }, + { code: 404, message: '404 Not found' }, + { code: 422, message: 'You can mark only pending migrations' } + ] + tags %w[migrations] + end + params do + optional :database, + type: String, + values: Gitlab::Database.all_database_names, + desc: 'The name of the database', + default: 'main' + requires :timestamp, + type: Integer, + desc: 'The migration version timestamp' + end + post do + response = Database::MarkMigrationService.new( + connection: base_model.connection, + version: params[:timestamp] + ).execute + + if response.success? + created! + elsif response.reason == :not_found + not_found! + else + render_api_error!('You can mark only pending migrations', 422) + end + end + end + end + + helpers do + def base_model + database = params[:database] || Gitlab::Database::MAIN_DATABASE_NAME + @base_model ||= Gitlab::Database.database_base_models[database] + end + end + end + end +end diff --git a/lib/api/api.rb b/lib/api/api.rb index 60a12ee7145..55bb549e5bc 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -182,6 +182,7 @@ module API mount ::API::Admin::BatchedBackgroundMigrations mount ::API::Admin::Ci::Variables mount ::API::Admin::InstanceClusters + mount ::API::Admin::Migrations mount ::API::Admin::PlanLimits mount ::API::AlertManagementAlerts mount ::API::Appearance @@ -327,7 +328,6 @@ module API mount ::API::Ci::PipelineSchedules mount ::API::Ci::SecureFiles mount ::API::Discussions - mount ::API::ErrorTracking::Collector mount ::API::GroupBoards mount ::API::GroupLabels mount ::API::GroupMilestones diff --git a/lib/api/entities/error_tracking.rb b/lib/api/entities/error_tracking.rb index 5e3b983c58c..180293a444d 100644 --- a/lib/api/entities/error_tracking.rb +++ b/lib/api/entities/error_tracking.rb @@ -21,7 +21,7 @@ module API expose :id, documentation: { type: 'integer', example: 1 } expose :active, documentation: { type: 'boolean' } expose :public_key, documentation: { type: 'string', example: 'glet_aa77551d849c083f76d0bc545ed053a3' } - expose :sentry_dsn, documentation: { type: 'string', example: 'https://glet_aa77551d849c083f76d0bc545ed053a3@gitlab.example.com/api/v4/error_tracking/collector/5' } + expose :sentry_dsn, documentation: { type: 'string', example: 'https://glet_aa77551d849c083f76d0bc545ed053a3@example.com/errortracking/api/v1/projects/5' } end end end diff --git a/lib/api/error_tracking/collector.rb b/lib/api/error_tracking/collector.rb deleted file mode 100644 index e10125e02c6..00000000000 --- a/lib/api/error_tracking/collector.rb +++ /dev/null @@ -1,156 +0,0 @@ -# frozen_string_literal: true - -module API - # This API is responsible for collecting error tracking information - # from sentry client. It allows us to use GitLab as an alternative to - # sentry backend. For more details see https://gitlab.com/gitlab-org/gitlab/-/issues/329596. - class ErrorTracking::Collector < ::API::Base - feature_category :error_tracking - urgency :low - - content_type :envelope, 'application/x-sentry-envelope' - content_type :json, 'application/json' - content_type :txt, 'text/plain' - default_format :envelope - - rescue_from Gitlab::ErrorTracking::ErrorRepository::DatabaseError do |e| - render_api_error!(e.message, 400) - end - - before do - not_found!('Project') unless project - not_found! unless feature_enabled? - not_found! unless active_client_key? - end - - helpers do - def project - @project ||= find_project(params[:id]) - end - - def feature_enabled? - Feature.enabled?(:integrated_error_tracking, project) && - project.error_tracking_setting&.integrated_enabled? - end - - def find_client_key(public_key) - return unless public_key.present? - - project.error_tracking_client_keys.active.find_by_public_key(public_key) - end - - def active_client_key? - public_key = extract_public_key - - find_client_key(public_key) - end - - def extract_public_key - # Some SDK send public_key as a param. In this case we don't need to parse headers. - return params[:sentry_key] if params[:sentry_key].present? - - begin - ::ErrorTracking::Collector::SentryAuthParser.parse(request)[:public_key] - rescue StandardError - bad_request!('Failed to parse sentry request') - end - end - - def validate_payload(payload) - unless ::ErrorTracking::Collector::PayloadValidator.new.valid?(payload) - bad_request!('Unsupported sentry payload') - end - end - end - - desc 'Submit error tracking event to the project as envelope' do - detail 'This feature was introduced in GitLab 14.1.' - end - params do - requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' - end - post 'error_tracking/collector/api/:id/envelope' do - # There is a reason why we have such uncommon path. - # We depend on a client side error tracking software which - # modifies URL for its own reasons. - # - # When we give user a URL like this - # HOST/api/v4/error_tracking/collector/123 - # - # Then error tracking software will convert it like this: - # HOST/api/v4/error_tracking/collector/api/123/envelope/ - - begin - parsed_request = ::ErrorTracking::Collector::SentryRequestParser.parse(request) - rescue StandardError - bad_request!('Failed to parse sentry request') - end - - type = parsed_request[:request_type] - - # Sentry sends 2 requests on each exception: transaction and event. - # Everything else is not a desired behavior. - unless type == 'transaction' || type == 'event' - render_api_error!('400 Bad Request', 400) - - break - end - - # We don't have use for transaction request yet, - # so we record only event one. - if type == 'event' - validate_payload(parsed_request[:event]) - - ::ErrorTracking::CollectErrorService - .new(project, nil, event: parsed_request[:event]) - .execute - end - - # Collector should never return any information back. - # Because DSN and public key are designed for public use, - # it is safe only for submission of new events. - # - # Some clients sdk require status 200 OK to work correctly. - # See https://gitlab.com/gitlab-org/gitlab/-/issues/343531. - status 200 - end - - desc 'Submit error tracking event to the project' do - detail 'This feature was introduced in GitLab 14.1.' - end - params do - requires :id, types: [String, Integer], desc: 'The ID or URL-encoded path of the project' - end - post 'error_tracking/collector/api/:id/store' do - # There is a reason why we have such uncommon path. - # We depend on a client side error tracking software which - # modifies URL for its own reasons. - # - # When we give user a URL like this - # HOST/api/v4/error_tracking/collector/123 - # - # Then error tracking software will convert it like this: - # HOST/api/v4/error_tracking/collector/api/123/store/ - - begin - parsed_body = Gitlab::Json.parse(request.body.read) - rescue StandardError - bad_request!('Failed to parse sentry request') - end - - validate_payload(parsed_body) - - ::ErrorTracking::CollectErrorService - .new(project, nil, event: parsed_body) - .execute - - # Collector should never return any information back. - # Because DSN and public key are designed for public use, - # it is safe only for submission of new events. - # - # Some clients sdk require status 200 OK to work correctly. - # See https://gitlab.com/gitlab-org/gitlab/-/issues/343531. - status 200 - end - end -end diff --git a/lib/generators/gitlab/analytics/internal_events_generator.rb b/lib/generators/gitlab/analytics/internal_events_generator.rb index e945b4de3db..a85cdd352d5 100644 --- a/lib/generators/gitlab/analytics/internal_events_generator.rb +++ b/lib/generators/gitlab/analytics/internal_events_generator.rb @@ -114,7 +114,6 @@ module Gitlab def known_event_entry <<~YML - name: #{event} - aggregation: weekly YML end diff --git a/lib/generators/gitlab/usage_metric/templates/instrumentation_class_spec.rb.template b/lib/generators/gitlab/usage_metric/templates/instrumentation_class_spec.rb.template index f8bd502ab77..915b91e43da 100644 --- a/lib/generators/gitlab/usage_metric/templates/instrumentation_class_spec.rb.template +++ b/lib/generators/gitlab/usage_metric/templates/instrumentation_class_spec.rb.template @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Usage::Metrics::Instrumentations::<%= class_name %>Metric do +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::<%= class_name %>Metric, feature_category: :service_ping do let(:expected_value) { 1 } it_behaves_like 'a correct instrumented metric value', { time_frame: 'all', data_source: 'database' } diff --git a/lib/gitlab/middleware/compressed_json.rb b/lib/gitlab/middleware/compressed_json.rb index cc485d8a5db..1f15f1d5857 100644 --- a/lib/gitlab/middleware/compressed_json.rb +++ b/lib/gitlab/middleware/compressed_json.rb @@ -3,7 +3,6 @@ module Gitlab module Middleware class CompressedJson - COLLECTOR_PATH = '/api/v4/error_tracking/collector' INSTANCE_PACKAGES_PATH = %r{ \A/api/v4/packages/npm/-/npm/v1/security/ (?:(?:advisories/bulk)|(?:audits/quick))\z (?# end) @@ -79,8 +78,7 @@ module Gitlab end def match_path?(env) - env['PATH_INFO'].start_with?((File.join(relative_url, COLLECTOR_PATH))) || - match_packages_path?(env) + match_packages_path?(env) end def match_packages_path?(env) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index badcda1def0..eaa4bf15fe1 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -3,18 +3,14 @@ module Gitlab module UsageDataCounters module HLLRedisCounter - DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks - DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days + KEY_EXPIRY_LENGTH = 6.weeks REDIS_SLOT = 'hll_counters' EventError = Class.new(StandardError) UnknownEvent = Class.new(EventError) - UnknownAggregation = Class.new(EventError) - AggregationMismatch = Class.new(EventError) InvalidContext = Class.new(EventError) KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__) - ALLOWED_AGGREGATIONS = %i(daily weekly).freeze # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id @@ -24,7 +20,6 @@ module Gitlab # Event example: # # - name: g_compliance_dashboard # Unique event name - # aggregation: weekly # Aggregation level, keys are stored weekly # # Usage: # @@ -63,8 +58,7 @@ module Gitlab # end_date - The end date of the time range. # context - Event context, plan level tracking. Available if set when tracking. def unique_events(event_names:, start_date:, end_date:, context: '') - count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do |events| - raise AggregationMismatch, events unless events_same_aggregation?(events) + count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do raise InvalidContext if context.present? && !context.in?(valid_context_list) end end @@ -78,9 +72,7 @@ module Gitlab end def calculate_events_union(event_names:, start_date:, end_date:) - count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) do |events| - raise AggregationMismatch, events unless events_same_aggregation?(events) - end + count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) end private @@ -94,12 +86,7 @@ module Gitlab return if event.blank? return unless Feature.enabled?(:redis_hll_tracking, type: :ops) - if event[:aggregation].to_sym == :daily - weekly_event = event.dup.tap { |e| e['aggregation'] = 'weekly' } - Gitlab::Redis::HLL.add(key: redis_key(weekly_event, time, context), value: values, expiry: expiry(weekly_event)) - end - - Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: expiry(event)) + Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: KEY_EXPIRY_LENGTH) rescue StandardError => e # Ignore any exceptions unless is dev or test env @@ -117,25 +104,18 @@ module Gitlab yield events if block_given? - aggregation = events.first[:aggregation] - - if Feature.disabled?(:revert_daily_hll_events_to_weekly_aggregation) - aggregation = 'weekly' - events = events.map { |e| e.merge(aggregation: 'weekly') } - end + keys = keys_for_aggregation(events: events, start_date: start_date, end_date: end_date, context: context) - keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date, context: context) return FALLBACK unless keys.any? redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } end - def keys_for_aggregation(aggregation, events:, start_date:, end_date:, context: '') - if aggregation.to_sym == :daily - daily_redis_keys(events: events, start_date: start_date, end_date: end_date, context: context) - else - weekly_redis_keys(events: events, start_date: start_date, end_date: end_date, context: context) - end + def keys_for_aggregation(events:, start_date:, end_date:, context: '') + end_date = end_date.end_of_week - 1.week + (start_date.to_date..end_date.to_date).map do |date| + events.map { |event| redis_key(event, date, context) } + end.flatten.uniq end def load_events(wildcard) @@ -152,15 +132,6 @@ module Gitlab known_events.map { |event| event[:name] } end - def events_same_aggregation?(events) - aggregation = events.first[:aggregation] - events.all? { |event| event[:aggregation] == aggregation } - end - - def expiry(event) - event[:aggregation].to_sym == :daily ? DEFAULT_DAILY_KEY_EXPIRY_LENGTH : DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH - end - def event_for(event_name) known_events.find { |event| event[:name] == event_name.to_s } end @@ -173,36 +144,13 @@ module Gitlab def redis_key(event, time, context = '') raise UnknownEvent, "Unknown event #{event[:name]}" unless known_events_names.include?(event[:name].to_s) - # ToDo: remove during https://gitlab.com/groups/gitlab-org/-/epics/9542 cleanup - raise UnknownAggregation, "Use :daily or :weekly aggregation" unless ALLOWED_AGGREGATIONS.include?(event[:aggregation].to_sym) - key = "{#{REDIS_SLOT}}_#{event[:name]}" - key = apply_time_aggregation(key, time, event) - key = "#{context}_#{key}" if context.present? - key - end - def apply_time_aggregation(key, time, event) - if event[:aggregation].to_sym == :daily - year_day = time.strftime('%G-%j') - "#{year_day}-#{key}" - else - year_week = time.strftime('%G-%V') - "#{key}-#{year_week}" - end - end + year_week = time.strftime('%G-%V') + key = "#{key}-#{year_week}" - def daily_redis_keys(events:, start_date:, end_date:, context: '') - (start_date.to_date..end_date.to_date).map do |date| - events.map { |event| redis_key(event, date, context) } - end.flatten - end - - def weekly_redis_keys(events:, start_date:, end_date:, context: '') - end_date = end_date.end_of_week - 1.week - (start_date.to_date..end_date.to_date).map do |date| - events.map { |event| redis_key(event, date, context) } - end.flatten.uniq + key = "#{context}_#{key}" if context.present? + key end end end diff --git a/lib/tasks/gitlab/packages/events.rake b/lib/tasks/gitlab/packages/events.rake index 1234ba039a3..b5dfd163dba 100644 --- a/lib/tasks/gitlab/packages/events.rake +++ b/lib/tasks/gitlab/packages/events.rake @@ -45,10 +45,7 @@ namespace :gitlab do events = event_pairs.each_with_object([]) do |(event_type, event_scope), events| Packages::Event::ORIGINATOR_TYPES.excluding(:guest).each do |originator_type| events_definition = Packages::Event.unique_counters_for(event_scope, event_type, originator_type).map do |event_name| - { - "name" => event_name, - "aggregation" => "weekly" - } + { "name" => event_name } end events.concat(events_definition) diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake index fcbec4b0dba..f5bf1a266e5 100644 --- a/lib/tasks/gitlab/usage_data.rake +++ b/lib/tasks/gitlab/usage_data.rake @@ -85,12 +85,13 @@ namespace :gitlab do end end + # rubocop:disable Gitlab/NoCodeCoverageComment + # :nocov: remove in https://gitlab.com/gitlab-org/gitlab/-/issues/299453 def ci_template_event(event_name) - { - 'name' => event_name, - 'aggregation' => 'weekly' - } + { 'name' => event_name } end + # :nocov: + # rubocop:enable Gitlab/NoCodeCoverageComment def implicit_auto_devops_event(expanded_template_name) event_name = Gitlab::UsageDataCounters::CiTemplateUniqueCounter.ci_template_event_name(expanded_template_name, :auto_devops_source) |