From 8f535d9082bb0f10f2ef6b9133dbc77691ace4db Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 1 Jun 2023 06:08:03 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- .../Geo Replicate a new Git repository type.md | 34 +++++++ .../Geo Replicate a new blob type.md | 34 +++++++ .rubocop_todo/layout/argument_alignment.yml | 12 --- .rubocop_todo/search/namespaced_class.yml | 4 + GITLAB_ELASTICSEARCH_INDEXER_VERSION | 2 +- Gemfile | 2 +- Gemfile.checksum | 2 +- Gemfile.lock | 4 +- .../javascripts/graphql_shared/possible_types.json | 1 + .../delete_stale_direct_uploads_worker.rb | 2 +- .../20230202211434_migrate_redis_slot_keys.rb | 102 +-------------------- ...daily_redis_hll_events_to_weekly_aggregation.rb | 89 +++++++++++++++++- .../20230328111013_re_migrate_redis_slot_keys.rb | 93 ++++++++++++++++++- doc/api/graphql/reference/index.md | 1 + doc/user/group/import/index.md | 28 +++++- lib/api/helpers.rb | 3 + .../saas/pull_container_registry_image_spec.rb | 31 +++++++ .../20230202211434_migrate_redis_slot_keys_spec.rb | 54 ----------- ...230302811133_re_migrate_redis_slot_keys_spec.rb | 19 ++-- ..._redis_hll_events_to_weekly_aggregation_spec.rb | 43 ++------- 20 files changed, 338 insertions(+), 222 deletions(-) create mode 100644 qa/qa/specs/features/browser_ui/5_package/container_registry/saas/pull_container_registry_image_spec.rb delete mode 100644 spec/migrations/20230202211434_migrate_redis_slot_keys_spec.rb diff --git a/.gitlab/issue_templates/Geo Replicate a new Git repository type.md b/.gitlab/issue_templates/Geo Replicate a new Git repository type.md index 95f92430eea..e93c5d79964 100644 --- a/.gitlab/issue_templates/Geo Replicate a new Git repository type.md +++ b/.gitlab/issue_templates/Geo Replicate a new Git repository type.md @@ -725,6 +725,40 @@ The GraphQL API is used by `Admin > Geo > Replication Details` views, and is dir } ``` +To allow the new replicable to resync and reverify via GraphQL: + +- [ ] Add the `CoolWidgetRegistryType` to the `GEO_REGISTRY_TYPE` constant in `ee/app/graphql/types/geo/registrable_type.rb`: + + ```ruby + GEO_REGISTRY_TYPES = { + ::Geo::CoolWidgetRegistry => Types::Geo::CoolWidgetRegistryType + } + ``` + +- [ ] Include the `CoolWidgetRegistry` in the `let(:registry_classes)` variable of `ee/spec/graphql/types/geo/registry_class_enum_spec.rb`: + + ```ruby + let(:registry_classes) do + %w[ + COOL_WIDGET_REGISTRY + ] + end + ``` + +- [ ] Include the new registry in the Rspec parameterized table of `ee/spec/support/shared_contexts/graphql/geo/registries_shared_context.rb`: + + ```ruby + # frozen_string_literal: true + + RSpec.shared_context 'with geo registries shared context' do + using RSpec::Parameterized::TableSyntax + + where(:registry_class, :registry_type, :registry_factory) do + Geo::CoolWidgetRegistry | Types::Geo::CoolWidgetRegistryType | :geo_cool_widget_registry + end + end + ``` + - [ ] Update the GraphQL reference documentation: ```shell diff --git a/.gitlab/issue_templates/Geo Replicate a new blob type.md b/.gitlab/issue_templates/Geo Replicate a new blob type.md index 7bf8dd6526e..2183ba9c384 100644 --- a/.gitlab/issue_templates/Geo Replicate a new blob type.md +++ b/.gitlab/issue_templates/Geo Replicate a new blob type.md @@ -693,6 +693,40 @@ The GraphQL API is used by `Admin > Geo > Replication Details` views, and is dir } ``` +To allow the new replicable to resync and reverify via GraphQL: + +- [ ] Add the `CoolWidgetRegistryType` to the `GEO_REGISTRY_TYPE` constant in `ee/app/graphql/types/geo/registrable_type.rb`: + + ```ruby + GEO_REGISTRY_TYPES = { + ::Geo::CoolWidgetRegistry => Types::Geo::CoolWidgetRegistryType + } + ``` + +- [ ] Include the `CoolWidgetRegistry` in the `let(:registry_classes)` variable of `ee/spec/graphql/types/geo/registry_class_enum_spec.rb`: + + ```ruby + let(:registry_classes) do + %w[ + COOL_WIDGET_REGISTRY + ] + end + ``` + +- [ ] Include the new registry in the Rspec parameterized table of `ee/spec/support/shared_contexts/graphql/geo/registries_shared_context.rb`: + + ```ruby + # frozen_string_literal: true + + RSpec.shared_context 'with geo registries shared context' do + using RSpec::Parameterized::TableSyntax + + where(:registry_class, :registry_type, :registry_factory) do + Geo::CoolWidgetRegistry | Types::Geo::CoolWidgetRegistryType | :geo_cool_widget_registry + end + end + ``` + - [ ] Update the GraphQL reference documentation: ```shell diff --git a/.rubocop_todo/layout/argument_alignment.yml b/.rubocop_todo/layout/argument_alignment.yml index 2ff4c3dc07f..3aaf0b09e53 100644 --- a/.rubocop_todo/layout/argument_alignment.yml +++ b/.rubocop_todo/layout/argument_alignment.yml @@ -977,18 +977,6 @@ Layout/ArgumentAlignment: - 'ee/app/models/group_merge_request_approval_setting.rb' - 'ee/app/models/incident_management/escalation_rule.rb' - 'ee/app/models/integrations/github/status_notifier.rb' - - 'ee/app/models/merge_requests/compliance_violation.rb' - - 'ee/app/models/merge_requests/external_status_check.rb' - - 'ee/app/models/package_metadata/package_version_license.rb' - - 'ee/app/models/product_analytics/dashboard.rb' - - 'ee/app/models/protected_environments/approval_rule.rb' - - 'ee/app/models/requirements_management/requirement.rb' - - 'ee/app/models/sbom/vulnerable_component_version.rb' - - 'ee/app/models/sca/license_policy.rb' - - 'ee/app/models/scim_identity.rb' - - 'ee/app/models/security/finding.rb' - - 'ee/app/models/security/orchestration_policy_rule_schedule.rb' - - 'ee/app/models/smartcard_identity.rb' - 'ee/app/models/status_page/project_setting.rb' - 'ee/app/models/vulnerabilities/external_issue_link.rb' - 'ee/app/models/vulnerabilities/feedback.rb' diff --git a/.rubocop_todo/search/namespaced_class.yml b/.rubocop_todo/search/namespaced_class.yml index ad27603ad47..8750d87a900 100644 --- a/.rubocop_todo/search/namespaced_class.yml +++ b/.rubocop_todo/search/namespaced_class.yml @@ -117,6 +117,8 @@ Search/NamespacedClass: - 'ee/lib/elastic/latest/snippet_instance_proxy.rb' - 'ee/lib/elastic/latest/state_filter.rb' - 'ee/lib/elastic/latest/user_class_proxy.rb' + - 'ee/lib/elastic/latest/wiki_class_proxy.rb' + - 'ee/lib/elastic/latest/wiki_instance_proxy.rb' - 'ee/lib/elastic/latest/user_config.rb' - 'ee/lib/elastic/latest/user_instance_proxy.rb' - 'ee/lib/elastic/latest/wiki_config.rb' @@ -148,6 +150,8 @@ Search/NamespacedClass: - 'ee/lib/elastic/v12p1/snippet_instance_proxy.rb' - 'ee/lib/elastic/v12p1/user_class_proxy.rb' - 'ee/lib/elastic/v12p1/user_instance_proxy.rb' + - 'ee/lib/elastic/v12p1/wiki_class_proxy.rb' + - 'ee/lib/elastic/v12p1/wiki_instance_proxy.rb' - 'ee/lib/gem_extensions/elasticsearch/model/adapter/active_record/importing.rb' - 'ee/lib/gem_extensions/elasticsearch/model/adapter/multiple/records.rb' - 'ee/lib/gem_extensions/elasticsearch/model/client.rb' diff --git a/GITLAB_ELASTICSEARCH_INDEXER_VERSION b/GITLAB_ELASTICSEARCH_INDEXER_VERSION index e91d9be2a86..eda862a98c1 100644 --- a/GITLAB_ELASTICSEARCH_INDEXER_VERSION +++ b/GITLAB_ELASTICSEARCH_INDEXER_VERSION @@ -1 +1 @@ -4.3.3 +4.3.4 diff --git a/Gemfile b/Gemfile index 5c621c4545e..70cc3276a9d 100644 --- a/Gemfile +++ b/Gemfile @@ -505,7 +505,7 @@ gem 'ssh_data', '~> 1.3' gem 'spamcheck', '~> 1.3.0' # Gitaly GRPC protocol definitions -gem 'gitaly', '~> 15.9.0-rc3' +gem 'gitaly', '~> 16.1.0-rc1' # KAS GRPC protocol definitions gem 'kas-grpc', '~> 0.1.0' diff --git a/Gemfile.checksum b/Gemfile.checksum index b23ee78f108..e2252689e1b 100644 --- a/Gemfile.checksum +++ b/Gemfile.checksum @@ -207,7 +207,7 @@ {"name":"gettext_i18n_rails","version":"1.8.0","platform":"ruby","checksum":"95e5cf8440b1e08705b27f2bccb56143272c5a7a0dabcf54ea1bd701140a496f"}, {"name":"gettext_i18n_rails_js","version":"1.3.0","platform":"ruby","checksum":"5d10afe4be3639bff78c50a56768c20f39aecdabc580c08aa45573911c2bd687"}, {"name":"git","version":"1.11.0","platform":"ruby","checksum":"7e95ba4da8298a0373ef1a6862aa22007d761f3c8274b675aa787966fecea0f1"}, -{"name":"gitaly","version":"15.9.0.pre.rc3","platform":"ruby","checksum":"6ac64320a70417131a4b97f5dd45d4e203d60703cc3cba156561e7f8c50a4abe"}, +{"name":"gitaly","version":"16.1.0.pre.rc1","platform":"ruby","checksum":"3f61ecd96baf283a3365943dac58c3fb81221fee74d1ccc7931e1adea5403405"}, {"name":"gitlab","version":"4.19.0","platform":"ruby","checksum":"3f645e3e195dbc24f0834fbf83e8ccfb2056d8e9712b01a640aad418a6949679"}, {"name":"gitlab-chronic","version":"0.10.5","platform":"ruby","checksum":"f80f18dc699b708870a80685243331290bc10cfeedb6b99c92219722f729c875"}, {"name":"gitlab-dangerfiles","version":"3.10.0","platform":"ruby","checksum":"df4cfe051f52529c0256346d89d06d5ef2bb630928754eb620b5233eb9b14041"}, diff --git a/Gemfile.lock b/Gemfile.lock index 655953117e8..cd96633ac2f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -581,7 +581,7 @@ GEM rails (>= 3.2.0) git (1.11.0) rchardet (~> 1.8) - gitaly (15.9.0.pre.rc3) + gitaly (16.1.0.pre.rc1) grpc (~> 1.0) gitlab (4.19.0) httparty (~> 0.20) @@ -1738,7 +1738,7 @@ DEPENDENCIES gettext (~> 3.3) gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails_js (~> 1.3) - gitaly (~> 15.9.0.pre.rc3) + gitaly (~> 16.1.0.pre.rc1) gitlab-chronic (~> 0.10.5) gitlab-dangerfiles (~> 3.10.0) gitlab-experiment (~> 0.7.1) diff --git a/app/assets/javascripts/graphql_shared/possible_types.json b/app/assets/javascripts/graphql_shared/possible_types.json index f35886716ee..c7a446b36f8 100644 --- a/app/assets/javascripts/graphql_shared/possible_types.json +++ b/app/assets/javascripts/graphql_shared/possible_types.json @@ -94,6 +94,7 @@ "ContainerRepositoryRegistry", "DependencyProxyBlobRegistry", "DependencyProxyManifestRegistry", + "DesignManagementRepositoryRegistry", "JobArtifactRegistry", "LfsObjectRegistry", "MergeRequestDiffRegistry", diff --git a/app/workers/object_storage/delete_stale_direct_uploads_worker.rb b/app/workers/object_storage/delete_stale_direct_uploads_worker.rb index 216df498cc2..081b58ca726 100644 --- a/app/workers/object_storage/delete_stale_direct_uploads_worker.rb +++ b/app/workers/object_storage/delete_stale_direct_uploads_worker.rb @@ -22,7 +22,7 @@ module ObjectStorage result = ObjectStorage::DeleteStaleDirectUploadsService.new.execute log_extra_metadata_on_done(:total_pending_entries, result[:total_pending_entries]) - log_extra_metadata_on_done(:total_stale_entries, result[:total_stale_entries]) + log_extra_metadata_on_done(:total_deleted_stale_entries, result[:total_deleted_stale_entries]) log_extra_metadata_on_done(:execution_timeout, result[:execution_timeout]) end end diff --git a/db/post_migrate/20230202211434_migrate_redis_slot_keys.rb b/db/post_migrate/20230202211434_migrate_redis_slot_keys.rb index e1d3787cbb3..126555ae391 100644 --- a/db/post_migrate/20230202211434_migrate_redis_slot_keys.rb +++ b/db/post_migrate/20230202211434_migrate_redis_slot_keys.rb @@ -4,110 +4,10 @@ class MigrateRedisSlotKeys < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up - BackupHLLRedisCounter.known_events.each do |event| - if event[:aggregation].to_sym == :daily - migrate_daily_aggregated(event) - else - migrate_weekly_aggregated(event) - end - end + # deleted as contained a bug end def down # no-op end - - private - - def migrate_daily_aggregated(event) - days_back = BackupHLLRedisCounter::DEFAULT_DAILY_KEY_EXPIRY_LENGTH - start_date = Date.today - days_back - 1.day - end_date = Date.today + 1.day - - (start_date..end_date).each do |date| - rename_key(event, date) - end - end - - def migrate_weekly_aggregated(event) - weeks_back = BackupHLLRedisCounter::DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH - start_date = (Date.today - weeks_back).beginning_of_week - 1.day - end_date = Date.today.end_of_week + 1.day - - (start_date..end_date).each { |date| rename_key(event, date) } - end - - def rename_key(event, date) - old_key = old_redis_key(event, date) - new_key = BackupHLLRedisCounter.redis_key(event, date) - - # cannot simply rename due to different slots - Gitlab::Redis::SharedState.with do |r| - break unless r.exists?(old_key) - - Gitlab::Redis::HLL.add( - key: new_key, - value: r.pfcount(old_key), - expiry: r.ttl(old_key) - ) - end - end - - def old_redis_key(event, time) - name_with_slot = if event[:redis_slot].present? - event[:name].to_s.gsub(event[:redis_slot], "{#{event[:redis_slot]}}") - else - "{#{event[:name]}}" - end - - BackupHLLRedisCounter.apply_time_aggregation(name_with_slot, time, event) - end - - # :nocov: Existing backed up class # rubocop:disable Gitlab/NoCodeCoverageComment - module BackupHLLRedisCounter - DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks - DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days - REDIS_SLOT = 'hll_counters' - - KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__) - ALLOWED_AGGREGATIONS = %i[daily weekly].freeze - - class << self - def known_events - @known_events ||= load_events(KNOWN_EVENTS_PATH) - end - - def load_events(wildcard) - Dir[wildcard].each_with_object([]) do |path, events| - events.push(*load_yaml_from_path(path)) - end - end - - def load_yaml_from_path(path) - YAML.safe_load(File.read(path))&.map(&:with_indifferent_access) - end - - def known_events_names - known_events.map { |event| event[:name] } # rubocop:disable Rails/Pluck - end - - def redis_key(event, time, context = '') - key = "{#{REDIS_SLOT}}_#{event[:name]}" - key = apply_time_aggregation(key, time, event) - key = "#{context}_#{key}" if context.present? - key - end - - def apply_time_aggregation(key, time, event) - if event[:aggregation].to_sym == :daily - year_day = time.strftime('%G-%j') - "#{year_day}-#{key}" - else - year_week = time.strftime('%G-%V') - "#{key}-#{year_week}" - end - end - end - end - # :nocov: # rubocop:disable Gitlab/NoCodeCoverageComment end diff --git a/db/post_migrate/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation.rb b/db/post_migrate/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation.rb index b4468c851ae..59bff26f964 100644 --- a/db/post_migrate/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation.rb +++ b/db/post_migrate/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation.rb @@ -3,6 +3,93 @@ class MigrateDailyRedisHllEventsToWeeklyAggregation < Gitlab::Database::Migration[2.1] disable_ddl_transaction! + DAILY_EVENTS = + %w[g_edit_by_web_ide + g_edit_by_sfe + g_edit_by_snippet_ide + g_edit_by_live_preview + wiki_action + design_action + project_action + git_write_action + merge_request_action + i_source_code_code_intelligence + g_project_management_issue_title_changed + g_project_management_issue_description_changed + g_project_management_issue_assignee_changed + g_project_management_issue_made_confidential + g_project_management_issue_made_visible + g_project_management_issue_created + g_project_management_issue_closed + g_project_management_issue_reopened + g_project_management_issue_label_changed + g_project_management_issue_milestone_changed + g_project_management_issue_cross_referenced + g_project_management_issue_moved + g_project_management_issue_related + g_project_management_issue_unrelated + g_project_management_issue_marked_as_duplicate + g_project_management_issue_locked + g_project_management_issue_unlocked + g_project_management_issue_designs_added + g_project_management_issue_designs_modified + g_project_management_issue_designs_removed + g_project_management_issue_due_date_changed + g_project_management_issue_design_comments_removed + g_project_management_issue_time_estimate_changed + g_project_management_issue_time_spent_changed + g_project_management_issue_comment_added + g_project_management_issue_comment_edited + g_project_management_issue_comment_removed + g_project_management_issue_cloned + g_geo_proxied_requests + approval_project_rule_created + g_project_management_issue_added_to_epic + g_project_management_issue_changed_epic + g_project_management_issue_health_status_changed + g_project_management_issue_iteration_changed + g_project_management_issue_removed_from_epic + g_project_management_issue_weight_changed + g_geo_proxied_requests + g_project_management_users_creating_epic_boards + g_project_management_users_viewing_epic_boards + g_project_management_users_updating_epic_board_names + g_project_management_epic_created + project_management_users_unchecking_epic_task + project_management_users_checking_epic_task + g_project_management_users_updating_epic_titles + g_project_management_users_updating_epic_descriptions + g_project_management_users_creating_epic_notes + g_project_management_users_updating_epic_notes + g_project_management_users_destroying_epic_notes + g_project_management_users_awarding_epic_emoji + g_project_management_users_removing_epic_emoji + g_project_management_users_setting_epic_start_date_as_fixed + g_project_management_users_updating_fixed_epic_start_date + g_project_management_users_setting_epic_start_date_as_inherited + g_project_management_users_setting_epic_due_date_as_fixed + g_project_management_users_updating_fixed_epic_due_date + g_project_management_users_setting_epic_due_date_as_inherited + g_project_management_epic_issue_added + g_project_management_epic_issue_removed + g_project_management_epic_issue_moved_from_project + g_project_management_users_updating_epic_parent + g_project_management_epic_closed + g_project_management_epic_reopened + g_project_management_issue_promoted_to_epic + g_project_management_users_setting_epic_confidential + g_project_management_users_setting_epic_visible + g_project_management_epic_users_changing_labels + g_project_management_epic_destroyed + g_project_management_epic_cross_referenced + g_project_management_users_epic_issue_added_from_epic + g_project_management_epic_related_added + g_project_management_epic_related_removed + g_project_management_epic_blocking_added + g_project_management_epic_blocking_removed + g_project_management_epic_blocked_added + g_project_management_epic_blocked_removed].freeze + def up days_back = 29.days start_date = Date.today - days_back - 1.day @@ -10,7 +97,7 @@ class MigrateDailyRedisHllEventsToWeeklyAggregation < Gitlab::Database::Migratio keys = {} Gitlab::UsageDataCounters::HLLRedisCounter.known_events.each do |event| - next unless event[:aggregation].to_sym == :daily + next unless DAILY_EVENTS.include?(event[:name].to_s) (start_date..end_date).each do |date| daily_key = redis_key(event, date, :daily) diff --git a/db/post_migrate/20230328111013_re_migrate_redis_slot_keys.rb b/db/post_migrate/20230328111013_re_migrate_redis_slot_keys.rb index f09888bd821..71ab958b601 100644 --- a/db/post_migrate/20230328111013_re_migrate_redis_slot_keys.rb +++ b/db/post_migrate/20230328111013_re_migrate_redis_slot_keys.rb @@ -3,9 +3,96 @@ class ReMigrateRedisSlotKeys < Gitlab::Database::Migration[2.1] disable_ddl_transaction! + DAILY_EVENTS = + %w[g_edit_by_web_ide + g_edit_by_sfe + g_edit_by_snippet_ide + g_edit_by_live_preview + wiki_action + design_action + project_action + git_write_action + merge_request_action + i_source_code_code_intelligence + g_project_management_issue_title_changed + g_project_management_issue_description_changed + g_project_management_issue_assignee_changed + g_project_management_issue_made_confidential + g_project_management_issue_made_visible + g_project_management_issue_created + g_project_management_issue_closed + g_project_management_issue_reopened + g_project_management_issue_label_changed + g_project_management_issue_milestone_changed + g_project_management_issue_cross_referenced + g_project_management_issue_moved + g_project_management_issue_related + g_project_management_issue_unrelated + g_project_management_issue_marked_as_duplicate + g_project_management_issue_locked + g_project_management_issue_unlocked + g_project_management_issue_designs_added + g_project_management_issue_designs_modified + g_project_management_issue_designs_removed + g_project_management_issue_due_date_changed + g_project_management_issue_design_comments_removed + g_project_management_issue_time_estimate_changed + g_project_management_issue_time_spent_changed + g_project_management_issue_comment_added + g_project_management_issue_comment_edited + g_project_management_issue_comment_removed + g_project_management_issue_cloned + g_geo_proxied_requests + approval_project_rule_created + g_project_management_issue_added_to_epic + g_project_management_issue_changed_epic + g_project_management_issue_health_status_changed + g_project_management_issue_iteration_changed + g_project_management_issue_removed_from_epic + g_project_management_issue_weight_changed + g_geo_proxied_requests + g_project_management_users_creating_epic_boards + g_project_management_users_viewing_epic_boards + g_project_management_users_updating_epic_board_names + g_project_management_epic_created + project_management_users_unchecking_epic_task + project_management_users_checking_epic_task + g_project_management_users_updating_epic_titles + g_project_management_users_updating_epic_descriptions + g_project_management_users_creating_epic_notes + g_project_management_users_updating_epic_notes + g_project_management_users_destroying_epic_notes + g_project_management_users_awarding_epic_emoji + g_project_management_users_removing_epic_emoji + g_project_management_users_setting_epic_start_date_as_fixed + g_project_management_users_updating_fixed_epic_start_date + g_project_management_users_setting_epic_start_date_as_inherited + g_project_management_users_setting_epic_due_date_as_fixed + g_project_management_users_updating_fixed_epic_due_date + g_project_management_users_setting_epic_due_date_as_inherited + g_project_management_epic_issue_added + g_project_management_epic_issue_removed + g_project_management_epic_issue_moved_from_project + g_project_management_users_updating_epic_parent + g_project_management_epic_closed + g_project_management_epic_reopened + g_project_management_issue_promoted_to_epic + g_project_management_users_setting_epic_confidential + g_project_management_users_setting_epic_visible + g_project_management_epic_users_changing_labels + g_project_management_epic_destroyed + g_project_management_epic_cross_referenced + g_project_management_users_epic_issue_added_from_epic + g_project_management_epic_related_added + g_project_management_epic_related_removed + g_project_management_epic_blocking_added + g_project_management_epic_blocking_removed + g_project_management_epic_blocked_added + g_project_management_epic_blocked_removed].freeze + def up Gitlab::UsageDataCounters::HLLRedisCounter.known_events.each do |event| - if event[:aggregation].to_sym == :daily + if DAILY_EVENTS.include?(event[:name].to_s) migrate_daily_aggregated(event) else migrate_weekly_aggregated(event) @@ -20,7 +107,7 @@ class ReMigrateRedisSlotKeys < Gitlab::Database::Migration[2.1] private def migrate_daily_aggregated(event) - days_back = Gitlab::UsageDataCounters::HLLRedisCounter::DEFAULT_DAILY_KEY_EXPIRY_LENGTH + days_back = 29.days start_date = Date.today - days_back - 1.day end_date = Date.today + 1.day @@ -77,7 +164,7 @@ class ReMigrateRedisSlotKeys < Gitlab::Database::Migration[2.1] end def apply_time_aggregation(key, time, event) - if event[:aggregation].to_sym == :daily + if DAILY_EVENTS.include?(event[:name].to_s) year_day = time.strftime('%G-%j') "#{year_day}-#{key}" else diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 093d8aa624d..cf3a5e3a541 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -27157,6 +27157,7 @@ One of: - [`ContainerRepositoryRegistry`](#containerrepositoryregistry) - [`DependencyProxyBlobRegistry`](#dependencyproxyblobregistry) - [`DependencyProxyManifestRegistry`](#dependencyproxymanifestregistry) +- [`DesignManagementRepositoryRegistry`](#designmanagementrepositoryregistry) - [`JobArtifactRegistry`](#jobartifactregistry) - [`LfsObjectRegistry`](#lfsobjectregistry) - [`MergeRequestDiffRegistry`](#mergerequestdiffregistry) diff --git a/doc/user/group/import/index.md b/doc/user/group/import/index.md index 7f55bf56102..2dd5347e5dc 100644 --- a/doc/user/group/import/index.md +++ b/doc/user/group/import/index.md @@ -220,6 +220,13 @@ Group items that are migrated to the destination GitLab instance include: - Already exists in the destination GitLab instance. - Has a public email in the source GitLab instance that matches a confirmed email in the destination GitLab instance. +#### Excluded items + +Some group items are excluded from migration because they either: + +- May contain sensitive information: CI/CD variables, webhooks, and deploy tokens. +- Are not supported: push rules. + ### Migrated project items (Beta) > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267945) in GitLab 14.4 [with a flag](../../feature_flags.md) named `bulk_import_projects`. Disabled by default. @@ -325,9 +332,24 @@ Setting-related project items that are migrated to the destination GitLab instan #### Excluded items -Project items excluded from migration because they contain sensitive information: - -- Pipeline triggers. +Some project items are excluded from migration because they either: + +- May contain sensitive information: + - CI/CD variables + - Deploy keys + - Deploy tokens + - Pipeline schedule variables + - Pipeline triggers + - Webhooks +- Are not supported: + - Agents + - Container Registry + - Environments + - Feature flags + - Infrastructure Registry + - Package Registry + - Pages domains + - Remote mirrors ### Troubleshooting diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 83641e824ae..98f6733a6f2 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -202,6 +202,7 @@ module API not_found!('Namespace') end + # find_namespace returns the namespace regardless of user access level on the namespace # rubocop: disable CodeReuse/ActiveRecord def find_namespace(id) if id.to_s =~ INTEGER_ID_REGEX @@ -212,6 +213,8 @@ module API end # rubocop: enable CodeReuse/ActiveRecord + # find_namespace! returns the namespace if the current user can read the given namespace + # Otherwise, returns a not_found! error def find_namespace!(id) check_namespace_access(find_namespace(id)) end diff --git a/qa/qa/specs/features/browser_ui/5_package/container_registry/saas/pull_container_registry_image_spec.rb b/qa/qa/specs/features/browser_ui/5_package/container_registry/saas/pull_container_registry_image_spec.rb new file mode 100644 index 00000000000..85a88b54cc2 --- /dev/null +++ b/qa/qa/specs/features/browser_ui/5_package/container_registry/saas/pull_container_registry_image_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module QA + RSpec.describe 'Package' do + describe 'SaaS Container Registry', only: { subdomain: %i[staging] }, product_group: :container_registry do + let(:project) do + Resource::Project.init do |project| + project.path_with_namespace = 'gitlab-qa/container-registry-sanity' + end.reload! + end + + it 'pulls an image from an existing repository', + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/412799' do + Flow::Login.sign_in + project.visit! + + Page::Project::Menu.perform(&:go_to_pipelines) + Page::Project::Pipeline::Index.perform(&:click_run_pipeline_button) + Page::Project::Pipeline::New.perform(&:click_run_pipeline_button) + + Page::Project::Pipeline::Show.perform do |pipeline| + pipeline.click_job('test') + end + + Page::Project::Job::Show.perform do |job| + expect(job).to be_successful(timeout: 800) + end + end + end + end +end diff --git a/spec/migrations/20230202211434_migrate_redis_slot_keys_spec.rb b/spec/migrations/20230202211434_migrate_redis_slot_keys_spec.rb deleted file mode 100644 index ca2c50241bf..00000000000 --- a/spec/migrations/20230202211434_migrate_redis_slot_keys_spec.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require_migration! - -RSpec.describe MigrateRedisSlotKeys, :migration, feature_category: :service_ping do - let(:date) { Date.yesterday.strftime('%G-%j') } - let(:week) { Date.yesterday.strftime('%G-%V') } - - before do - allow(described_class::BackupHLLRedisCounter).to receive(:known_events).and_return([{ - redis_slot: 'analytics', - aggregation: 'daily', - name: 'users_viewing_analytics_group_devops_adoption' - }, { - aggregation: 'weekly', - name: 'wiki_action' - }]) - end - - describe "#up" do - it 'rename keys', :aggregate_failures do - expiry_daily = described_class::BackupHLLRedisCounter::DEFAULT_DAILY_KEY_EXPIRY_LENGTH - expiry_weekly = described_class::BackupHLLRedisCounter::DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH - - default_slot = described_class::BackupHLLRedisCounter::REDIS_SLOT - - old_slot_a = "#{date}-users_viewing_{analytics}_group_devops_adoption" - old_slot_b = "{wiki_action}-#{week}" - - new_slot_a = "#{date}-{#{default_slot}}_users_viewing_analytics_group_devops_adoption" - new_slot_b = "{#{default_slot}}_wiki_action-#{week}" - - Gitlab::Redis::HLL.add(key: old_slot_a, value: 1, expiry: expiry_daily) - Gitlab::Redis::HLL.add(key: old_slot_b, value: 1, expiry: expiry_weekly) - - # check that we merge values during migration - # i.e. we dont drop keys created after code deploy but before the migration - Gitlab::Redis::HLL.add(key: new_slot_a, value: 2, expiry: expiry_daily) - Gitlab::Redis::HLL.add(key: new_slot_b, value: 2, expiry: expiry_weekly) - - migrate! - - expect(Gitlab::Redis::HLL.count(keys: new_slot_a)).to eq(2) - expect(Gitlab::Redis::HLL.count(keys: new_slot_b)).to eq(2) - expect(with_redis { |r| r.ttl(new_slot_a) }).to be_within(600).of(expiry_daily) - expect(with_redis { |r| r.ttl(new_slot_b) }).to be_within(600).of(expiry_weekly) - end - end - - def with_redis(&block) - Gitlab::Redis::SharedState.with(&block) - end -end diff --git a/spec/migrations/20230302811133_re_migrate_redis_slot_keys_spec.rb b/spec/migrations/20230302811133_re_migrate_redis_slot_keys_spec.rb index 4c6d4907c29..d33a80b5e64 100644 --- a/spec/migrations/20230302811133_re_migrate_redis_slot_keys_spec.rb +++ b/spec/migrations/20230302811133_re_migrate_redis_slot_keys_spec.rb @@ -9,13 +9,14 @@ RSpec.describe ReMigrateRedisSlotKeys, :migration, feature_category: :service_pi let(:known_events) do [ { - redis_slot: 'analytics', + redis_slot: 'management', aggregation: 'daily', - name: 'users_viewing_analytics_group_devops_adoption' + name: 'g_project_management_epic_closed' }, { aggregation: 'weekly', - name: 'wiki_action' - }, { + name: 'incident_management_incident_assigned' + }, + { aggregation: 'weekly', name: 'non_existing_event' }, { @@ -30,17 +31,17 @@ RSpec.describe ReMigrateRedisSlotKeys, :migration, feature_category: :service_pi allow(Gitlab::UsageDataCounters::HLLRedisCounter).to receive(:known_events) .and_return(known_events) - expiry_daily = Gitlab::UsageDataCounters::HLLRedisCounter::DEFAULT_DAILY_KEY_EXPIRY_LENGTH + expiry_daily = 29.days expiry_weekly = Gitlab::UsageDataCounters::HLLRedisCounter::DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH default_slot = Gitlab::UsageDataCounters::HLLRedisCounter::REDIS_SLOT - old_slot_a = "#{date}-users_viewing_{analytics}_group_devops_adoption" - old_slot_b = "{wiki_action}-#{week}" + old_slot_a = "#{date}-g_project_{management}_epic_closed" + old_slot_b = "{incident_management_incident_assigned}-#{week}" old_slot_without_expiry = "{event_without_expiry}-#{week}" - new_slot_a = "#{date}-{#{default_slot}}_users_viewing_analytics_group_devops_adoption" - new_slot_b = "{#{default_slot}}_wiki_action-#{week}" + new_slot_a = "#{date}-{#{default_slot}}_g_project_management_epic_closed" + new_slot_b = "{#{default_slot}}_incident_management_incident_assigned-#{week}" new_slot_without_expiry = "{#{default_slot}}_event_without_expiry-#{week}" Gitlab::Redis::HLL.add(key: old_slot_a, value: 1, expiry: expiry_daily) diff --git a/spec/migrations/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation_spec.rb b/spec/migrations/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation_spec.rb index 86787273fbc..21cdb6afa24 100644 --- a/spec/migrations/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation_spec.rb +++ b/spec/migrations/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation_spec.rb @@ -25,10 +25,11 @@ RSpec.describe MigrateDailyRedisHllEventsToWeeklyAggregation, :migration, :clean end context 'with weekly aggregation' do - let(:event) { { aggregation: 'weekly', name: 'wiki_action' } } + let(:date_formatted) { date.strftime('%G-%V') } + let(:event) { { aggregation: 'weekly', name: 'weekly_action' } } it 'returns correct key' do - existing_key = Gitlab::UsageDataCounters::HLLRedisCounter.send(:redis_key, event, date) + existing_key = "{hll_counters}_weekly_action-#{date_formatted}" expect(described_class.new.redis_key(event, date, event[:aggregation])).to eq(existing_key) end @@ -36,7 +37,7 @@ RSpec.describe MigrateDailyRedisHllEventsToWeeklyAggregation, :migration, :clean end context 'with weekly events' do - let(:events) { [{ aggregation: 'weekly', name: 'wiki_action' }] } + let(:events) { [{ aggregation: 'weekly', name: 'weekly_action' }] } before do allow(Gitlab::UsageDataCounters::HLLRedisCounter).to receive(:known_events).and_return(events) @@ -53,47 +54,23 @@ RSpec.describe MigrateDailyRedisHllEventsToWeeklyAggregation, :migration, :clean end context 'with daily events' do - let(:daily_expiry) { Gitlab::UsageDataCounters::HLLRedisCounter::DEFAULT_DAILY_KEY_EXPIRY_LENGTH } + let(:daily_expiry) { 29.days } let(:weekly_expiry) { Gitlab::UsageDataCounters::HLLRedisCounter::DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH } - it 'doesnt override events from migrated keys (code deployed before migration)' do - events = [{ aggregation: 'daily', name: 'users_viewing_analytics' }, - { aggregation: 'weekly', name: 'users_viewing_analytics' }] - allow(Gitlab::UsageDataCounters::HLLRedisCounter).to receive(:known_events).and_return(events) - - day = (Date.today - 1.week).beginning_of_week - daily_event = events.first - key_daily1 = Gitlab::UsageDataCounters::HLLRedisCounter.send(:redis_key, daily_event, day) - Gitlab::Redis::HLL.add(key: key_daily1, value: 1, expiry: daily_expiry) - key_daily2 = Gitlab::UsageDataCounters::HLLRedisCounter.send(:redis_key, daily_event, day + 2.days) - Gitlab::Redis::HLL.add(key: key_daily2, value: 2, expiry: daily_expiry) - key_daily3 = Gitlab::UsageDataCounters::HLLRedisCounter.send(:redis_key, daily_event, day + 5.days) - Gitlab::Redis::HLL.add(key: key_daily3, value: 3, expiry: daily_expiry) - - # the same event but with weekly aggregation and pre-Redis migration - weekly_event = events.second - key_weekly = Gitlab::UsageDataCounters::HLLRedisCounter.send(:redis_key, weekly_event, day + 5.days) - Gitlab::Redis::HLL.add(key: key_weekly, value: 3, expiry: weekly_expiry) - - migrate! - - expect(Gitlab::Redis::HLL.count(keys: key_weekly)).to eq(3) - end - it 'migrates with correct parameters', :aggregate_failures do - events = [{ aggregation: 'daily', name: 'users_viewing_analytics_group_devops_adoption' }] + events = [{ aggregation: 'daily', name: 'g_project_management_epic_blocked_removed' }] allow(Gitlab::UsageDataCounters::HLLRedisCounter).to receive(:known_events).and_return(events) event = events.first.dup.tap { |e| e[:aggregation] = 'weekly' } # For every day in the last 30 days, add a value to the daily key with daily expiry (including today) 31.times do |i| - key = Gitlab::UsageDataCounters::HLLRedisCounter.send(:redis_key, event, Date.today - i.days) + key = described_class.new.send(:redis_key, event, Date.today - i.days, :weekly) Gitlab::Redis::HLL.add(key: key, value: i, expiry: daily_expiry) end migrate! - new_key = Gitlab::UsageDataCounters::HLLRedisCounter.send(:redis_key, event, Date.today) + new_key = described_class.new.send(:redis_key, event, Date.today, :weekly) # for the current week we should have value eq to the day of the week (ie. 1 for Monday, 2 for Tuesday, etc.) first_week_days = Date.today.cwday expect(Gitlab::Redis::HLL.count(keys: new_key)).to eq(first_week_days) @@ -102,7 +79,7 @@ RSpec.describe MigrateDailyRedisHllEventsToWeeklyAggregation, :migration, :clean full_weeks = (31 - first_week_days) / 7 # for the next full weeks we should have value eq to 7 (ie. 7 days in a week) (1..full_weeks).each do |i| - new_key = Gitlab::UsageDataCounters::HLLRedisCounter.send(:redis_key, event, Date.today - i.weeks) + new_key = described_class.new.send(:redis_key, event, Date.today - i.weeks, :weekly) expect(Gitlab::Redis::HLL.count(keys: new_key)).to eq(7) expect(with_redis { |r| r.ttl(new_key) }).to be_within(600).of(weekly_expiry) end @@ -111,7 +88,7 @@ RSpec.describe MigrateDailyRedisHllEventsToWeeklyAggregation, :migration, :clean last_week_days = 31 - ((full_weeks * 7) + first_week_days) unless last_week_days.zero? last_week = full_weeks + 1 - new_key = Gitlab::UsageDataCounters::HLLRedisCounter.send(:redis_key, event, Date.today - last_week.weeks) + new_key = described_class.new.send(:redis_key, event, Date.today - last_week.weeks, :weekly) expect(Gitlab::Redis::HLL.count(keys: new_key)).to eq(last_week_days) expect(with_redis { |r| r.ttl(new_key) }).to be_within(600).of(weekly_expiry) end -- cgit v1.2.3