Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2023-06-01 09:08:03 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2023-06-01 09:08:03 +0300
commit8f535d9082bb0f10f2ef6b9133dbc77691ace4db (patch)
tree5e8df6512d9c5a5fee8e499b10e5b0817799d4aa
parent4346c3ae00a177d8afe73a898514b0edeb43e07d (diff)
Add latest changes from gitlab-org/gitlab@master
-rw-r--r--.gitlab/issue_templates/Geo Replicate a new Git repository type.md34
-rw-r--r--.gitlab/issue_templates/Geo Replicate a new blob type.md34
-rw-r--r--.rubocop_todo/layout/argument_alignment.yml12
-rw-r--r--.rubocop_todo/search/namespaced_class.yml4
-rw-r--r--GITLAB_ELASTICSEARCH_INDEXER_VERSION2
-rw-r--r--Gemfile2
-rw-r--r--Gemfile.checksum2
-rw-r--r--Gemfile.lock4
-rw-r--r--app/assets/javascripts/graphql_shared/possible_types.json1
-rw-r--r--app/workers/object_storage/delete_stale_direct_uploads_worker.rb2
-rw-r--r--db/post_migrate/20230202211434_migrate_redis_slot_keys.rb102
-rw-r--r--db/post_migrate/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation.rb89
-rw-r--r--db/post_migrate/20230328111013_re_migrate_redis_slot_keys.rb93
-rw-r--r--doc/api/graphql/reference/index.md1
-rw-r--r--doc/user/group/import/index.md28
-rw-r--r--lib/api/helpers.rb3
-rw-r--r--qa/qa/specs/features/browser_ui/5_package/container_registry/saas/pull_container_registry_image_spec.rb31
-rw-r--r--spec/migrations/20230202211434_migrate_redis_slot_keys_spec.rb54
-rw-r--r--spec/migrations/20230302811133_re_migrate_redis_slot_keys_spec.rb19
-rw-r--r--spec/migrations/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation_spec.rb43
20 files changed, 338 insertions, 222 deletions
diff --git a/.gitlab/issue_templates/Geo Replicate a new Git repository type.md b/.gitlab/issue_templates/Geo Replicate a new Git repository type.md
index 95f92430eea..e93c5d79964 100644
--- a/.gitlab/issue_templates/Geo Replicate a new Git repository type.md
+++ b/.gitlab/issue_templates/Geo Replicate a new Git repository type.md
@@ -725,6 +725,40 @@ The GraphQL API is used by `Admin > Geo > Replication Details` views, and is dir
}
```
+To allow the new replicable to resync and reverify via GraphQL:
+
+- [ ] Add the `CoolWidgetRegistryType` to the `GEO_REGISTRY_TYPE` constant in `ee/app/graphql/types/geo/registrable_type.rb`:
+
+ ```ruby
+ GEO_REGISTRY_TYPES = {
+ ::Geo::CoolWidgetRegistry => Types::Geo::CoolWidgetRegistryType
+ }
+ ```
+
+- [ ] Include the `CoolWidgetRegistry` in the `let(:registry_classes)` variable of `ee/spec/graphql/types/geo/registry_class_enum_spec.rb`:
+
+ ```ruby
+ let(:registry_classes) do
+ %w[
+ COOL_WIDGET_REGISTRY
+ ]
+ end
+ ```
+
+- [ ] Include the new registry in the Rspec parameterized table of `ee/spec/support/shared_contexts/graphql/geo/registries_shared_context.rb`:
+
+ ```ruby
+ # frozen_string_literal: true
+
+ RSpec.shared_context 'with geo registries shared context' do
+ using RSpec::Parameterized::TableSyntax
+
+ where(:registry_class, :registry_type, :registry_factory) do
+ Geo::CoolWidgetRegistry | Types::Geo::CoolWidgetRegistryType | :geo_cool_widget_registry
+ end
+ end
+ ```
+
- [ ] Update the GraphQL reference documentation:
```shell
diff --git a/.gitlab/issue_templates/Geo Replicate a new blob type.md b/.gitlab/issue_templates/Geo Replicate a new blob type.md
index 7bf8dd6526e..2183ba9c384 100644
--- a/.gitlab/issue_templates/Geo Replicate a new blob type.md
+++ b/.gitlab/issue_templates/Geo Replicate a new blob type.md
@@ -693,6 +693,40 @@ The GraphQL API is used by `Admin > Geo > Replication Details` views, and is dir
}
```
+To allow the new replicable to resync and reverify via GraphQL:
+
+- [ ] Add the `CoolWidgetRegistryType` to the `GEO_REGISTRY_TYPE` constant in `ee/app/graphql/types/geo/registrable_type.rb`:
+
+ ```ruby
+ GEO_REGISTRY_TYPES = {
+ ::Geo::CoolWidgetRegistry => Types::Geo::CoolWidgetRegistryType
+ }
+ ```
+
+- [ ] Include the `CoolWidgetRegistry` in the `let(:registry_classes)` variable of `ee/spec/graphql/types/geo/registry_class_enum_spec.rb`:
+
+ ```ruby
+ let(:registry_classes) do
+ %w[
+ COOL_WIDGET_REGISTRY
+ ]
+ end
+ ```
+
+- [ ] Include the new registry in the Rspec parameterized table of `ee/spec/support/shared_contexts/graphql/geo/registries_shared_context.rb`:
+
+ ```ruby
+ # frozen_string_literal: true
+
+ RSpec.shared_context 'with geo registries shared context' do
+ using RSpec::Parameterized::TableSyntax
+
+ where(:registry_class, :registry_type, :registry_factory) do
+ Geo::CoolWidgetRegistry | Types::Geo::CoolWidgetRegistryType | :geo_cool_widget_registry
+ end
+ end
+ ```
+
- [ ] Update the GraphQL reference documentation:
```shell
diff --git a/.rubocop_todo/layout/argument_alignment.yml b/.rubocop_todo/layout/argument_alignment.yml
index 2ff4c3dc07f..3aaf0b09e53 100644
--- a/.rubocop_todo/layout/argument_alignment.yml
+++ b/.rubocop_todo/layout/argument_alignment.yml
@@ -977,18 +977,6 @@ Layout/ArgumentAlignment:
- 'ee/app/models/group_merge_request_approval_setting.rb'
- 'ee/app/models/incident_management/escalation_rule.rb'
- 'ee/app/models/integrations/github/status_notifier.rb'
- - 'ee/app/models/merge_requests/compliance_violation.rb'
- - 'ee/app/models/merge_requests/external_status_check.rb'
- - 'ee/app/models/package_metadata/package_version_license.rb'
- - 'ee/app/models/product_analytics/dashboard.rb'
- - 'ee/app/models/protected_environments/approval_rule.rb'
- - 'ee/app/models/requirements_management/requirement.rb'
- - 'ee/app/models/sbom/vulnerable_component_version.rb'
- - 'ee/app/models/sca/license_policy.rb'
- - 'ee/app/models/scim_identity.rb'
- - 'ee/app/models/security/finding.rb'
- - 'ee/app/models/security/orchestration_policy_rule_schedule.rb'
- - 'ee/app/models/smartcard_identity.rb'
- 'ee/app/models/status_page/project_setting.rb'
- 'ee/app/models/vulnerabilities/external_issue_link.rb'
- 'ee/app/models/vulnerabilities/feedback.rb'
diff --git a/.rubocop_todo/search/namespaced_class.yml b/.rubocop_todo/search/namespaced_class.yml
index ad27603ad47..8750d87a900 100644
--- a/.rubocop_todo/search/namespaced_class.yml
+++ b/.rubocop_todo/search/namespaced_class.yml
@@ -117,6 +117,8 @@ Search/NamespacedClass:
- 'ee/lib/elastic/latest/snippet_instance_proxy.rb'
- 'ee/lib/elastic/latest/state_filter.rb'
- 'ee/lib/elastic/latest/user_class_proxy.rb'
+ - 'ee/lib/elastic/latest/wiki_class_proxy.rb'
+ - 'ee/lib/elastic/latest/wiki_instance_proxy.rb'
- 'ee/lib/elastic/latest/user_config.rb'
- 'ee/lib/elastic/latest/user_instance_proxy.rb'
- 'ee/lib/elastic/latest/wiki_config.rb'
@@ -148,6 +150,8 @@ Search/NamespacedClass:
- 'ee/lib/elastic/v12p1/snippet_instance_proxy.rb'
- 'ee/lib/elastic/v12p1/user_class_proxy.rb'
- 'ee/lib/elastic/v12p1/user_instance_proxy.rb'
+ - 'ee/lib/elastic/v12p1/wiki_class_proxy.rb'
+ - 'ee/lib/elastic/v12p1/wiki_instance_proxy.rb'
- 'ee/lib/gem_extensions/elasticsearch/model/adapter/active_record/importing.rb'
- 'ee/lib/gem_extensions/elasticsearch/model/adapter/multiple/records.rb'
- 'ee/lib/gem_extensions/elasticsearch/model/client.rb'
diff --git a/GITLAB_ELASTICSEARCH_INDEXER_VERSION b/GITLAB_ELASTICSEARCH_INDEXER_VERSION
index e91d9be2a86..eda862a98c1 100644
--- a/GITLAB_ELASTICSEARCH_INDEXER_VERSION
+++ b/GITLAB_ELASTICSEARCH_INDEXER_VERSION
@@ -1 +1 @@
-4.3.3
+4.3.4
diff --git a/Gemfile b/Gemfile
index 5c621c4545e..70cc3276a9d 100644
--- a/Gemfile
+++ b/Gemfile
@@ -505,7 +505,7 @@ gem 'ssh_data', '~> 1.3'
gem 'spamcheck', '~> 1.3.0'
# Gitaly GRPC protocol definitions
-gem 'gitaly', '~> 15.9.0-rc3'
+gem 'gitaly', '~> 16.1.0-rc1'
# KAS GRPC protocol definitions
gem 'kas-grpc', '~> 0.1.0'
diff --git a/Gemfile.checksum b/Gemfile.checksum
index b23ee78f108..e2252689e1b 100644
--- a/Gemfile.checksum
+++ b/Gemfile.checksum
@@ -207,7 +207,7 @@
{"name":"gettext_i18n_rails","version":"1.8.0","platform":"ruby","checksum":"95e5cf8440b1e08705b27f2bccb56143272c5a7a0dabcf54ea1bd701140a496f"},
{"name":"gettext_i18n_rails_js","version":"1.3.0","platform":"ruby","checksum":"5d10afe4be3639bff78c50a56768c20f39aecdabc580c08aa45573911c2bd687"},
{"name":"git","version":"1.11.0","platform":"ruby","checksum":"7e95ba4da8298a0373ef1a6862aa22007d761f3c8274b675aa787966fecea0f1"},
-{"name":"gitaly","version":"15.9.0.pre.rc3","platform":"ruby","checksum":"6ac64320a70417131a4b97f5dd45d4e203d60703cc3cba156561e7f8c50a4abe"},
+{"name":"gitaly","version":"16.1.0.pre.rc1","platform":"ruby","checksum":"3f61ecd96baf283a3365943dac58c3fb81221fee74d1ccc7931e1adea5403405"},
{"name":"gitlab","version":"4.19.0","platform":"ruby","checksum":"3f645e3e195dbc24f0834fbf83e8ccfb2056d8e9712b01a640aad418a6949679"},
{"name":"gitlab-chronic","version":"0.10.5","platform":"ruby","checksum":"f80f18dc699b708870a80685243331290bc10cfeedb6b99c92219722f729c875"},
{"name":"gitlab-dangerfiles","version":"3.10.0","platform":"ruby","checksum":"df4cfe051f52529c0256346d89d06d5ef2bb630928754eb620b5233eb9b14041"},
diff --git a/Gemfile.lock b/Gemfile.lock
index 655953117e8..cd96633ac2f 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -581,7 +581,7 @@ GEM
rails (>= 3.2.0)
git (1.11.0)
rchardet (~> 1.8)
- gitaly (15.9.0.pre.rc3)
+ gitaly (16.1.0.pre.rc1)
grpc (~> 1.0)
gitlab (4.19.0)
httparty (~> 0.20)
@@ -1738,7 +1738,7 @@ DEPENDENCIES
gettext (~> 3.3)
gettext_i18n_rails (~> 1.8.0)
gettext_i18n_rails_js (~> 1.3)
- gitaly (~> 15.9.0.pre.rc3)
+ gitaly (~> 16.1.0.pre.rc1)
gitlab-chronic (~> 0.10.5)
gitlab-dangerfiles (~> 3.10.0)
gitlab-experiment (~> 0.7.1)
diff --git a/app/assets/javascripts/graphql_shared/possible_types.json b/app/assets/javascripts/graphql_shared/possible_types.json
index f35886716ee..c7a446b36f8 100644
--- a/app/assets/javascripts/graphql_shared/possible_types.json
+++ b/app/assets/javascripts/graphql_shared/possible_types.json
@@ -94,6 +94,7 @@
"ContainerRepositoryRegistry",
"DependencyProxyBlobRegistry",
"DependencyProxyManifestRegistry",
+ "DesignManagementRepositoryRegistry",
"JobArtifactRegistry",
"LfsObjectRegistry",
"MergeRequestDiffRegistry",
diff --git a/app/workers/object_storage/delete_stale_direct_uploads_worker.rb b/app/workers/object_storage/delete_stale_direct_uploads_worker.rb
index 216df498cc2..081b58ca726 100644
--- a/app/workers/object_storage/delete_stale_direct_uploads_worker.rb
+++ b/app/workers/object_storage/delete_stale_direct_uploads_worker.rb
@@ -22,7 +22,7 @@ module ObjectStorage
result = ObjectStorage::DeleteStaleDirectUploadsService.new.execute
log_extra_metadata_on_done(:total_pending_entries, result[:total_pending_entries])
- log_extra_metadata_on_done(:total_stale_entries, result[:total_stale_entries])
+ log_extra_metadata_on_done(:total_deleted_stale_entries, result[:total_deleted_stale_entries])
log_extra_metadata_on_done(:execution_timeout, result[:execution_timeout])
end
end
diff --git a/db/post_migrate/20230202211434_migrate_redis_slot_keys.rb b/db/post_migrate/20230202211434_migrate_redis_slot_keys.rb
index e1d3787cbb3..126555ae391 100644
--- a/db/post_migrate/20230202211434_migrate_redis_slot_keys.rb
+++ b/db/post_migrate/20230202211434_migrate_redis_slot_keys.rb
@@ -4,110 +4,10 @@ class MigrateRedisSlotKeys < Gitlab::Database::Migration[2.1]
disable_ddl_transaction!
def up
- BackupHLLRedisCounter.known_events.each do |event|
- if event[:aggregation].to_sym == :daily
- migrate_daily_aggregated(event)
- else
- migrate_weekly_aggregated(event)
- end
- end
+ # deleted as contained a bug
end
def down
# no-op
end
-
- private
-
- def migrate_daily_aggregated(event)
- days_back = BackupHLLRedisCounter::DEFAULT_DAILY_KEY_EXPIRY_LENGTH
- start_date = Date.today - days_back - 1.day
- end_date = Date.today + 1.day
-
- (start_date..end_date).each do |date|
- rename_key(event, date)
- end
- end
-
- def migrate_weekly_aggregated(event)
- weeks_back = BackupHLLRedisCounter::DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH
- start_date = (Date.today - weeks_back).beginning_of_week - 1.day
- end_date = Date.today.end_of_week + 1.day
-
- (start_date..end_date).each { |date| rename_key(event, date) }
- end
-
- def rename_key(event, date)
- old_key = old_redis_key(event, date)
- new_key = BackupHLLRedisCounter.redis_key(event, date)
-
- # cannot simply rename due to different slots
- Gitlab::Redis::SharedState.with do |r|
- break unless r.exists?(old_key)
-
- Gitlab::Redis::HLL.add(
- key: new_key,
- value: r.pfcount(old_key),
- expiry: r.ttl(old_key)
- )
- end
- end
-
- def old_redis_key(event, time)
- name_with_slot = if event[:redis_slot].present?
- event[:name].to_s.gsub(event[:redis_slot], "{#{event[:redis_slot]}}")
- else
- "{#{event[:name]}}"
- end
-
- BackupHLLRedisCounter.apply_time_aggregation(name_with_slot, time, event)
- end
-
- # :nocov: Existing backed up class # rubocop:disable Gitlab/NoCodeCoverageComment
- module BackupHLLRedisCounter
- DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks
- DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days
- REDIS_SLOT = 'hll_counters'
-
- KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__)
- ALLOWED_AGGREGATIONS = %i[daily weekly].freeze
-
- class << self
- def known_events
- @known_events ||= load_events(KNOWN_EVENTS_PATH)
- end
-
- def load_events(wildcard)
- Dir[wildcard].each_with_object([]) do |path, events|
- events.push(*load_yaml_from_path(path))
- end
- end
-
- def load_yaml_from_path(path)
- YAML.safe_load(File.read(path))&.map(&:with_indifferent_access)
- end
-
- def known_events_names
- known_events.map { |event| event[:name] } # rubocop:disable Rails/Pluck
- end
-
- def redis_key(event, time, context = '')
- key = "{#{REDIS_SLOT}}_#{event[:name]}"
- key = apply_time_aggregation(key, time, event)
- key = "#{context}_#{key}" if context.present?
- key
- end
-
- def apply_time_aggregation(key, time, event)
- if event[:aggregation].to_sym == :daily
- year_day = time.strftime('%G-%j')
- "#{year_day}-#{key}"
- else
- year_week = time.strftime('%G-%V')
- "#{key}-#{year_week}"
- end
- end
- end
- end
- # :nocov: # rubocop:disable Gitlab/NoCodeCoverageComment
end
diff --git a/db/post_migrate/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation.rb b/db/post_migrate/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation.rb
index b4468c851ae..59bff26f964 100644
--- a/db/post_migrate/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation.rb
+++ b/db/post_migrate/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation.rb
@@ -3,6 +3,93 @@
class MigrateDailyRedisHllEventsToWeeklyAggregation < Gitlab::Database::Migration[2.1]
disable_ddl_transaction!
+ DAILY_EVENTS =
+ %w[g_edit_by_web_ide
+ g_edit_by_sfe
+ g_edit_by_snippet_ide
+ g_edit_by_live_preview
+ wiki_action
+ design_action
+ project_action
+ git_write_action
+ merge_request_action
+ i_source_code_code_intelligence
+ g_project_management_issue_title_changed
+ g_project_management_issue_description_changed
+ g_project_management_issue_assignee_changed
+ g_project_management_issue_made_confidential
+ g_project_management_issue_made_visible
+ g_project_management_issue_created
+ g_project_management_issue_closed
+ g_project_management_issue_reopened
+ g_project_management_issue_label_changed
+ g_project_management_issue_milestone_changed
+ g_project_management_issue_cross_referenced
+ g_project_management_issue_moved
+ g_project_management_issue_related
+ g_project_management_issue_unrelated
+ g_project_management_issue_marked_as_duplicate
+ g_project_management_issue_locked
+ g_project_management_issue_unlocked
+ g_project_management_issue_designs_added
+ g_project_management_issue_designs_modified
+ g_project_management_issue_designs_removed
+ g_project_management_issue_due_date_changed
+ g_project_management_issue_design_comments_removed
+ g_project_management_issue_time_estimate_changed
+ g_project_management_issue_time_spent_changed
+ g_project_management_issue_comment_added
+ g_project_management_issue_comment_edited
+ g_project_management_issue_comment_removed
+ g_project_management_issue_cloned
+ g_geo_proxied_requests
+ approval_project_rule_created
+ g_project_management_issue_added_to_epic
+ g_project_management_issue_changed_epic
+ g_project_management_issue_health_status_changed
+ g_project_management_issue_iteration_changed
+ g_project_management_issue_removed_from_epic
+ g_project_management_issue_weight_changed
+ g_geo_proxied_requests
+ g_project_management_users_creating_epic_boards
+ g_project_management_users_viewing_epic_boards
+ g_project_management_users_updating_epic_board_names
+ g_project_management_epic_created
+ project_management_users_unchecking_epic_task
+ project_management_users_checking_epic_task
+ g_project_management_users_updating_epic_titles
+ g_project_management_users_updating_epic_descriptions
+ g_project_management_users_creating_epic_notes
+ g_project_management_users_updating_epic_notes
+ g_project_management_users_destroying_epic_notes
+ g_project_management_users_awarding_epic_emoji
+ g_project_management_users_removing_epic_emoji
+ g_project_management_users_setting_epic_start_date_as_fixed
+ g_project_management_users_updating_fixed_epic_start_date
+ g_project_management_users_setting_epic_start_date_as_inherited
+ g_project_management_users_setting_epic_due_date_as_fixed
+ g_project_management_users_updating_fixed_epic_due_date
+ g_project_management_users_setting_epic_due_date_as_inherited
+ g_project_management_epic_issue_added
+ g_project_management_epic_issue_removed
+ g_project_management_epic_issue_moved_from_project
+ g_project_management_users_updating_epic_parent
+ g_project_management_epic_closed
+ g_project_management_epic_reopened
+ g_project_management_issue_promoted_to_epic
+ g_project_management_users_setting_epic_confidential
+ g_project_management_users_setting_epic_visible
+ g_project_management_epic_users_changing_labels
+ g_project_management_epic_destroyed
+ g_project_management_epic_cross_referenced
+ g_project_management_users_epic_issue_added_from_epic
+ g_project_management_epic_related_added
+ g_project_management_epic_related_removed
+ g_project_management_epic_blocking_added
+ g_project_management_epic_blocking_removed
+ g_project_management_epic_blocked_added
+ g_project_management_epic_blocked_removed].freeze
+
def up
days_back = 29.days
start_date = Date.today - days_back - 1.day
@@ -10,7 +97,7 @@ class MigrateDailyRedisHllEventsToWeeklyAggregation < Gitlab::Database::Migratio
keys = {}
Gitlab::UsageDataCounters::HLLRedisCounter.known_events.each do |event|
- next unless event[:aggregation].to_sym == :daily
+ next unless DAILY_EVENTS.include?(event[:name].to_s)
(start_date..end_date).each do |date|
daily_key = redis_key(event, date, :daily)
diff --git a/db/post_migrate/20230328111013_re_migrate_redis_slot_keys.rb b/db/post_migrate/20230328111013_re_migrate_redis_slot_keys.rb
index f09888bd821..71ab958b601 100644
--- a/db/post_migrate/20230328111013_re_migrate_redis_slot_keys.rb
+++ b/db/post_migrate/20230328111013_re_migrate_redis_slot_keys.rb
@@ -3,9 +3,96 @@
class ReMigrateRedisSlotKeys < Gitlab::Database::Migration[2.1]
disable_ddl_transaction!
+ DAILY_EVENTS =
+ %w[g_edit_by_web_ide
+ g_edit_by_sfe
+ g_edit_by_snippet_ide
+ g_edit_by_live_preview
+ wiki_action
+ design_action
+ project_action
+ git_write_action
+ merge_request_action
+ i_source_code_code_intelligence
+ g_project_management_issue_title_changed
+ g_project_management_issue_description_changed
+ g_project_management_issue_assignee_changed
+ g_project_management_issue_made_confidential
+ g_project_management_issue_made_visible
+ g_project_management_issue_created
+ g_project_management_issue_closed
+ g_project_management_issue_reopened
+ g_project_management_issue_label_changed
+ g_project_management_issue_milestone_changed
+ g_project_management_issue_cross_referenced
+ g_project_management_issue_moved
+ g_project_management_issue_related
+ g_project_management_issue_unrelated
+ g_project_management_issue_marked_as_duplicate
+ g_project_management_issue_locked
+ g_project_management_issue_unlocked
+ g_project_management_issue_designs_added
+ g_project_management_issue_designs_modified
+ g_project_management_issue_designs_removed
+ g_project_management_issue_due_date_changed
+ g_project_management_issue_design_comments_removed
+ g_project_management_issue_time_estimate_changed
+ g_project_management_issue_time_spent_changed
+ g_project_management_issue_comment_added
+ g_project_management_issue_comment_edited
+ g_project_management_issue_comment_removed
+ g_project_management_issue_cloned
+ g_geo_proxied_requests
+ approval_project_rule_created
+ g_project_management_issue_added_to_epic
+ g_project_management_issue_changed_epic
+ g_project_management_issue_health_status_changed
+ g_project_management_issue_iteration_changed
+ g_project_management_issue_removed_from_epic
+ g_project_management_issue_weight_changed
+ g_geo_proxied_requests
+ g_project_management_users_creating_epic_boards
+ g_project_management_users_viewing_epic_boards
+ g_project_management_users_updating_epic_board_names
+ g_project_management_epic_created
+ project_management_users_unchecking_epic_task
+ project_management_users_checking_epic_task
+ g_project_management_users_updating_epic_titles
+ g_project_management_users_updating_epic_descriptions
+ g_project_management_users_creating_epic_notes
+ g_project_management_users_updating_epic_notes
+ g_project_management_users_destroying_epic_notes
+ g_project_management_users_awarding_epic_emoji
+ g_project_management_users_removing_epic_emoji
+ g_project_management_users_setting_epic_start_date_as_fixed
+ g_project_management_users_updating_fixed_epic_start_date
+ g_project_management_users_setting_epic_start_date_as_inherited
+ g_project_management_users_setting_epic_due_date_as_fixed
+ g_project_management_users_updating_fixed_epic_due_date
+ g_project_management_users_setting_epic_due_date_as_inherited
+ g_project_management_epic_issue_added
+ g_project_management_epic_issue_removed
+ g_project_management_epic_issue_moved_from_project
+ g_project_management_users_updating_epic_parent
+ g_project_management_epic_closed
+ g_project_management_epic_reopened
+ g_project_management_issue_promoted_to_epic
+ g_project_management_users_setting_epic_confidential
+ g_project_management_users_setting_epic_visible
+ g_project_management_epic_users_changing_labels
+ g_project_management_epic_destroyed
+ g_project_management_epic_cross_referenced
+ g_project_management_users_epic_issue_added_from_epic
+ g_project_management_epic_related_added
+ g_project_management_epic_related_removed
+ g_project_management_epic_blocking_added
+ g_project_management_epic_blocking_removed
+ g_project_management_epic_blocked_added
+ g_project_management_epic_blocked_removed].freeze
+
def up
Gitlab::UsageDataCounters::HLLRedisCounter.known_events.each do |event|
- if event[:aggregation].to_sym == :daily
+ if DAILY_EVENTS.include?(event[:name].to_s)
migrate_daily_aggregated(event)
else
migrate_weekly_aggregated(event)
@@ -20,7 +107,7 @@ class ReMigrateRedisSlotKeys < Gitlab::Database::Migration[2.1]
private
def migrate_daily_aggregated(event)
- days_back = Gitlab::UsageDataCounters::HLLRedisCounter::DEFAULT_DAILY_KEY_EXPIRY_LENGTH
+ days_back = 29.days
start_date = Date.today - days_back - 1.day
end_date = Date.today + 1.day
@@ -77,7 +164,7 @@ class ReMigrateRedisSlotKeys < Gitlab::Database::Migration[2.1]
end
def apply_time_aggregation(key, time, event)
- if event[:aggregation].to_sym == :daily
+ if DAILY_EVENTS.include?(event[:name].to_s)
year_day = time.strftime('%G-%j')
"#{year_day}-#{key}"
else
diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md
index 093d8aa624d..cf3a5e3a541 100644
--- a/doc/api/graphql/reference/index.md
+++ b/doc/api/graphql/reference/index.md
@@ -27157,6 +27157,7 @@ One of:
- [`ContainerRepositoryRegistry`](#containerrepositoryregistry)
- [`DependencyProxyBlobRegistry`](#dependencyproxyblobregistry)
- [`DependencyProxyManifestRegistry`](#dependencyproxymanifestregistry)
+- [`DesignManagementRepositoryRegistry`](#designmanagementrepositoryregistry)
- [`JobArtifactRegistry`](#jobartifactregistry)
- [`LfsObjectRegistry`](#lfsobjectregistry)
- [`MergeRequestDiffRegistry`](#mergerequestdiffregistry)
diff --git a/doc/user/group/import/index.md b/doc/user/group/import/index.md
index 7f55bf56102..2dd5347e5dc 100644
--- a/doc/user/group/import/index.md
+++ b/doc/user/group/import/index.md
@@ -220,6 +220,13 @@ Group items that are migrated to the destination GitLab instance include:
- Already exists in the destination GitLab instance.
- Has a public email in the source GitLab instance that matches a confirmed email in the destination GitLab instance.
+#### Excluded items
+
+Some group items are excluded from migration because they either:
+
+- May contain sensitive information: CI/CD variables, webhooks, and deploy tokens.
+- Are not supported: push rules.
+
### Migrated project items (Beta)
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/267945) in GitLab 14.4 [with a flag](../../feature_flags.md) named `bulk_import_projects`. Disabled by default.
@@ -325,9 +332,24 @@ Setting-related project items that are migrated to the destination GitLab instan
#### Excluded items
-Project items excluded from migration because they contain sensitive information:
-
-- Pipeline triggers.
+Some project items are excluded from migration because they either:
+
+- May contain sensitive information:
+ - CI/CD variables
+ - Deploy keys
+ - Deploy tokens
+ - Pipeline schedule variables
+ - Pipeline triggers
+ - Webhooks
+- Are not supported:
+ - Agents
+ - Container Registry
+ - Environments
+ - Feature flags
+ - Infrastructure Registry
+ - Package Registry
+ - Pages domains
+ - Remote mirrors
### Troubleshooting
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 83641e824ae..98f6733a6f2 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -202,6 +202,7 @@ module API
not_found!('Namespace')
end
+ # find_namespace returns the namespace regardless of user access level on the namespace
# rubocop: disable CodeReuse/ActiveRecord
def find_namespace(id)
if id.to_s =~ INTEGER_ID_REGEX
@@ -212,6 +213,8 @@ module API
end
# rubocop: enable CodeReuse/ActiveRecord
+ # find_namespace! returns the namespace if the current user can read the given namespace
+ # Otherwise, returns a not_found! error
def find_namespace!(id)
check_namespace_access(find_namespace(id))
end
diff --git a/qa/qa/specs/features/browser_ui/5_package/container_registry/saas/pull_container_registry_image_spec.rb b/qa/qa/specs/features/browser_ui/5_package/container_registry/saas/pull_container_registry_image_spec.rb
new file mode 100644
index 00000000000..85a88b54cc2
--- /dev/null
+++ b/qa/qa/specs/features/browser_ui/5_package/container_registry/saas/pull_container_registry_image_spec.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module QA
+ RSpec.describe 'Package' do
+ describe 'SaaS Container Registry', only: { subdomain: %i[staging] }, product_group: :container_registry do
+ let(:project) do
+ Resource::Project.init do |project|
+ project.path_with_namespace = 'gitlab-qa/container-registry-sanity'
+ end.reload!
+ end
+
+ it 'pulls an image from an existing repository',
+ testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/412799' do
+ Flow::Login.sign_in
+ project.visit!
+
+ Page::Project::Menu.perform(&:go_to_pipelines)
+ Page::Project::Pipeline::Index.perform(&:click_run_pipeline_button)
+ Page::Project::Pipeline::New.perform(&:click_run_pipeline_button)
+
+ Page::Project::Pipeline::Show.perform do |pipeline|
+ pipeline.click_job('test')
+ end
+
+ Page::Project::Job::Show.perform do |job|
+ expect(job).to be_successful(timeout: 800)
+ end
+ end
+ end
+ end
+end
diff --git a/spec/migrations/20230202211434_migrate_redis_slot_keys_spec.rb b/spec/migrations/20230202211434_migrate_redis_slot_keys_spec.rb
deleted file mode 100644
index ca2c50241bf..00000000000
--- a/spec/migrations/20230202211434_migrate_redis_slot_keys_spec.rb
+++ /dev/null
@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-require_migration!
-
-RSpec.describe MigrateRedisSlotKeys, :migration, feature_category: :service_ping do
- let(:date) { Date.yesterday.strftime('%G-%j') }
- let(:week) { Date.yesterday.strftime('%G-%V') }
-
- before do
- allow(described_class::BackupHLLRedisCounter).to receive(:known_events).and_return([{
- redis_slot: 'analytics',
- aggregation: 'daily',
- name: 'users_viewing_analytics_group_devops_adoption'
- }, {
- aggregation: 'weekly',
- name: 'wiki_action'
- }])
- end
-
- describe "#up" do
- it 'rename keys', :aggregate_failures do
- expiry_daily = described_class::BackupHLLRedisCounter::DEFAULT_DAILY_KEY_EXPIRY_LENGTH
- expiry_weekly = described_class::BackupHLLRedisCounter::DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH
-
- default_slot = described_class::BackupHLLRedisCounter::REDIS_SLOT
-
- old_slot_a = "#{date}-users_viewing_{analytics}_group_devops_adoption"
- old_slot_b = "{wiki_action}-#{week}"
-
- new_slot_a = "#{date}-{#{default_slot}}_users_viewing_analytics_group_devops_adoption"
- new_slot_b = "{#{default_slot}}_wiki_action-#{week}"
-
- Gitlab::Redis::HLL.add(key: old_slot_a, value: 1, expiry: expiry_daily)
- Gitlab::Redis::HLL.add(key: old_slot_b, value: 1, expiry: expiry_weekly)
-
- # check that we merge values during migration
- # i.e. we dont drop keys created after code deploy but before the migration
- Gitlab::Redis::HLL.add(key: new_slot_a, value: 2, expiry: expiry_daily)
- Gitlab::Redis::HLL.add(key: new_slot_b, value: 2, expiry: expiry_weekly)
-
- migrate!
-
- expect(Gitlab::Redis::HLL.count(keys: new_slot_a)).to eq(2)
- expect(Gitlab::Redis::HLL.count(keys: new_slot_b)).to eq(2)
- expect(with_redis { |r| r.ttl(new_slot_a) }).to be_within(600).of(expiry_daily)
- expect(with_redis { |r| r.ttl(new_slot_b) }).to be_within(600).of(expiry_weekly)
- end
- end
-
- def with_redis(&block)
- Gitlab::Redis::SharedState.with(&block)
- end
-end
diff --git a/spec/migrations/20230302811133_re_migrate_redis_slot_keys_spec.rb b/spec/migrations/20230302811133_re_migrate_redis_slot_keys_spec.rb
index 4c6d4907c29..d33a80b5e64 100644
--- a/spec/migrations/20230302811133_re_migrate_redis_slot_keys_spec.rb
+++ b/spec/migrations/20230302811133_re_migrate_redis_slot_keys_spec.rb
@@ -9,13 +9,14 @@ RSpec.describe ReMigrateRedisSlotKeys, :migration, feature_category: :service_pi
let(:known_events) do
[
{
- redis_slot: 'analytics',
+ redis_slot: 'management',
aggregation: 'daily',
- name: 'users_viewing_analytics_group_devops_adoption'
+ name: 'g_project_management_epic_closed'
}, {
aggregation: 'weekly',
- name: 'wiki_action'
- }, {
+ name: 'incident_management_incident_assigned'
+ },
+ {
aggregation: 'weekly',
name: 'non_existing_event'
}, {
@@ -30,17 +31,17 @@ RSpec.describe ReMigrateRedisSlotKeys, :migration, feature_category: :service_pi
allow(Gitlab::UsageDataCounters::HLLRedisCounter).to receive(:known_events)
.and_return(known_events)
- expiry_daily = Gitlab::UsageDataCounters::HLLRedisCounter::DEFAULT_DAILY_KEY_EXPIRY_LENGTH
+ expiry_daily = 29.days
expiry_weekly = Gitlab::UsageDataCounters::HLLRedisCounter::DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH
default_slot = Gitlab::UsageDataCounters::HLLRedisCounter::REDIS_SLOT
- old_slot_a = "#{date}-users_viewing_{analytics}_group_devops_adoption"
- old_slot_b = "{wiki_action}-#{week}"
+ old_slot_a = "#{date}-g_project_{management}_epic_closed"
+ old_slot_b = "{incident_management_incident_assigned}-#{week}"
old_slot_without_expiry = "{event_without_expiry}-#{week}"
- new_slot_a = "#{date}-{#{default_slot}}_users_viewing_analytics_group_devops_adoption"
- new_slot_b = "{#{default_slot}}_wiki_action-#{week}"
+ new_slot_a = "#{date}-{#{default_slot}}_g_project_management_epic_closed"
+ new_slot_b = "{#{default_slot}}_incident_management_incident_assigned-#{week}"
new_slot_without_expiry = "{#{default_slot}}_event_without_expiry-#{week}"
Gitlab::Redis::HLL.add(key: old_slot_a, value: 1, expiry: expiry_daily)
diff --git a/spec/migrations/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation_spec.rb b/spec/migrations/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation_spec.rb
index 86787273fbc..21cdb6afa24 100644
--- a/spec/migrations/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation_spec.rb
+++ b/spec/migrations/20230317004428_migrate_daily_redis_hll_events_to_weekly_aggregation_spec.rb
@@ -25,10 +25,11 @@ RSpec.describe MigrateDailyRedisHllEventsToWeeklyAggregation, :migration, :clean
end
context 'with weekly aggregation' do
- let(:event) { { aggregation: 'weekly', name: 'wiki_action' } }
+ let(:date_formatted) { date.strftime('%G-%V') }
+ let(:event) { { aggregation: 'weekly', name: 'weekly_action' } }
it 'returns correct key' do
- existing_key = Gitlab::UsageDataCounters::HLLRedisCounter.send(:redis_key, event, date)
+ existing_key = "{hll_counters}_weekly_action-#{date_formatted}"
expect(described_class.new.redis_key(event, date, event[:aggregation])).to eq(existing_key)
end
@@ -36,7 +37,7 @@ RSpec.describe MigrateDailyRedisHllEventsToWeeklyAggregation, :migration, :clean
end
context 'with weekly events' do
- let(:events) { [{ aggregation: 'weekly', name: 'wiki_action' }] }
+ let(:events) { [{ aggregation: 'weekly', name: 'weekly_action' }] }
before do
allow(Gitlab::UsageDataCounters::HLLRedisCounter).to receive(:known_events).and_return(events)
@@ -53,47 +54,23 @@ RSpec.describe MigrateDailyRedisHllEventsToWeeklyAggregation, :migration, :clean
end
context 'with daily events' do
- let(:daily_expiry) { Gitlab::UsageDataCounters::HLLRedisCounter::DEFAULT_DAILY_KEY_EXPIRY_LENGTH }
+ let(:daily_expiry) { 29.days }
let(:weekly_expiry) { Gitlab::UsageDataCounters::HLLRedisCounter::DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH }
- it 'doesnt override events from migrated keys (code deployed before migration)' do
- events = [{ aggregation: 'daily', name: 'users_viewing_analytics' },
- { aggregation: 'weekly', name: 'users_viewing_analytics' }]
- allow(Gitlab::UsageDataCounters::HLLRedisCounter).to receive(:known_events).and_return(events)
-
- day = (Date.today - 1.week).beginning_of_week
- daily_event = events.first
- key_daily1 = Gitlab::UsageDataCounters::HLLRedisCounter.send(:redis_key, daily_event, day)
- Gitlab::Redis::HLL.add(key: key_daily1, value: 1, expiry: daily_expiry)
- key_daily2 = Gitlab::UsageDataCounters::HLLRedisCounter.send(:redis_key, daily_event, day + 2.days)
- Gitlab::Redis::HLL.add(key: key_daily2, value: 2, expiry: daily_expiry)
- key_daily3 = Gitlab::UsageDataCounters::HLLRedisCounter.send(:redis_key, daily_event, day + 5.days)
- Gitlab::Redis::HLL.add(key: key_daily3, value: 3, expiry: daily_expiry)
-
- # the same event but with weekly aggregation and pre-Redis migration
- weekly_event = events.second
- key_weekly = Gitlab::UsageDataCounters::HLLRedisCounter.send(:redis_key, weekly_event, day + 5.days)
- Gitlab::Redis::HLL.add(key: key_weekly, value: 3, expiry: weekly_expiry)
-
- migrate!
-
- expect(Gitlab::Redis::HLL.count(keys: key_weekly)).to eq(3)
- end
-
it 'migrates with correct parameters', :aggregate_failures do
- events = [{ aggregation: 'daily', name: 'users_viewing_analytics_group_devops_adoption' }]
+ events = [{ aggregation: 'daily', name: 'g_project_management_epic_blocked_removed' }]
allow(Gitlab::UsageDataCounters::HLLRedisCounter).to receive(:known_events).and_return(events)
event = events.first.dup.tap { |e| e[:aggregation] = 'weekly' }
# For every day in the last 30 days, add a value to the daily key with daily expiry (including today)
31.times do |i|
- key = Gitlab::UsageDataCounters::HLLRedisCounter.send(:redis_key, event, Date.today - i.days)
+ key = described_class.new.send(:redis_key, event, Date.today - i.days, :weekly)
Gitlab::Redis::HLL.add(key: key, value: i, expiry: daily_expiry)
end
migrate!
- new_key = Gitlab::UsageDataCounters::HLLRedisCounter.send(:redis_key, event, Date.today)
+ new_key = described_class.new.send(:redis_key, event, Date.today, :weekly)
# for the current week we should have value eq to the day of the week (ie. 1 for Monday, 2 for Tuesday, etc.)
first_week_days = Date.today.cwday
expect(Gitlab::Redis::HLL.count(keys: new_key)).to eq(first_week_days)
@@ -102,7 +79,7 @@ RSpec.describe MigrateDailyRedisHllEventsToWeeklyAggregation, :migration, :clean
full_weeks = (31 - first_week_days) / 7
# for the next full weeks we should have value eq to 7 (ie. 7 days in a week)
(1..full_weeks).each do |i|
- new_key = Gitlab::UsageDataCounters::HLLRedisCounter.send(:redis_key, event, Date.today - i.weeks)
+ new_key = described_class.new.send(:redis_key, event, Date.today - i.weeks, :weekly)
expect(Gitlab::Redis::HLL.count(keys: new_key)).to eq(7)
expect(with_redis { |r| r.ttl(new_key) }).to be_within(600).of(weekly_expiry)
end
@@ -111,7 +88,7 @@ RSpec.describe MigrateDailyRedisHllEventsToWeeklyAggregation, :migration, :clean
last_week_days = 31 - ((full_weeks * 7) + first_week_days)
unless last_week_days.zero?
last_week = full_weeks + 1
- new_key = Gitlab::UsageDataCounters::HLLRedisCounter.send(:redis_key, event, Date.today - last_week.weeks)
+ new_key = described_class.new.send(:redis_key, event, Date.today - last_week.weeks, :weekly)
expect(Gitlab::Redis::HLL.count(keys: new_key)).to eq(last_week_days)
expect(with_redis { |r| r.ttl(new_key) }).to be_within(600).of(weekly_expiry)
end