diff options
32 files changed, 457 insertions, 133 deletions
diff --git a/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue b/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue index 7474f8f3298..764c78ff581 100644 --- a/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue +++ b/app/assets/javascripts/design_management/components/design_notes/design_reply_form.vue @@ -233,7 +233,7 @@ export default { </template> </markdown-field> <slot name="resolve-checkbox"></slot> - <div class="note-form-actions gl-display-flex"> + <div class="note-form-actions gl-display-flex gl-mt-4!"> <gl-button ref="submitButton" :disabled="!hasValue" diff --git a/app/assets/javascripts/projects/settings_service_desk/components/service_desk_root.vue b/app/assets/javascripts/projects/settings_service_desk/components/service_desk_root.vue index 650b60cba4f..baf2ce2aa9c 100644 --- a/app/assets/javascripts/projects/settings_service_desk/components/service_desk_root.vue +++ b/app/assets/javascripts/projects/settings_service_desk/components/service_desk_root.vue @@ -7,7 +7,7 @@ import { __, sprintf } from '~/locale'; import ServiceDeskSetting from './service_desk_setting.vue'; export default { - customEmailHelpPath: helpPagePath('/user/project/service_desk.html', { + serviceDeskEmailHelpPath: helpPagePath('/user/project/service_desk.html', { anchor: 'use-a-custom-email-address', }), components: { @@ -32,10 +32,10 @@ export default { initialIncomingEmail: { default: '', }, - customEmail: { + serviceDeskEmail: { default: '', }, - customEmailEnabled: { + serviceDeskEmailEnabled: { default: false, }, selectedTemplate: { @@ -65,7 +65,7 @@ export default { alertVariant: 'danger', alertMessage: '', incomingEmail: this.initialIncomingEmail, - updatedCustomEmail: this.customEmail, + updatedServiceDeskEmail: this.serviceDeskEmail, }; }, methods: { @@ -110,7 +110,7 @@ export default { return axios .put(this.endpoint, body) .then(({ data }) => { - this.updatedCustomEmail = data?.service_desk_address; + this.updatedServiceDeskEmail = data?.service_desk_address; this.showAlert(__('Changes saved.'), 'success'); }) .catch((err) => { @@ -155,7 +155,7 @@ export default { " > <template #link="{ content }"> - <gl-link :href="$options.customEmailHelpPath" target="_blank"> + <gl-link :href="$options.serviceDeskEmailHelpPath" target="_blank"> {{ content }} </gl-link> </template> @@ -168,8 +168,8 @@ export default { :is-enabled="isEnabled" :is-issue-tracker-enabled="isIssueTrackerEnabled" :incoming-email="incomingEmail" - :custom-email="updatedCustomEmail" - :custom-email-enabled="customEmailEnabled" + :service-desk-email="updatedServiceDeskEmail" + :service-desk-email-enabled="serviceDeskEmailEnabled" :initial-selected-template="selectedTemplate" :initial-selected-file-template-project-id="selectedFileTemplateProjectId" :initial-outgoing-name="outgoingName" diff --git a/app/assets/javascripts/projects/settings_service_desk/components/service_desk_setting.vue b/app/assets/javascripts/projects/settings_service_desk/components/service_desk_setting.vue index 38a2c12d137..36a49625116 100644 --- a/app/assets/javascripts/projects/settings_service_desk/components/service_desk_setting.vue +++ b/app/assets/javascripts/projects/settings_service_desk/components/service_desk_setting.vue @@ -49,12 +49,12 @@ export default { required: false, default: '', }, - customEmail: { + serviceDeskEmail: { type: String, required: false, default: '', }, - customEmailEnabled: { + serviceDeskEmailEnabled: { type: Boolean, required: false, }, @@ -101,20 +101,20 @@ export default { }, computed: { hasProjectKeySupport() { - return Boolean(this.customEmailEnabled); + return Boolean(this.serviceDeskEmailEnabled); }, email() { - return this.customEmail || this.incomingEmail; + return this.serviceDeskEmail || this.incomingEmail; }, - hasCustomEmail() { - return this.customEmail && this.customEmail !== this.incomingEmail; + hasServiceDeskEmail() { + return this.serviceDeskEmail && this.serviceDeskEmail !== this.incomingEmail; }, emailSuffixHelpUrl() { return helpPagePath('user/project/service_desk.html', { anchor: 'configure-a-custom-email-address-suffix', }); }, - customEmailAddressHelpUrl() { + serviceDeskEmailAddressHelpUrl() { return helpPagePath('user/project/service_desk.html', { anchor: 'use-a-custom-email-address', }); @@ -204,7 +204,7 @@ export default { <clipboard-button :title="__('Copy')" :text="email" css-class="input-group-text" /> </template> </gl-form-input-group> - <template v-if="email && hasCustomEmail" #description> + <template v-if="email && hasServiceDeskEmail" #description> <span class="gl-mt-2 gl-display-inline-block"> <gl-sprintf :message="__('Emails sent to %{email} are also supported.')"> <template #email> @@ -260,7 +260,7 @@ export default { > <template #link="{ content }"> <gl-link - :href="customEmailAddressHelpUrl" + :href="serviceDeskEmailAddressHelpUrl" target="_blank" class="gl-text-blue-600 font-size-inherit" >{{ content }} diff --git a/app/assets/javascripts/projects/settings_service_desk/index.js b/app/assets/javascripts/projects/settings_service_desk/index.js index 84229175c0b..0f4c747a7b6 100644 --- a/app/assets/javascripts/projects/settings_service_desk/index.js +++ b/app/assets/javascripts/projects/settings_service_desk/index.js @@ -10,8 +10,8 @@ export default () => { } const { - customEmail, - customEmailEnabled, + serviceDeskEmail, + serviceDeskEmailEnabled, enabled, issueTrackerEnabled, endpoint, @@ -27,8 +27,8 @@ export default () => { return new Vue({ el, provide: { - customEmail, - customEmailEnabled: parseBoolean(customEmailEnabled), + serviceDeskEmail, + serviceDeskEmailEnabled: parseBoolean(serviceDeskEmailEnabled), endpoint, initialIncomingEmail: incomingEmail, initialIsEnabled: parseBoolean(enabled), diff --git a/app/assets/stylesheets/page_bundles/design_management.scss b/app/assets/stylesheets/page_bundles/design_management.scss index c19561a5e5e..87ca94096cb 100644 --- a/app/assets/stylesheets/page_bundles/design_management.scss +++ b/app/assets/stylesheets/page_bundles/design_management.scss @@ -170,16 +170,12 @@ $t-gray-a-16-design-pin: rgba($black, 0.16); } .reply-wrapper { - padding: $gl-padding-8 $gl-padding-8 $gl-padding-4; + padding: $gl-padding-8; background: $gray-10; border-radius: 0 0 $border-radius-default $border-radius-default; } } - .reply-wrapper { - border-top: 1px solid var(--border-color, $border-color); - } - .new-discussion-disclaimer { line-height: 20px; } diff --git a/app/views/projects/_service_desk_settings.html.haml b/app/views/projects/_service_desk_settings.html.haml index 14991ce3824..0a83efdb3b8 100644 --- a/app/views/projects/_service_desk_settings.html.haml +++ b/app/views/projects/_service_desk_settings.html.haml @@ -12,8 +12,8 @@ enabled: "#{@project.service_desk_enabled}", issue_tracker_enabled: "#{@project.project_feature.issues_enabled?}", incoming_email: (@project.service_desk_incoming_address if @project.service_desk_enabled), - custom_email: (@project.service_desk_custom_address if @project.service_desk_enabled), - custom_email_enabled: "#{Gitlab::Email::ServiceDeskEmail.enabled?}", + service_desk_email: (@project.service_desk_custom_address if @project.service_desk_enabled), + service_desk_email_enabled: "#{Gitlab::Email::ServiceDeskEmail.enabled?}", selected_template: "#{@project.service_desk_setting&.issue_template_key}", selected_file_template_project_id: "#{@project.service_desk_setting&.file_template_project_id}", outgoing_name: "#{@project.service_desk_setting&.outgoing_name}", diff --git a/config/feature_flags/development/allow_protected_branches_for_group.yml b/config/feature_flags/development/allow_protected_branches_for_group.yml index 16b06182f7c..88695ddf245 100644 --- a/config/feature_flags/development/allow_protected_branches_for_group.yml +++ b/config/feature_flags/development/allow_protected_branches_for_group.yml @@ -4,5 +4,5 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/116779 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/383178 milestone: '15.11' type: development -group: group::compliance +group: group::source code default_enabled: false diff --git a/config/feature_flags/development/group_protected_branches.yml b/config/feature_flags/development/group_protected_branches.yml index 4d580734dc4..5c1a39a0e3b 100644 --- a/config/feature_flags/development/group_protected_branches.yml +++ b/config/feature_flags/development/group_protected_branches.yml @@ -4,5 +4,5 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/372816 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/383178 milestone: '15.7' type: development -group: group::compliance +group: group::source code default_enabled: false diff --git a/config/feature_flags/development/support_group_level_merge_checks_setting.yml b/config/feature_flags/development/support_group_level_merge_checks_setting.yml index 66cb9830261..282fa289c91 100644 --- a/config/feature_flags/development/support_group_level_merge_checks_setting.yml +++ b/config/feature_flags/development/support_group_level_merge_checks_setting.yml @@ -4,5 +4,5 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/102864 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/377723 milestone: '15.8' type: development -group: group::compliance +group: group::code review default_enabled: false diff --git a/db/docs/u2f_registrations.yml b/db/docs/deleted_tables/u2f_registrations.yml index b1aaa8148bd..a41f0f092b7 100644 --- a/db/docs/u2f_registrations.yml +++ b/db/docs/deleted_tables/u2f_registrations.yml @@ -7,4 +7,6 @@ feature_categories: description: TODO introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/791cc9138be6ea1783e3c3853370cf0290f4d41e milestone: '8.9' +removed_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/114576 +removed_in_milestone: '16.1' gitlab_schema: gitlab_main diff --git a/db/post_migrate/20230314094213_remove_foreign_keys_from_u2f_registrations_table.rb b/db/post_migrate/20230314094213_remove_foreign_keys_from_u2f_registrations_table.rb new file mode 100644 index 00000000000..d6185d6674d --- /dev/null +++ b/db/post_migrate/20230314094213_remove_foreign_keys_from_u2f_registrations_table.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class RemoveForeignKeysFromU2fRegistrationsTable < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + def up + with_lock_retries do + remove_foreign_key :u2f_registrations, :users + end + end + + def down + add_concurrent_foreign_key :u2f_registrations, :users, column: :user_id + end +end diff --git a/db/post_migrate/20230314094215_drop_u2f_registrations_table.rb b/db/post_migrate/20230314094215_drop_u2f_registrations_table.rb new file mode 100644 index 00000000000..9a57c424592 --- /dev/null +++ b/db/post_migrate/20230314094215_drop_u2f_registrations_table.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class DropU2fRegistrationsTable < Gitlab::Database::Migration[2.1] + enable_lock_retries! + + def up + drop_table :u2f_registrations + end + + def down + create_table :u2f_registrations do |t| # rubocop: disable Migration/SchemaAdditionMethodsNoPost + t.text :certificate + t.string :key_handle + t.string :public_key + t.integer :counter + t.references :user, foreign_key: false + t.datetime_with_timezone :created_at, null: false + t.datetime_with_timezone :updated_at, null: false + t.string :name + t.index [:key_handle], name: 'index_u2f_registrations_on_key_handle' + end + end +end diff --git a/db/post_migrate/20230615121103_replace_p_ci_builds_metadata_foreign_key_v3.rb b/db/post_migrate/20230615121103_replace_p_ci_builds_metadata_foreign_key_v3.rb new file mode 100644 index 00000000000..dca891fc064 --- /dev/null +++ b/db/post_migrate/20230615121103_replace_p_ci_builds_metadata_foreign_key_v3.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class ReplacePCiBuildsMetadataForeignKeyV3 < Gitlab::Database::Migration[2.1] + include Gitlab::Database::PartitioningMigrationHelpers + include Gitlab::Database::MigrationHelpers::WraparoundAutovacuum + + disable_ddl_transaction! + + def up + return unless should_run? + + add_concurrent_partitioned_foreign_key :p_ci_builds_metadata, :p_ci_builds, + name: :temp_fk_e20479742e_p, + column: [:partition_id, :build_id], + target_column: [:partition_id, :id], + on_update: :cascade, + on_delete: :cascade, + validate: true, + reverse_lock_order: true + end + + def down + return unless should_run? + + with_lock_retries do + remove_foreign_key_if_exists :p_ci_builds_metadata, :p_ci_builds, + name: :temp_fk_e20479742e_p, + reverse_lock_order: true + end + end + + private + + def should_run? + can_execute_on?(:ci_builds_metadata, :ci_builds) + end +end diff --git a/db/post_migrate/20230615121122_replace_p_ci_runner_machine_builds_foreign_key_v3.rb b/db/post_migrate/20230615121122_replace_p_ci_runner_machine_builds_foreign_key_v3.rb new file mode 100644 index 00000000000..854d7358e5e --- /dev/null +++ b/db/post_migrate/20230615121122_replace_p_ci_runner_machine_builds_foreign_key_v3.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class ReplacePCiRunnerMachineBuildsForeignKeyV3 < Gitlab::Database::Migration[2.1] + include Gitlab::Database::PartitioningMigrationHelpers + include Gitlab::Database::MigrationHelpers::WraparoundAutovacuum + + disable_ddl_transaction! + + def up + return unless should_run? + + add_concurrent_partitioned_foreign_key :p_ci_runner_machine_builds, :p_ci_builds, + name: :temp_fk_bb490f12fe_p, + column: [:partition_id, :build_id], + target_column: [:partition_id, :id], + on_update: :cascade, + on_delete: :cascade, + validate: true, + reverse_lock_order: true + end + + def down + return unless should_run? + + with_lock_retries do + remove_foreign_key_if_exists :p_ci_runner_machine_builds, :p_ci_builds, + name: :temp_fk_bb490f12fe_p, + reverse_lock_order: true + end + end + + private + + def should_run? + can_execute_on?(:ci_builds) + end +end diff --git a/db/schema_migrations/20230314094213 b/db/schema_migrations/20230314094213 new file mode 100644 index 00000000000..e2741f5ea30 --- /dev/null +++ b/db/schema_migrations/20230314094213 @@ -0,0 +1 @@ +ccf7548dc1916a60da14f9545039d7ac3666e44e96ccd59c0f6db4117b15515a
\ No newline at end of file diff --git a/db/schema_migrations/20230314094215 b/db/schema_migrations/20230314094215 new file mode 100644 index 00000000000..a030527fac4 --- /dev/null +++ b/db/schema_migrations/20230314094215 @@ -0,0 +1 @@ +5549548b53710624294aed31af6ed8aeb26263e538d2e2d7eac254a5c105d08a
\ No newline at end of file diff --git a/db/schema_migrations/20230615121103 b/db/schema_migrations/20230615121103 new file mode 100644 index 00000000000..9c77e1512c1 --- /dev/null +++ b/db/schema_migrations/20230615121103 @@ -0,0 +1 @@ +37634ac8f82ed0fa043036b1aab2dffbd7a4f9a2fc51f18d53511f8a834eb312
\ No newline at end of file diff --git a/db/schema_migrations/20230615121122 b/db/schema_migrations/20230615121122 new file mode 100644 index 00000000000..0704b5d0c8d --- /dev/null +++ b/db/schema_migrations/20230615121122 @@ -0,0 +1 @@ +7014d18db20d925d1bc41b4104b1ce57f2033f7c8debed8fd31e6afbba8efd78
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 34927b960c3..ebd2c9a0424 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -23330,27 +23330,6 @@ CREATE SEQUENCE trending_projects_id_seq ALTER SEQUENCE trending_projects_id_seq OWNED BY trending_projects.id; -CREATE TABLE u2f_registrations ( - id integer NOT NULL, - certificate text, - key_handle character varying, - public_key character varying, - counter integer, - user_id integer, - created_at timestamp without time zone NOT NULL, - updated_at timestamp without time zone NOT NULL, - name character varying -); - -CREATE SEQUENCE u2f_registrations_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE u2f_registrations_id_seq OWNED BY u2f_registrations.id; - CREATE TABLE upcoming_reconciliations ( id bigint NOT NULL, namespace_id bigint, @@ -25882,8 +25861,6 @@ ALTER TABLE ONLY topics ALTER COLUMN id SET DEFAULT nextval('topics_id_seq'::reg ALTER TABLE ONLY trending_projects ALTER COLUMN id SET DEFAULT nextval('trending_projects_id_seq'::regclass); -ALTER TABLE ONLY u2f_registrations ALTER COLUMN id SET DEFAULT nextval('u2f_registrations_id_seq'::regclass); - ALTER TABLE ONLY upcoming_reconciliations ALTER COLUMN id SET DEFAULT nextval('upcoming_reconciliations_id_seq'::regclass); ALTER TABLE ONLY upload_states ALTER COLUMN upload_id SET DEFAULT nextval('upload_states_upload_id_seq'::regclass); @@ -28384,9 +28361,6 @@ ALTER TABLE ONLY topics ALTER TABLE ONLY trending_projects ADD CONSTRAINT trending_projects_pkey PRIMARY KEY (id); -ALTER TABLE ONLY u2f_registrations - ADD CONSTRAINT u2f_registrations_pkey PRIMARY KEY (id); - ALTER TABLE ONLY upcoming_reconciliations ADD CONSTRAINT upcoming_reconciliations_pkey PRIMARY KEY (id); @@ -32992,10 +32966,6 @@ CREATE INDEX index_topics_total_projects_count ON topics USING btree (total_proj CREATE UNIQUE INDEX index_trending_projects_on_project_id ON trending_projects USING btree (project_id); -CREATE INDEX index_u2f_registrations_on_key_handle ON u2f_registrations USING btree (key_handle); - -CREATE INDEX index_u2f_registrations_on_user_id ON u2f_registrations USING btree (user_id); - CREATE UNIQUE INDEX index_uniq_ci_runners_on_token ON ci_runners USING btree (token); CREATE UNIQUE INDEX index_uniq_ci_runners_on_token_encrypted ON ci_runners USING btree (token_encrypted); @@ -37951,9 +37921,6 @@ ALTER TABLE ONLY timelogs ALTER TABLE ONLY timelogs ADD CONSTRAINT fk_timelogs_note_id FOREIGN KEY (note_id) REFERENCES notes(id) ON DELETE SET NULL; -ALTER TABLE ONLY u2f_registrations - ADD CONSTRAINT fk_u2f_registrations_user_id FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; - ALTER TABLE issue_search_data ADD CONSTRAINT issue_search_data_issue_id_fkey FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE; @@ -37963,8 +37930,11 @@ ALTER TABLE issue_search_data ALTER TABLE product_analytics_events_experimental ADD CONSTRAINT product_analytics_events_experimental_project_id_fkey FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; -ALTER TABLE ONLY ci_builds_metadata - ADD CONSTRAINT temp_fk_e20479742e_p FOREIGN KEY (partition_id, build_id) REFERENCES p_ci_builds(partition_id, id) ON UPDATE CASCADE ON DELETE CASCADE NOT VALID; +ALTER TABLE p_ci_runner_machine_builds + ADD CONSTRAINT temp_fk_bb490f12fe_p FOREIGN KEY (partition_id, build_id) REFERENCES p_ci_builds(partition_id, id) ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE p_ci_builds_metadata + ADD CONSTRAINT temp_fk_e20479742e_p FOREIGN KEY (partition_id, build_id) REFERENCES p_ci_builds(partition_id, id) ON UPDATE CASCADE ON DELETE CASCADE; ALTER TABLE ONLY user_follow_users ADD CONSTRAINT user_follow_users_followee_id_fkey FOREIGN KEY (followee_id) REFERENCES users(id) ON DELETE CASCADE; diff --git a/doc/architecture/blueprints/container_registry_metadata_database_self_managed_rollout/index.md b/doc/architecture/blueprints/container_registry_metadata_database_self_managed_rollout/index.md new file mode 100644 index 00000000000..6cd64a42e85 --- /dev/null +++ b/doc/architecture/blueprints/container_registry_metadata_database_self_managed_rollout/index.md @@ -0,0 +1,238 @@ +--- +status: proposed +creation-date: "2023-06-09" +authors: [ "@hswimelar" ] +coach: "@grzesiek" +approvers: [ "@trizzi ", "@sgoldstein" ] +owning-stage: "~devops::package" +participating-stages: [] +--- + +<!-- Blueprints often contain forward-looking statements --> +<!-- vale gitlab.FutureTense = NO --> + +# Container Registry Self-Managed Database Rollout + +## Summary + +The latest iteration of the [Container Registry](https://gitlab.com/gitlab-org/container-registry) +has been rearchitected to use a PostgreSQL database and deployed on GitLab.com. +Now we must bring the advantages provided by the database to self-managed users. +While the container registry retains the capacity to run without the new database, +many new and highly desired features cannot be implemented without it. +Additionally, unifying the registry used for GitLab.com and for self-managed +allows us to provide a cohesive user experience and reduces the burden +associated with maintaining the old registry implementation. To accomplish this, +we plan to eventually require all self-managed to migrate to the new registry +database, so that we may deprecate and remove support for the old object storage +metadata subsystem. + +This document seeks to describe how we may use the proven core migration +functionality, which was used to migrate millions of container images on GitLab.com, +to enable self-managed users to enjoy the benefits of the metadata database. + +## Motivation + +Enabling self-managed users to migrate to the new metadata database allows these +users to take advantage of the new features that require the database. Additionally, +the greater adoption of the database allows the container registry team to focus +our knowledge and capacity, and will eventually allow us to fully remove the old +registry metadata subsystem, greatly improving the maintainability and stability +of the container registry for both GitLab.com and for self-managed users. + +### Goals + +- Progressively rollout the new dependency of a PostgreSQL database instance for the registry for charts and omnibus deployments. +- Progressively rollout automation for the registry PostgreSQL database instance for charts and omnibus deployments. +- Develop processes and tools that self-managed admins can use to migrate existing registry deployments to the metadata database. +- Develop processes and tools that self-managed admins can use spin up fresh installs of the Container Registry which use the metadata database. +- Create a plan which will eventually allow us to fully drop support for original object storage metadata subsystem. + +### Non-Goals + +- Developing new Container Registry features outside the scope of enabling admins to migrate to the metadata database. +- Determining lifecycle support decisions, such as when to default to the database, and when to end support for non-database registries. + +## Proposal + +There are two main components that must be further developed in order for +self-managed admins to move to the registry database: the deployment environment and +the registry migration tooling. + +For the deployment environments need to document what the user needs to do to set up their +deployment such that the registry has access to a suitable database given the +expected registry workload. As well as develop tooling and automation to ease +the setup and maintenance of the registry database for new and existing deploys. + +For the registry, we need to develop and validate import tooling which +coordinates with the core import functionality which was used to migrate all +container images on GitLab.com. Additionally, we must validate that each supported +storage driver works as expected with the import process and provide estimated +import times for admins. + +We can structure our work to meet the standards outlined in support for +Experiment, Beta, and Alpha features. Doing so will help to prioritize core +functionality and to allow users who wish to be early adopters to begin using +the database and providing us with invaluable feedback. + +These levels of support could be advertised to self-managed users via a simple +chart, allowing them to tell at a glance the status of this project as it relates +to their situation. + +| Installation | GCS | AWS | Filesystem | Azure | OSS | Swift| +| ------ | ------ |------ | ------ | ------ |------ | ------ | +| Omnibus | GA | GA | Beta | Experimental | Experimental | Experimental | +| Charts | GA | GA |Beta | Experimental | Experimental | Experimental | + +### Justification of Structuring Support by Driver + +It's possible that we could simplify the proposed support matrix by structuring +it only by deployment environment and not differentiate by storage driver. The +following two sections briefly summarize several points for and against. + +#### Arguments Opposed to Structuring Support by Driver + +Each storage driver is well abstracted in the code, specifically the import process +makes use of the following Methods: + +- Walk +- List +- GetContent +- Stat +- Reader + +Each of the methods is a read method we do not need to create or delete data via +the object storage methods. Additionally, all of these methods are standard API +methods. + +Given that we're not mutating data via object storage as part of the import +process, we should not need to double-check these drivers or try to predict +potential errors. Relying on user feedback during the beta to direct any efforts +we should be making here could prevent us from scheduling unnecessary work. + +#### Arguments in Favor of Structuring Support by Driver + +Our experience with enhancing and supporting offline garbage collection has +shown that while the storage driver implementation should not matter, it does. +The drivers have proven to have important differences in performance and +reliability. Many of the planned possible driver-related improvements are +related to testing and stability, rather than outright new work for each driver. + +In particular, retries and error reporting across storage drivers are not as +standardized as one would hope for, and therefore there is a potential that a +long-running import process could be interrupted by an error that could have +been retried. + +Creating import estimates based on combinations of the registry size and storage +driver, would also be of use to self-managed admins, looking to schedule their +migration. There will be a difference here between local filesystem storage and +object storage and there could be a difference between the object storage +providers as well. + +Also, we could work with the importer to smooth out the differences in the +storage drivers. Even without unified retryable error reporting from the storage +drivers, we could have the importer retry more time and for more errors. There's +a risk we would retry several times on non-retryable errors, but since no writes +are being made to object storage, this should not ultimately be harmful. + +Additionally, implementing [Validate Self-Managed Imports](https://gitlab.com/gitlab-org/container-registry/-/issues/938) +would perform a consistency check against a sample of images before and after +import which would lead to greater consistency across all storage driver implementations. + +## Design and Implementation Details + +### The Import Tool + +The [import tool](https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/database-import-tool.md) +is a well-validated component of the Container Registry project that we have used +from the beginning as a way to perform local testing. This tool is a thin wrapper +over the core import functionality — the code which handles the import logic has +been extensively validated. + +While the core import functionality is solid, we must ensure that this tool and +the surrounding process will enable non-expert users to import their registries +with both minimal risk and with minimal support from GitLab team members. +Therefore, the most important work remaining is crafting the UX of this tooling +such that those goals are met. This +[epic](https://gitlab.com/groups/gitlab-org/-/epics/8602) captures many of the +proposed improvements. + +#### Design + +The tool is designed such that a single execution flow can support both users +with large registries with strict uptime requirements who can take advantage of +a more involved process to reduce read-only time to the absolute minimum as well +as users with small registries who benefit from a streamlined workflow. This is +achieved via the same pre import, then full import cycle that was used on +GitLab.com, along with an additional step to catalog all unreferenced blobs held +in common storage. + +##### One-Shot Import + +In most cases, a user can simply choose to run the import tool while the registry +is offline or read-only in mode. This will be similar to what admins must +already do in order to run offline garbage collection. Each step completes in +sequence, moving directly to the next. The command exits when the import process +is complete and the registry is ready to make full use of the metadata database. + +##### Minimal Downtime Import + +For users with large registries and who are interested in the minimum possible +downtime, each step can be ran independently when the tool is passed the appropriate +flag. The user will first run the pre-import step while the registry is +performing its normal workload. Once that has completed, and the user is ready +to stop writes to the registry, the tag import step can be ran. As with the GitLab.com +migration, importing tags requires that the registry be offline or in +read-only mode. This step does the minimum possible work to achieve fast and +efficient tag imports and will always be the fastest of the three steps, reducing +the downtime component to a fraction of the total import time. The user can then +bring up the registry configured to use the metadata database. After that, the +user is free to run the third step during normal registry operations. This step +makes any dangling blobs in common storage visible to the database and therefore +the online garbage collection process. + +### Distribution Paths + +Tooling, process, and documentation will need to be developed in order to +support users who wish to use the metadata database, especially in regards to +providing a foundation for the new database instance required for the migration. + +For new deployments, we should wait until we've moved to general support, have +automation in place for the registry database and migration, and have a major +GitLab version bump before enabling the database by default for self-managed. + +#### Omnibus + +#### Charts + +## Alternative Solutions + +### Do Nothing + +#### Pros + +- The database and associated features are generally most useful for large-scale, high-availability deployments. +- Eliminate the need to support an additional logical or physical database for self-managed deployments. + +#### Cons + +- The registry on GitLab.com and the registry used by self-managed will greatly diverge in supported features over time. +- The maintenance burden of supporting two registry implementations will reduce the velocity at which new registry features can be released. +- The registry on GitLab.com stops being an effective way to validate changes before they are released to self-managed. +- Large self-managed users continue to not be able to scale the registry to suit their needs. + +### Gradual Migration + +This approach would be to exactly replicate the GitLab.com migration on +self-managed. + +#### Pros + +- Replicate an already successful process. +- Scope downtime by repository, rather than instance. + +#### Cons + +- Dramatically increased complexity in all aspects of the migration process. +- Greatly increased possibility of data consistency issues. +- Less clear demarcation of registry migration progress. diff --git a/doc/development/documentation/styleguide/word_list.md b/doc/development/documentation/styleguide/word_list.md index 0bbd679abe5..a4eb457643e 100644 --- a/doc/development/documentation/styleguide/word_list.md +++ b/doc/development/documentation/styleguide/word_list.md @@ -862,7 +862,23 @@ Do not use `master`. Use `main` when you need a sample [default branch name](#de ## may, might -**Might** means something has the probability of occurring. **May** gives permission to do something. Consider **can** instead of **may**. +**Might** means something has the probability of occurring. Might is often used in troubleshooting documentation. + +**May** gives permission to do something. Consider **can** instead of **may**. + +Consider rewording phrases that use these terms. These terms often indicate possibility and doubt, and technical writing strives to be precise. + +See also [you can](#you-can). + +Use: + +- The `committed_date` and `authored_date` fields are generated from different sources, and might not be identical. +- A typical pipeline consists of four stages, executed in the following order: + +Instead of: + +- The `committed_date` and `authored_date` fields are generated from different sources, and may not be identical. +- A typical pipeline might consist of four stages, executed in the following order: ## MB, megabytes @@ -1616,6 +1632,7 @@ For example: - Use code review analytics to view merge request data. - Create a board to organize your team tasks. - Configure variables to restrict pushes to a repository. +- Add links to external accounts you have, like Skype and Twitter. Use **you can** for optional actions. For example: diff --git a/doc/user/application_security/policies/scan-execution-policies.md b/doc/user/application_security/policies/scan-execution-policies.md index e18d4d1787e..0e51cb93da3 100644 --- a/doc/user/application_security/policies/scan-execution-policies.md +++ b/doc/user/application_security/policies/scan-execution-policies.md @@ -91,6 +91,7 @@ This rule enforces the defined actions and schedules a scan on the provided date | `branches` | `array` of `string` | `*` or the branch's name | The branch the given policy applies to (supports wildcard). This field is required if the `agents` field is not set. Cannot be used with the `branch_type` field. | | `branch_type` | `string` | `default`, `protected` or `all` | The types of branches the given policy applies to. Cannot be used with the `branches` field. | | `cadence` | `string` | CRON expression (for example, `0 0 * * *`) | A whitespace-separated string containing five fields that represents the scheduled time. Minimum of 15 minute intervals when used together with the `branches` field. | +| `timezone` | `string` | Time zone identifier (for example, `America/New_York`) | Time zone to apply to the cadence. Value must be an IANA Time Zone Database identifier. | | `agents` | `object` | | The name of the [GitLab agents](../../clusters/agent/index.md) where [Operational Container Scanning](../../clusters/agent/vulnerabilities.md) runs. The object key is the name of the Kubernetes agent configured for your project in GitLab. This field is required if the `branches` field is not set. | GitLab supports the following types of CRON syntax for the `cadence` field: diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 0bd6acd65e3..c66bcb2093e 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -41546,6 +41546,9 @@ msgstr "" msgid "SecurityOrchestration|This view only shows scan results for the agent %{agent}. You can view scan results for all agents in the %{linkStart}Operational Vulnerabilities tab of the vulnerability report%{linkEnd}." msgstr "" +msgid "SecurityOrchestration|Timezone is invalid" +msgstr "" + msgid "SecurityOrchestration|To widen your search, change filters above or select a different security policy project." msgstr "" diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/maven/maven_project_level_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/maven/maven_project_level_spec.rb index 91d8f0dc181..d87be118dfe 100644 --- a/qa/qa/specs/features/browser_ui/5_package/package_registry/maven/maven_project_level_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/maven/maven_project_level_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Package', :object_storage, except: { job: 'relative-url' } do + RSpec.describe 'Package', :object_storage, :skip_live_env, except: { job: 'relative-url' } do describe 'Maven project level endpoint', product_group: :package_registry do include Runtime::Fixtures include Support::Helpers::MaskToken diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/maven_gradle_repository_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/maven_gradle_repository_spec.rb index 032c77b2519..e961bc41dc0 100644 --- a/qa/qa/specs/features/browser_ui/5_package/package_registry/maven_gradle_repository_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/maven_gradle_repository_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Package', :object_storage, except: { job: 'relative-url' }, product_group: :package_registry do + RSpec.describe 'Package', :object_storage, :skip_live_env, except: { job: 'relative-url' }, product_group: :package_registry do describe 'Maven Repository with Gradle' do using RSpec::Parameterized::TableSyntax include Runtime::Fixtures diff --git a/qa/qa/specs/features/browser_ui/5_package/package_registry/pypi_repository_spec.rb b/qa/qa/specs/features/browser_ui/5_package/package_registry/pypi_repository_spec.rb index 63549113517..f29a9691849 100644 --- a/qa/qa/specs/features/browser_ui/5_package/package_registry/pypi_repository_spec.rb +++ b/qa/qa/specs/features/browser_ui/5_package/package_registry/pypi_repository_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module QA - RSpec.describe 'Package', :object_storage, except: { job: 'relative-url' }, product_group: :package_registry do + RSpec.describe 'Package', :object_storage, :skip_live_env, except: { job: 'relative-url' }, product_group: :package_registry do describe 'PyPI Repository' do include Runtime::Fixtures include Support::Helpers::MaskToken diff --git a/spec/frontend/monitoring/components/dashboards_dropdown_spec.js b/spec/frontend/monitoring/components/dashboards_dropdown_spec.js index 3ccaa2d28ac..b769faa2d16 100644 --- a/spec/frontend/monitoring/components/dashboards_dropdown_spec.js +++ b/spec/frontend/monitoring/components/dashboards_dropdown_spec.js @@ -1,4 +1,4 @@ -import { GlDropdownItem, GlIcon } from '@gitlab/ui'; +import { GlDropdownItem, GlIcon, GlSearchBoxByType } from '@gitlab/ui'; import { shallowMount } from '@vue/test-utils'; import { nextTick } from 'vue'; @@ -38,9 +38,8 @@ describe('DashboardsDropdown', () => { const findSearchInput = () => wrapper.findComponent({ ref: 'monitorDashboardsDropdownSearch' }); const findNoItemsMsg = () => wrapper.findComponent({ ref: 'monitorDashboardsDropdownMsg' }); const findStarredListDivider = () => wrapper.findComponent({ ref: 'starredListDivider' }); - // setData usage is discouraged. See https://gitlab.com/groups/gitlab-org/-/epics/7330 for details - // eslint-disable-next-line no-restricted-syntax - const setSearchTerm = (searchTerm) => wrapper.setData({ searchTerm }); + const setSearchTerm = (searchTerm) => + wrapper.findComponent(GlSearchBoxByType).vm.$emit('input', searchTerm); beforeEach(() => { mockDashboards = dashboardGitResponse; diff --git a/spec/frontend/notes/components/comment_form_spec.js b/spec/frontend/notes/components/comment_form_spec.js index 6c774a1ecd0..0179d2afa9e 100644 --- a/spec/frontend/notes/components/comment_form_spec.js +++ b/spec/frontend/notes/components/comment_form_spec.js @@ -327,9 +327,8 @@ describe('issue_comment_form component', () => { jest.spyOn(wrapper.vm, 'stopPolling'); jest.spyOn(wrapper.vm, 'saveNote').mockResolvedValue(); - // setData usage is discouraged. See https://gitlab.com/groups/gitlab-org/-/epics/7330 for details - // eslint-disable-next-line no-restricted-syntax - await wrapper.setData({ note: 'hello world' }); + findMarkdownEditor().vm.$emit('input', 'hello world'); + await nextTick(); await findCommentButton().trigger('click'); @@ -459,9 +458,8 @@ describe('issue_comment_form component', () => { it('should enable comment button if it has note', async () => { mountComponent(); - // setData usage is discouraged. See https://gitlab.com/groups/gitlab-org/-/epics/7330 for details - // eslint-disable-next-line no-restricted-syntax - await wrapper.setData({ note: 'Foo' }); + findMarkdownEditor().vm.$emit('input', 'Foo'); + await nextTick(); expect(findCommentTypeDropdown().props('disabled')).toBe(false); }); diff --git a/spec/frontend/projects/settings_service_desk/components/service_desk_root_spec.js b/spec/frontend/projects/settings_service_desk/components/service_desk_root_spec.js index 7f6ecbac748..784a8dec975 100644 --- a/spec/frontend/projects/settings_service_desk/components/service_desk_root_spec.js +++ b/spec/frontend/projects/settings_service_desk/components/service_desk_root_spec.js @@ -13,8 +13,8 @@ describe('ServiceDeskRoot', () => { let spy; const provideData = { - customEmail: 'custom.email@example.com', - customEmailEnabled: true, + serviceDeskEmail: 'custom.email@example.com', + serviceDeskEmailEnabled: true, endpoint: '/gitlab-org/gitlab-test/service_desk', initialIncomingEmail: 'servicedeskaddress@example.com', initialIsEnabled: true, @@ -52,8 +52,8 @@ describe('ServiceDeskRoot', () => { wrapper = createComponent(); expect(wrapper.findComponent(ServiceDeskSetting).props()).toEqual({ - customEmail: provideData.customEmail, - customEmailEnabled: provideData.customEmailEnabled, + serviceDeskEmail: provideData.serviceDeskEmail, + serviceDeskEmailEnabled: provideData.serviceDeskEmailEnabled, incomingEmail: provideData.initialIncomingEmail, initialOutgoingName: provideData.outgoingName, initialProjectKey: provideData.projectKey, diff --git a/spec/frontend/projects/settings_service_desk/components/service_desk_setting_spec.js b/spec/frontend/projects/settings_service_desk/components/service_desk_setting_spec.js index 5631927cc2f..260fd200f03 100644 --- a/spec/frontend/projects/settings_service_desk/components/service_desk_setting_spec.js +++ b/spec/frontend/projects/settings_service_desk/components/service_desk_setting_spec.js @@ -134,26 +134,26 @@ describe('ServiceDeskSetting', () => { }); }); - describe('with customEmail', () => { - describe('customEmail is different than incomingEmail', () => { + describe('with serviceDeskEmail', () => { + describe('serviceDeskEmail is different than incomingEmail', () => { const incomingEmail = 'foo@bar.com'; - const customEmail = 'custom@bar.com'; + const serviceDeskEmail = 'servicedesk@bar.com'; beforeEach(() => { wrapper = createComponent({ - props: { incomingEmail, customEmail }, + props: { incomingEmail, serviceDeskEmail }, }); }); - it('should see custom email', () => { - expect(findIncomingEmail().element.value).toEqual(customEmail); + it('should see service desk email', () => { + expect(findIncomingEmail().element.value).toEqual(serviceDeskEmail); }); }); describe('project suffix', () => { it('input is hidden', () => { wrapper = createComponent({ - props: { customEmailEnabled: false }, + props: { serviceDeskEmailEnabled: false }, }); const input = wrapper.findByTestId('project-suffix'); @@ -163,7 +163,7 @@ describe('ServiceDeskSetting', () => { it('input is enabled', () => { wrapper = createComponent({ - props: { customEmailEnabled: true }, + props: { serviceDeskEmailEnabled: true }, }); const input = wrapper.findByTestId('project-suffix'); @@ -174,7 +174,7 @@ describe('ServiceDeskSetting', () => { it('shows error when value contains uppercase or special chars', async () => { wrapper = createComponent({ - props: { email: 'foo@bar.com', customEmailEnabled: true }, + props: { email: 'foo@bar.com', serviceDeskEmailEnabled: true }, }); const input = wrapper.findByTestId('project-suffix'); @@ -189,16 +189,16 @@ describe('ServiceDeskSetting', () => { }); }); - describe('customEmail is the same as incomingEmail', () => { + describe('serviceDeskEmail is the same as incomingEmail', () => { const email = 'foo@bar.com'; beforeEach(() => { wrapper = createComponent({ - props: { incomingEmail: email, customEmail: email }, + props: { incomingEmail: email, serviceDeskEmail: email }, }); }); - it('should see custom email', () => { + it('should see service desk email', () => { expect(findIncomingEmail().element.value).toEqual(email); }); }); diff --git a/spec/frontend/vue_shared/components/filtered_search_bar/filtered_search_bar_root_spec.js b/spec/frontend/vue_shared/components/filtered_search_bar/filtered_search_bar_root_spec.js index c0cb17f0d16..00a412d9de8 100644 --- a/spec/frontend/vue_shared/components/filtered_search_bar/filtered_search_bar_root_spec.js +++ b/spec/frontend/vue_shared/components/filtered_search_bar/filtered_search_bar_root_spec.js @@ -125,46 +125,23 @@ describe('FilteredSearchBarRoot', () => { }); describe('sortDirectionIcon', () => { - it('returns string "sort-lowest" when `selectedSortDirection` is "ascending"', () => { - // setData usage is discouraged. See https://gitlab.com/groups/gitlab-org/-/epics/7330 for details - // eslint-disable-next-line no-restricted-syntax - wrapper.setData({ - selectedSortDirection: SORT_DIRECTION.ascending, - }); - - expect(wrapper.vm.sortDirectionIcon).toBe('sort-lowest'); - }); - - it('returns string "sort-highest" when `selectedSortDirection` is "descending"', () => { - // setData usage is discouraged. See https://gitlab.com/groups/gitlab-org/-/epics/7330 for details - // eslint-disable-next-line no-restricted-syntax - wrapper.setData({ - selectedSortDirection: SORT_DIRECTION.descending, + it('renders `sort-highest` descending icon by default', () => { + expect(findGlButton().props('icon')).toBe('sort-highest'); + expect(findGlButton().attributes()).toMatchObject({ + 'aria-label': 'Sort direction: Descending', + title: 'Sort direction: Descending', }); - - expect(wrapper.vm.sortDirectionIcon).toBe('sort-highest'); }); - }); - describe('sortDirectionTooltip', () => { - it('returns string "Sort direction: Ascending" when `selectedSortDirection` is "ascending"', () => { - // setData usage is discouraged. See https://gitlab.com/groups/gitlab-org/-/epics/7330 for details - // eslint-disable-next-line no-restricted-syntax - wrapper.setData({ - selectedSortDirection: SORT_DIRECTION.ascending, - }); - - expect(wrapper.vm.sortDirectionTooltip).toBe('Sort direction: Ascending'); - }); + it('renders `sort-lowest` ascending icon when the sort button is clicked', async () => { + findGlButton().vm.$emit('click'); + await nextTick(); - it('returns string "Sort direction: Descending" when `selectedSortDirection` is "descending"', () => { - // setData usage is discouraged. See https://gitlab.com/groups/gitlab-org/-/epics/7330 for details - // eslint-disable-next-line no-restricted-syntax - wrapper.setData({ - selectedSortDirection: SORT_DIRECTION.descending, + expect(findGlButton().props('icon')).toBe('sort-lowest'); + expect(findGlButton().attributes()).toMatchObject({ + 'aria-label': 'Sort direction: Ascending', + title: 'Sort direction: Ascending', }); - - expect(wrapper.vm.sortDirectionTooltip).toBe('Sort direction: Descending'); }); }); diff --git a/spec/support/webmock.rb b/spec/support/webmock.rb index 171c7ace2d2..1df92ce70cf 100644 --- a/spec/support/webmock.rb +++ b/spec/support/webmock.rb @@ -26,7 +26,14 @@ end def allowed_host_and_ip(url) host = URI.parse(url).host ip_address = Addrinfo.ip(host).ip_address - [host, ip_address] + + allowed = [host, ip_address] + + # Sometimes IPv6 address has square brackets around it + parsed_ip = IPAddr.new(ip_address) + allowed << "[#{ip_address}]" if parsed_ip.ipv6? + + allowed end def with_net_connect_allowed |