diff options
Diffstat (limited to 'lib/gitlab/background_migration')
17 files changed, 253 insertions, 313 deletions
diff --git a/lib/gitlab/background_migration/backfill_namespace_settings.rb b/lib/gitlab/background_migration/backfill_namespace_settings.rb new file mode 100644 index 00000000000..a391d5f4ebe --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_settings.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfillnamespace_settings for a range of namespaces + class BackfillNamespaceSettings + def perform(start_id, end_id) + ActiveRecord::Base.connection.execute <<~SQL + INSERT INTO namespace_settings (namespace_id, created_at, updated_at) + SELECT namespaces.id, now(), now() + FROM namespaces + WHERE namespaces.id BETWEEN #{start_id} AND #{end_id} + ON CONFLICT (namespace_id) DO NOTHING; + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb b/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb index 54f77f184d5..91b50c1a493 100644 --- a/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb +++ b/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb @@ -2,7 +2,7 @@ module Gitlab module BackgroundMigration - # Base class for cleaning up concurrent schema changes. + # Base class for background migration for rename/type changes. class CleanupConcurrentSchemaChange include Database::MigrationHelpers @@ -10,7 +10,7 @@ module Gitlab # old_column - The name of the old (to drop) column. # new_column - The name of the new column. def perform(table, old_column, new_column) - return unless column_exists?(table, new_column) + return unless column_exists?(table, new_column) && column_exists?(table, old_column) rows_to_migrate = define_model_for(table) .where(new_column => nil) @@ -28,6 +28,10 @@ module Gitlab end end + def cleanup_concurrent_schema_change(_table, _old_column, _new_column) + raise NotImplementedError + end + # These methods are necessary so we can re-use the migration helpers in # this class. def connection diff --git a/lib/gitlab/background_migration/digest_column.rb b/lib/gitlab/background_migration/digest_column.rb deleted file mode 100644 index 22a3bb8f8f3..00000000000 --- a/lib/gitlab/background_migration/digest_column.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -# rubocop:disable Style/Documentation -module Gitlab - module BackgroundMigration - class DigestColumn - class PersonalAccessToken < ActiveRecord::Base - self.table_name = 'personal_access_tokens' - end - - def perform(model, attribute_from, attribute_to, start_id, stop_id) - model = model.constantize if model.is_a?(String) - - model.transaction do - relation = model.where(id: start_id..stop_id).where.not(attribute_from => nil).lock - - relation.each do |instance| - instance.update_columns(attribute_to => Gitlab::CryptoHelper.sha256(instance.read_attribute(attribute_from)), - attribute_from => nil) - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/encrypt_columns.rb b/lib/gitlab/background_migration/encrypt_columns.rb deleted file mode 100644 index 173543b7c25..00000000000 --- a/lib/gitlab/background_migration/encrypt_columns.rb +++ /dev/null @@ -1,104 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # EncryptColumn migrates data from an unencrypted column - `foo`, say - to - # an encrypted column - `encrypted_foo`, say. - # - # To avoid depending on a particular version of the model in app/, add a - # model to `lib/gitlab/background_migration/models/encrypt_columns` and use - # it in the migration that enqueues the jobs, so code can be shared. - # - # For this background migration to work, the table that is migrated _has_ to - # have an `id` column as the primary key. Additionally, the encrypted column - # should be managed by attr_encrypted, and map to an attribute with the same - # name as the unencrypted column (i.e., the unencrypted column should be - # shadowed), unless you want to define specific methods / accessors in the - # temporary model in `/models/encrypt_columns/your_model.rb`. - # - class EncryptColumns - def perform(model, attributes, from, to) - model = model.constantize if model.is_a?(String) - - # If sidekiq hasn't undergone a restart, its idea of what columns are - # present may be inaccurate, so ensure this is as fresh as possible - model.reset_column_information - model.define_attribute_methods - - attributes = expand_attributes(model, Array(attributes).map(&:to_sym)) - - model.transaction do - # Use SELECT ... FOR UPDATE to prevent the value being changed while - # we are encrypting it - relation = model.where(id: from..to).lock - - relation.each do |instance| - encrypt!(instance, attributes) - end - end - end - - def clear_migrated_values? - true - end - - private - - # Build a hash of { attribute => encrypted column name } - def expand_attributes(klass, attributes) - expanded = attributes.flat_map do |attribute| - attr_config = klass.encrypted_attributes[attribute] - crypt_column_name = attr_config&.fetch(:attribute) - - raise "Couldn't determine encrypted column for #{klass}##{attribute}" if - crypt_column_name.nil? - - raise "#{klass} source column: #{attribute} is missing" unless - klass.column_names.include?(attribute.to_s) - - # Running the migration without the destination column being present - # leads to data loss - raise "#{klass} destination column: #{crypt_column_name} is missing" unless - klass.column_names.include?(crypt_column_name.to_s) - - [attribute, crypt_column_name] - end - - Hash[*expanded] - end - - # Generate ciphertext for each column and update the database - def encrypt!(instance, attributes) - to_clear = attributes - .map { |plain, crypt| apply_attribute!(instance, plain, crypt) } - .compact - .flat_map { |plain| [plain, nil] } - - to_clear = Hash[*to_clear] - - if instance.changed? - instance.save! - - if clear_migrated_values? - instance.update_columns(to_clear) - end - end - end - - def apply_attribute!(instance, plain_column, crypt_column) - plaintext = instance[plain_column] - ciphertext = instance[crypt_column] - - # No need to do anything if the plaintext is nil, or an encrypted - # value already exists - return unless plaintext.present? - return if ciphertext.present? - - # attr_encrypted will calculate and set the expected value for us - instance.public_send("#{plain_column}=", plaintext) # rubocop:disable GitlabSecurity/PublicSend - - plain_column - end - end - end -end diff --git a/lib/gitlab/background_migration/encrypt_runners_tokens.rb b/lib/gitlab/background_migration/encrypt_runners_tokens.rb deleted file mode 100644 index ec64a73542e..00000000000 --- a/lib/gitlab/background_migration/encrypt_runners_tokens.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # EncryptColumn migrates data from an unencrypted column - `foo`, say - to - # an encrypted column - `encrypted_foo`, say. - # - # We only create a subclass here because we want to isolate this migration - # (migrating unencrypted runner registration tokens to encrypted columns) - # from other `EncryptColumns` migration. This class name is going to be - # serialized and stored in Redis and later picked by Sidekiq, so we need to - # create a separate class name in order to isolate these migration tasks. - # - # We can solve this differently, see tech debt issue: - # - # https://gitlab.com/gitlab-org/gitlab-foss/issues/54328 - # - class EncryptRunnersTokens < EncryptColumns - def perform(model, from, to) - resource = "::Gitlab::BackgroundMigration::Models::EncryptColumns::#{model.to_s.capitalize}" - model = resource.constantize - attributes = model.encrypted_attributes.keys - - super(model, attributes, from, to) - end - - def clear_migrated_values? - false - end - end - end -end diff --git a/lib/gitlab/background_migration/fix_pages_access_level.rb b/lib/gitlab/background_migration/fix_pages_access_level.rb index 0d49f3dd8c5..31d2e78b2d2 100644 --- a/lib/gitlab/background_migration/fix_pages_access_level.rb +++ b/lib/gitlab/background_migration/fix_pages_access_level.rb @@ -16,7 +16,7 @@ module Gitlab end # Namespace - class Namespace < ApplicationRecord + class Namespace < ActiveRecord::Base self.table_name = 'namespaces' self.inheritance_column = :_type_disabled diff --git a/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb b/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb new file mode 100644 index 00000000000..c096dae0631 --- /dev/null +++ b/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + module Mailers + class UnconfirmMailer < ::Notify + prepend_view_path(File.join(__dir__, 'views')) + + def unconfirm_notification_email(user) + @user = user + @verification_from_mail = Gitlab.config.gitlab.email_from + + mail( + template_path: 'unconfirm_mailer', + template_name: 'unconfirm_notification_email', + to: @user.notification_email, + subject: subject('GitLab email verification request') + ) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/mailers/views/unconfirm_mailer/unconfirm_notification_email.html.haml b/lib/gitlab/background_migration/mailers/views/unconfirm_mailer/unconfirm_notification_email.html.haml new file mode 100644 index 00000000000..d8f7466a1ca --- /dev/null +++ b/lib/gitlab/background_migration/mailers/views/unconfirm_mailer/unconfirm_notification_email.html.haml @@ -0,0 +1,19 @@ +-# haml-lint:disable NoPlainNodes +%p + Dear GitLab user, + +%p + As part of our commitment to keeping GitLab secure, we have identified and addressed a vulnerability in GitLab that allowed some users to bypass the email verification process in a #{link_to("recent security release", "https://about.gitlab.com/releases/2020/05/27/security-release-13-0-1-released", target: '_blank')}. + +%p + As a precautionary measure, you will need to re-verify some of your account's email addresses before continuing to use GitLab. Sorry for the inconvenience! + +%p + We have already sent the re-verification email with a subject line of "Confirmation instructions" from #{@verification_from_mail}. Please feel free to contribute any questions or comments to #{link_to("this issue", "https://gitlab.com/gitlab-com/www-gitlab-com/-/issues/7942", target: '_blank')}. + +%p + If you are not "#{@user.username}", please #{link_to 'report this to our administrator', new_abuse_report_url(user_id: @user.id)} + +%p + Thank you for being a GitLab user! +-# haml-lint:enable NoPlainNodes diff --git a/lib/gitlab/background_migration/mailers/views/unconfirm_mailer/unconfirm_notification_email.text.erb b/lib/gitlab/background_migration/mailers/views/unconfirm_mailer/unconfirm_notification_email.text.erb new file mode 100644 index 00000000000..d20af9b9803 --- /dev/null +++ b/lib/gitlab/background_migration/mailers/views/unconfirm_mailer/unconfirm_notification_email.text.erb @@ -0,0 +1,14 @@ +Dear GitLab user, + +As part of our commitment to keeping GitLab secure, we have identified and addressed a vulnerability in GitLab that allowed some users to bypass the email verification process in a recent security release. + +Security release: https://about.gitlab.com/releases/2020/05/27/security-release-13-0-1-released + +As a precautionary measure, you will need to re-verify some of your account's email addresses before continuing to use GitLab. Sorry for the inconvenience! + +We have already sent the re-verification email with a subject line of "Confirmation instructions" from <%= @verification_from_mail %>. +Please feel free to contribute any questions or comments to this issue: https://gitlab.com/gitlab-com/www-gitlab-com/-/issues/7942 + +If you are not "<%= @user.username %>", please report this to our administrator. Report link: <%= new_abuse_report_url(user_id: @user.id) %> + +Thank you for being a GitLab user! diff --git a/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb b/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb deleted file mode 100644 index 41f18979d76..00000000000 --- a/lib/gitlab/background_migration/models/encrypt_columns/namespace.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module Models - module EncryptColumns - # This model is shared between synchronous and background migrations to - # encrypt the `runners_token` column in `namespaces` table. - # - class Namespace < ActiveRecord::Base - include ::EachBatch - - self.table_name = 'namespaces' - self.inheritance_column = :_type_disabled - - def runners_token=(value) - self.runners_token_encrypted = - ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) - end - - def self.encrypted_attributes - { runners_token: { attribute: :runners_token_encrypted } } - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/project.rb b/lib/gitlab/background_migration/models/encrypt_columns/project.rb deleted file mode 100644 index bfeae14584d..00000000000 --- a/lib/gitlab/background_migration/models/encrypt_columns/project.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module Models - module EncryptColumns - # This model is shared between synchronous and background migrations to - # encrypt the `runners_token` column in `projects` table. - # - class Project < ActiveRecord::Base - include ::EachBatch - - self.table_name = 'projects' - self.inheritance_column = :_type_disabled - - def runners_token=(value) - self.runners_token_encrypted = - ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) - end - - def self.encrypted_attributes - { runners_token: { attribute: :runners_token_encrypted } } - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/runner.rb b/lib/gitlab/background_migration/models/encrypt_columns/runner.rb deleted file mode 100644 index 14ddce4b147..00000000000 --- a/lib/gitlab/background_migration/models/encrypt_columns/runner.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module Models - module EncryptColumns - # This model is shared between synchronous and background migrations to - # encrypt the `token` column in `ci_runners` table. - # - class Runner < ActiveRecord::Base - include ::EachBatch - - self.table_name = 'ci_runners' - self.inheritance_column = :_type_disabled - - def token=(value) - self.token_encrypted = - ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) - end - - def self.encrypted_attributes - { token: { attribute: :token_encrypted } } - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/settings.rb b/lib/gitlab/background_migration/models/encrypt_columns/settings.rb deleted file mode 100644 index 08ae35c0671..00000000000 --- a/lib/gitlab/background_migration/models/encrypt_columns/settings.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module Models - module EncryptColumns - # This model is shared between synchronous and background migrations to - # encrypt the `runners_token` column in `application_settings` table. - # - class Settings < ActiveRecord::Base - include ::EachBatch - include ::CacheableAttributes - - self.table_name = 'application_settings' - self.inheritance_column = :_type_disabled - - after_commit do - ::ApplicationSetting.expire - end - - def runners_registration_token=(value) - self.runners_registration_token_encrypted = - ::Gitlab::CryptoHelper.aes256_gcm_encrypt(value) - end - - def self.encrypted_attributes - { - runners_registration_token: { - attribute: :runners_registration_token_encrypted - } - } - end - end - end - end - end -end diff --git a/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb b/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb deleted file mode 100644 index 34e72fd9f34..00000000000 --- a/lib/gitlab/background_migration/models/encrypt_columns/web_hook.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module Models - module EncryptColumns - # This model is shared between synchronous and background migrations to - # encrypt the `token` and `url` columns - class WebHook < ActiveRecord::Base - include ::EachBatch - - self.table_name = 'web_hooks' - self.inheritance_column = :_type_disabled - - attr_encrypted :token, - mode: :per_attribute_iv, - algorithm: 'aes-256-gcm', - key: ::Settings.attr_encrypted_db_key_base_32 - - attr_encrypted :url, - mode: :per_attribute_iv, - algorithm: 'aes-256-gcm', - key: ::Settings.attr_encrypted_db_key_base_32 - end - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_project_snippet_statistics.rb b/lib/gitlab/background_migration/populate_project_snippet_statistics.rb new file mode 100644 index 00000000000..7659b63271f --- /dev/null +++ b/lib/gitlab/background_migration/populate_project_snippet_statistics.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class creates/updates those project snippets statistics + # that haven't been created nor initialized. + # It also updates the related project statistics and its root storage namespace stats + class PopulateProjectSnippetStatistics + def perform(snippet_ids) + project_snippets(snippet_ids).group_by(&:namespace_id).each do |namespace_id, namespace_snippets| + namespace_snippets.group_by(&:project).each do |project, snippets| + upsert_snippet_statistics(snippets) + update_project_statistics(project) + rescue + error_message("Error updating statistics for project #{project.id}") + end + + update_namespace_statistics(namespace_snippets.first.project.root_namespace) + rescue => e + error_message("Error updating statistics for namespace #{namespace_id}: #{e.message}") + end + end + + private + + def project_snippets(snippet_ids) + ProjectSnippet + .select('snippets.*, projects.namespace_id') + .where(id: snippet_ids) + .joins(:project) + .includes(:statistics) + .includes(snippet_repository: :shard) + .includes(project: [:route, :statistics, :namespace]) + end + + def upsert_snippet_statistics(snippets) + snippets.each do |snippet| + response = Snippets::UpdateStatisticsService.new(snippet).execute + + error_message("#{response.message} snippet: #{snippet.id}") if response.error? + end + end + + def logger + @logger ||= Gitlab::BackgroundMigration::Logger.build + end + + def error_message(message) + logger.error(message: "Snippet Statistics Migration: #{message}") + end + + def update_project_statistics(project) + project.statistics&.refresh!(only: [:snippets_size]) + end + + def update_namespace_statistics(namespace) + Namespaces::StatisticsRefresherService.new.execute(namespace) + end + end + end +end diff --git a/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb b/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb new file mode 100644 index 00000000000..bfe9f673b53 --- /dev/null +++ b/lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class UpdateVulnerabilitiesFromDismissalFeedback + def perform(project_id) + end + end + end +end + +Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback') diff --git a/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb b/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb new file mode 100644 index 00000000000..5f63cf5836e --- /dev/null +++ b/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class WrongfullyConfirmedEmailUnconfirmer + class UserModel < ActiveRecord::Base + alias_method :reset, :reload + + self.table_name = 'users' + + scope :active, -> { where(state: 'active', user_type: nil) } # only humans, skip bots + + devise :confirmable + end + + class EmailModel < ActiveRecord::Base + alias_method :reset, :reload + + self.table_name = 'emails' + + belongs_to :user + + devise :confirmable + + def self.wrongfully_confirmed_emails(start_id, stop_id) + joins(:user) + .merge(UserModel.active) + .where(id: (start_id..stop_id)) + .where('emails.confirmed_at IS NOT NULL') + .where('emails.confirmed_at = users.confirmed_at') + .where('emails.email <> users.email') + end + end + + def perform(start_id, stop_id) + email_records = EmailModel + .wrongfully_confirmed_emails(start_id, stop_id) + .to_a + + user_ids = email_records.map(&:user_id).uniq + + ActiveRecord::Base.transaction do + update_email_records(start_id, stop_id) + update_user_records(user_ids) + end + + # Refind the records with the "real" Email model so devise will notice that the user / email is unconfirmed + unconfirmed_email_records = ::Email.where(id: email_records.map(&:id)) + ActiveRecord::Associations::Preloader.new.preload(unconfirmed_email_records, [:user]) + + send_emails(unconfirmed_email_records) + end + + private + + def update_email_records(start_id, stop_id) + EmailModel.connection.execute <<-SQL + WITH md5_strings as ( + #{email_query_for_update(start_id, stop_id).to_sql} + ) + UPDATE #{EmailModel.connection.quote_table_name(EmailModel.table_name)} + SET confirmed_at = NULL, + confirmation_token = md5_strings.md5_string, + confirmation_sent_at = NOW() + FROM md5_strings + WHERE id = md5_strings.email_id + SQL + end + + def update_user_records(user_ids) + UserModel + .where(id: user_ids) + .update_all("confirmed_at = NULL, confirmation_sent_at = NOW(), unconfirmed_email = NULL, confirmation_token=md5(users.id::varchar || users.created_at || users.encrypted_password || '#{Integer(Time.now.to_i)}')") + end + + def email_query_for_update(start_id, stop_id) + EmailModel + .wrongfully_confirmed_emails(start_id, stop_id) + .select('emails.id as email_id', "md5(emails.id::varchar || emails.created_at || users.encrypted_password || '#{Integer(Time.now.to_i)}') as md5_string") + end + + def send_emails(email_records) + user_records = email_records.map(&:user).uniq + + user_records.each do |user| + Gitlab::BackgroundMigration::Mailers::UnconfirmMailer.unconfirm_notification_email(user).deliver_later + DeviseMailer.confirmation_instructions(user, user.confirmation_token).deliver_later(wait: 1.minute) + end + + email_records.each do |email| + DeviseMailer.confirmation_instructions(email, email.confirmation_token).deliver_later(wait: 1.minute) + end + end + end + end +end |