From 57a5544f883ad9687c38270519edc7914912af5d Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Fri, 28 Jul 2017 10:44:33 +0100 Subject: Remove events column from notification settings This was migrated to separate columns in 9.4, and now just needs to be removed for real. --- .../20170728101014_remove_events_from_notification_settings.rb | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 db/post_migrate/20170728101014_remove_events_from_notification_settings.rb (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170728101014_remove_events_from_notification_settings.rb b/db/post_migrate/20170728101014_remove_events_from_notification_settings.rb new file mode 100644 index 00000000000..cd533391d8d --- /dev/null +++ b/db/post_migrate/20170728101014_remove_events_from_notification_settings.rb @@ -0,0 +1,9 @@ +class RemoveEventsFromNotificationSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + remove_column :notification_settings, :events, :text + end +end -- cgit v1.2.3 From f2d50af917b878a98e06b994ac32c0718f3d0b78 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Mon, 3 Jul 2017 15:48:59 +0100 Subject: Migrate MR commits and diffs to new tables Previously, we stored these as serialised fields - `st_{commits,diffs}` - on the `merge_request_diffs` table. These now have their own tables - `merge_request_diff_{commits,diffs}` - with a column for each attribute of the serialised data. Add a background migration to go through the existing MR diffs and migrate them to the new format. Ignore any contents that cannot be displayed. Assuming that we have 5 million rows to migrate, and each batch of 2,500 rows can be completed in 5 minutes, this will take about 7 days to migrate everything. --- ...30158_schedule_merge_request_diff_migrations.rb | 33 ++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 db/post_migrate/20170703130158_schedule_merge_request_diff_migrations.rb (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170703130158_schedule_merge_request_diff_migrations.rb b/db/post_migrate/20170703130158_schedule_merge_request_diff_migrations.rb new file mode 100644 index 00000000000..17a9dc293f1 --- /dev/null +++ b/db/post_migrate/20170703130158_schedule_merge_request_diff_migrations.rb @@ -0,0 +1,33 @@ +class ScheduleMergeRequestDiffMigrations < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BATCH_SIZE = 2500 + MIGRATION = 'DeserializeMergeRequestDiffsAndCommits' + + disable_ddl_transaction! + + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + + include ::EachBatch + end + + # Assuming that there are 5 million rows affected (which is more than on + # GitLab.com), and that each batch of 2,500 rows takes up to 5 minutes, then + # we can migrate all the rows in 7 days. + # + # On staging, plucking the IDs themselves takes 5 seconds. + def up + non_empty = 'st_commits IS NOT NULL OR st_diffs IS NOT NULL' + + MergeRequestDiff.where(non_empty).each_batch(of: BATCH_SIZE) do |relation, index| + range = relation.pluck('MIN(id)', 'MAX(id)').first + + BackgroundMigrationWorker.perform_in(index * 5.minutes, MIGRATION, range) + end + end + + def down + end +end -- cgit v1.2.3 From 0f9bde41fc5d51ef227021444e2185fb5e5162f1 Mon Sep 17 00:00:00 2001 From: Jarka Kadlecova Date: Tue, 1 Aug 2017 08:46:43 +0200 Subject: Store & use ConvDev percentages returned by Version app --- ...3090603_calculate_conv_dev_index_percentages.rb | 30 ++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 db/post_migrate/20170803090603_calculate_conv_dev_index_percentages.rb (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170803090603_calculate_conv_dev_index_percentages.rb b/db/post_migrate/20170803090603_calculate_conv_dev_index_percentages.rb new file mode 100644 index 00000000000..9af76c94bf3 --- /dev/null +++ b/db/post_migrate/20170803090603_calculate_conv_dev_index_percentages.rb @@ -0,0 +1,30 @@ +class CalculateConvDevIndexPercentages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + DOWNTIME = false + + class ConversationalDevelopmentIndexMetric < ActiveRecord::Base + self.table_name = 'conversational_development_index_metrics' + + METRICS = %w[boards ci_pipelines deployments environments issues merge_requests milestones notes + projects_prometheus_active service_desk_issues] + end + + def up + ConversationalDevelopmentIndexMetric.find_each do |conv_dev_index| + update = [] + + ConversationalDevelopmentIndexMetric::METRICS.each do |metric| + instance_score = conv_dev_index["instance_#{metric}"].to_f + leader_score = conv_dev_index["leader_#{metric}"].to_f + + percentage = leader_score.zero? ? 0.0 : (instance_score / leader_score) * 100 + update << "percentage_#{metric} = '#{percentage}'" + end + + execute("UPDATE conversational_development_index_metrics SET #{update.join(',')} WHERE id = #{conv_dev_index.id}") + end + end + + def down + end +end -- cgit v1.2.3 From 16cffa97f64f55a16a50cf861e133021f31c828f Mon Sep 17 00:00:00 2001 From: Oswaldo Ferreira Date: Mon, 7 Aug 2017 16:34:57 -0300 Subject: Move locked_at removal to post-deployment migration --- ...70807160457_remove_locked_at_column_from_merge_requests.rb | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 db/post_migrate/20170807160457_remove_locked_at_column_from_merge_requests.rb (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170807160457_remove_locked_at_column_from_merge_requests.rb b/db/post_migrate/20170807160457_remove_locked_at_column_from_merge_requests.rb new file mode 100644 index 00000000000..3949cf8261a --- /dev/null +++ b/db/post_migrate/20170807160457_remove_locked_at_column_from_merge_requests.rb @@ -0,0 +1,11 @@ +class RemoveLockedAtColumnFromMergeRequests < ActiveRecord::Migration + DOWNTIME = false + + def up + remove_column :merge_requests, :locked_at + end + + def down + # nothing to do to recover the values + end +end -- cgit v1.2.3 From 43b03e9c5a6918a6106819d8819ac1088c2e7af9 Mon Sep 17 00:00:00 2001 From: Oswaldo Ferreira Date: Mon, 7 Aug 2017 18:09:53 -0300 Subject: Re-add column locked_at on migration rollback --- .../20170807160457_remove_locked_at_column_from_merge_requests.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170807160457_remove_locked_at_column_from_merge_requests.rb b/db/post_migrate/20170807160457_remove_locked_at_column_from_merge_requests.rb index 3949cf8261a..ea3d1fb3e02 100644 --- a/db/post_migrate/20170807160457_remove_locked_at_column_from_merge_requests.rb +++ b/db/post_migrate/20170807160457_remove_locked_at_column_from_merge_requests.rb @@ -6,6 +6,6 @@ class RemoveLockedAtColumnFromMergeRequests < ActiveRecord::Migration end def down - # nothing to do to recover the values + add_column :merge_requests, :locked_at, :datetime_with_timezone end end -- cgit v1.2.3 From c946ee1282655d332da4ba99c448d6f68cf87cee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Wed, 9 Aug 2017 11:52:22 +0200 Subject: Enable the Layout/SpaceBeforeBlockBraces cop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- .../20170502101023_cleanup_namespaceless_pending_delete_projects.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170502101023_cleanup_namespaceless_pending_delete_projects.rb b/db/post_migrate/20170502101023_cleanup_namespaceless_pending_delete_projects.rb index c1e64f20109..5238a2ba1b7 100644 --- a/db/post_migrate/20170502101023_cleanup_namespaceless_pending_delete_projects.rb +++ b/db/post_migrate/20170502101023_cleanup_namespaceless_pending_delete_projects.rb @@ -30,7 +30,7 @@ class CleanupNamespacelessPendingDeleteProjects < ActiveRecord::Migration private def pending_delete_batch - connection.exec_query(find_batch).map{ |row| row['id'].to_i } + connection.exec_query(find_batch).map { |row| row['id'].to_i } end BATCH_SIZE = 5000 -- cgit v1.2.3 From 0395c47193b3bbf6b4f060f28c9f632580313a35 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 10 Jul 2017 17:43:57 +0200 Subject: Migrate events into a new format This commit migrates events data in such a way that push events are stored much more efficiently. This is done by creating a shadow table called "events_for_migration", and a table called "push_event_payloads" which is used for storing push data of push events. The background migration in this commit will copy events from the "events" table into the "events_for_migration" table, push events in will also have a row created in "push_event_payloads". This approach allows us to reclaim space in the next release by simply swapping the "events" and "events_for_migration" tables, then dropping the old events (now "events_for_migration") table. The new table structure is also optimised for storage space, and does not include the unused "title" column nor the "data" column (since this data is moved to "push_event_payloads"). == Newly Created Events Newly created events are inserted into both "events" and "events_for_migration", both using the exact same primary key value. The table "push_event_payloads" in turn has a foreign key to the _shadow_ table. This removes the need for recreating and validating the foreign key after swapping the tables. Since the shadow table also has a foreign key to "projects.id" we also don't have to worry about orphaned rows. This approach however does require some additional storage as we're duplicating a portion of the events data for at least 1 release. The exact amount is hard to estimate, but for GitLab.com this is expected to be between 10 and 20 GB at most. The background migration in this commit deliberately does _not_ update the "events" table as doing so would put a lot of pressure on PostgreSQL's auto vacuuming system. == Supporting Both Old And New Events Application code has also been adjusted to support push events using both the old and new data formats. This is done by creating a PushEvent class which extends the regular Event class. Using Rails' Single Table Inheritance system we can ensure the right class is used for the right data, which in this case is based on the value of `events.action`. To support displaying old and new data at the same time the PushEvent class re-defines a few methods of the Event class, falling back to their original implementations for push events in the old format. Once all existing events have been migrated the various push event related methods can be removed from the Event model, and the calls to `super` can be removed from the methods in the PushEvent model. The UI and event atom feed have also been slightly changed to better handle this new setup, fortunately only a few changes were necessary to make this work. == API Changes The API only displays push data of events in the new format. Supporting both formats in the API is a bit more difficult compared to the UI. Since the old push data was not really well documented (apart from one example that used an incorrect "action" nmae) I decided that supporting both was not worth the effort, especially since events will be migrated in a few days _and_ new events are created in the correct format. --- .../20170627101016_schedule_event_migrations.rb | 40 ++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 db/post_migrate/20170627101016_schedule_event_migrations.rb (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170627101016_schedule_event_migrations.rb b/db/post_migrate/20170627101016_schedule_event_migrations.rb new file mode 100644 index 00000000000..1f34375ff0d --- /dev/null +++ b/db/post_migrate/20170627101016_schedule_event_migrations.rb @@ -0,0 +1,40 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class ScheduleEventMigrations < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + BUFFER_SIZE = 1000 + + disable_ddl_transaction! + + class Event < ActiveRecord::Base + include EachBatch + + self.table_name = 'events' + end + + def up + jobs = [] + + Event.each_batch(of: 1000) do |relation| + min, max = relation.pluck('MIN(id), MAX(id)').first + + if jobs.length == BUFFER_SIZE + # We push multiple jobs at a time to reduce the time spent in + # Sidekiq/Redis operations. We're using this buffer based approach so we + # don't need to run additional queries for every range. + BackgroundMigrationWorker.perform_bulk(jobs) + jobs.clear + end + + jobs << ['MigrateEventsToPushEventPayloads', [min, max]] + end + + BackgroundMigrationWorker.perform_bulk(jobs) unless jobs.empty? + end + + def down + end +end -- cgit v1.2.3 From f0f4506775d0e06b28ee55d591cb223115e3975a Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Wed, 2 Aug 2017 15:31:39 +0200 Subject: Don't update upload paths twice This will be done in 20170717150329_enqueue_migrate_system_uploads_to_new_folder.rb instead. --- ...20170317162059_update_upload_paths_to_system.rb | 57 ---------------------- 1 file changed, 57 deletions(-) delete mode 100644 db/post_migrate/20170317162059_update_upload_paths_to_system.rb (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170317162059_update_upload_paths_to_system.rb b/db/post_migrate/20170317162059_update_upload_paths_to_system.rb deleted file mode 100644 index ca2912f8dce..00000000000 --- a/db/post_migrate/20170317162059_update_upload_paths_to_system.rb +++ /dev/null @@ -1,57 +0,0 @@ -# See http://doc.gitlab.com/ce/development/migration_style_guide.html -# for more information on how to write migrations for GitLab. - -class UpdateUploadPathsToSystem < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - AFFECTED_MODELS = %w(User Project Note Namespace Appearance) - - disable_ddl_transaction! - - def up - update_column_in_batches(:uploads, :path, replace_sql(arel_table[:path], base_directory, new_upload_dir)) do |_table, query| - query.where(uploads_to_switch_to_new_path) - end - end - - def down - update_column_in_batches(:uploads, :path, replace_sql(arel_table[:path], new_upload_dir, base_directory)) do |_table, query| - query.where(uploads_to_switch_to_old_path) - end - end - - # "SELECT \"uploads\".* FROM \"uploads\" WHERE \"uploads\".\"model_type\" IN ('User', 'Project', 'Note', 'Namespace', 'Appearance') AND (\"uploads\".\"path\" ILIKE 'uploads/%' AND NOT (\"uploads\".\"path\" ILIKE 'uploads/system/%'))" - def uploads_to_switch_to_new_path - affected_uploads.and(starting_with_base_directory).and(starting_with_new_upload_directory.not) - end - - # "SELECT \"uploads\".* FROM \"uploads\" WHERE \"uploads\".\"model_type\" IN ('User', 'Project', 'Note', 'Namespace', 'Appearance') AND (\"uploads\".\"path\" ILIKE 'uploads/%' AND \"uploads\".\"path\" ILIKE 'uploads/system/%')" - def uploads_to_switch_to_old_path - affected_uploads.and(starting_with_new_upload_directory) - end - - def starting_with_base_directory - arel_table[:path].matches("#{base_directory}/%") - end - - def starting_with_new_upload_directory - arel_table[:path].matches("#{new_upload_dir}/%") - end - - def affected_uploads - arel_table[:model_type].in(AFFECTED_MODELS) - end - - def base_directory - "uploads" - end - - def new_upload_dir - File.join(base_directory, "system") - end - - def arel_table - Arel::Table.new(:uploads) - end -end -- cgit v1.2.3 From b8ae15397f0f21b87ea2f91efb470e7e51ba8964 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Wed, 2 Aug 2017 15:42:36 +0200 Subject: Update migrations to move directly into the `-/system` folder --- ...20170317162059_update_upload_paths_to_system.rb | 57 ++++++++++++++++++++++ .../20170406111121_clean_upload_symlinks.rb | 2 +- ...20170606202615_move_appearance_to_system_dir.rb | 2 +- .../20170612071012_move_personal_snippets_files.rb | 4 +- 4 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 db/post_migrate/20170317162059_update_upload_paths_to_system.rb (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170317162059_update_upload_paths_to_system.rb b/db/post_migrate/20170317162059_update_upload_paths_to_system.rb new file mode 100644 index 00000000000..92e33848bf0 --- /dev/null +++ b/db/post_migrate/20170317162059_update_upload_paths_to_system.rb @@ -0,0 +1,57 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class UpdateUploadPathsToSystem < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + AFFECTED_MODELS = %w(User Project Note Namespace Appearance) + + disable_ddl_transaction! + + def up + update_column_in_batches(:uploads, :path, replace_sql(arel_table[:path], base_directory, new_upload_dir)) do |_table, query| + query.where(uploads_to_switch_to_new_path) + end + end + + def down + update_column_in_batches(:uploads, :path, replace_sql(arel_table[:path], new_upload_dir, base_directory)) do |_table, query| + query.where(uploads_to_switch_to_old_path) + end + end + + # "SELECT \"uploads\".* FROM \"uploads\" WHERE \"uploads\".\"model_type\" IN ('User', 'Project', 'Note', 'Namespace', 'Appearance') AND (\"uploads\".\"path\" ILIKE 'uploads/%' AND NOT (\"uploads\".\"path\" ILIKE 'uploads/system/%'))" + def uploads_to_switch_to_new_path + affected_uploads.and(starting_with_base_directory).and(starting_with_new_upload_directory.not) + end + + # "SELECT \"uploads\".* FROM \"uploads\" WHERE \"uploads\".\"model_type\" IN ('User', 'Project', 'Note', 'Namespace', 'Appearance') AND (\"uploads\".\"path\" ILIKE 'uploads/%' AND \"uploads\".\"path\" ILIKE 'uploads/system/%')" + def uploads_to_switch_to_old_path + affected_uploads.and(starting_with_new_upload_directory) + end + + def starting_with_base_directory + arel_table[:path].matches("#{base_directory}/%") + end + + def starting_with_new_upload_directory + arel_table[:path].matches("#{new_upload_dir}/%") + end + + def affected_uploads + arel_table[:model_type].in(AFFECTED_MODELS) + end + + def base_directory + "uploads" + end + + def new_upload_dir + File.join(base_directory, "-", "system") + end + + def arel_table + Arel::Table.new(:uploads) + end +end diff --git a/db/post_migrate/20170406111121_clean_upload_symlinks.rb b/db/post_migrate/20170406111121_clean_upload_symlinks.rb index fc3a4acc0bb..f2ce25d4524 100644 --- a/db/post_migrate/20170406111121_clean_upload_symlinks.rb +++ b/db/post_migrate/20170406111121_clean_upload_symlinks.rb @@ -47,6 +47,6 @@ class CleanUploadSymlinks < ActiveRecord::Migration end def new_upload_dir - File.join(base_directory, "public", "uploads", "system") + File.join(base_directory, "public", "uploads", "-", "system") end end diff --git a/db/post_migrate/20170606202615_move_appearance_to_system_dir.rb b/db/post_migrate/20170606202615_move_appearance_to_system_dir.rb index 561de59ec69..07935ab8a52 100644 --- a/db/post_migrate/20170606202615_move_appearance_to_system_dir.rb +++ b/db/post_migrate/20170606202615_move_appearance_to_system_dir.rb @@ -52,6 +52,6 @@ class MoveAppearanceToSystemDir < ActiveRecord::Migration end def new_upload_dir - File.join(base_directory, "public", "uploads", "system") + File.join(base_directory, "public", "uploads", "-", "system") end end diff --git a/db/post_migrate/20170612071012_move_personal_snippets_files.rb b/db/post_migrate/20170612071012_move_personal_snippets_files.rb index 33043364bde..2b79a87ccd8 100644 --- a/db/post_migrate/20170612071012_move_personal_snippets_files.rb +++ b/db/post_migrate/20170612071012_move_personal_snippets_files.rb @@ -10,7 +10,7 @@ class MovePersonalSnippetsFiles < ActiveRecord::Migration return unless file_storage? @source_relative_location = File.join('/uploads', 'personal_snippet') - @destination_relative_location = File.join('/uploads', 'system', 'personal_snippet') + @destination_relative_location = File.join('/uploads', '-', 'system', 'personal_snippet') move_personal_snippet_files end @@ -18,7 +18,7 @@ class MovePersonalSnippetsFiles < ActiveRecord::Migration def down return unless file_storage? - @source_relative_location = File.join('/uploads', 'system', 'personal_snippet') + @source_relative_location = File.join('/uploads', '-', 'system', 'personal_snippet') @destination_relative_location = File.join('/uploads', 'personal_snippet') move_personal_snippet_files -- cgit v1.2.3 From 2ea8442ff3398a788b1005a825c1d13f61f91c2d Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Mon, 7 Aug 2017 20:01:45 +0200 Subject: Move the personal snippet uploads from `system` to `-/system` Update the markdown unconditionally since the move might have been done before, but the markdown not updated. --- ...e_personal_snippet_files_into_correct_folder.rb | 29 ++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 db/post_migrate/20170807190736_move_personal_snippet_files_into_correct_folder.rb (limited to 'db/post_migrate') diff --git a/db/post_migrate/20170807190736_move_personal_snippet_files_into_correct_folder.rb b/db/post_migrate/20170807190736_move_personal_snippet_files_into_correct_folder.rb new file mode 100644 index 00000000000..e3d2446b897 --- /dev/null +++ b/db/post_migrate/20170807190736_move_personal_snippet_files_into_correct_folder.rb @@ -0,0 +1,29 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class MovePersonalSnippetFilesIntoCorrectFolder < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + disable_ddl_transaction! + + DOWNTIME = false + NEW_DIRECTORY = File.join('/uploads', '-', 'system', 'personal_snippet') + OLD_DIRECTORY = File.join('/uploads', 'system', 'personal_snippet') + + def up + return unless file_storage? + + BackgroundMigrationWorker.perform_async('MovePersonalSnippetFiles', + [OLD_DIRECTORY, NEW_DIRECTORY]) + end + + def down + return unless file_storage? + + BackgroundMigrationWorker.perform_async('MovePersonalSnippetFiles', + [NEW_DIRECTORY, OLD_DIRECTORY]) + end + + def file_storage? + CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File + end +end -- cgit v1.2.3