diff options
Diffstat (limited to 'db')
38 files changed, 583 insertions, 261 deletions
diff --git a/db/fixtures/development/14_pipelines.rb b/db/fixtures/development/14_pipelines.rb index d3a63aa2a78..5535c4a14e5 100644 --- a/db/fixtures/development/14_pipelines.rb +++ b/db/fixtures/development/14_pipelines.rb @@ -41,7 +41,7 @@ class Gitlab::Seeder::Pipelines when: 'manual', status: :skipped }, # notify stage - { name: 'slack', stage: 'notify', when: 'manual', status: :created }, + { name: 'slack', stage: 'notify', when: 'manual', status: :success }, ] EXTERNAL_JOBS = [ { name: 'jenkins', stage: 'test', status: :success, @@ -54,16 +54,10 @@ class Gitlab::Seeder::Pipelines def seed! pipelines.each do |pipeline| - begin - BUILDS.each { |opts| build_create!(pipeline, opts) } - EXTERNAL_JOBS.each { |opts| commit_status_create!(pipeline, opts) } - print '.' - rescue ActiveRecord::RecordInvalid - print 'F' - ensure - pipeline.update_duration - pipeline.update_status - end + BUILDS.each { |opts| build_create!(pipeline, opts) } + EXTERNAL_JOBS.each { |opts| commit_status_create!(pipeline, opts) } + pipeline.update_duration + pipeline.update_status end end @@ -87,7 +81,9 @@ class Gitlab::Seeder::Pipelines branch = merge_request.source_branch merge_request.commits.last(4).map do |commit| - create_pipeline!(project, branch, commit) + create_pipeline!(project, branch, commit).tap do |pipeline| + merge_request.update!(head_pipeline_id: pipeline.id) + end end end @@ -98,7 +94,7 @@ class Gitlab::Seeder::Pipelines def create_pipeline!(project, ref, commit) - project.pipelines.create(sha: commit.id, ref: ref, source: :push) + project.pipelines.create!(sha: commit.id, ref: ref, source: :push) end def build_create!(pipeline, opts = {}) @@ -111,24 +107,39 @@ class Gitlab::Seeder::Pipelines # block directly to `Ci::Build#create!`. setup_artifacts(build) + setup_test_reports(build) setup_build_log(build) build.project.environments. find_or_create_by(name: build.expanded_environment_name) - build.save + build.save! end end def setup_artifacts(build) - return unless %w[build test].include?(build.stage) + return unless build.stage == "build" artifacts_cache_file(artifacts_archive_path) do |file| - build.job_artifacts.build(project: build.project, file_type: :archive, file: file) + build.job_artifacts.build(project: build.project, file_type: :archive, file_format: :zip, file: file) end artifacts_cache_file(artifacts_metadata_path) do |file| - build.job_artifacts.build(project: build.project, file_type: :metadata, file: file) + build.job_artifacts.build(project: build.project, file_type: :metadata, file_format: :gzip, file: file) + end + end + + def setup_test_reports(build) + return unless build.stage == "test" && build.name == "rspec:osx" + + if build.ref == build.project.default_branch + artifacts_cache_file(test_reports_pass_path) do |file| + build.job_artifacts.build(project: build.project, file_type: :junit, file_format: :gzip, file: file) + end + else + artifacts_cache_file(test_reports_failed_path) do |file| + build.job_artifacts.build(project: build.project, file_type: :junit, file_format: :gzip, file: file) + end end end @@ -171,13 +182,21 @@ class Gitlab::Seeder::Pipelines Rails.root + 'spec/fixtures/ci_build_artifacts_metadata.gz' end + def test_reports_pass_path + Rails.root + 'spec/fixtures/junit/junit_ant.xml.gz' + end + + def test_reports_failed_path + Rails.root + 'spec/fixtures/junit/junit.xml.gz' + end + def artifacts_cache_file(file_path) - cache_path = file_path.to_s.gsub('ci_', "p#{@project.id}_") + file = Tempfile.new("artifacts") + file.close - FileUtils.copy(file_path, cache_path) - File.open(cache_path) do |file| - yield file - end + FileUtils.copy(file_path, file.path) + + yield(UploadedFile.new(file.path, filename: File.basename(file_path))) end end diff --git a/db/migrate/20160712171823_remove_award_emojis_with_no_user.rb b/db/migrate/20160712171823_remove_award_emojis_with_no_user.rb index 668c22bb51c..8ebf1a5234d 100644 --- a/db/migrate/20160712171823_remove_award_emojis_with_no_user.rb +++ b/db/migrate/20160712171823_remove_award_emojis_with_no_user.rb @@ -16,6 +16,6 @@ class RemoveAwardEmojisWithNoUser < ActiveRecord::Migration # disable_ddl_transaction! def up - AwardEmoji.joins('LEFT JOIN users ON users.id = user_id').where('users.id IS NULL').destroy_all + AwardEmoji.joins('LEFT JOIN users ON users.id = user_id').where('users.id IS NULL').destroy_all # rubocop: disable DestroyAll end end diff --git a/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb b/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb index 1199073ed3a..12352d98a62 100644 --- a/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb +++ b/db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb @@ -106,11 +106,11 @@ class ProjectForeignKeysWithCascadingDeletes < ActiveRecord::Migration # Disables statement timeouts for the current connection. This is # necessary as removing of orphaned data might otherwise exceed the # statement timeout. - disable_statement_timeout + disable_statement_timeout do + remove_orphans(*queue.pop) until queue.empty? - remove_orphans(*queue.pop) until queue.empty? - - steal_from_queues(queues - [queue]) + steal_from_queues(queues - [queue]) + end end end end diff --git a/db/migrate/20170724214302_add_lower_path_index_to_redirect_routes.rb b/db/migrate/20170724214302_add_lower_path_index_to_redirect_routes.rb index db60c2087b9..a770ff63b4e 100644 --- a/db/migrate/20170724214302_add_lower_path_index_to_redirect_routes.rb +++ b/db/migrate/20170724214302_add_lower_path_index_to_redirect_routes.rb @@ -25,8 +25,9 @@ class AddLowerPathIndexToRedirectRoutes < ActiveRecord::Migration # trivial to write a query that checks for an index. BUT there is a # convenient `IF EXISTS` parameter for `DROP INDEX`. if supports_drop_index_concurrently? - disable_statement_timeout - execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME};" + disable_statement_timeout do + execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME};" + end else execute "DROP INDEX IF EXISTS #{INDEX_NAME};" end diff --git a/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb b/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb index d1a039ed551..130b24fe6f0 100644 --- a/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb +++ b/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb @@ -8,25 +8,25 @@ class AddIndexOnNamespacesLowerName < ActiveRecord::Migration def up return unless Gitlab::Database.postgresql? - disable_statement_timeout - - if Gitlab::Database.version.to_f >= 9.5 - # Allow us to hot-patch the index manually ahead of the migration - execute "CREATE INDEX CONCURRENTLY IF NOT EXISTS #{INDEX_NAME} ON namespaces (lower(name));" - else - execute "CREATE INDEX CONCURRENTLY #{INDEX_NAME} ON namespaces (lower(name));" + disable_statement_timeout do + if Gitlab::Database.version.to_f >= 9.5 + # Allow us to hot-patch the index manually ahead of the migration + execute "CREATE INDEX CONCURRENTLY IF NOT EXISTS #{INDEX_NAME} ON namespaces (lower(name));" + else + execute "CREATE INDEX CONCURRENTLY #{INDEX_NAME} ON namespaces (lower(name));" + end end end def down return unless Gitlab::Database.postgresql? - disable_statement_timeout - - if Gitlab::Database.version.to_f >= 9.2 - execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME};" - else - execute "DROP INDEX IF EXISTS #{INDEX_NAME};" + disable_statement_timeout do + if Gitlab::Database.version.to_f >= 9.2 + execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME};" + else + execute "DROP INDEX IF EXISTS #{INDEX_NAME};" + end end end end diff --git a/db/migrate/20180113220114_rework_redirect_routes_indexes.rb b/db/migrate/20180113220114_rework_redirect_routes_indexes.rb index ab9971be074..53f82a31203 100644 --- a/db/migrate/20180113220114_rework_redirect_routes_indexes.rb +++ b/db/migrate/20180113220114_rework_redirect_routes_indexes.rb @@ -18,51 +18,51 @@ class ReworkRedirectRoutesIndexes < ActiveRecord::Migration OLD_INDEX_NAME_PATH_LOWER = "index_on_redirect_routes_lower_path" def up - disable_statement_timeout - - # this is a plain btree on a single boolean column. It'll never be - # selective enough to be valuable. This class is called by - # setup_postgresql.rake so it needs to be able to handle this - # index not existing. - if index_exists?(:redirect_routes, :permanent) - remove_concurrent_index(:redirect_routes, :permanent) - end + disable_statement_timeout do + # this is a plain btree on a single boolean column. It'll never be + # selective enough to be valuable. This class is called by + # setup_postgresql.rake so it needs to be able to handle this + # index not existing. + if index_exists?(:redirect_routes, :permanent) + remove_concurrent_index(:redirect_routes, :permanent) + end - # If we're on MySQL then the existing index on path is ok. But on - # Postgres we need to clean things up: - return unless Gitlab::Database.postgresql? + # If we're on MySQL then the existing index on path is ok. But on + # Postgres we need to clean things up: + break unless Gitlab::Database.postgresql? - if_not_exists = Gitlab::Database.version.to_f >= 9.5 ? "IF NOT EXISTS" : "" + if_not_exists = Gitlab::Database.version.to_f >= 9.5 ? "IF NOT EXISTS" : "" - # Unique index on lower(path) across both types of redirect_routes: - execute("CREATE UNIQUE INDEX CONCURRENTLY #{if_not_exists} #{INDEX_NAME_UNIQUE} ON redirect_routes (lower(path) varchar_pattern_ops);") + # Unique index on lower(path) across both types of redirect_routes: + execute("CREATE UNIQUE INDEX CONCURRENTLY #{if_not_exists} #{INDEX_NAME_UNIQUE} ON redirect_routes (lower(path) varchar_pattern_ops);") - # Make two indexes on path -- one for permanent and one for temporary routes: - execute("CREATE INDEX CONCURRENTLY #{if_not_exists} #{INDEX_NAME_PERM} ON redirect_routes (lower(path) varchar_pattern_ops) where (permanent);") - execute("CREATE INDEX CONCURRENTLY #{if_not_exists} #{INDEX_NAME_TEMP} ON redirect_routes (lower(path) varchar_pattern_ops) where (not permanent or permanent is null) ;") + # Make two indexes on path -- one for permanent and one for temporary routes: + execute("CREATE INDEX CONCURRENTLY #{if_not_exists} #{INDEX_NAME_PERM} ON redirect_routes (lower(path) varchar_pattern_ops) where (permanent);") + execute("CREATE INDEX CONCURRENTLY #{if_not_exists} #{INDEX_NAME_TEMP} ON redirect_routes (lower(path) varchar_pattern_ops) where (not permanent or permanent is null) ;") - # Remove the old indexes: + # Remove the old indexes: - # This one needed to be on lower(path) but wasn't so it's replaced with the two above - execute "DROP INDEX CONCURRENTLY IF EXISTS #{OLD_INDEX_NAME_PATH_TPOPS};" + # This one needed to be on lower(path) but wasn't so it's replaced with the two above + execute "DROP INDEX CONCURRENTLY IF EXISTS #{OLD_INDEX_NAME_PATH_TPOPS};" - # This one isn't needed because we only ever do = and LIKE on this - # column so the varchar_pattern_ops index is sufficient - execute "DROP INDEX CONCURRENTLY IF EXISTS #{OLD_INDEX_NAME_PATH_LOWER};" + # This one isn't needed because we only ever do = and LIKE on this + # column so the varchar_pattern_ops index is sufficient + execute "DROP INDEX CONCURRENTLY IF EXISTS #{OLD_INDEX_NAME_PATH_LOWER};" + end end def down - disable_statement_timeout + disable_statement_timeout do + add_concurrent_index(:redirect_routes, :permanent) - add_concurrent_index(:redirect_routes, :permanent) + break unless Gitlab::Database.postgresql? - return unless Gitlab::Database.postgresql? + execute("CREATE INDEX CONCURRENTLY #{OLD_INDEX_NAME_PATH_TPOPS} ON redirect_routes (path varchar_pattern_ops);") + execute("CREATE INDEX CONCURRENTLY #{OLD_INDEX_NAME_PATH_LOWER} ON redirect_routes (LOWER(path));") - execute("CREATE INDEX CONCURRENTLY #{OLD_INDEX_NAME_PATH_TPOPS} ON redirect_routes (path varchar_pattern_ops);") - execute("CREATE INDEX CONCURRENTLY #{OLD_INDEX_NAME_PATH_LOWER} ON redirect_routes (LOWER(path));") - - execute("DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_UNIQUE};") - execute("DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_PERM};") - execute("DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_TEMP};") + execute("DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_UNIQUE};") + execute("DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_PERM};") + execute("DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_TEMP};") + end end end diff --git a/db/migrate/20180308125206_add_user_internal_regex_to_application_setting.rb b/db/migrate/20180308125206_add_user_internal_regex_to_application_setting.rb new file mode 100644 index 00000000000..fe50e909563 --- /dev/null +++ b/db/migrate/20180308125206_add_user_internal_regex_to_application_setting.rb @@ -0,0 +1,13 @@ +class AddUserInternalRegexToApplicationSetting < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + add_column :application_settings, :user_default_internal_regex, :string, null: true + end + + def down + remove_column :application_settings, :user_default_internal_regex + end +end diff --git a/db/migrate/20180403035759_create_project_ci_cd_settings.rb b/db/migrate/20180403035759_create_project_ci_cd_settings.rb index 06856af6204..173e662cffc 100644 --- a/db/migrate/20180403035759_create_project_ci_cd_settings.rb +++ b/db/migrate/20180403035759_create_project_ci_cd_settings.rb @@ -13,16 +13,16 @@ class CreateProjectCiCdSettings < ActiveRecord::Migration end end - disable_statement_timeout + disable_statement_timeout do + # This particular INSERT will take between 10 and 20 seconds. + execute 'INSERT INTO project_ci_cd_settings (project_id) SELECT id FROM projects' - # This particular INSERT will take between 10 and 20 seconds. - execute 'INSERT INTO project_ci_cd_settings (project_id) SELECT id FROM projects' + # We add the index and foreign key separately so the above INSERT statement + # takes as little time as possible. + add_concurrent_index(:project_ci_cd_settings, :project_id, unique: true) - # We add the index and foreign key separately so the above INSERT statement - # takes as little time as possible. - add_concurrent_index(:project_ci_cd_settings, :project_id, unique: true) - - add_foreign_key_with_retry + add_foreign_key_with_retry + end end def down diff --git a/db/migrate/20180420010616_cleanup_build_stage_migration.rb b/db/migrate/20180420010616_cleanup_build_stage_migration.rb index 24777294101..5e9fe756efd 100644 --- a/db/migrate/20180420010616_cleanup_build_stage_migration.rb +++ b/db/migrate/20180420010616_cleanup_build_stage_migration.rb @@ -14,48 +14,50 @@ class CleanupBuildStageMigration < ActiveRecord::Migration end def up - disable_statement_timeout - - ## - # We steal from the background migrations queue to catch up with the - # scheduled migrations set. - # - Gitlab::BackgroundMigration.steal('MigrateBuildStage') - - ## - # We add temporary index, to make iteration over batches more performant. - # Conditional here is to avoid the need of doing that in a separate - # migration file to make this operation idempotent. - # - unless index_exists_by_name?(:ci_builds, TMP_INDEX) - add_concurrent_index(:ci_builds, :id, where: 'stage_id IS NULL', name: TMP_INDEX) - end - - ## - # We check if there are remaining rows that should be migrated (for example - # if Sidekiq / Redis fails / is restarted, what could result in not all - # background migrations being executed correctly. - # - # We migrate remaining rows synchronously in a blocking way, to make sure - # that when this migration is done we are confident that all rows are - # already migrated. - # - Build.where('stage_id IS NULL').each_batch(of: 50) do |batch| - range = batch.pluck('MIN(id)', 'MAX(id)').first - - Gitlab::BackgroundMigration::MigrateBuildStage.new.perform(*range) + disable_statement_timeout do + ## + # We steal from the background migrations queue to catch up with the + # scheduled migrations set. + # + Gitlab::BackgroundMigration.steal('MigrateBuildStage') + + ## + # We add temporary index, to make iteration over batches more performant. + # Conditional here is to avoid the need of doing that in a separate + # migration file to make this operation idempotent. + # + unless index_exists_by_name?(:ci_builds, TMP_INDEX) + add_concurrent_index(:ci_builds, :id, where: 'stage_id IS NULL', name: TMP_INDEX) + end + + ## + # We check if there are remaining rows that should be migrated (for example + # if Sidekiq / Redis fails / is restarted, what could result in not all + # background migrations being executed correctly. + # + # We migrate remaining rows synchronously in a blocking way, to make sure + # that when this migration is done we are confident that all rows are + # already migrated. + # + Build.where('stage_id IS NULL').each_batch(of: 50) do |batch| + range = batch.pluck('MIN(id)', 'MAX(id)').first + + Gitlab::BackgroundMigration::MigrateBuildStage.new.perform(*range) + end + + ## + # We remove temporary index, because it is not required during standard + # operations and runtime. + # + remove_concurrent_index_by_name(:ci_builds, TMP_INDEX) end - - ## - # We remove temporary index, because it is not required during standard - # operations and runtime. - # - remove_concurrent_index_by_name(:ci_builds, TMP_INDEX) end def down if index_exists_by_name?(:ci_builds, TMP_INDEX) - remove_concurrent_index_by_name(:ci_builds, TMP_INDEX) + disable_statement_timeout do + remove_concurrent_index_by_name(:ci_builds, TMP_INDEX) + end end end end diff --git a/db/migrate/20180504195842_project_name_lower_index.rb b/db/migrate/20180504195842_project_name_lower_index.rb index d6f25d3d4ab..74f3673bb03 100644 --- a/db/migrate/20180504195842_project_name_lower_index.rb +++ b/db/migrate/20180504195842_project_name_lower_index.rb @@ -13,20 +13,20 @@ class ProjectNameLowerIndex < ActiveRecord::Migration def up return unless Gitlab::Database.postgresql? - disable_statement_timeout - - execute "CREATE INDEX CONCURRENTLY #{INDEX_NAME} ON projects (LOWER(name))" + disable_statement_timeout do + execute "CREATE INDEX CONCURRENTLY #{INDEX_NAME} ON projects (LOWER(name))" + end end def down return unless Gitlab::Database.postgresql? - disable_statement_timeout - - if supports_drop_index_concurrently? - execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME}" - else - execute "DROP INDEX IF EXISTS #{INDEX_NAME}" + disable_statement_timeout do + if supports_drop_index_concurrently? + execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME}" + else + execute "DROP INDEX IF EXISTS #{INDEX_NAME}" + end end end end diff --git a/db/migrate/20180702124358_remove_orphaned_routes.rb b/db/migrate/20180702124358_remove_orphaned_routes.rb index 6f6e289ba87..4068e479b6c 100644 --- a/db/migrate/20180702124358_remove_orphaned_routes.rb +++ b/db/migrate/20180702124358_remove_orphaned_routes.rb @@ -28,16 +28,16 @@ class RemoveOrphanedRoutes < ActiveRecord::Migration # which is pretty close to our 15 second statement timeout. To ensure a # smooth deployment procedure we disable the statement timeouts for this # migration, just in case. - disable_statement_timeout - - # On GitLab.com there are around 4000 orphaned project routes, and around - # 150 orphaned namespace routes. - [ - Route.orphaned_project_routes, - Route.orphaned_namespace_routes - ].each do |relation| - relation.each_batch(of: 1_000) do |batch| - batch.delete_all + disable_statement_timeout do + # On GitLab.com there are around 4000 orphaned project routes, and around + # 150 orphaned namespace routes. + [ + Route.orphaned_project_routes, + Route.orphaned_namespace_routes + ].each do |relation| + relation.each_batch(of: 1_000) do |batch| + batch.delete_all + end end end end diff --git a/db/migrate/20180711103851_drop_duplicate_protected_tags.rb b/db/migrate/20180711103851_drop_duplicate_protected_tags.rb new file mode 100644 index 00000000000..8fa2137551e --- /dev/null +++ b/db/migrate/20180711103851_drop_duplicate_protected_tags.rb @@ -0,0 +1,45 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class DropDuplicateProtectedTags < ActiveRecord::Migration + DOWNTIME = false + + disable_ddl_transaction! + + BATCH_SIZE = 1000 + + class Project < ActiveRecord::Base + self.table_name = 'projects' + + include ::EachBatch + end + + class ProtectedTag < ActiveRecord::Base + self.table_name = 'protected_tags' + end + + def up + Project.each_batch(of: BATCH_SIZE) do |projects| + ids = ProtectedTag + .where(project_id: projects) + .group(:name, :project_id) + .select('max(id)') + + tags = ProtectedTag + .where(project_id: projects) + .where.not(id: ids) + + if Gitlab::Database.postgresql? + tags.delete_all + else + # Workaround needed for MySQL + sql = "SELECT id FROM (#{tags.to_sql}) protected_tags" + + ProtectedTag.where("id IN (#{sql})").delete_all # rubocop:disable GitlabSecurity/SqlInjection + end + end + end + + def down + end +end diff --git a/db/migrate/20180711103922_add_protected_tags_index.rb b/db/migrate/20180711103922_add_protected_tags_index.rb new file mode 100644 index 00000000000..7ed2258ebaf --- /dev/null +++ b/db/migrate/20180711103922_add_protected_tags_index.rb @@ -0,0 +1,18 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddProtectedTagsIndex < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :protected_tags, [:project_id, :name], unique: true + end + + def down + remove_concurrent_index :protected_tags, [:project_id, :name] + end +end diff --git a/db/migrate/20180808162000_add_user_show_add_ssh_key_message_to_application_settings.rb b/db/migrate/20180808162000_add_user_show_add_ssh_key_message_to_application_settings.rb new file mode 100644 index 00000000000..e3019af2cc9 --- /dev/null +++ b/db/migrate/20180808162000_add_user_show_add_ssh_key_message_to_application_settings.rb @@ -0,0 +1,19 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddUserShowAddSshKeyMessageToApplicationSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column_with_default :application_settings, :user_show_add_ssh_key_message, :boolean, default: true, allow_null: false + end + + def down + remove_column :application_settings, :user_show_add_ssh_key_message + end +end diff --git a/db/migrate/20180815160409_add_file_location_to_ci_job_artifacts.rb b/db/migrate/20180815160409_add_file_location_to_ci_job_artifacts.rb new file mode 100644 index 00000000000..620342005fe --- /dev/null +++ b/db/migrate/20180815160409_add_file_location_to_ci_job_artifacts.rb @@ -0,0 +1,9 @@ +class AddFileLocationToCiJobArtifacts < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :ci_job_artifacts, :file_location, :integer, limit: 2 + end +end diff --git a/db/migrate/20180815170510_add_partial_index_to_ci_builds_artifacts_file.rb b/db/migrate/20180815170510_add_partial_index_to_ci_builds_artifacts_file.rb new file mode 100644 index 00000000000..5e041ea6559 --- /dev/null +++ b/db/migrate/20180815170510_add_partial_index_to_ci_builds_artifacts_file.rb @@ -0,0 +1,16 @@ +class AddPartialIndexToCiBuildsArtifactsFile < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'partial_index_ci_builds_on_id_with_legacy_artifacts'.freeze + + disable_ddl_transaction! + + def up + add_concurrent_index(:ci_builds, :id, where: "artifacts_file <> ''", name: INDEX_NAME) + end + + def down + remove_concurrent_index_by_name(:ci_builds, INDEX_NAME) + end +end diff --git a/db/migrate/20180815175440_add_index_on_list_type.rb b/db/migrate/20180815175440_add_index_on_list_type.rb new file mode 100644 index 00000000000..aad805e436e --- /dev/null +++ b/db/migrate/20180815175440_add_index_on_list_type.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true +class AddIndexOnListType < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :lists, :list_type + end + + def down + remove_concurrent_index :lists, :list_type + end +end diff --git a/db/optional_migrations/composite_primary_keys.rb b/db/optional_migrations/composite_primary_keys.rb index d45705021b0..b330da13d43 100644 --- a/db/optional_migrations/composite_primary_keys.rb +++ b/db/optional_migrations/composite_primary_keys.rb @@ -29,18 +29,20 @@ class CompositePrimaryKeysMigration < ActiveRecord::Migration def up return unless Gitlab::Database.postgresql? - disable_statement_timeout - TABLES.each do |index| - add_primary_key(index) + disable_statement_timeout do + TABLES.each do |index| + add_primary_key(index) + end end end def down return unless Gitlab::Database.postgresql? - disable_statement_timeout - TABLES.each do |index| - remove_primary_key(index) + disable_statement_timeout do + TABLES.each do |index| + remove_primary_key(index) + end end end diff --git a/db/post_migrate/20170502070007_enable_auto_cancel_pending_pipelines_for_all.rb b/db/post_migrate/20170502070007_enable_auto_cancel_pending_pipelines_for_all.rb index bba37e32c01..845c6f0557f 100644 --- a/db/post_migrate/20170502070007_enable_auto_cancel_pending_pipelines_for_all.rb +++ b/db/post_migrate/20170502070007_enable_auto_cancel_pending_pipelines_for_all.rb @@ -8,9 +8,9 @@ class EnableAutoCancelPendingPipelinesForAll < ActiveRecord::Migration DOWNTIME = false def up - disable_statement_timeout - - update_column_in_batches(:projects, :auto_cancel_pending_pipelines, 1) + disable_statement_timeout do + update_column_in_batches(:projects, :auto_cancel_pending_pipelines, 1) + end end def down diff --git a/db/post_migrate/20170503004427_update_retried_for_ci_build.rb b/db/post_migrate/20170503004427_update_retried_for_ci_build.rb index b0b58ab3011..079f0e7511f 100644 --- a/db/post_migrate/20170503004427_update_retried_for_ci_build.rb +++ b/db/post_migrate/20170503004427_update_retried_for_ci_build.rb @@ -7,12 +7,12 @@ class UpdateRetriedForCiBuild < ActiveRecord::Migration disable_ddl_transaction! def up - disable_statement_timeout - if Gitlab::Database.mysql? up_mysql else - up_postgres + disable_statement_timeout do + up_postgres + end end end diff --git a/db/post_migrate/20170508170547_add_head_pipeline_for_each_merge_request.rb b/db/post_migrate/20170508170547_add_head_pipeline_for_each_merge_request.rb index 81e9d050668..5df3ab71648 100644 --- a/db/post_migrate/20170508170547_add_head_pipeline_for_each_merge_request.rb +++ b/db/post_migrate/20170508170547_add_head_pipeline_for_each_merge_request.rb @@ -7,20 +7,20 @@ class AddHeadPipelineForEachMergeRequest < ActiveRecord::Migration disable_ddl_transaction! def up - disable_statement_timeout - pipelines = Arel::Table.new(:ci_pipelines) merge_requests = Arel::Table.new(:merge_requests) - head_id = pipelines - .project(Arel::Nodes::NamedFunction.new('max', [pipelines[:id]])) - .from(pipelines) - .where(pipelines[:ref].eq(merge_requests[:source_branch])) - .where(pipelines[:project_id].eq(merge_requests[:source_project_id])) + disable_statement_timeout do + head_id = pipelines + .project(Arel::Nodes::NamedFunction.new('max', [pipelines[:id]])) + .from(pipelines) + .where(pipelines[:ref].eq(merge_requests[:source_branch])) + .where(pipelines[:project_id].eq(merge_requests[:source_project_id])) - sub_query = Arel::Nodes::SqlLiteral.new(Arel::Nodes::Grouping.new(head_id).to_sql) + sub_query = Arel::Nodes::SqlLiteral.new(Arel::Nodes::Grouping.new(head_id).to_sql) - update_column_in_batches(:merge_requests, :head_pipeline_id, sub_query) + update_column_in_batches(:merge_requests, :head_pipeline_id, sub_query) + end end def down diff --git a/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb b/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb index 2125cc046e5..c996ddbec84 100644 --- a/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb +++ b/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb @@ -87,16 +87,16 @@ class RenameAllReservedPathsAgain < ActiveRecord::Migration ].freeze def up - disable_statement_timeout - - TOP_LEVEL_ROUTES.each { |route| rename_root_paths(route) } - PROJECT_WILDCARD_ROUTES.each { |route| rename_wildcard_paths(route) } - GROUP_ROUTES.each { |route| rename_child_paths(route) } + disable_statement_timeout do + TOP_LEVEL_ROUTES.each { |route| rename_root_paths(route) } + PROJECT_WILDCARD_ROUTES.each { |route| rename_wildcard_paths(route) } + GROUP_ROUTES.each { |route| rename_child_paths(route) } + end end def down - disable_statement_timeout - - revert_renames + disable_statement_timeout do + revert_renames + end end end diff --git a/db/post_migrate/20170526185842_migrate_pipeline_stages.rb b/db/post_migrate/20170526185842_migrate_pipeline_stages.rb index afd4db183c2..736aff77f02 100644 --- a/db/post_migrate/20170526185842_migrate_pipeline_stages.rb +++ b/db/post_migrate/20170526185842_migrate_pipeline_stages.rb @@ -6,17 +6,17 @@ class MigratePipelineStages < ActiveRecord::Migration disable_ddl_transaction! def up - disable_statement_timeout - - execute <<-SQL.strip_heredoc - INSERT INTO ci_stages (project_id, pipeline_id, name) - SELECT project_id, commit_id, stage FROM ci_builds - WHERE stage IS NOT NULL - AND stage_id IS NULL - AND EXISTS (SELECT 1 FROM projects WHERE projects.id = ci_builds.project_id) - AND EXISTS (SELECT 1 FROM ci_pipelines WHERE ci_pipelines.id = ci_builds.commit_id) - GROUP BY project_id, commit_id, stage - ORDER BY MAX(stage_idx) - SQL + disable_statement_timeout do + execute <<-SQL.strip_heredoc + INSERT INTO ci_stages (project_id, pipeline_id, name) + SELECT project_id, commit_id, stage FROM ci_builds + WHERE stage IS NOT NULL + AND stage_id IS NULL + AND EXISTS (SELECT 1 FROM projects WHERE projects.id = ci_builds.project_id) + AND EXISTS (SELECT 1 FROM ci_pipelines WHERE ci_pipelines.id = ci_builds.commit_id) + GROUP BY project_id, commit_id, stage + ORDER BY MAX(stage_idx) + SQL + end end end diff --git a/db/post_migrate/20170526190000_migrate_build_stage_reference_again.rb b/db/post_migrate/20170526190000_migrate_build_stage_reference_again.rb index 31a73bb3b27..a7bfba0ab2b 100644 --- a/db/post_migrate/20170526190000_migrate_build_stage_reference_again.rb +++ b/db/post_migrate/20170526190000_migrate_build_stage_reference_again.rb @@ -7,22 +7,22 @@ class MigrateBuildStageReferenceAgain < ActiveRecord::Migration disable_ddl_transaction! def up - disable_statement_timeout - stage_id = Arel.sql <<-SQL.strip_heredoc (SELECT id FROM ci_stages WHERE ci_stages.pipeline_id = ci_builds.commit_id AND ci_stages.name = ci_builds.stage) SQL - update_column_in_batches(:ci_builds, :stage_id, stage_id) do |table, query| - query.where(table[:stage_id].eq(nil)) + disable_statement_timeout do + update_column_in_batches(:ci_builds, :stage_id, stage_id) do |table, query| + query.where(table[:stage_id].eq(nil)) + end end end def down - disable_statement_timeout - - update_column_in_batches(:ci_builds, :stage_id, nil) + disable_statement_timeout do + update_column_in_batches(:ci_builds, :stage_id, nil) + end end end diff --git a/db/post_migrate/20170711145558_migrate_stages_statuses.rb b/db/post_migrate/20170711145558_migrate_stages_statuses.rb index 65755c0e824..265f7317b9b 100644 --- a/db/post_migrate/20170711145558_migrate_stages_statuses.rb +++ b/db/post_migrate/20170711145558_migrate_stages_statuses.rb @@ -26,9 +26,9 @@ class MigrateStagesStatuses < ActiveRecord::Migration end def down - disable_statement_timeout - - # rubocop:disable Migration/UpdateLargeTable - update_column_in_batches(:ci_stages, :status, nil) + disable_statement_timeout do + # rubocop:disable Migration/UpdateLargeTable + update_column_in_batches(:ci_stages, :status, nil) + end end end diff --git a/db/post_migrate/20171207150343_remove_soft_removed_objects.rb b/db/post_migrate/20171207150343_remove_soft_removed_objects.rb index 3e2dedfdd6a..3109b6dbf8e 100644 --- a/db/post_migrate/20171207150343_remove_soft_removed_objects.rb +++ b/db/post_migrate/20171207150343_remove_soft_removed_objects.rb @@ -78,12 +78,12 @@ class RemoveSoftRemovedObjects < ActiveRecord::Migration MODELS = [Issue, MergeRequest, CiPipelineSchedule, CiTrigger].freeze def up - disable_statement_timeout - - remove_personal_routes - remove_personal_namespaces - remove_group_namespaces - remove_simple_soft_removed_rows + disable_statement_timeout do + remove_personal_routes + remove_personal_namespaces + remove_group_namespaces + remove_simple_soft_removed_rows + end end def down diff --git a/db/post_migrate/20180119121225_remove_redundant_pipeline_stages.rb b/db/post_migrate/20180119121225_remove_redundant_pipeline_stages.rb index 61ea85eb2a7..269f1287f91 100644 --- a/db/post_migrate/20180119121225_remove_redundant_pipeline_stages.rb +++ b/db/post_migrate/20180119121225_remove_redundant_pipeline_stages.rb @@ -38,29 +38,29 @@ class RemoveRedundantPipelineStages < ActiveRecord::Migration end def remove_redundant_pipeline_stages! - disable_statement_timeout - - redundant_stages_ids = <<~SQL - SELECT id FROM ci_stages WHERE (pipeline_id, name) IN ( - SELECT pipeline_id, name FROM ci_stages - GROUP BY pipeline_id, name HAVING COUNT(*) > 1 - ) - SQL - - execute <<~SQL - UPDATE ci_builds SET stage_id = NULL WHERE stage_id IN (#{redundant_stages_ids}) - SQL - - if Gitlab::Database.postgresql? - execute <<~SQL - DELETE FROM ci_stages WHERE id IN (#{redundant_stages_ids}) + disable_statement_timeout do + redundant_stages_ids = <<~SQL + SELECT id FROM ci_stages WHERE (pipeline_id, name) IN ( + SELECT pipeline_id, name FROM ci_stages + GROUP BY pipeline_id, name HAVING COUNT(*) > 1 + ) SQL - else # We can't modify a table we are selecting from on MySQL + execute <<~SQL - DELETE a FROM ci_stages AS a, ci_stages AS b - WHERE a.pipeline_id = b.pipeline_id AND a.name = b.name - AND a.id <> b.id + UPDATE ci_builds SET stage_id = NULL WHERE stage_id IN (#{redundant_stages_ids}) SQL + + if Gitlab::Database.postgresql? + execute <<~SQL + DELETE FROM ci_stages WHERE id IN (#{redundant_stages_ids}) + SQL + else # We can't modify a table we are selecting from on MySQL + execute <<~SQL + DELETE a FROM ci_stages AS a, ci_stages AS b + WHERE a.pipeline_id = b.pipeline_id AND a.name = b.name + AND a.id <> b.id + SQL + end end end end diff --git a/db/post_migrate/20180305100050_remove_permanent_from_redirect_routes.rb b/db/post_migrate/20180305100050_remove_permanent_from_redirect_routes.rb index db5165dbe70..aa19732ca1c 100644 --- a/db/post_migrate/20180305100050_remove_permanent_from_redirect_routes.rb +++ b/db/post_migrate/20180305100050_remove_permanent_from_redirect_routes.rb @@ -15,10 +15,10 @@ class RemovePermanentFromRedirectRoutes < ActiveRecord::Migration # ReworkRedirectRoutesIndexes: # https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/16211 if Gitlab::Database.postgresql? - disable_statement_timeout - - execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_PERM};" - execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_TEMP};" + disable_statement_timeout do + execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_PERM};" + execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME_TEMP};" + end end remove_column(:redirect_routes, :permanent) @@ -28,10 +28,10 @@ class RemovePermanentFromRedirectRoutes < ActiveRecord::Migration add_column(:redirect_routes, :permanent, :boolean) if Gitlab::Database.postgresql? - disable_statement_timeout - - execute("CREATE INDEX CONCURRENTLY #{INDEX_NAME_PERM} ON redirect_routes (lower(path) varchar_pattern_ops) where (permanent);") - execute("CREATE INDEX CONCURRENTLY #{INDEX_NAME_TEMP} ON redirect_routes (lower(path) varchar_pattern_ops) where (not permanent or permanent is null) ;") + disable_statement_timeout do + execute("CREATE INDEX CONCURRENTLY #{INDEX_NAME_PERM} ON redirect_routes (lower(path) varchar_pattern_ops) where (permanent);") + execute("CREATE INDEX CONCURRENTLY #{INDEX_NAME_TEMP} ON redirect_routes (lower(path) varchar_pattern_ops) where (not permanent or permanent is null) ;") + end end end end diff --git a/db/post_migrate/20180306164012_add_path_index_to_redirect_routes.rb b/db/post_migrate/20180306164012_add_path_index_to_redirect_routes.rb index d6fb4f06695..ca9212fae27 100644 --- a/db/post_migrate/20180306164012_add_path_index_to_redirect_routes.rb +++ b/db/post_migrate/20180306164012_add_path_index_to_redirect_routes.rb @@ -20,10 +20,10 @@ class AddPathIndexToRedirectRoutes < ActiveRecord::Migration def up return unless Gitlab::Database.postgresql? - disable_statement_timeout - - unless index_exists_by_name?(:redirect_routes, INDEX_NAME) - execute("CREATE UNIQUE INDEX CONCURRENTLY #{INDEX_NAME} ON redirect_routes (lower(path) varchar_pattern_ops);") + disable_statement_timeout do + unless index_exists_by_name?(:redirect_routes, INDEX_NAME) + execute("CREATE UNIQUE INDEX CONCURRENTLY #{INDEX_NAME} ON redirect_routes (lower(path) varchar_pattern_ops);") + end end end diff --git a/db/post_migrate/20180405101928_reschedule_builds_stages_migration.rb b/db/post_migrate/20180405101928_reschedule_builds_stages_migration.rb index e19387bce1e..c32123454f9 100644 --- a/db/post_migrate/20180405101928_reschedule_builds_stages_migration.rb +++ b/db/post_migrate/20180405101928_reschedule_builds_stages_migration.rb @@ -17,13 +17,13 @@ class RescheduleBuildsStagesMigration < ActiveRecord::Migration end def up - disable_statement_timeout - - Build.where('stage_id IS NULL').tap do |relation| - queue_background_migration_jobs_by_range_at_intervals(relation, - MIGRATION, - 5.minutes, - batch_size: BATCH_SIZE) + disable_statement_timeout do + Build.where('stage_id IS NULL').tap do |relation| + queue_background_migration_jobs_by_range_at_intervals(relation, + MIGRATION, + 5.minutes, + batch_size: BATCH_SIZE) + end end end diff --git a/db/post_migrate/20180420080616_schedule_stages_index_migration.rb b/db/post_migrate/20180420080616_schedule_stages_index_migration.rb index 1d0daad002f..eb82f098639 100644 --- a/db/post_migrate/20180420080616_schedule_stages_index_migration.rb +++ b/db/post_migrate/20180420080616_schedule_stages_index_migration.rb @@ -13,13 +13,13 @@ class ScheduleStagesIndexMigration < ActiveRecord::Migration end def up - disable_statement_timeout - - Stage.all.tap do |relation| - queue_background_migration_jobs_by_range_at_intervals(relation, - MIGRATION, - 5.minutes, - batch_size: BATCH_SIZE) + disable_statement_timeout do + Stage.all.tap do |relation| + queue_background_migration_jobs_by_range_at_intervals(relation, + MIGRATION, + 5.minutes, + batch_size: BATCH_SIZE) + end end end diff --git a/db/post_migrate/20180604123514_cleanup_stages_position_migration.rb b/db/post_migrate/20180604123514_cleanup_stages_position_migration.rb index 73c23dffca0..5418f442e79 100644 --- a/db/post_migrate/20180604123514_cleanup_stages_position_migration.rb +++ b/db/post_migrate/20180604123514_cleanup_stages_position_migration.rb @@ -12,32 +12,34 @@ class CleanupStagesPositionMigration < ActiveRecord::Migration end def up - disable_statement_timeout + disable_statement_timeout do + Gitlab::BackgroundMigration.steal('MigrateStageIndex') - Gitlab::BackgroundMigration.steal('MigrateStageIndex') - - unless index_exists_by_name?(:ci_stages, TMP_INDEX_NAME) - add_concurrent_index(:ci_stages, :id, where: 'position IS NULL', name: TMP_INDEX_NAME) - end + unless index_exists_by_name?(:ci_stages, TMP_INDEX_NAME) + add_concurrent_index(:ci_stages, :id, where: 'position IS NULL', name: TMP_INDEX_NAME) + end - migratable = <<~SQL - position IS NULL AND EXISTS ( - SELECT 1 FROM ci_builds WHERE stage_id = ci_stages.id AND stage_idx IS NOT NULL - ) - SQL + migratable = <<~SQL + position IS NULL AND EXISTS ( + SELECT 1 FROM ci_builds WHERE stage_id = ci_stages.id AND stage_idx IS NOT NULL + ) + SQL - Stages.where(migratable).each_batch(of: 1000) do |batch| - batch.pluck(:id).each do |stage| - Gitlab::BackgroundMigration::MigrateStageIndex.new.perform(stage, stage) + Stages.where(migratable).each_batch(of: 1000) do |batch| + batch.pluck(:id).each do |stage| + Gitlab::BackgroundMigration::MigrateStageIndex.new.perform(stage, stage) + end end - end - remove_concurrent_index_by_name(:ci_stages, TMP_INDEX_NAME) + remove_concurrent_index_by_name(:ci_stages, TMP_INDEX_NAME) + end end def down if index_exists_by_name?(:ci_stages, TMP_INDEX_NAME) - remove_concurrent_index_by_name(:ci_stages, TMP_INDEX_NAME) + disable_statement_timeout do + remove_concurrent_index_by_name(:ci_stages, TMP_INDEX_NAME) + end end end end diff --git a/db/post_migrate/20180723130817_delete_inconsistent_internal_id_records.rb b/db/post_migrate/20180723130817_delete_inconsistent_internal_id_records.rb new file mode 100644 index 00000000000..3b9b95ec9ca --- /dev/null +++ b/db/post_migrate/20180723130817_delete_inconsistent_internal_id_records.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true +class DeleteInconsistentInternalIdRecords < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + # This migration cleans up any inconsistent records in internal_ids. + # + # That is, it deletes records that track a `last_value` that is + # smaller than the maximum internal id (usually `iid`) found in + # the corresponding model records. + + def up + disable_statement_timeout do + delete_internal_id_records('issues', 'project_id') + delete_internal_id_records('merge_requests', 'project_id', 'target_project_id') + delete_internal_id_records('deployments', 'project_id') + delete_internal_id_records('milestones', 'project_id') + delete_internal_id_records('milestones', 'namespace_id', 'group_id') + delete_internal_id_records('ci_pipelines', 'project_id') + end + end + + class InternalId < ActiveRecord::Base + self.table_name = 'internal_ids' + enum usage: { issues: 0, merge_requests: 1, deployments: 2, milestones: 3, epics: 4, ci_pipelines: 5 } + end + + private + + def delete_internal_id_records(base_table, scope_column_name, base_scope_column_name = scope_column_name) + sql = <<~SQL + SELECT id FROM ( -- workaround for MySQL + SELECT internal_ids.id FROM ( + SELECT #{base_scope_column_name} AS #{scope_column_name}, max(iid) as maximum_iid from #{base_table} GROUP BY #{scope_column_name} + ) maxima JOIN internal_ids USING (#{scope_column_name}) + WHERE internal_ids.usage=#{InternalId.usages.fetch(base_table)} AND maxima.maximum_iid > internal_ids.last_value + ) internal_ids + SQL + + InternalId.where("id IN (#{sql})").tap do |ids| # rubocop:disable GitlabSecurity/SqlInjection + say "Deleting internal_id records for #{base_table}: #{ids.pluck(:project_id, :last_value)}" unless ids.empty? + end.delete_all + end +end diff --git a/db/post_migrate/20180809195358_migrate_null_wiki_access_levels.rb b/db/post_migrate/20180809195358_migrate_null_wiki_access_levels.rb new file mode 100644 index 00000000000..0a0a33299e4 --- /dev/null +++ b/db/post_migrate/20180809195358_migrate_null_wiki_access_levels.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class MigrateNullWikiAccessLevels < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + class ProjectFeature < ActiveRecord::Base + include EachBatch + + self.table_name = 'project_features' + end + + def up + ProjectFeature.where(wiki_access_level: nil).each_batch do |relation| + relation.update_all(wiki_access_level: 20) + end + + # We need to re-count wikis as previous attempt was not considering the NULLs. + transaction do + execute('SET LOCAL statement_timeout TO 0') if Gitlab::Database.postgresql? # see https://gitlab.com/gitlab-org/gitlab-ce/issues/48967 + + execute("UPDATE site_statistics SET wikis_count = (SELECT COUNT(*) FROM project_features WHERE wiki_access_level != 0)") + end + end + + def down + # there is no way to rollback this change, there are no downsides in keeping migrated data. + end +end diff --git a/db/post_migrate/20180816161409_migrate_legacy_artifacts_to_job_artifacts.rb b/db/post_migrate/20180816161409_migrate_legacy_artifacts_to_job_artifacts.rb new file mode 100644 index 00000000000..2dd711e9c10 --- /dev/null +++ b/db/post_migrate/20180816161409_migrate_legacy_artifacts_to_job_artifacts.rb @@ -0,0 +1,32 @@ +class MigrateLegacyArtifactsToJobArtifacts < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + MIGRATION = 'MigrateLegacyArtifacts'.freeze + BATCH_SIZE = 100 + + disable_ddl_transaction! + + class Build < ActiveRecord::Base + include EachBatch + + self.table_name = 'ci_builds' + self.inheritance_column = :_type_disabled + + scope :with_legacy_artifacts, -> { where("artifacts_file <> ''") } + end + + def up + MigrateLegacyArtifactsToJobArtifacts::Build + .with_legacy_artifacts.tap do |relation| + queue_background_migration_jobs_by_range_at_intervals(relation, + MIGRATION, + 5.minutes, + batch_size: BATCH_SIZE) + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20180816193530_rename_login_root_namespaces.rb b/db/post_migrate/20180816193530_rename_login_root_namespaces.rb new file mode 100644 index 00000000000..60cec24eed6 --- /dev/null +++ b/db/post_migrate/20180816193530_rename_login_root_namespaces.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true +class RenameLoginRootNamespaces < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + include Gitlab::Database::RenameReservedPathsMigration::V1 + + DOWNTIME = false + + # We're taking over the /login namespace as part of a fix for the Jira integration + def up + rename_root_paths 'login' + end + + def down + revert_renames + end +end diff --git a/db/post_migrate/20180826111825_recalculate_site_statistics.rb b/db/post_migrate/20180826111825_recalculate_site_statistics.rb new file mode 100644 index 00000000000..741035a444f --- /dev/null +++ b/db/post_migrate/20180826111825_recalculate_site_statistics.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class RecalculateSiteStatistics < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + transaction do + execute('SET LOCAL statement_timeout TO 0') if Gitlab::Database.postgresql? # see https://gitlab.com/gitlab-org/gitlab-ce/issues/48967 + + execute("UPDATE site_statistics SET repositories_count = (SELECT COUNT(*) FROM projects)") + end + + transaction do + execute('SET LOCAL statement_timeout TO 0') if Gitlab::Database.postgresql? # see https://gitlab.com/gitlab-org/gitlab-ce/issues/48967 + + execute("UPDATE site_statistics SET wikis_count = (SELECT COUNT(*) FROM project_features WHERE wiki_access_level != 0)") + end + end + + def down + # No downside in keeping the counter up-to-date + end +end diff --git a/db/schema.rb b/db/schema.rb index 16ae2565c93..f48df68b785 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180807153545) do +ActiveRecord::Schema.define(version: 20180826111825) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -164,12 +164,14 @@ ActiveRecord::Schema.define(version: 20180807153545) do t.boolean "authorized_keys_enabled", default: true, null: false t.string "auto_devops_domain" t.boolean "pages_domain_verification_enabled", default: true, null: false + t.string "user_default_internal_regex" t.boolean "allow_local_requests_from_hooks_and_services", default: false, null: false t.boolean "enforce_terms", default: false t.boolean "mirror_available", default: true, null: false t.boolean "hide_third_party_offers", default: false, null: false t.boolean "instance_statistics_visibility_private", default: false, null: false t.boolean "web_ide_clientside_preview_enabled", default: false, null: false + t.boolean "user_show_add_ssh_key_message", default: true, null: false end create_table "audit_events", force: :cascade do |t| @@ -340,6 +342,7 @@ ActiveRecord::Schema.define(version: 20180807153545) do add_index "ci_builds", ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree add_index "ci_builds", ["commit_id", "type", "name", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_name_and_ref", using: :btree add_index "ci_builds", ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree + add_index "ci_builds", ["id"], name: "partial_index_ci_builds_on_id_with_legacy_artifacts", where: "(artifacts_file <> ''::text)", using: :btree add_index "ci_builds", ["project_id", "id"], name: "index_ci_builds_on_project_id_and_id", using: :btree add_index "ci_builds", ["protected"], name: "index_ci_builds_on_protected", using: :btree add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree @@ -395,6 +398,7 @@ ActiveRecord::Schema.define(version: 20180807153545) do t.string "file" t.binary "file_sha256" t.integer "file_format", limit: 2 + t.integer "file_location", limit: 2 end add_index "ci_job_artifacts", ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id", using: :btree @@ -1134,6 +1138,7 @@ ActiveRecord::Schema.define(version: 20180807153545) do add_index "lists", ["board_id", "label_id"], name: "index_lists_on_board_id_and_label_id", unique: true, using: :btree add_index "lists", ["label_id"], name: "index_lists_on_label_id", using: :btree + add_index "lists", ["list_type"], name: "index_lists_on_list_type", using: :btree create_table "members", force: :cascade do |t| t.integer "access_level", null: false @@ -1739,6 +1744,7 @@ ActiveRecord::Schema.define(version: 20180807153545) do t.datetime "updated_at", null: false end + add_index "protected_tags", ["project_id", "name"], name: "index_protected_tags_on_project_id_and_name", unique: true, using: :btree add_index "protected_tags", ["project_id"], name: "index_protected_tags_on_project_id", using: :btree create_table "push_event_payloads", id: false, force: :cascade do |t| |