diff options
author | Kamil Trzciński <ayufan@ayufan.eu> | 2018-02-28 23:28:43 +0300 |
---|---|---|
committer | Kamil Trzciński <ayufan@ayufan.eu> | 2018-02-28 23:28:43 +0300 |
commit | a2f375e8f74870dcdcfa1c7886bd1c14c80a684e (patch) | |
tree | 6b6e3a4f7554f4671edc17d87869dd6916984404 /db | |
parent | a22f6fa6e50bb31921415b01fd345d6802581390 (diff) | |
parent | 81852d1f902c2923c239e9c33cab77f5fd6ca8d8 (diff) |
Merge remote-tracking branch 'origin/master' into object-storage-ee-to-ce-backport
Diffstat (limited to 'db')
17 files changed, 271 insertions, 33 deletions
diff --git a/db/fixtures/development/01_admin.rb b/db/fixtures/development/01_admin.rb index 6f241f6fa4a..dfb50c195c1 100644 --- a/db/fixtures/development/01_admin.rb +++ b/db/fixtures/development/01_admin.rb @@ -9,7 +9,6 @@ Gitlab::Seeder.quiet do s.username = 'root' s.password = '5iveL!fe' s.admin = true - s.projects_limit = 100 s.confirmed_at = DateTime.now end end diff --git a/db/migrate/20170929131201_populate_fork_networks.rb b/db/migrate/20170929131201_populate_fork_networks.rb index 1214962770f..ddbf27e1852 100644 --- a/db/migrate/20170929131201_populate_fork_networks.rb +++ b/db/migrate/20170929131201_populate_fork_networks.rb @@ -6,22 +6,8 @@ class PopulateForkNetworks < ActiveRecord::Migration DOWNTIME = false - MIGRATION = 'PopulateForkNetworksRange'.freeze - BATCH_SIZE = 100 - DELAY_INTERVAL = 15.seconds - - disable_ddl_transaction! - - class ForkedProjectLink < ActiveRecord::Base - include EachBatch - - self.table_name = 'forked_project_links' - end - def up - say 'Populating the `fork_networks` based on existing `forked_project_links`' - - queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE) + say 'Fork networks will be populated in 20171205190711 - RescheduleForkNetworkCreationCaller' end def down diff --git a/db/migrate/20180119160751_optimize_ci_job_artifacts.rb b/db/migrate/20180119160751_optimize_ci_job_artifacts.rb new file mode 100644 index 00000000000..9b4340ed7b7 --- /dev/null +++ b/db/migrate/20180119160751_optimize_ci_job_artifacts.rb @@ -0,0 +1,23 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class OptimizeCiJobArtifacts < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + def up + # job_id is just here to be a covering index for index only scans + # since we'll almost always be joining against ci_builds on job_id + add_concurrent_index(:ci_job_artifacts, [:expire_at, :job_id]) + add_concurrent_index(:ci_builds, [:artifacts_expire_at], where: "artifacts_file <> ''") + end + + def down + remove_concurrent_index(:ci_job_artifacts, [:expire_at, :job_id]) + remove_concurrent_index(:ci_builds, [:artifacts_expire_at], where: "artifacts_file <> ''") + end +end diff --git a/db/migrate/20180208183958_schedule_populate_untracked_uploads_if_needed.rb b/db/migrate/20180208183958_schedule_populate_untracked_uploads_if_needed.rb new file mode 100644 index 00000000000..e46e793d9d2 --- /dev/null +++ b/db/migrate/20180208183958_schedule_populate_untracked_uploads_if_needed.rb @@ -0,0 +1,47 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class SchedulePopulateUntrackedUploadsIfNeeded < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze + + class UntrackedFile < ActiveRecord::Base + include EachBatch + + self.table_name = 'untracked_files_for_uploads' + end + + def up + if table_exists?(:untracked_files_for_uploads) + process_or_remove_table + end + end + + def down + # nothing + end + + private + + def process_or_remove_table + if UntrackedFile.all.empty? + drop_temp_table + else + schedule_populate_untracked_uploads_jobs + end + end + + def drop_temp_table + drop_table(:untracked_files_for_uploads, if_exists: true) + end + + def schedule_populate_untracked_uploads_jobs + say "Scheduling #{FOLLOW_UP_MIGRATION} background migration jobs since there are rows in untracked_files_for_uploads." + + bulk_queue_background_migration_jobs_by_range( + UntrackedFile, FOLLOW_UP_MIGRATION) + end +end diff --git a/db/migrate/20180213131630_add_partial_index_to_projects_for_index_only_scans.rb b/db/migrate/20180213131630_add_partial_index_to_projects_for_index_only_scans.rb new file mode 100644 index 00000000000..cedf2510dda --- /dev/null +++ b/db/migrate/20180213131630_add_partial_index_to_projects_for_index_only_scans.rb @@ -0,0 +1,21 @@ +class AddPartialIndexToProjectsForIndexOnlyScans < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'index_projects_on_id_partial_for_visibility' + + disable_ddl_transaction! + + # Adds a partial index to leverage index-only scans when looking up project ids + def up + unless index_exists?(:projects, :id, name: INDEX_NAME) + add_concurrent_index :projects, :id, name: INDEX_NAME, unique: true, where: 'visibility_level IN (10,20)' + end + end + + def down + if index_exists?(:projects, :id, name: INDEX_NAME) + remove_concurrent_index_by_name :projects, INDEX_NAME + end + end +end diff --git a/db/migrate/20180215181245_users_name_lower_index.rb b/db/migrate/20180215181245_users_name_lower_index.rb new file mode 100644 index 00000000000..d3f68cb7d45 --- /dev/null +++ b/db/migrate/20180215181245_users_name_lower_index.rb @@ -0,0 +1,29 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class UsersNameLowerIndex < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + INDEX_NAME = 'index_on_users_name_lower' + + disable_ddl_transaction! + + def up + return unless Gitlab::Database.postgresql? + + # On GitLab.com this produces an index with a size of roughly 60 MB. + execute "CREATE INDEX CONCURRENTLY #{INDEX_NAME} ON users (LOWER(name))" + end + + def down + return unless Gitlab::Database.postgresql? + + if supports_drop_index_concurrently? + execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME}" + else + execute "DROP INDEX IF EXISTS #{INDEX_NAME}" + end + end +end diff --git a/db/migrate/20180216120000_add_pages_domain_verification.rb b/db/migrate/20180216120000_add_pages_domain_verification.rb new file mode 100644 index 00000000000..8b7cae92285 --- /dev/null +++ b/db/migrate/20180216120000_add_pages_domain_verification.rb @@ -0,0 +1,8 @@ +class AddPagesDomainVerification < ActiveRecord::Migration + DOWNTIME = false + + def change + add_column :pages_domains, :verified_at, :datetime_with_timezone + add_column :pages_domains, :verification_code, :string + end +end diff --git a/db/migrate/20180216120010_add_pages_domain_verified_at_index.rb b/db/migrate/20180216120010_add_pages_domain_verified_at_index.rb new file mode 100644 index 00000000000..825dfb52dce --- /dev/null +++ b/db/migrate/20180216120010_add_pages_domain_verified_at_index.rb @@ -0,0 +1,15 @@ +class AddPagesDomainVerifiedAtIndex < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :pages_domains, :verified_at + end + + def down + remove_concurrent_index :pages_domains, :verified_at + end +end diff --git a/db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb b/db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb new file mode 100644 index 00000000000..06d458028b3 --- /dev/null +++ b/db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb @@ -0,0 +1,7 @@ +class AllowDomainVerificationToBeDisabled < ActiveRecord::Migration + DOWNTIME = false + + def change + add_column :application_settings, :pages_domain_verification_enabled, :boolean, default: true, null: false + end +end diff --git a/db/migrate/20180216120030_add_pages_domain_enabled_until.rb b/db/migrate/20180216120030_add_pages_domain_enabled_until.rb new file mode 100644 index 00000000000..b40653044dd --- /dev/null +++ b/db/migrate/20180216120030_add_pages_domain_enabled_until.rb @@ -0,0 +1,7 @@ +class AddPagesDomainEnabledUntil < ActiveRecord::Migration + DOWNTIME = false + + def change + add_column :pages_domains, :enabled_until, :datetime_with_timezone + end +end diff --git a/db/migrate/20180216120040_add_pages_domain_enabled_until_index.rb b/db/migrate/20180216120040_add_pages_domain_enabled_until_index.rb new file mode 100644 index 00000000000..00f6e4979da --- /dev/null +++ b/db/migrate/20180216120040_add_pages_domain_enabled_until_index.rb @@ -0,0 +1,17 @@ +class AddPagesDomainEnabledUntilIndex < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :pages_domains, [:project_id, :enabled_until] + add_concurrent_index :pages_domains, [:verified_at, :enabled_until] + end + + def down + remove_concurrent_index :pages_domains, [:verified_at, :enabled_until] + remove_concurrent_index :pages_domains, [:project_id, :enabled_until] + end +end diff --git a/db/migrate/20180216120050_pages_domains_verification_grace_period.rb b/db/migrate/20180216120050_pages_domains_verification_grace_period.rb new file mode 100644 index 00000000000..d7f8634b536 --- /dev/null +++ b/db/migrate/20180216120050_pages_domains_verification_grace_period.rb @@ -0,0 +1,26 @@ +class PagesDomainsVerificationGracePeriod < ActiveRecord::Migration + DOWNTIME = false + + class PagesDomain < ActiveRecord::Base + include EachBatch + end + + # Allow this migration to resume if it fails partway through + disable_ddl_transaction! + + def up + now = Time.now + grace = now + 30.days + + PagesDomain.each_batch do |relation| + relation.update_all(verified_at: now, enabled_until: grace) + + # Sleep 2 minutes between batches to not overload the DB with dead tuples + sleep(2.minutes) unless relation.reorder(:id).last == PagesDomain.reorder(:id).last + end + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb b/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb index 26b99b61424..c48f1c938d0 100644 --- a/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb +++ b/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb @@ -20,7 +20,7 @@ class CleanupMoveSystemUploadFolderSymlink < ActiveRecord::Migration def down if File.directory?(new_directory) say "Symlinking #{old_directory} -> #{new_directory}" - FileUtils.ln_s(new_directory, old_directory) + FileUtils.ln_s(new_directory, old_directory) unless File.exist?(old_directory) else say "#{new_directory} doesn't exist, skipping." end diff --git a/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb b/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb index 05430efe1f6..26f917d5a1e 100644 --- a/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb +++ b/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb @@ -3,22 +3,8 @@ class RescheduleForkNetworkCreation < ActiveRecord::Migration DOWNTIME = false - MIGRATION = 'PopulateForkNetworksRange'.freeze - BATCH_SIZE = 100 - DELAY_INTERVAL = 15.seconds - - disable_ddl_transaction! - - class ForkedProjectLink < ActiveRecord::Base - include EachBatch - - self.table_name = 'forked_project_links' - end - def up - say 'Populating the `fork_networks` based on existing `forked_project_links`' - - queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE) + say 'Fork networks will be populated in 20171205190711 - RescheduleForkNetworkCreationCaller' end def down diff --git a/db/post_migrate/20180216121020_fill_pages_domain_verification_code.rb b/db/post_migrate/20180216121020_fill_pages_domain_verification_code.rb new file mode 100644 index 00000000000..d423673d2a5 --- /dev/null +++ b/db/post_migrate/20180216121020_fill_pages_domain_verification_code.rb @@ -0,0 +1,41 @@ +class FillPagesDomainVerificationCode < ActiveRecord::Migration + DOWNTIME = false + + class PagesDomain < ActiveRecord::Base + include EachBatch + end + + # Allow this migration to resume if it fails partway through + disable_ddl_transaction! + + def up + PagesDomain.where(verification_code: [nil, '']).each_batch do |relation| + connection.execute(set_codes_sql(relation)) + + # Sleep 2 minutes between batches to not overload the DB with dead tuples + sleep(2.minutes) unless relation.reorder(:id).last == PagesDomain.reorder(:id).last + end + + change_column_null(:pages_domains, :verification_code, false) + end + + def down + change_column_null(:pages_domains, :verification_code, true) + end + + private + + def set_codes_sql(relation) + ids = relation.pluck(:id) + whens = ids.map { |id| "WHEN #{id} THEN '#{SecureRandom.hex(16)}'" } + + <<~SQL + UPDATE pages_domains + SET verification_code = + CASE id + #{whens.join("\n")} + END + WHERE id IN(#{ids.join(',')}) + SQL + end +end diff --git a/db/post_migrate/20180216121030_enqueue_verify_pages_domain_workers.rb b/db/post_migrate/20180216121030_enqueue_verify_pages_domain_workers.rb new file mode 100644 index 00000000000..bf9bf4e660f --- /dev/null +++ b/db/post_migrate/20180216121030_enqueue_verify_pages_domain_workers.rb @@ -0,0 +1,16 @@ +class EnqueueVerifyPagesDomainWorkers < ActiveRecord::Migration + class PagesDomain < ActiveRecord::Base + include EachBatch + end + + def up + PagesDomain.each_batch do |relation| + ids = relation.pluck(:id).map { |id| [id] } + PagesDomainVerificationWorker.bulk_perform_async(ids) + end + end + + def down + # no-op + end +end diff --git a/db/schema.rb b/db/schema.rb index e381fc9fb8a..fba0c710705 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180206200543) do +ActiveRecord::Schema.define(version: 20180216121030) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -156,6 +156,7 @@ ActiveRecord::Schema.define(version: 20180206200543) do t.integer "gitaly_timeout_fast", default: 10, null: false t.boolean "authorized_keys_enabled", default: true, null: false t.string "auto_devops_domain" + t.boolean "pages_domain_verification_enabled", default: true, null: false end create_table "audit_events", force: :cascade do |t| @@ -295,6 +296,7 @@ ActiveRecord::Schema.define(version: 20180206200543) do t.integer "failure_reason" end + add_index "ci_builds", ["artifacts_expire_at"], name: "index_ci_builds_on_artifacts_expire_at", where: "(artifacts_file <> ''::text)", using: :btree add_index "ci_builds", ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree add_index "ci_builds", ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree @@ -336,6 +338,7 @@ ActiveRecord::Schema.define(version: 20180206200543) do t.string "file" end + add_index "ci_job_artifacts", ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id", using: :btree add_index "ci_job_artifacts", ["job_id", "file_type"], name: "index_ci_job_artifacts_on_job_id_and_file_type", unique: true, using: :btree add_index "ci_job_artifacts", ["project_id"], name: "index_ci_job_artifacts_on_project_id", using: :btree @@ -1315,10 +1318,16 @@ ActiveRecord::Schema.define(version: 20180206200543) do t.string "encrypted_key_iv" t.string "encrypted_key_salt" t.string "domain" + t.datetime_with_timezone "verified_at" + t.string "verification_code", null: false + t.datetime_with_timezone "enabled_until" end add_index "pages_domains", ["domain"], name: "index_pages_domains_on_domain", unique: true, using: :btree + add_index "pages_domains", ["project_id", "enabled_until"], name: "index_pages_domains_on_project_id_and_enabled_until", using: :btree add_index "pages_domains", ["project_id"], name: "index_pages_domains_on_project_id", using: :btree + add_index "pages_domains", ["verified_at", "enabled_until"], name: "index_pages_domains_on_verified_at_and_enabled_until", using: :btree + add_index "pages_domains", ["verified_at"], name: "index_pages_domains_on_verified_at", using: :btree create_table "personal_access_tokens", force: :cascade do |t| t.integer "user_id", null: false @@ -1470,6 +1479,7 @@ ActiveRecord::Schema.define(version: 20180206200543) do add_index "projects", ["created_at"], name: "index_projects_on_created_at", using: :btree add_index "projects", ["creator_id"], name: "index_projects_on_creator_id", using: :btree add_index "projects", ["description"], name: "index_projects_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"} + add_index "projects", ["id"], name: "index_projects_on_id_partial_for_visibility", unique: true, where: "(visibility_level = ANY (ARRAY[10, 20]))", using: :btree add_index "projects", ["last_activity_at"], name: "index_projects_on_last_activity_at", using: :btree add_index "projects", ["last_repository_check_failed"], name: "index_projects_on_last_repository_check_failed", using: :btree add_index "projects", ["last_repository_updated_at"], name: "index_projects_on_last_repository_updated_at", using: :btree |