diff options
author | Dylan Griffith <dyl.griffith@gmail.com> | 2018-04-26 04:45:01 +0300 |
---|---|---|
committer | Dylan Griffith <dyl.griffith@gmail.com> | 2018-04-26 04:45:01 +0300 |
commit | c80e6b9c1229776cf5af065954b99c182cc046c2 (patch) | |
tree | 5b9d1ca0c9f0101fd663fd83d68012c49f0ab54b /db | |
parent | 51cc01b6c9468056f40cdc8deb0e4b6414a3ccb9 (diff) | |
parent | 55f07cc32e7684b21e0c1662c70128df14c6abf7 (diff) |
Merge branch 'master' into siemens-runner-per-group
Diffstat (limited to 'db')
6 files changed, 155 insertions, 18 deletions
diff --git a/db/migrate/20161220141214_remove_dot_git_from_group_names.rb b/db/migrate/20161220141214_remove_dot_git_from_group_names.rb index bddc234db25..17357b67ab7 100644 --- a/db/migrate/20161220141214_remove_dot_git_from_group_names.rb +++ b/db/migrate/20161220141214_remove_dot_git_from_group_names.rb @@ -59,17 +59,17 @@ class RemoveDotGitFromGroupNames < ActiveRecord::Migration end def move_namespace(group_id, path_was, path) - repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{group_id}").map do |row| - Gitlab.config.repositories.storages[row['repository_storage']].legacy_disk_path + repository_storages = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{group_id}").map do |row| + row['repository_storage'] end.compact # Move the namespace directory in all storages paths used by member projects - repository_storage_paths.each do |repository_storage_path| + repository_storages.each do |repository_storage| # Ensure old directory exists before moving it - gitlab_shell.add_namespace(repository_storage_path, path_was) + gitlab_shell.add_namespace(repository_storage, path_was) - unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path) - Rails.logger.error "Exception moving path #{repository_storage_path} from #{path_was} to #{path}" + unless gitlab_shell.mv_namespace(repository_storage, path_was, path) + Rails.logger.error "Exception moving on shard #{repository_storage} from #{path_was} to #{path}" # if we cannot move namespace directory we should rollback # db changes in order to prevent out of sync between db and fs diff --git a/db/migrate/20161226122833_remove_dot_git_from_usernames.rb b/db/migrate/20161226122833_remove_dot_git_from_usernames.rb index 7c28d934c29..8986cd8cb4b 100644 --- a/db/migrate/20161226122833_remove_dot_git_from_usernames.rb +++ b/db/migrate/20161226122833_remove_dot_git_from_usernames.rb @@ -53,8 +53,8 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration select_all("SELECT id, path FROM routes WHERE path = '#{quote_string(path)}'").present? end - def path_exists?(path, repository_storage_path) - repository_storage_path && gitlab_shell.exists?(repository_storage_path, path) + def path_exists?(shard, repository_storage_path) + repository_storage_path && gitlab_shell.exists?(shard, repository_storage_path) end # Accepts invalid path like test.git and returns test_git or @@ -70,8 +70,8 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration def check_routes(base, counter, path) route_exists = route_exists?(path) - Gitlab.config.repositories.storages.each_value do |storage| - if route_exists || path_exists?(path, storage.legacy_disk_path) + Gitlab.config.repositories.storages.each do |shard, storage| + if route_exists || path_exists?(shard, storage.legacy_disk_path) counter += 1 path = "#{base}#{counter}" @@ -83,17 +83,17 @@ class RemoveDotGitFromUsernames < ActiveRecord::Migration end def move_namespace(namespace_id, path_was, path) - repository_storage_paths = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{namespace_id}").map do |row| - Gitlab.config.repositories.storages[row['repository_storage']].legacy_disk_path + repository_storages = select_all("SELECT distinct(repository_storage) FROM projects WHERE namespace_id = #{namespace_id}").map do |row| + row['repository_storage'] end.compact - # Move the namespace directory in all storages paths used by member projects - repository_storage_paths.each do |repository_storage_path| + # Move the namespace directory in all storages used by member projects + repository_storages.each do |repository_storage| # Ensure old directory exists before moving it - gitlab_shell.add_namespace(repository_storage_path, path_was) + gitlab_shell.add_namespace(repository_storage, path_was) - unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path) - Rails.logger.error "Exception moving path #{repository_storage_path} from #{path_was} to #{path}" + unless gitlab_shell.mv_namespace(repository_storage, path_was, path) + Rails.logger.error "Exception moving on shard #{repository_storage} from #{path_was} to #{path}" # if we cannot move namespace directory we should rollback # db changes in order to prevent out of sync between db and fs diff --git a/db/migrate/20180403035759_create_project_ci_cd_settings.rb b/db/migrate/20180403035759_create_project_ci_cd_settings.rb new file mode 100644 index 00000000000..06856af6204 --- /dev/null +++ b/db/migrate/20180403035759_create_project_ci_cd_settings.rb @@ -0,0 +1,68 @@ +class CreateProjectCiCdSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + unless table_exists?(:project_ci_cd_settings) + create_table(:project_ci_cd_settings) do |t| + t.integer(:project_id, null: false) + t.boolean(:group_runners_enabled, default: true, null: false) + end + end + + disable_statement_timeout + + # This particular INSERT will take between 10 and 20 seconds. + execute 'INSERT INTO project_ci_cd_settings (project_id) SELECT id FROM projects' + + # We add the index and foreign key separately so the above INSERT statement + # takes as little time as possible. + add_concurrent_index(:project_ci_cd_settings, :project_id, unique: true) + + add_foreign_key_with_retry + end + + def down + drop_table :project_ci_cd_settings + end + + def add_foreign_key_with_retry + if Gitlab::Database.mysql? + # When using MySQL we don't support online upgrades, thus projects can't + # be deleted while we are running this migration. + return add_project_id_foreign_key + end + + # Between the initial INSERT and the addition of the foreign key some + # projects may have been removed, leaving orphaned rows in our new settings + # table. + loop do + remove_orphaned_settings + + begin + add_project_id_foreign_key + break + rescue ActiveRecord::InvalidForeignKey + say 'project_ci_cd_settings contains some orphaned rows, retrying...' + end + end + end + + def add_project_id_foreign_key + add_concurrent_foreign_key(:project_ci_cd_settings, :projects, column: :project_id) + end + + def remove_orphaned_settings + execute <<~SQL + DELETE FROM project_ci_cd_settings + WHERE NOT EXISTS ( + SELECT 1 + FROM projects + WHERE projects.id = project_ci_cd_settings.project_id + ) + SQL + end +end diff --git a/db/migrate/20180425131009_assure_commits_count_for_merge_request_diff.rb b/db/migrate/20180425131009_assure_commits_count_for_merge_request_diff.rb new file mode 100644 index 00000000000..0e991c23bfa --- /dev/null +++ b/db/migrate/20180425131009_assure_commits_count_for_merge_request_diff.rb @@ -0,0 +1,27 @@ +class AssureCommitsCountForMergeRequestDiff < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + + include ::EachBatch + end + + def up + Gitlab::BackgroundMigration.steal('AddMergeRequestDiffCommitsCount') + + MergeRequestDiff.where(commits_count: nil).each_batch(of: 50) do |batch| + range = batch.pluck('MIN(id)', 'MAX(id)').first + + Gitlab::BackgroundMigration::AddMergeRequestDiffCommitsCount.new.perform(*range) + end + end + + def down + # noop + end +end diff --git a/db/post_migrate/20180409170809_populate_missing_project_ci_cd_settings.rb b/db/post_migrate/20180409170809_populate_missing_project_ci_cd_settings.rb new file mode 100644 index 00000000000..3b0fdb3aeea --- /dev/null +++ b/db/post_migrate/20180409170809_populate_missing_project_ci_cd_settings.rb @@ -0,0 +1,34 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class PopulateMissingProjectCiCdSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + # MySQL does not support online upgrades, thus there can't be any missing + # rows. + return if Gitlab::Database.mysql? + + # Projects created after the initial migration but before the code started + # using ProjectCiCdSetting won't have a corresponding row in + # project_ci_cd_settings, so let's fix that. + execute <<~SQL + INSERT INTO project_ci_cd_settings (project_id) + SELECT id + FROM projects + WHERE NOT EXISTS ( + SELECT 1 + FROM project_ci_cd_settings + WHERE project_ci_cd_settings.project_id = projects.id + ) + SQL + end + + def down + # There's nothing to revert for this migration. + end +end diff --git a/db/schema.rb b/db/schema.rb index 9bb5c9d3c97..7eebe94a8a6 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180418053107) do +ActiveRecord::Schema.define(version: 20180425131009) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -1449,6 +1449,13 @@ ActiveRecord::Schema.define(version: 20180418053107) do add_index "project_auto_devops", ["project_id"], name: "index_project_auto_devops_on_project_id", unique: true, using: :btree + create_table "project_ci_cd_settings", force: :cascade do |t| + t.integer "project_id", null: false + t.boolean "group_runners_enabled", default: true, null: false + end + + add_index "project_ci_cd_settings", ["project_id"], name: "index_project_ci_cd_settings_on_project_id", unique: true, using: :btree + create_table "project_custom_attributes", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -2186,6 +2193,7 @@ ActiveRecord::Schema.define(version: 20180418053107) do add_foreign_key "project_authorizations", "projects", on_delete: :cascade add_foreign_key "project_authorizations", "users", on_delete: :cascade add_foreign_key "project_auto_devops", "projects", on_delete: :cascade + add_foreign_key "project_ci_cd_settings", "projects", name: "fk_24c15d2f2e", on_delete: :cascade add_foreign_key "project_custom_attributes", "projects", on_delete: :cascade add_foreign_key "project_deploy_tokens", "deploy_tokens", on_delete: :cascade add_foreign_key "project_deploy_tokens", "projects", on_delete: :cascade |