Rename forbidden paths in a single migration

4 files changed, 39 insertions, 574 deletions

diff --git a/db/post_migrate/20170403121055_rename_forbidden_root_namespaces.rb b/db/post_migrate/20170403121055_rename_forbidden_root_namespaces.rb
deleted file mode 100644
index fb475cae465..00000000000
--- a/db/post_migrate/20170403121055_rename_forbidden_root_namespaces.rb
+++ /dev/null
@@ -1,247 +0,0 @@
-# See http://doc.gitlab.com/ce/development/migration_style_guide.html
-# for more information on how to write migrations for GitLab.
-
-class RenameForbiddenRootNamespaces < ActiveRecord::Migration
-  include Gitlab::Database::MigrationHelpers
-  include Gitlab::ShellAdapter
-  disable_ddl_transaction!
-
-  class Namespace < ActiveRecord::Base
-    self.table_name = 'namespaces'
-    belongs_to :parent, class_name: "Namespace"
-    has_one :route, as: :source, autosave: true
-    has_many :children, class_name: "Namespace", foreign_key: :parent_id
-    has_many :projects
-    belongs_to :owner, class_name: "User"
-
-    def full_path
-      if route && route.path.present?
-        @full_path ||= route.path
-      else
-        update_route if persisted?
-
-        build_full_path
-      end
-    end
-
-    def build_full_path
-      if parent && path
-        parent.full_path + '/' + path
-      else
-        path
-      end
-    end
-
-    def update_route
-      prepare_route
-      route.save
-    end
-
-    def prepare_route
-      route || build_route(source: self)
-      route.path = build_full_path
-      route.name = build_full_name
-      @full_path = nil
-      @full_name = nil
-    end
-
-    def build_full_name
-      if parent && name
-        parent.human_name + ' / ' + name
-      else
-        name
-      end
-    end
-
-    def human_name
-      owner&.name
-    end
-  end
-
-  class Route < ActiveRecord::Base
-    self.table_name = 'routes'
-    belongs_to :source, polymorphic: true
-
-    validates :source, presence: true
-
-    validates :path,
-              length: { within: 1..255 },
-              presence: true,
-              uniqueness: { case_sensitive: false }
-  end
-
-  class Project < ActiveRecord::Base
-    self.table_name = 'projects'
-
-    def repository_storage_path
-      Gitlab.config.repositories.storages[repository_storage]['path']
-    end
-  end
-
-  DOWNTIME = false
-  DISALLOWED_PATHS = %w[
-    api
-    autocomplete
-    search
-    member
-    explore
-    uploads
-    import
-    notification_settings
-    abuse_reports
-    invites
-    help
-    koding
-    health_check
-    jwt
-    oauth
-    sent_notifications
-  ]
-
-  def up
-    DISALLOWED_PATHS.each do |path|
-      say "Renaming namespaces called #{path}"
-      forbidden_namespaces_with_path(path).each do |namespace|
-        rename_namespace(namespace)
-      end
-    end
-  end
-
-  def down
-    # nothing to do
-  end
-
-  def rename_namespace(namespace)
-    old_path = namespace.path
-    old_full_path = namespace.full_path
-    # Only remove the last occurrence of the path name to get the parent namespace path
-    namespace_path = remove_last_occurrence(old_full_path, old_path)
-    new_path = rename_path(namespace_path, old_path)
-    new_full_path = if namespace_path.present?
-                      File.join(namespace_path, new_path)
-                    else
-                      new_path
-                    end
-
-    Namespace.where(id: namespace).update_all(path: new_path) # skips callbacks & validations
-
-    replace_statement = replace_sql(Route.arel_table[:path], old_full_path, new_full_path)
-
-    update_column_in_batches(:routes, :path, replace_statement)  do |table, query|
-      query.where(Route.arel_table[:path].matches("#{old_full_path}%"))
-    end
-
-    clear_cache_for_namespace(namespace)
-
-    # tasks here are based on `Namespace#move_dir`
-    move_repositories(namespace, old_full_path, new_full_path)
-    move_namespace_folders(uploads_dir, old_full_path, new_full_path) if file_storage?
-    move_namespace_folders(pages_dir, old_full_path, new_full_path)
-  end
-
-  # This will replace the first occurance of a string in a column with
-  # the replacement
-  # On postgresql we can use `regexp_replace` for that.
-  # On mysql we remove the pattern from the beginning of the string, and
-  # concatenate the remaining part tot the replacement.
-  def replace_sql(column, pattern, replacement)
-    if Gitlab::Database.mysql?
-      substr = Arel::Nodes::NamedFunction.new("substring", [column, pattern.to_s.size + 1])
-      concat = Arel::Nodes::NamedFunction.new("concat", [Arel::Nodes::Quoted.new(replacement.to_s), substr])
-      Arel::Nodes::SqlLiteral.new(concat.to_sql)
-    else
-      replace = Arel::Nodes::NamedFunction.new("regexp_replace", [column, Arel::Nodes::Quoted.new(pattern.to_s), Arel::Nodes::Quoted.new(replacement.to_s)])
-      Arel::Nodes::SqlLiteral.new(replace.to_sql)
-    end
-  end
-
-  def remove_last_occurrence(string, pattern)
-    string.reverse.sub(pattern.reverse, "").reverse
-  end
-
-  def move_namespace_folders(directory, old_relative_path, new_relative_path)
-    old_path = File.join(directory, old_relative_path)
-    return unless File.directory?(old_path)
-
-    new_path = File.join(directory, new_relative_path)
-    FileUtils.mv(old_path, new_path)
-  end
-
-  def move_repositories(namespace, old_full_path, new_full_path)
-    repo_paths_for_namespace(namespace).each do |repository_storage_path|
-      # Ensure old directory exists before moving it
-      gitlab_shell.add_namespace(repository_storage_path, old_full_path)
-
-      unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path)
-        say "Exception moving path #{repository_storage_path} from #{old_full_path} to #{new_full_path}"
-      end
-    end
-  end
-
-  def rename_path(namespace_path, path_was)
-    counter = 0
-    path = "#{path_was}#{counter}"
-
-    while route_exists?(File.join(namespace_path, path))
-      counter += 1
-      path = "#{path_was}#{counter}"
-    end
-
-    path
-  end
-
-  def route_exists?(full_path)
-    Route.where(Route.arel_table[:path].matches(full_path)).any?
-  end
-
-  def forbidden_namespaces_with_path(name)
-    Namespace.where(arel_table[:path].matches(name).and(arel_table[:parent_id].eq(nil)))
-  end
-
-  def clear_cache_for_namespace(namespace)
-    project_ids = project_ids_for_namespace(namespace)
-    scopes = { "Project" => { id: project_ids },
-               "Issue" => { project_id: project_ids },
-               "MergeRequest" => { target_project_id: project_ids },
-               "Note" => { project_id: project_ids } }
-
-    ClearDatabaseCacheWorker.perform_async(scopes)
-  rescue => e
-    Rails.logger.error ["Couldn't clear the markdown cache: #{e.message}", e.backtrace.join("\n")].join("\n")
-  end
-
-  def project_ids_for_namespace(namespace)
-    namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id])
-    namespace_or_children = Project.arel_table[:namespace_id].in(namespace_ids)
-    Project.unscoped.where(namespace_or_children).pluck(:id)
-  end
-
-  # This won't scale to huge trees, but it should do for a handful of namespaces
-  def child_ids_for_parent(namespace, ids: [])
-    namespace.children.each do |child|
-      ids << child.id
-      child_ids_for_parent(child, ids: ids) if child.children.any?
-    end
-    ids
-  end
-
-  def repo_paths_for_namespace(namespace)
-    namespace.projects.unscoped.select('distinct(repository_storage)').to_a.map(&:repository_storage_path)
-  end
-
-  def uploads_dir
-    File.join(Rails.root, "public", "uploads")
-  end
-
-  def pages_dir
-    Settings.pages.path
-  end
-
-  def file_storage?
-    CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
-  end
-
-  def arel_table
-    Namespace.arel_table
-  end
-end
diff --git a/db/post_migrate/20170404152317_rename_forbidden_child_namespaces.rb b/db/post_migrate/20170404152317_rename_forbidden_child_namespaces.rb
deleted file mode 100644
index 8b082a892d4..00000000000
--- a/db/post_migrate/20170404152317_rename_forbidden_child_namespaces.rb
+++ /dev/null
@@ -1,242 +0,0 @@
-# See http://doc.gitlab.com/ce/development/migration_style_guide.html
-# for more information on how to write migrations for GitLab.
-
-class RenameForbiddenChildNamespaces < ActiveRecord::Migration
-  include Gitlab::Database::MigrationHelpers
-  include Gitlab::ShellAdapter
-  disable_ddl_transaction!
-
-  class Namespace < ActiveRecord::Base
-    self.table_name = 'namespaces'
-    belongs_to :parent, class_name: "Namespace"
-    has_one :route, as: :source, autosave: true
-    has_many :children, class_name: "Namespace", foreign_key: :parent_id
-    has_many :projects
-    belongs_to :owner, class_name: "User"
-
-    def full_path
-      if route && route.path.present?
-        @full_path ||= route.path
-      else
-        update_route if persisted?
-
-        build_full_path
-      end
-    end
-
-    def build_full_path
-      if parent && path
-        parent.full_path + '/' + path
-      else
-        path
-      end
-    end
-
-    def update_route
-      prepare_route
-      route.save
-    end
-
-    def prepare_route
-      route || build_route(source: self)
-      route.path = build_full_path
-      route.name = build_full_name
-      @full_path = nil
-      @full_name = nil
-    end
-
-    def build_full_name
-      if parent && name
-        parent.human_name + ' / ' + name
-      else
-        name
-      end
-    end
-
-    def human_name
-      owner&.name
-    end
-  end
-
-  class Route < ActiveRecord::Base
-    self.table_name = 'routes'
-    belongs_to :source, polymorphic: true
-
-    validates :source, presence: true
-
-    validates :path,
-              length: { within: 1..255 },
-              presence: true,
-              uniqueness: { case_sensitive: false }
-  end
-
-  class Project < ActiveRecord::Base
-    self.table_name = 'projects'
-
-    def repository_storage_path
-      Gitlab.config.repositories.storages[repository_storage]['path']
-    end
-  end
-
-  DOWNTIME = false
-  DISALLOWED_PATHS = %w[info git-upload-pack
-                        git-receive-pack gitlab-lfs autocomplete_sources
-                        templates avatar commit pages compare network snippets
-                        services mattermost deploy_keys forks import merge_requests
-                        branches merged_branches tags protected_branches variables
-                        triggers pipelines environments cycle_analytics builds
-                        hooks container_registry milestones labels issues
-                        project_members group_links notes noteable boards todos
-                        uploads runners runner_projects settings repository
-                        transfer remove_fork archive unarchive housekeeping
-                        toggle_star preview_markdown export remove_export
-                        generate_new_export download_export activity
-                        new_issue_address]
-
-  def up
-    DISALLOWED_PATHS.each do |path|
-      say "Renaming namespaces called #{path}"
-      forbidden_namespaces_with_path(path).each do |namespace|
-        rename_namespace(namespace)
-      end
-    end
-  end
-
-  def down
-    # nothing to do
-  end
-
-  def rename_namespace(namespace)
-    old_path = namespace.path
-    old_full_path = namespace.full_path
-    # Only remove the last occurrence of the path name to get the parent namespace path
-    namespace_path = remove_last_occurrence(old_full_path, old_path)
-    new_path = rename_path(namespace_path, old_path)
-    new_full_path = if namespace_path.present?
-                      File.join(namespace_path, new_path)
-                    else
-                      new_path
-                    end
-
-    Namespace.where(id: namespace).update_all(path: new_path) # skips callbacks & validations
-
-    replace_statement = replace_sql(Route.arel_table[:path], old_full_path, new_full_path)
-
-    update_column_in_batches(:routes, :path, replace_statement)  do |table, query|
-      query.where(Route.arel_table[:path].matches("#{old_full_path}%"))
-    end
-
-    clear_cache_for_namespace(namespace)
-
-    # tasks here are based on `Namespace#move_dir`
-    move_repositories(namespace, old_full_path, new_full_path)
-    move_namespace_folders(uploads_dir, old_full_path, new_full_path) if file_storage?
-    move_namespace_folders(pages_dir, old_full_path, new_full_path)
-  end
-
-  # This will replace the first occurance of a string in a column with
-  # the replacement
-  # On postgresql we can use `regexp_replace` for that.
-  # On mysql we remove the pattern from the beginning of the string, and
-  # concatenate the remaining part tot the replacement.
-  def replace_sql(column, pattern, replacement)
-    if Gitlab::Database.mysql?
-      substr = Arel::Nodes::NamedFunction.new("substring", [column, pattern.to_s.size + 1])
-      concat = Arel::Nodes::NamedFunction.new("concat", [Arel::Nodes::Quoted.new(replacement.to_s), substr])
-      Arel::Nodes::SqlLiteral.new(concat.to_sql)
-    else
-      replace = Arel::Nodes::NamedFunction.new("regexp_replace", [column, Arel::Nodes::Quoted.new(pattern.to_s), Arel::Nodes::Quoted.new(replacement.to_s)])
-      Arel::Nodes::SqlLiteral.new(replace.to_sql)
-    end
-  end
-
-  def remove_last_occurrence(string, pattern)
-    string.reverse.sub(pattern.reverse, "").reverse
-  end
-
-  def move_namespace_folders(directory, old_relative_path, new_relative_path)
-    old_path = File.join(directory, old_relative_path)
-    return unless File.directory?(old_path)
-
-    new_path = File.join(directory, new_relative_path)
-    FileUtils.mv(old_path, new_path)
-  end
-
-  def move_repositories(namespace, old_full_path, new_full_path)
-    repo_paths_for_namespace(namespace).each do |repository_storage_path|
-      # Ensure old directory exists before moving it
-      gitlab_shell.add_namespace(repository_storage_path, old_full_path)
-
-      unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path)
-        say "Exception moving path #{repository_storage_path} from #{old_full_path} to #{new_full_path}"
-      end
-    end
-  end
-
-  def rename_path(namespace_path, path_was)
-    counter = 0
-    path = "#{path_was}#{counter}"
-
-    while route_exists?(File.join(namespace_path, path))
-      counter += 1
-      path = "#{path_was}#{counter}"
-    end
-
-    path
-  end
-
-  def route_exists?(full_path)
-    Route.where(Route.arel_table[:path].matches(full_path)).any?
-  end
-
-  def forbidden_namespaces_with_path(path)
-    Namespace.where(arel_table[:parent_id].eq(nil).not).where(arel_table[:path].matches(path))
-  end
-
-  def clear_cache_for_namespace(namespace)
-    project_ids = project_ids_for_namespace(namespace)
-    scopes = { "Project" => { id: project_ids },
-               "Issue" => { project_id: project_ids },
-               "MergeRequest" => { target_project_id: project_ids },
-               "Note" => { project_id: project_ids } }
-
-    ClearDatabaseCacheWorker.perform_async(scopes)
-  rescue => e
-    Rails.logger.error ["Couldn't clear the markdown cache: #{e.message}", e.backtrace.join("\n")].join("\n")
-  end
-
-  def project_ids_for_namespace(namespace)
-    namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id])
-    namespace_or_children = Project.arel_table[:namespace_id].in(namespace_ids)
-    Project.unscoped.where(namespace_or_children).pluck(:id)
-  end
-
-  # This won't scale to huge trees, but it should do for a handful of namespaces
-  def child_ids_for_parent(namespace, ids: [])
-    namespace.children.each do |child|
-      ids << child.id
-      child_ids_for_parent(child, ids: ids) if child.children.any?
-    end
-    ids
-  end
-
-  def repo_paths_for_namespace(namespace)
-    namespace.projects.unscoped.select('distinct(repository_storage)').to_a.map(&:repository_storage_path)
-  end
-
-  def uploads_dir
-    File.join(Rails.root, "public", "uploads")
-  end
-
-  def pages_dir
-    Settings.pages.path
-  end
-
-  def file_storage?
-    CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
-  end
-
-  def arel_table
-    Namespace.arel_table
-  end
-end
diff --git a/db/post_migrate/20170405111106_rename_wildcard_project_names.rb b/db/post_migrate/20170405111106_rename_wildcard_project_names.rb
deleted file mode 100644
index 1b8d2a40e99..00000000000
--- a/db/post_migrate/20170405111106_rename_wildcard_project_names.rb
+++ /dev/null
@@ -1,85 +0,0 @@
-# See http://doc.gitlab.com/ce/development/migration_style_guide.html
-# for more information on how to write migrations for GitLab.
-
-class RenameWildcardProjectNames < ActiveRecord::Migration
-  include Gitlab::Database::MigrationHelpers
-  include Gitlab::ShellAdapter
-  disable_ddl_transaction!
-
-  DOWNTIME = false
-  KNOWN_PATHS = %w[info git-upload-pack
-                   git-receive-pack gitlab-lfs autocomplete_sources
-                   templates avatar commit pages compare network snippets
-                   services mattermost deploy_keys forks import merge_requests
-                   branches merged_branches tags protected_branches variables
-                   triggers pipelines environments cycle_analytics builds
-                   hooks container_registry milestones labels issues
-                   project_members group_links notes noteable boards todos
-                   uploads runners runner_projects settings repository
-                   transfer remove_fork archive unarchive housekeeping
-                   toggle_star preview_markdown export remove_export
-                   generate_new_export download_export activity
-                   new_issue_address].freeze
-
-  def up
-    reserved_projects.find_in_batches(batch_size: 100) do |slice|
-      rename_projects(slice)
-    end
-  end
-
-  def down
-    # nothing to do here
-  end
-
-  private
-
-  def reserved_projects
-    Project.unscoped.
-      includes(:namespace).
-      where('EXISTS (SELECT 1 FROM namespaces WHERE projects.namespace_id = namespaces.id)').
-      where('projects.path' => KNOWN_PATHS)
-  end
-
-  def route_exists?(full_path)
-    quoted_path = ActiveRecord::Base.connection.quote_string(full_path.downcase)
-
-    ActiveRecord::Base.connection.
-      select_all("SELECT id, path FROM routes WHERE lower(path) = '#{quoted_path}'").present?
-  end
-
-  # Adds number to the end of the path that is not taken by other route
-  def rename_path(namespace_path, path_was)
-    counter = 0
-    path = "#{path_was}#{counter}"
-
-    while route_exists?("#{namespace_path}/#{path}")
-      counter += 1
-      path = "#{path_was}#{counter}"
-    end
-
-    path
-  end
-
-  def rename_projects(projects)
-    projects.each do |project|
-      id = project.id
-      path_was = project.path
-      namespace_path = project.namespace.path
-      path = rename_path(namespace_path, path_was)
-
-      begin
-        # Because project path update is quite complex operation we can't safely
-        # copy-paste all code from GitLab. As exception we use Rails code here
-        project.rename_repo if rename_project_row(project, path)
-      rescue Exception => e # rubocop: disable Lint/RescueException
-        Rails.logger.error "Exception when renaming project #{id}: #{e.message}"
-      end
-    end
-  end
-
-  def rename_project_row(project, path)
-    project.respond_to?(:update_attributes) &&
-      project.update_attributes(path: path) &&
-      project.respond_to?(:rename_repo)
-  end
-end
diff --git a/db/post_migrate/20170412174900_rename_reserved_dynamic_paths.rb b/db/post_migrate/20170412174900_rename_reserved_dynamic_paths.rb
new file mode 100644
index 00000000000..fcab298eb09
--- /dev/null
+++ b/db/post_migrate/20170412174900_rename_reserved_dynamic_paths.rb
@@ -0,0 +1,39 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RenameReservedDynamicPaths < ActiveRecord::Migration
+  include Gitlab::Database::RenameReservedPathsMigration
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  DISALLOWED_ROOT_PATHS = %w[
+    api
+    autocomplete
+    member
+    explore
+    uploads
+    import
+    notification_settings
+    abuse_reports
+    invites
+    koding
+    health_check
+    jwt
+    oauth
+    sent_notifications
+    -
+  ]
+
+  DISALLOWED_WILDCARD_PATHS = %w[objects folders file]
+
+  def up
+    rename_root_paths(DISALLOWED_ROOT_PATHS)
+    rename_wildcard_paths(DISALLOWED_WILDCARD_PATHS)
+  end
+
+  def down
+    # nothing to do
+  end
+end