Merge branch 'master' into backstage/gb/migrate-stages-statuses

* master: (319 commits)
  remove redundant changelog entries
  Merge branch '24570-use-re2-for-user-supplied-regexp-9-3' into 'security-9-3'
  Merge branch '33303-404-for-unauthorized-project' into 'security-9-3'
  Merge branch '33359-pers-snippet-files-location' into 'security-9-3'
  Merge branch 'bvl-remove-appearance-symlink' into 'security-9-3'
  Hide description about protected branches to non-member
  Update CHANGELOG.md for 9.0.11
  Update CHANGELOG.md for 9.1.8
  Update CHANGELOG.md for 8.17.7
  Update CHANGELOG.md for 9.2.8
  Update CHANGELOG.md for 9.3.8
  Respect blockquote line breaks in markdown
  35209 Add wip message to new navigation preference section
  Add github imported projects count to usage data
  Add versions to Prometheus metrics doc
  Add Bulgarian translations of Pipeline Schedules
  Add Esperanto translations of Pipeline Schedules
  Add Traditional Chinese in HongKong translations of Pipeline Schedules
  Add Simplified Chinese translations of Pipeline Schedules
  Resolve "Clarify k8s service keys"
  ...

Conflicts:
	db/schema.rb

7 files changed, 221 insertions, 3 deletions

diff --git a/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb b/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb
index 397a9a2d28e..cb1b4f1855d 100644
--- a/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb
+++ b/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb
@@ -56,7 +56,7 @@ class MigrateUserActivitiesToUsersLastActivityOn < ActiveRecord::Migration
   end
 
   def activities(from, to, page: 1)
-    Gitlab::Redis.with do |redis|
+    Gitlab::Redis::SharedState.with do |redis|
       redis.zrangebyscore(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i,
         with_scores: true,
         limit: limit(page))
@@ -64,7 +64,7 @@ class MigrateUserActivitiesToUsersLastActivityOn < ActiveRecord::Migration
   end
 
   def activities_count(from, to)
-    Gitlab::Redis.with do |redis|
+    Gitlab::Redis::SharedState.with do |redis|
       redis.zcount(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i)
     end
   end
diff --git a/db/post_migrate/20170406111121_clean_upload_symlinks.rb b/db/post_migrate/20170406111121_clean_upload_symlinks.rb
index 3ac9a6c10bc..fc3a4acc0bb 100644
--- a/db/post_migrate/20170406111121_clean_upload_symlinks.rb
+++ b/db/post_migrate/20170406111121_clean_upload_symlinks.rb
@@ -6,7 +6,7 @@ class CleanUploadSymlinks < ActiveRecord::Migration
   disable_ddl_transaction!
 
   DOWNTIME = false
-  DIRECTORIES_TO_MOVE = %w(user project note group appeareance)
+  DIRECTORIES_TO_MOVE = %w(user project note group appearance)
 
   def up
     return unless file_storage?
diff --git a/db/post_migrate/20170612071012_move_personal_snippets_files.rb b/db/post_migrate/20170612071012_move_personal_snippets_files.rb
new file mode 100644
index 00000000000..33043364bde
--- /dev/null
+++ b/db/post_migrate/20170612071012_move_personal_snippets_files.rb
@@ -0,0 +1,91 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+class MovePersonalSnippetsFiles < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  def up
+    return unless file_storage?
+
+    @source_relative_location = File.join('/uploads', 'personal_snippet')
+    @destination_relative_location = File.join('/uploads', 'system', 'personal_snippet')
+
+    move_personal_snippet_files
+  end
+
+  def down
+    return unless file_storage?
+
+    @source_relative_location = File.join('/uploads', 'system', 'personal_snippet')
+    @destination_relative_location = File.join('/uploads', 'personal_snippet')
+
+    move_personal_snippet_files
+  end
+
+  def move_personal_snippet_files
+    query = "SELECT uploads.path, uploads.model_id, snippets.description FROM uploads "\
+            "INNER JOIN snippets ON snippets.id = uploads.model_id WHERE uploader = 'PersonalFileUploader'"
+    select_all(query).each do |upload|
+      secret = upload['path'].split('/')[0]
+      file_name = upload['path'].split('/')[1]
+
+      next unless move_file(upload['model_id'], secret, file_name)
+      update_markdown(upload['model_id'], secret, file_name, upload['description'])
+    end
+  end
+
+  def move_file(snippet_id, secret, file_name)
+    source_dir = File.join(base_directory, @source_relative_location, snippet_id.to_s, secret)
+    destination_dir = File.join(base_directory, @destination_relative_location, snippet_id.to_s, secret)
+
+    source_file_path = File.join(source_dir, file_name)
+    destination_file_path = File.join(destination_dir, file_name)
+
+    unless File.exist?(source_file_path)
+      say "Source file `#{source_file_path}` doesn't exist. Skipping."
+      return
+    end
+
+    say "Moving file #{source_file_path} -> #{destination_file_path}"
+
+    FileUtils.mkdir_p(destination_dir)
+    FileUtils.move(source_file_path, destination_file_path)
+
+    true
+  end
+
+  def update_markdown(snippet_id, secret, file_name, description)
+    source_markdown_path = File.join(@source_relative_location, snippet_id.to_s, secret, file_name)
+    destination_markdown_path = File.join(@destination_relative_location, snippet_id.to_s, secret, file_name)
+
+    source_markdown = "](#{source_markdown_path})"
+    destination_markdown = "](#{destination_markdown_path})"
+
+    if description.present?
+      description = description.gsub(source_markdown, destination_markdown)
+      quoted_description = quote_string(description)
+
+      execute("UPDATE snippets SET description = '#{quoted_description}', description_html = NULL "\
+              "WHERE id = #{snippet_id}")
+    end
+
+    query = "SELECT id, note FROM notes WHERE noteable_id = #{snippet_id} "\
+            "AND noteable_type = 'Snippet' AND note IS NOT NULL"
+    select_all(query).each do |note|
+      text = note['note'].gsub(source_markdown, destination_markdown)
+      quoted_text = quote_string(text)
+
+      execute("UPDATE notes SET note = '#{quoted_text}', note_html = NULL WHERE id = #{note['id']}")
+    end
+  end
+
+  def base_directory
+    File.join(Rails.root, 'public')
+  end
+
+  def file_storage?
+    CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
+  end
+end
diff --git a/db/post_migrate/20170613111224_clean_appearance_symlinks.rb b/db/post_migrate/20170613111224_clean_appearance_symlinks.rb
new file mode 100644
index 00000000000..acb895e426f
--- /dev/null
+++ b/db/post_migrate/20170613111224_clean_appearance_symlinks.rb
@@ -0,0 +1,52 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CleanAppearanceSymlinks < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+
+  def up
+    return unless file_storage?
+
+    symlink_location = File.join(old_upload_dir, dir)
+
+    return unless File.symlink?(symlink_location)
+    say "removing symlink: #{symlink_location}"
+    FileUtils.rm(symlink_location)
+  end
+
+  def down
+    return unless file_storage?
+
+    symlink = File.join(old_upload_dir, dir)
+    destination = File.join(new_upload_dir, dir)
+
+    return if File.directory?(symlink)
+    return unless File.directory?(destination)
+
+    say "Creating symlink #{symlink} -> #{destination}"
+    FileUtils.ln_s(destination, symlink)
+  end
+
+  def file_storage?
+    CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
+  end
+
+  def dir
+    'appearance'
+  end
+
+  def base_directory
+    Rails.root
+  end
+
+  def old_upload_dir
+    File.join(base_directory, "public", "uploads")
+  end
+
+  def new_upload_dir
+    File.join(base_directory, "public", "uploads", "system")
+  end
+end
diff --git a/db/post_migrate/20170629180131_cleanup_application_settings_signin_enabled_rename.rb b/db/post_migrate/20170629180131_cleanup_application_settings_signin_enabled_rename.rb
new file mode 100644
index 00000000000..52a773ddfee
--- /dev/null
+++ b/db/post_migrate/20170629180131_cleanup_application_settings_signin_enabled_rename.rb
@@ -0,0 +1,15 @@
+class CleanupApplicationSettingsSigninEnabledRename < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    cleanup_concurrent_column_rename :application_settings, :signin_enabled, :password_authentication_enabled
+  end
+
+  def down
+    rename_column_concurrently :application_settings, :password_authentication_enabled, :signin_enabled
+  end
+end
diff --git a/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb b/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb
new file mode 100644
index 00000000000..26b99b61424
--- /dev/null
+++ b/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb
@@ -0,0 +1,40 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CleanupMoveSystemUploadFolderSymlink < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    if File.symlink?(old_directory)
+      say "Removing #{old_directory} -> #{new_directory} symlink"
+      FileUtils.rm(old_directory)
+    else
+      say "Symlink #{old_directory} non existant, nothing to do."
+    end
+  end
+
+  def down
+    if File.directory?(new_directory)
+      say "Symlinking #{old_directory} -> #{new_directory}"
+      FileUtils.ln_s(new_directory, old_directory)
+    else
+      say "#{new_directory} doesn't exist, skipping."
+    end
+  end
+
+  def new_directory
+    File.join(base_directory, '-', 'system')
+  end
+
+  def old_directory
+    File.join(base_directory, 'system')
+  end
+
+  def base_directory
+    File.join(Rails.root, 'public', 'uploads')
+  end
+end
diff --git a/db/post_migrate/20170717150329_enqueue_migrate_system_uploads_to_new_folder.rb b/db/post_migrate/20170717150329_enqueue_migrate_system_uploads_to_new_folder.rb
new file mode 100644
index 00000000000..87069dce006
--- /dev/null
+++ b/db/post_migrate/20170717150329_enqueue_migrate_system_uploads_to_new_folder.rb
@@ -0,0 +1,20 @@
+class EnqueueMigrateSystemUploadsToNewFolder < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  OLD_FOLDER = 'uploads/system/'
+  NEW_FOLDER = 'uploads/-/system/'
+
+  disable_ddl_transaction!
+
+  def up
+    BackgroundMigrationWorker.perform_async('MigrateSystemUploadsToNewFolder',
+                                            [OLD_FOLDER, NEW_FOLDER])
+  end
+
+  def down
+    BackgroundMigrationWorker.perform_async('MigrateSystemUploadsToNewFolder',
+                                            [NEW_FOLDER, OLD_FOLDER])
+  end
+end