Merge remote-tracking branch 'origin/master' into object-storage-ee-to-ce-backport

17 files changed, 271 insertions, 33 deletions

diff --git a/db/fixtures/development/01_admin.rb b/db/fixtures/development/01_admin.rb
index 6f241f6fa4a..dfb50c195c1 100644
--- a/db/fixtures/development/01_admin.rb
+++ b/db/fixtures/development/01_admin.rb
@@ -9,7 +9,6 @@ Gitlab::Seeder.quiet do
     s.username = 'root'
     s.password = '5iveL!fe'
     s.admin = true
-    s.projects_limit = 100
     s.confirmed_at = DateTime.now
   end
 end
diff --git a/db/migrate/20170929131201_populate_fork_networks.rb b/db/migrate/20170929131201_populate_fork_networks.rb
index 1214962770f..ddbf27e1852 100644
--- a/db/migrate/20170929131201_populate_fork_networks.rb
+++ b/db/migrate/20170929131201_populate_fork_networks.rb
@@ -6,22 +6,8 @@ class PopulateForkNetworks < ActiveRecord::Migration
 
   DOWNTIME = false
 
-  MIGRATION = 'PopulateForkNetworksRange'.freeze
-  BATCH_SIZE = 100
-  DELAY_INTERVAL = 15.seconds
-
-  disable_ddl_transaction!
-
-  class ForkedProjectLink < ActiveRecord::Base
-    include EachBatch
-
-    self.table_name = 'forked_project_links'
-  end
-
   def up
-    say 'Populating the `fork_networks` based on existing `forked_project_links`'
-
-    queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+    say 'Fork networks will be populated in 20171205190711 - RescheduleForkNetworkCreationCaller'
   end
 
   def down
diff --git a/db/migrate/20180119160751_optimize_ci_job_artifacts.rb b/db/migrate/20180119160751_optimize_ci_job_artifacts.rb
new file mode 100644
index 00000000000..9b4340ed7b7
--- /dev/null
+++ b/db/migrate/20180119160751_optimize_ci_job_artifacts.rb
@@ -0,0 +1,23 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class OptimizeCiJobArtifacts < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    # job_id is just here to be a covering index for index only scans
+    # since we'll almost always be joining against ci_builds on job_id
+    add_concurrent_index(:ci_job_artifacts, [:expire_at, :job_id])
+    add_concurrent_index(:ci_builds, [:artifacts_expire_at], where: "artifacts_file <> ''")
+  end
+
+  def down
+    remove_concurrent_index(:ci_job_artifacts, [:expire_at, :job_id])
+    remove_concurrent_index(:ci_builds, [:artifacts_expire_at], where: "artifacts_file <> ''")
+  end
+end
diff --git a/db/migrate/20180208183958_schedule_populate_untracked_uploads_if_needed.rb b/db/migrate/20180208183958_schedule_populate_untracked_uploads_if_needed.rb
new file mode 100644
index 00000000000..e46e793d9d2
--- /dev/null
+++ b/db/migrate/20180208183958_schedule_populate_untracked_uploads_if_needed.rb
@@ -0,0 +1,47 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class SchedulePopulateUntrackedUploadsIfNeeded < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze
+
+  class UntrackedFile < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'untracked_files_for_uploads'
+  end
+
+  def up
+    if table_exists?(:untracked_files_for_uploads)
+      process_or_remove_table
+    end
+  end
+
+  def down
+    # nothing
+  end
+
+  private
+
+  def process_or_remove_table
+    if UntrackedFile.all.empty?
+      drop_temp_table
+    else
+      schedule_populate_untracked_uploads_jobs
+    end
+  end
+
+  def drop_temp_table
+    drop_table(:untracked_files_for_uploads, if_exists: true)
+  end
+
+  def schedule_populate_untracked_uploads_jobs
+    say "Scheduling #{FOLLOW_UP_MIGRATION} background migration jobs since there are rows in untracked_files_for_uploads."
+
+    bulk_queue_background_migration_jobs_by_range(
+      UntrackedFile, FOLLOW_UP_MIGRATION)
+  end
+end
diff --git a/db/migrate/20180213131630_add_partial_index_to_projects_for_index_only_scans.rb b/db/migrate/20180213131630_add_partial_index_to_projects_for_index_only_scans.rb
new file mode 100644
index 00000000000..cedf2510dda
--- /dev/null
+++ b/db/migrate/20180213131630_add_partial_index_to_projects_for_index_only_scans.rb
@@ -0,0 +1,21 @@
+class AddPartialIndexToProjectsForIndexOnlyScans < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX_NAME = 'index_projects_on_id_partial_for_visibility'
+
+  disable_ddl_transaction!
+
+  # Adds a partial index to leverage index-only scans when looking up project ids
+  def up
+    unless index_exists?(:projects, :id, name: INDEX_NAME)
+      add_concurrent_index :projects, :id, name: INDEX_NAME, unique: true, where: 'visibility_level IN (10,20)'
+    end
+  end
+
+  def down
+    if index_exists?(:projects, :id, name: INDEX_NAME)
+      remove_concurrent_index_by_name :projects, INDEX_NAME
+    end
+  end
+end
diff --git a/db/migrate/20180215181245_users_name_lower_index.rb b/db/migrate/20180215181245_users_name_lower_index.rb
new file mode 100644
index 00000000000..d3f68cb7d45
--- /dev/null
+++ b/db/migrate/20180215181245_users_name_lower_index.rb
@@ -0,0 +1,29 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class UsersNameLowerIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+  INDEX_NAME = 'index_on_users_name_lower'
+
+  disable_ddl_transaction!
+
+  def up
+    return unless Gitlab::Database.postgresql?
+
+    # On GitLab.com this produces an index with a size of roughly 60 MB.
+    execute "CREATE INDEX CONCURRENTLY #{INDEX_NAME} ON users (LOWER(name))"
+  end
+
+  def down
+    return unless Gitlab::Database.postgresql?
+
+    if supports_drop_index_concurrently?
+      execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME}"
+    else
+      execute "DROP INDEX IF EXISTS #{INDEX_NAME}"
+    end
+  end
+end
diff --git a/db/migrate/20180216120000_add_pages_domain_verification.rb b/db/migrate/20180216120000_add_pages_domain_verification.rb
new file mode 100644
index 00000000000..8b7cae92285
--- /dev/null
+++ b/db/migrate/20180216120000_add_pages_domain_verification.rb
@@ -0,0 +1,8 @@
+class AddPagesDomainVerification < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :pages_domains, :verified_at, :datetime_with_timezone
+    add_column :pages_domains, :verification_code, :string
+  end
+end
diff --git a/db/migrate/20180216120010_add_pages_domain_verified_at_index.rb b/db/migrate/20180216120010_add_pages_domain_verified_at_index.rb
new file mode 100644
index 00000000000..825dfb52dce
--- /dev/null
+++ b/db/migrate/20180216120010_add_pages_domain_verified_at_index.rb
@@ -0,0 +1,15 @@
+class AddPagesDomainVerifiedAtIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :pages_domains, :verified_at
+  end
+
+  def down
+    remove_concurrent_index :pages_domains, :verified_at
+  end
+end
diff --git a/db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb b/db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb
new file mode 100644
index 00000000000..06d458028b3
--- /dev/null
+++ b/db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb
@@ -0,0 +1,7 @@
+class AllowDomainVerificationToBeDisabled < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :application_settings, :pages_domain_verification_enabled, :boolean, default: true, null: false
+  end
+end
diff --git a/db/migrate/20180216120030_add_pages_domain_enabled_until.rb b/db/migrate/20180216120030_add_pages_domain_enabled_until.rb
new file mode 100644
index 00000000000..b40653044dd
--- /dev/null
+++ b/db/migrate/20180216120030_add_pages_domain_enabled_until.rb
@@ -0,0 +1,7 @@
+class AddPagesDomainEnabledUntil < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :pages_domains, :enabled_until, :datetime_with_timezone
+  end
+end
diff --git a/db/migrate/20180216120040_add_pages_domain_enabled_until_index.rb b/db/migrate/20180216120040_add_pages_domain_enabled_until_index.rb
new file mode 100644
index 00000000000..00f6e4979da
--- /dev/null
+++ b/db/migrate/20180216120040_add_pages_domain_enabled_until_index.rb
@@ -0,0 +1,17 @@
+class AddPagesDomainEnabledUntilIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :pages_domains, [:project_id, :enabled_until]
+    add_concurrent_index :pages_domains, [:verified_at, :enabled_until]
+  end
+
+  def down
+    remove_concurrent_index :pages_domains, [:verified_at, :enabled_until]
+    remove_concurrent_index :pages_domains, [:project_id, :enabled_until]
+  end
+end
diff --git a/db/migrate/20180216120050_pages_domains_verification_grace_period.rb b/db/migrate/20180216120050_pages_domains_verification_grace_period.rb
new file mode 100644
index 00000000000..d7f8634b536
--- /dev/null
+++ b/db/migrate/20180216120050_pages_domains_verification_grace_period.rb
@@ -0,0 +1,26 @@
+class PagesDomainsVerificationGracePeriod < ActiveRecord::Migration
+  DOWNTIME = false
+
+  class PagesDomain < ActiveRecord::Base
+    include EachBatch
+  end
+
+  # Allow this migration to resume if it fails partway through
+  disable_ddl_transaction!
+
+  def up
+    now = Time.now
+    grace = now + 30.days
+
+    PagesDomain.each_batch do |relation|
+      relation.update_all(verified_at: now, enabled_until: grace)
+
+      # Sleep 2 minutes between batches to not overload the DB with dead tuples
+      sleep(2.minutes) unless relation.reorder(:id).last == PagesDomain.reorder(:id).last
+    end
+  end
+
+  def down
+    # no-op
+  end
+end
diff --git a/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb b/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb
index 26b99b61424..c48f1c938d0 100644
--- a/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb
+++ b/db/post_migrate/20170717111152_cleanup_move_system_upload_folder_symlink.rb
@@ -20,7 +20,7 @@ class CleanupMoveSystemUploadFolderSymlink < ActiveRecord::Migration
   def down
     if File.directory?(new_directory)
       say "Symlinking #{old_directory} -> #{new_directory}"
-      FileUtils.ln_s(new_directory, old_directory)
+      FileUtils.ln_s(new_directory, old_directory) unless File.exist?(old_directory)
     else
       say "#{new_directory} doesn't exist, skipping."
     end
diff --git a/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb b/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb
index 05430efe1f6..26f917d5a1e 100644
--- a/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb
+++ b/db/post_migrate/20171124150326_reschedule_fork_network_creation.rb
@@ -3,22 +3,8 @@ class RescheduleForkNetworkCreation < ActiveRecord::Migration
 
   DOWNTIME = false
 
-  MIGRATION = 'PopulateForkNetworksRange'.freeze
-  BATCH_SIZE = 100
-  DELAY_INTERVAL = 15.seconds
-
-  disable_ddl_transaction!
-
-  class ForkedProjectLink < ActiveRecord::Base
-    include EachBatch
-
-    self.table_name = 'forked_project_links'
-  end
-
   def up
-    say 'Populating the `fork_networks` based on existing `forked_project_links`'
-
-    queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+    say 'Fork networks will be populated in 20171205190711 - RescheduleForkNetworkCreationCaller'
   end
 
   def down
diff --git a/db/post_migrate/20180216121020_fill_pages_domain_verification_code.rb b/db/post_migrate/20180216121020_fill_pages_domain_verification_code.rb
new file mode 100644
index 00000000000..d423673d2a5
--- /dev/null
+++ b/db/post_migrate/20180216121020_fill_pages_domain_verification_code.rb
@@ -0,0 +1,41 @@
+class FillPagesDomainVerificationCode < ActiveRecord::Migration
+  DOWNTIME = false
+
+  class PagesDomain < ActiveRecord::Base
+    include EachBatch
+  end
+
+  # Allow this migration to resume if it fails partway through
+  disable_ddl_transaction!
+
+  def up
+    PagesDomain.where(verification_code: [nil, '']).each_batch do |relation|
+      connection.execute(set_codes_sql(relation))
+
+      # Sleep 2 minutes between batches to not overload the DB with dead tuples
+      sleep(2.minutes) unless relation.reorder(:id).last == PagesDomain.reorder(:id).last
+    end
+
+    change_column_null(:pages_domains, :verification_code, false)
+  end
+
+  def down
+    change_column_null(:pages_domains, :verification_code, true)
+  end
+
+  private
+
+  def set_codes_sql(relation)
+    ids = relation.pluck(:id)
+    whens = ids.map { |id| "WHEN #{id} THEN '#{SecureRandom.hex(16)}'" }
+
+    <<~SQL
+      UPDATE pages_domains
+      SET verification_code =
+        CASE id
+        #{whens.join("\n")}
+        END
+      WHERE id IN(#{ids.join(',')})
+    SQL
+  end
+end
diff --git a/db/post_migrate/20180216121030_enqueue_verify_pages_domain_workers.rb b/db/post_migrate/20180216121030_enqueue_verify_pages_domain_workers.rb
new file mode 100644
index 00000000000..bf9bf4e660f
--- /dev/null
+++ b/db/post_migrate/20180216121030_enqueue_verify_pages_domain_workers.rb
@@ -0,0 +1,16 @@
+class EnqueueVerifyPagesDomainWorkers < ActiveRecord::Migration
+  class PagesDomain < ActiveRecord::Base
+    include EachBatch
+  end
+
+  def up
+    PagesDomain.each_batch do |relation|
+      ids = relation.pluck(:id).map { |id| [id] }
+      PagesDomainVerificationWorker.bulk_perform_async(ids)
+    end
+  end
+
+  def down
+    # no-op
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index e381fc9fb8a..fba0c710705 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180206200543) do
+ActiveRecord::Schema.define(version: 20180216121030) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -156,6 +156,7 @@ ActiveRecord::Schema.define(version: 20180206200543) do
     t.integer "gitaly_timeout_fast", default: 10, null: false
     t.boolean "authorized_keys_enabled", default: true, null: false
     t.string "auto_devops_domain"
+    t.boolean "pages_domain_verification_enabled", default: true, null: false
   end
 
   create_table "audit_events", force: :cascade do |t|
@@ -295,6 +296,7 @@ ActiveRecord::Schema.define(version: 20180206200543) do
     t.integer "failure_reason"
   end
 
+  add_index "ci_builds", ["artifacts_expire_at"], name: "index_ci_builds_on_artifacts_expire_at", where: "(artifacts_file <> ''::text)", using: :btree
   add_index "ci_builds", ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree
   add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree
   add_index "ci_builds", ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree
@@ -336,6 +338,7 @@ ActiveRecord::Schema.define(version: 20180206200543) do
     t.string "file"
   end
 
+  add_index "ci_job_artifacts", ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id", using: :btree
   add_index "ci_job_artifacts", ["job_id", "file_type"], name: "index_ci_job_artifacts_on_job_id_and_file_type", unique: true, using: :btree
   add_index "ci_job_artifacts", ["project_id"], name: "index_ci_job_artifacts_on_project_id", using: :btree
 
@@ -1315,10 +1318,16 @@ ActiveRecord::Schema.define(version: 20180206200543) do
     t.string "encrypted_key_iv"
     t.string "encrypted_key_salt"
     t.string "domain"
+    t.datetime_with_timezone "verified_at"
+    t.string "verification_code", null: false
+    t.datetime_with_timezone "enabled_until"
   end
 
   add_index "pages_domains", ["domain"], name: "index_pages_domains_on_domain", unique: true, using: :btree
+  add_index "pages_domains", ["project_id", "enabled_until"], name: "index_pages_domains_on_project_id_and_enabled_until", using: :btree
   add_index "pages_domains", ["project_id"], name: "index_pages_domains_on_project_id", using: :btree
+  add_index "pages_domains", ["verified_at", "enabled_until"], name: "index_pages_domains_on_verified_at_and_enabled_until", using: :btree
+  add_index "pages_domains", ["verified_at"], name: "index_pages_domains_on_verified_at", using: :btree
 
   create_table "personal_access_tokens", force: :cascade do |t|
     t.integer "user_id", null: false
@@ -1470,6 +1479,7 @@ ActiveRecord::Schema.define(version: 20180206200543) do
   add_index "projects", ["created_at"], name: "index_projects_on_created_at", using: :btree
   add_index "projects", ["creator_id"], name: "index_projects_on_creator_id", using: :btree
   add_index "projects", ["description"], name: "index_projects_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"}
+  add_index "projects", ["id"], name: "index_projects_on_id_partial_for_visibility", unique: true, where: "(visibility_level = ANY (ARRAY[10, 20]))", using: :btree
   add_index "projects", ["last_activity_at"], name: "index_projects_on_last_activity_at", using: :btree
   add_index "projects", ["last_repository_check_failed"], name: "index_projects_on_last_repository_check_failed", using: :btree
   add_index "projects", ["last_repository_updated_at"], name: "index_projects_on_last_repository_updated_at", using: :btree