Merge branch 'master' into digitalmoksha/gitlab-ce-feature/verify_secondary_emails

# Conflicts:
#	app/controllers/admin/users_controller.rb
#	app/controllers/confirmations_controller.rb
#	app/controllers/profiles/emails_controller.rb
#	app/models/user.rb
#	app/services/emails/base_service.rb
#	app/services/emails/destroy_service.rb
#	app/views/devise/mailer/confirmation_instructions.html.haml
#	lib/api/users.rb
#	spec/services/emails/destroy_service_spec.rb

15 files changed, 316 insertions, 5 deletions

diff --git a/db/migrate/20141126120926_add_merge_request_rebase_enabled_to_projects.rb b/db/migrate/20141126120926_add_merge_request_rebase_enabled_to_projects.rb
new file mode 100644
index 00000000000..3dafdf0fde4
--- /dev/null
+++ b/db/migrate/20141126120926_add_merge_request_rebase_enabled_to_projects.rb
@@ -0,0 +1,17 @@
+# rubocop:disable all
+class AddMergeRequestRebaseEnabledToProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default(:projects, :merge_requests_rebase_enabled, :boolean, default: false)
+  end
+
+  def down
+    remove_column(:projects, :merge_requests_rebase_enabled)
+  end
+end
diff --git a/db/migrate/20150827121444_add_fast_forward_option_to_project.rb b/db/migrate/20150827121444_add_fast_forward_option_to_project.rb
new file mode 100644
index 00000000000..6f22641077d
--- /dev/null
+++ b/db/migrate/20150827121444_add_fast_forward_option_to_project.rb
@@ -0,0 +1,19 @@
+# rubocop:disable all
+class AddFastForwardOptionToProject < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default(:projects, :merge_requests_ff_only_enabled, :boolean, default: false)
+  end
+
+  def down
+    if column_exists?(:projects, :merge_requests_ff_only_enabled)
+      remove_column(:projects, :merge_requests_ff_only_enabled)
+    end
+  end
+end
diff --git a/db/migrate/20160713200638_add_repository_read_only_to_projects.rb b/db/migrate/20160713200638_add_repository_read_only_to_projects.rb
new file mode 100644
index 00000000000..8ee8b55f210
--- /dev/null
+++ b/db/migrate/20160713200638_add_repository_read_only_to_projects.rb
@@ -0,0 +1,9 @@
+class AddRepositoryReadOnlyToProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :projects, :repository_read_only, :boolean
+  end
+end
diff --git a/db/migrate/20170720122741_create_user_custom_attributes.rb b/db/migrate/20170720122741_create_user_custom_attributes.rb
new file mode 100644
index 00000000000..b1c0bebc633
--- /dev/null
+++ b/db/migrate/20170720122741_create_user_custom_attributes.rb
@@ -0,0 +1,17 @@
+class CreateUserCustomAttributes < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :user_custom_attributes do |t|
+      t.timestamps_with_timezone null: false
+      t.references :user, null: false, foreign_key: { on_delete: :cascade }
+      t.string :key, null: false
+      t.string :value, null: false
+
+      t.index [:user_id, :key], unique: true
+      t.index [:key, :value]
+    end
+  end
+end
diff --git a/db/migrate/20170828135939_migrate_user_external_mail_data.rb b/db/migrate/20170828135939_migrate_user_external_mail_data.rb
index 395181a3b22..f7ac87374b6 100644
--- a/db/migrate/20170828135939_migrate_user_external_mail_data.rb
+++ b/db/migrate/20170828135939_migrate_user_external_mail_data.rb
@@ -33,7 +33,7 @@ class MigrateUserExternalMailData < ActiveRecord::Migration
           SELECT true
           FROM user_synced_attributes_metadata
           WHERE user_id = users.id
-          AND provider = users.email_provider OR (provider IS NULL AND users.email_provider IS NULL)
+          AND (provider = users.email_provider OR (provider IS NULL AND users.email_provider IS NULL))
         )
         AND id BETWEEN #{start_id} AND #{end_id}
       EOF
diff --git a/db/migrate/20170919211300_remove_temporary_ci_builds_index.rb b/db/migrate/20170919211300_remove_temporary_ci_builds_index.rb
new file mode 100644
index 00000000000..b2009b282e9
--- /dev/null
+++ b/db/migrate/20170919211300_remove_temporary_ci_builds_index.rb
@@ -0,0 +1,27 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveTemporaryCiBuildsIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  # To use create/remove index concurrently
+  disable_ddl_transaction!
+
+  def up
+    return unless index_exists?(:ci_builds, :id, name: 'index_for_ci_builds_retried_migration')
+    remove_concurrent_index(:ci_builds, :id, name: "index_for_ci_builds_retried_migration")
+  end
+
+  def down
+    # this was a temporary index for a migration that was never
+    # present previously so this probably shouldn't be here but it's
+    # easier to test the drop if we have a way to create it.
+    add_concurrent_index("ci_builds", ["id"],
+                         name: "index_for_ci_builds_retried_migration",
+                         where: "(retried IS NULL)",
+                         using: :btree)
+  end
+end
diff --git a/db/migrate/20170927095921_add_ci_builds_index_for_jobscontroller.rb b/db/migrate/20170927095921_add_ci_builds_index_for_jobscontroller.rb
new file mode 100644
index 00000000000..c2cb1df2586
--- /dev/null
+++ b/db/migrate/20170927095921_add_ci_builds_index_for_jobscontroller.rb
@@ -0,0 +1,39 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddCiBuildsIndexForJobscontroller < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  # When a migration requires downtime you **must** uncomment the following
+  # constant and define a short and easy to understand explanation as to why the
+  # migration requires downtime.
+  # DOWNTIME_REASON = ''
+
+  # When using the methods "add_concurrent_index", "remove_concurrent_index" or
+  # "add_column_with_default" you must disable the use of transactions
+  # as these methods can not run in an existing transaction.
+  # When using "add_concurrent_index" or "remove_concurrent_index" methods make sure
+  # that either of them is the _only_ method called in the migration,
+  # any other changes should go in a separate migration.
+  # This ensures that upon failure _only_ the index creation or removing fails
+  # and can be retried or reverted easily.
+  #
+  # To disable transactions uncomment the following line and remove these
+  # comments:
+  # disable_ddl_transaction!
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :ci_builds, [:project_id, :id]  unless index_exists? :ci_builds, [:project_id, :id]
+    remove_concurrent_index :ci_builds, :project_id if index_exists? :ci_builds, :project_id
+  end
+
+  def down
+    add_concurrent_index :ci_builds, :project_id unless index_exists? :ci_builds, :project_id
+    remove_concurrent_index :ci_builds, [:project_id, :id] if index_exists? :ci_builds, [:project_id, :id]
+  end
+end
diff --git a/db/migrate/20170927122209_add_partial_index_for_labels_template.rb b/db/migrate/20170927122209_add_partial_index_for_labels_template.rb
new file mode 100644
index 00000000000..c3e5077ba20
--- /dev/null
+++ b/db/migrate/20170927122209_add_partial_index_for_labels_template.rb
@@ -0,0 +1,45 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddPartialIndexForLabelsTemplate < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  # When a migration requires downtime you **must** uncomment the following
+  # constant and define a short and easy to understand explanation as to why the
+  # migration requires downtime.
+  # DOWNTIME_REASON = ''
+
+  # When using the methods "add_concurrent_index", "remove_concurrent_index" or
+  # "add_column_with_default" you must disable the use of transactions
+  # as these methods can not run in an existing transaction.
+  # When using "add_concurrent_index" or "remove_concurrent_index" methods make sure
+  # that either of them is the _only_ method called in the migration,
+  # any other changes should go in a separate migration.
+  # This ensures that upon failure _only_ the index creation or removing fails
+  # and can be retried or reverted easily.
+  #
+  # To disable transactions uncomment the following line and remove these
+  # comments:
+
+  disable_ddl_transaction!
+
+  # Note this is a partial index in Postgres but MySQL will ignore the
+  # partial index clause. By making it an index on "template" this
+  # means the index will still accomplish the same goal of optimizing
+  # a query with "where template = true" on MySQL -- it'll just take
+  # more space. In this case the number of records with template=true
+  # is expected to be very small (small enough to display on a single
+  # web page) so it's ok to filter or sort them without the index
+  # anyways.
+
+  def up
+    add_concurrent_index "labels", ["template"], where: "template"
+  end
+
+  def down
+    remove_concurrent_index "labels", ["template"], where: "template"
+  end
+end
diff --git a/db/migrate/20170928100231_add_composite_index_on_merge_requests_merge_commit_sha.rb b/db/migrate/20170928100231_add_composite_index_on_merge_requests_merge_commit_sha.rb
new file mode 100644
index 00000000000..9f02daf04c1
--- /dev/null
+++ b/db/migrate/20170928100231_add_composite_index_on_merge_requests_merge_commit_sha.rb
@@ -0,0 +1,33 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddCompositeIndexOnMergeRequestsMergeCommitSha < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  # The default index name is too long for PostgreSQL and would thus be
+  # truncated.
+  INDEX_NAME = 'index_merge_requests_on_tp_id_and_merge_commit_sha_and_id'
+
+  COLUMNS = [:target_project_id, :merge_commit_sha, :id]
+
+  disable_ddl_transaction!
+
+  def up
+    return if index_is_present?
+
+    add_concurrent_index(:merge_requests, COLUMNS, name: INDEX_NAME)
+  end
+
+  def down
+    return unless index_is_present?
+
+    remove_concurrent_index(:merge_requests, COLUMNS, name: INDEX_NAME)
+  end
+
+  def index_is_present?
+    index_exists?(:merge_requests, COLUMNS, name: INDEX_NAME)
+  end
+end
diff --git a/db/migrate/20171004121444_make_sure_fast_forward_option_exists.rb b/db/migrate/20171004121444_make_sure_fast_forward_option_exists.rb
new file mode 100644
index 00000000000..ac266c3e22e
--- /dev/null
+++ b/db/migrate/20171004121444_make_sure_fast_forward_option_exists.rb
@@ -0,0 +1,25 @@
+# rubocop:disable all
+class MakeSureFastForwardOptionExists < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    # We had to fix the migration db/migrate/20150827121444_add_fast_forward_option_to_project.rb
+    # And this is why it's possible that someone has ran the migrations but does
+    # not have the merge_requests_ff_only_enabled column. This migration makes sure it will
+    # be added
+    unless column_exists?(:projects, :merge_requests_ff_only_enabled)
+      add_column_with_default(:projects, :merge_requests_ff_only_enabled, :boolean, default: false)
+    end
+  end
+
+  def down
+    if column_exists?(:projects, :merge_requests_ff_only_enabled)
+      remove_column(:projects, :merge_requests_ff_only_enabled)
+    end
+  end
+end
diff --git a/db/post_migrate/20170828170502_post_deploy_migrate_user_external_mail_data.rb b/db/post_migrate/20170828170502_post_deploy_migrate_user_external_mail_data.rb
index a475b242921..fd1437b07f5 100644
--- a/db/post_migrate/20170828170502_post_deploy_migrate_user_external_mail_data.rb
+++ b/db/post_migrate/20170828170502_post_deploy_migrate_user_external_mail_data.rb
@@ -33,7 +33,7 @@ class PostDeployMigrateUserExternalMailData < ActiveRecord::Migration
           SELECT true
           FROM user_synced_attributes_metadata
           WHERE user_id = users.id
-          AND provider = users.email_provider OR (provider IS NULL AND users.email_provider IS NULL)
+          AND (provider = users.email_provider OR (provider IS NULL AND users.email_provider IS NULL))
         )
         AND id BETWEEN #{start_id} AND #{end_id}
       EOF
diff --git a/db/post_migrate/20170926150348_schedule_merge_request_diff_migrations_take_two.rb b/db/post_migrate/20170926150348_schedule_merge_request_diff_migrations_take_two.rb
new file mode 100644
index 00000000000..5732cb85ea5
--- /dev/null
+++ b/db/post_migrate/20170926150348_schedule_merge_request_diff_migrations_take_two.rb
@@ -0,0 +1,32 @@
+class ScheduleMergeRequestDiffMigrationsTakeTwo < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  BATCH_SIZE = 500
+  MIGRATION = 'DeserializeMergeRequestDiffsAndCommits'
+  DELAY_INTERVAL = 10.minutes
+
+  disable_ddl_transaction!
+
+  class MergeRequestDiff < ActiveRecord::Base
+    self.table_name = 'merge_request_diffs'
+
+    include ::EachBatch
+
+    default_scope { where('st_commits IS NOT NULL OR st_diffs IS NOT NULL') }
+  end
+
+  # By this point, we assume ScheduleMergeRequestDiffMigrations - the first
+  # version of this - has already run. On GitLab.com, we have ~220k un-migrated
+  # rows, but these rows will, in general, take a long time.
+  #
+  # With a gap of 10 minutes per batch, and 500 rows per batch, these migrations
+  # are scheduled over 220_000 / 500 / 6 ~= 74 hours, which is a little over
+  # three days.
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(MergeRequestDiff, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+  end
+
+  def down
+  end
+end
diff --git a/db/post_migrate/20170927112318_update_legacy_diff_notes_type_for_import.rb b/db/post_migrate/20170927112318_update_legacy_diff_notes_type_for_import.rb
new file mode 100644
index 00000000000..a238216253b
--- /dev/null
+++ b/db/post_migrate/20170927112318_update_legacy_diff_notes_type_for_import.rb
@@ -0,0 +1,16 @@
+class UpdateLegacyDiffNotesTypeForImport < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    update_column_in_batches(:notes, :type, 'LegacyDiffNote') do |table, query|
+      query.where(table[:type].eq('Github::Import::LegacyDiffNote'))
+    end
+  end
+
+  def down
+  end
+end
diff --git a/db/post_migrate/20170927112319_update_notes_type_for_import.rb b/db/post_migrate/20170927112319_update_notes_type_for_import.rb
new file mode 100644
index 00000000000..1e70acd9868
--- /dev/null
+++ b/db/post_migrate/20170927112319_update_notes_type_for_import.rb
@@ -0,0 +1,16 @@
+class UpdateNotesTypeForImport < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    update_column_in_batches(:notes, :type, 'Note') do |table, query|
+      query.where(table[:type].eq('Github::Import::Note'))
+    end
+  end
+
+  def down
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 17f59145472..17be774e9de 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20170921115009) do
+ActiveRecord::Schema.define(version: 20171004121444) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -256,8 +256,7 @@ ActiveRecord::Schema.define(version: 20170921115009) do
   add_index "ci_builds", ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree
   add_index "ci_builds", ["commit_id", "type", "name", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_name_and_ref", using: :btree
   add_index "ci_builds", ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree
-  add_index "ci_builds", ["id"], name: "index_for_ci_builds_retried_migration", where: "(retried IS NULL)", using: :btree
-  add_index "ci_builds", ["project_id"], name: "index_ci_builds_on_project_id", using: :btree
+  add_index "ci_builds", ["project_id", "id"], name: "index_ci_builds_on_project_id_and_id", using: :btree
   add_index "ci_builds", ["protected"], name: "index_ci_builds_on_protected", using: :btree
   add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree
   add_index "ci_builds", ["stage_id"], name: "index_ci_builds_on_stage_id", using: :btree
@@ -735,6 +734,7 @@ ActiveRecord::Schema.define(version: 20170921115009) do
 
   add_index "labels", ["group_id", "project_id", "title"], name: "index_labels_on_group_id_and_project_id_and_title", unique: true, using: :btree
   add_index "labels", ["project_id"], name: "index_labels_on_project_id", using: :btree
+  add_index "labels", ["template"], name: "index_labels_on_template", where: "template", using: :btree
   add_index "labels", ["title"], name: "index_labels_on_title", using: :btree
   add_index "labels", ["type", "project_id"], name: "index_labels_on_type_and_project_id", using: :btree
 
@@ -900,6 +900,7 @@ ActiveRecord::Schema.define(version: 20170921115009) do
   add_index "merge_requests", ["source_project_id", "source_branch"], name: "index_merge_requests_on_source_project_id_and_source_branch", using: :btree
   add_index "merge_requests", ["target_branch"], name: "index_merge_requests_on_target_branch", using: :btree
   add_index "merge_requests", ["target_project_id", "iid"], name: "index_merge_requests_on_target_project_id_and_iid", unique: true, using: :btree
+  add_index "merge_requests", ["target_project_id", "merge_commit_sha", "id"], name: "index_merge_requests_on_tp_id_and_merge_commit_sha_and_id", using: :btree
   add_index "merge_requests", ["title"], name: "index_merge_requests_on_title", using: :btree
   add_index "merge_requests", ["title"], name: "index_merge_requests_on_title_trigram", using: :gin, opclasses: {"title"=>"gin_trgm_ops"}
 
@@ -1220,6 +1221,9 @@ ActiveRecord::Schema.define(version: 20170921115009) do
     t.datetime "last_repository_updated_at"
     t.integer "storage_version", limit: 2
     t.boolean "resolve_outdated_diff_discussions"
+    t.boolean "repository_read_only"
+    t.boolean "merge_requests_ff_only_enabled", default: false
+    t.boolean "merge_requests_rebase_enabled", default: false, null: false
   end
 
   add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree
@@ -1539,6 +1543,17 @@ ActiveRecord::Schema.define(version: 20170921115009) do
 
   add_index "user_agent_details", ["subject_id", "subject_type"], name: "index_user_agent_details_on_subject_id_and_subject_type", using: :btree
 
+  create_table "user_custom_attributes", force: :cascade do |t|
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.integer "user_id", null: false
+    t.string "key", null: false
+    t.string "value", null: false
+  end
+
+  add_index "user_custom_attributes", ["key", "value"], name: "index_user_custom_attributes_on_key_and_value", using: :btree
+  add_index "user_custom_attributes", ["user_id", "key"], name: "index_user_custom_attributes_on_user_id_and_key", unique: true, using: :btree
+
   create_table "user_synced_attributes_metadata", force: :cascade do |t|
     t.boolean "name_synced", default: false
     t.boolean "email_synced", default: false
@@ -1765,6 +1780,7 @@ ActiveRecord::Schema.define(version: 20170921115009) do
   add_foreign_key "todos", "projects", name: "fk_45054f9c45", on_delete: :cascade
   add_foreign_key "trending_projects", "projects", on_delete: :cascade
   add_foreign_key "u2f_registrations", "users"
+  add_foreign_key "user_custom_attributes", "users", on_delete: :cascade
   add_foreign_key "user_synced_attributes_metadata", "users", on_delete: :cascade
   add_foreign_key "users_star_projects", "projects", name: "fk_22cd27ddfc", on_delete: :cascade
   add_foreign_key "web_hook_logs", "web_hooks", on_delete: :cascade