Merge remote-tracking branch 'upstream/master' into add-index-for-auto_canceled_by_id-mysql

* upstream/master: (224 commits)
  Added balsamiq jasmine integration test
  Add support for deltas_only under Gitaly
  Codestyle
  Update CHANGELOG.md for 9.1.4
  Update CHANGELOG.md for 9.1.4
  Update CHANGELOG.md for 9.1.4
  Update CHANGELOG.md for 9.1.4
  Minor cosmetic fixes in hooks admin screen
  Documentation for repository_update_events
  Changelog
  Update SystemHooks API to expose and handle new repository_update_events
  Make the new repository_update_events configurable in System Hooks UI
  Added repository_update hook
  Wait for requests after each Spinach scenario instead of feature
  Remove trailing comma in dependency linker to satisfy Rubocop
  Fix specs
  New branch new mr docs
  Relax rake backup regex to handle CE and EE RCs
  Fix conflict resolution from corrupted upstream
  Removed all instances of Object.assign by using es6 classes, also includes some …
  ...

9 files changed, 144 insertions, 5 deletions

diff --git a/db/migrate/20160810142633_remove_redundant_indexes.rb b/db/migrate/20160810142633_remove_redundant_indexes.rb
index d7ab022d7bc..ea7d1f9a436 100644
--- a/db/migrate/20160810142633_remove_redundant_indexes.rb
+++ b/db/migrate/20160810142633_remove_redundant_indexes.rb
@@ -69,7 +69,7 @@ class RemoveRedundantIndexes < ActiveRecord::Migration
       [:namespaces, 'index_namespaces_on_created_at_and_id'],
       [:notes, 'index_notes_on_created_at_and_id'],
       [:projects, 'index_projects_on_created_at_and_id'],
-      [:users, 'index_users_on_created_at_and_id'],
+      [:users, 'index_users_on_created_at_and_id']
     ]
 
     transaction do
diff --git a/db/migrate/20160829114652_add_markdown_cache_columns.rb b/db/migrate/20160829114652_add_markdown_cache_columns.rb
index 9cb44dfa9f9..6ad7237f4cd 100644
--- a/db/migrate/20160829114652_add_markdown_cache_columns.rb
+++ b/db/migrate/20160829114652_add_markdown_cache_columns.rb
@@ -25,7 +25,7 @@ class AddMarkdownCacheColumns < ActiveRecord::Migration
     notes: [:note],
     projects: [:description],
     releases: [:description],
-    snippets: [:title, :content],
+    snippets: [:title, :content]
   }.freeze
 
   def change
diff --git a/db/migrate/20170503004426_add_retried_to_ci_build.rb b/db/migrate/20170503004426_add_retried_to_ci_build.rb
new file mode 100644
index 00000000000..2851e3de473
--- /dev/null
+++ b/db/migrate/20170503004426_add_retried_to_ci_build.rb
@@ -0,0 +1,9 @@
+class AddRetriedToCiBuild < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column(:ci_builds, :retried, :boolean)
+  end
+end
diff --git a/db/migrate/20170503023315_add_repository_update_events_to_web_hooks.rb b/db/migrate/20170503023315_add_repository_update_events_to_web_hooks.rb
new file mode 100644
index 00000000000..0faea87a962
--- /dev/null
+++ b/db/migrate/20170503023315_add_repository_update_events_to_web_hooks.rb
@@ -0,0 +1,15 @@
+class AddRepositoryUpdateEventsToWebHooks < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default :web_hooks, :repository_update_events, :boolean, default: false, allow_null: false
+  end
+
+  def down
+    remove_column :web_hooks, :repository_update_events
+  end
+end
diff --git a/db/migrate/20170508153950_add_not_null_contraints_to_ci_variables.rb b/db/migrate/20170508153950_add_not_null_contraints_to_ci_variables.rb
new file mode 100644
index 00000000000..41c687a4f6e
--- /dev/null
+++ b/db/migrate/20170508153950_add_not_null_contraints_to_ci_variables.rb
@@ -0,0 +1,12 @@
+class AddNotNullContraintsToCiVariables < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def up
+    change_column(:ci_variables, :key, :string, null: false)
+    change_column(:ci_variables, :project_id, :integer, null: false)
+  end
+
+  def down
+    # no op
+  end
+end
diff --git a/db/migrate/20170508190732_add_foreign_key_to_ci_variables.rb b/db/migrate/20170508190732_add_foreign_key_to_ci_variables.rb
new file mode 100644
index 00000000000..20ecaa2c36c
--- /dev/null
+++ b/db/migrate/20170508190732_add_foreign_key_to_ci_variables.rb
@@ -0,0 +1,24 @@
+class AddForeignKeyToCiVariables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    execute <<~SQL
+      DELETE FROM ci_variables
+      WHERE NOT EXISTS (
+        SELECT true
+        FROM projects
+        WHERE projects.id = ci_variables.project_id
+      )
+    SQL
+
+    add_concurrent_foreign_key(:ci_variables, :projects, column: :project_id)
+  end
+
+  def down
+    remove_foreign_key(:ci_variables, column: :project_id)
+  end
+end
diff --git a/db/post_migrate/20170502101023_cleanup_namespaceless_pending_delete_projects.rb b/db/post_migrate/20170502101023_cleanup_namespaceless_pending_delete_projects.rb
new file mode 100644
index 00000000000..ce52de91cdd
--- /dev/null
+++ b/db/post_migrate/20170502101023_cleanup_namespaceless_pending_delete_projects.rb
@@ -0,0 +1,47 @@
+# This is the counterpart of RequeuePendingDeleteProjects and cleans all
+# projects with `pending_delete = true` and that do not have a namespace.
+class CleanupNamespacelessPendingDeleteProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    @offset = 0
+
+    loop do
+      ids = pending_delete_batch
+
+      break if ids.empty?
+
+      args = ids.map { |id| Array(id) }
+
+      NamespacelessProjectDestroyWorker.bulk_perform_async(args)
+
+      @offset += 1
+    end
+  end
+
+  def down
+    # noop
+  end
+
+  private
+
+  def pending_delete_batch
+    connection.exec_query(find_batch).map{ |row| row['id'].to_i }
+  end
+
+  BATCH_SIZE = 5000
+
+  def find_batch
+    projects = Arel::Table.new(:projects)
+    projects.project(projects[:id]).
+      where(projects[:pending_delete].eq(true)).
+      where(projects[:namespace_id].eq(nil)).
+      skip(@offset * BATCH_SIZE).
+      take(BATCH_SIZE).
+      to_sql
+  end
+end
diff --git a/db/post_migrate/20170503004427_upate_retried_for_ci_build.rb b/db/post_migrate/20170503004427_upate_retried_for_ci_build.rb
new file mode 100644
index 00000000000..80215d662e4
--- /dev/null
+++ b/db/post_migrate/20170503004427_upate_retried_for_ci_build.rb
@@ -0,0 +1,29 @@
+class UpateRetriedForCiBuild < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  
+  disable_ddl_transaction!
+
+  def up
+    disable_statement_timeout
+
+    latest_id = <<-SQL.strip_heredoc
+      SELECT MAX(ci_builds2.id)
+        FROM ci_builds ci_builds2
+        WHERE ci_builds.commit_id=ci_builds2.commit_id
+          AND ci_builds.name=ci_builds2.name
+    SQL
+    
+    # This is slow update as it does single-row query
+    # This is designed to be run as idle, or a post deployment migration
+    is_retried = Arel.sql("((#{latest_id}) != ci_builds.id)")
+
+    update_column_in_batches(:ci_builds, :retried, is_retried) do |table, query|
+      query.where(table[:retried].eq(nil))
+    end
+  end
+
+  def down
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 6934f7a793c..d76432fa1e7 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20170508170547) do
+ActiveRecord::Schema.define(version: 20170508190732) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -232,6 +232,7 @@ ActiveRecord::Schema.define(version: 20170508170547) do
     t.integer "lock_version"
     t.string "coverage_regex"
     t.integer "auto_canceled_by_id"
+    t.boolean "retried"
   end
 
   add_index "ci_builds", ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree
@@ -348,12 +349,12 @@ ActiveRecord::Schema.define(version: 20170508170547) do
   add_index "ci_triggers", ["project_id"], name: "index_ci_triggers_on_project_id", using: :btree
 
   create_table "ci_variables", force: :cascade do |t|
-    t.string "key"
+    t.string "key", null: false
     t.text "value"
     t.text "encrypted_value"
     t.string "encrypted_value_salt"
     t.string "encrypted_value_iv"
-    t.integer "project_id"
+    t.integer "project_id", null: false
   end
 
   add_index "ci_variables", ["project_id"], name: "index_ci_variables_on_project_id", using: :btree
@@ -1405,6 +1406,7 @@ ActiveRecord::Schema.define(version: 20170508170547) do
     t.string "token"
     t.boolean "pipeline_events", default: false, null: false
     t.boolean "confidential_issues_events", default: false, null: false
+    t.boolean "repository_update_events", default: false, null: false
   end
 
   add_index "web_hooks", ["project_id"], name: "index_web_hooks_on_project_id", using: :btree
@@ -1418,6 +1420,7 @@ ActiveRecord::Schema.define(version: 20170508170547) do
   add_foreign_key "ci_pipelines", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_262d4c2d19", on_delete: :nullify
   add_foreign_key "ci_trigger_requests", "ci_triggers", column: "trigger_id", name: "fk_b8ec8b7245", on_delete: :cascade
   add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade
+  add_foreign_key "ci_variables", "projects", name: "fk_ada5eb64b3", on_delete: :cascade
   add_foreign_key "container_repositories", "projects"
   add_foreign_key "issue_assignees", "issues", on_delete: :cascade
   add_foreign_key "issue_assignees", "users", on_delete: :cascade