Merge remote-tracking branch 'upstream/master' into 25680-CI_ENVIRONMENT_URL

* upstream/master: (251 commits)
  Don't match email addresses or foo@bar as user references
  Revert "Update GITLAB_SHELL_VERSION"
  Update GITLAB_SHELL_VERSION
  Add feature toggles through Flipper
  Change no_limits to limits
  Move includes call to scope
  Add GitLab Resources to University
  Add Documentation for GIT_CHECKOUT variable
  Remove entry variable
  Do not try to preload Commits when using Note.includes(:noteable)
  Ui improvements for count badges and permission badges
  Rename the other jobs
  Update jobs_spec for changes from builds_spec
  Introduce source to pipeline entity
  Update docs related to protected actions
  Add changelog for protected branches abilities fix
  Ask for an example project for bug reports
  Center loading spinner in issuable filters
  Fix chat commands specs related to protected actions
  Fix builds controller specs related to protected actions
  ...

19 files changed, 271 insertions, 14 deletions

diff --git a/db/fixtures/development/14_pipelines.rb b/db/fixtures/development/14_pipelines.rb
index 3c42f7db6d5..68767f0e585 100644
--- a/db/fixtures/development/14_pipelines.rb
+++ b/db/fixtures/development/14_pipelines.rb
@@ -98,7 +98,7 @@ class Gitlab::Seeder::Pipelines
 
 
   def create_pipeline!(project, ref, commit)
-    project.pipelines.create(sha: commit.id, ref: ref)
+    project.pipelines.create(sha: commit.id, ref: ref, source: :push)
   end
 
   def build_create!(pipeline, opts = {})
diff --git a/db/fixtures/development/17_cycle_analytics.rb b/db/fixtures/development/17_cycle_analytics.rb
index 0d7eb1a7c93..75457b2d369 100644
--- a/db/fixtures/development/17_cycle_analytics.rb
+++ b/db/fixtures/development/17_cycle_analytics.rb
@@ -190,7 +190,7 @@ class Gitlab::Seeder::CycleAnalytics
       service = Ci::CreatePipelineService.new(merge_request.project,
                                               @user,
                                               ref: "refs/heads/#{merge_request.source_branch}")
-      pipeline = service.execute(ignore_skip_ci: true, save_on_errors: false)
+      pipeline = service.execute(:push, ignore_skip_ci: true, save_on_errors: false)
 
       pipeline.run!
       Timecop.travel rand(1..6).hours.from_now
diff --git a/db/migrate/20160615191922_set_missing_stage_on_ci_builds.rb b/db/migrate/20160615191922_set_missing_stage_on_ci_builds.rb
index bd0463886bc..4d6a61bd614 100644
--- a/db/migrate/20160615191922_set_missing_stage_on_ci_builds.rb
+++ b/db/migrate/20160615191922_set_missing_stage_on_ci_builds.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Migration/UpdateColumnInBatches
 class SetMissingStageOnCiBuilds < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/migrate/20160721081015_drop_and_readd_has_external_wiki_in_projects.rb b/db/migrate/20160721081015_drop_and_readd_has_external_wiki_in_projects.rb
index 1eb99feb40c..b2a2ce41391 100644
--- a/db/migrate/20160721081015_drop_and_readd_has_external_wiki_in_projects.rb
+++ b/db/migrate/20160721081015_drop_and_readd_has_external_wiki_in_projects.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Migration/UpdateColumnInBatches
 class DropAndReaddHasExternalWikiInProjects < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/migrate/20160901141443_set_confidential_issues_events_on_webhooks.rb b/db/migrate/20160901141443_set_confidential_issues_events_on_webhooks.rb
index f1a1f001cb3..febd2c0e65e 100644
--- a/db/migrate/20160901141443_set_confidential_issues_events_on_webhooks.rb
+++ b/db/migrate/20160901141443_set_confidential_issues_events_on_webhooks.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Migration/UpdateColumnInBatches
 class SetConfidentialIssuesEventsOnWebhooks < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/migrate/20160919144305_add_type_to_labels.rb b/db/migrate/20160919144305_add_type_to_labels.rb
index 66172bda6ff..2d2725ccf59 100644
--- a/db/migrate/20160919144305_add_type_to_labels.rb
+++ b/db/migrate/20160919144305_add_type_to_labels.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Migration/UpdateColumnInBatches
 class AddTypeToLabels < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/migrate/20161018124658_make_project_owners_masters.rb b/db/migrate/20161018124658_make_project_owners_masters.rb
index a576bb7b622..fe11699c196 100644
--- a/db/migrate/20161018124658_make_project_owners_masters.rb
+++ b/db/migrate/20161018124658_make_project_owners_masters.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Migration/UpdateColumnInBatches
 class MakeProjectOwnersMasters < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/migrate/20161227192806_rename_slack_and_mattermost_notification_services.rb b/db/migrate/20161227192806_rename_slack_and_mattermost_notification_services.rb
index 50ad7437227..c7cada6dfc5 100644
--- a/db/migrate/20161227192806_rename_slack_and_mattermost_notification_services.rb
+++ b/db/migrate/20161227192806_rename_slack_and_mattermost_notification_services.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Migration/UpdateColumnInBatches
 class RenameSlackAndMattermostNotificationServices < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/migrate/20170320173259_migrate_assignees.rb b/db/migrate/20170320173259_migrate_assignees.rb
index 23e7500a32d..7b61e811317 100644
--- a/db/migrate/20170320173259_migrate_assignees.rb
+++ b/db/migrate/20170320173259_migrate_assignees.rb
@@ -1,6 +1,4 @@
-# See http://doc.gitlab.com/ce/development/migration_style_guide.html
-# for more information on how to write migrations for GitLab.
-
+# rubocop:disable Migration/UpdateColumnInBatches
 class MigrateAssignees < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/migrate/20170503140201_reschedule_project_authorizations.rb b/db/migrate/20170503140201_reschedule_project_authorizations.rb
new file mode 100644
index 00000000000..fa45adadbae
--- /dev/null
+++ b/db/migrate/20170503140201_reschedule_project_authorizations.rb
@@ -0,0 +1,44 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RescheduleProjectAuthorizations < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class User < ActiveRecord::Base
+    self.table_name = 'users'
+  end
+
+  def up
+    offset = 0
+    batch = 5000
+    start = Time.now
+
+    loop do
+      relation = User.where('id > ?', offset)
+      user_ids = relation.limit(batch).reorder(id: :asc).pluck(:id)
+
+      break if user_ids.empty?
+
+      offset = user_ids.last
+
+      # This will schedule each batch 5 minutes after the previous batch was
+      # scheduled. This smears out the load over time, instead of immediately
+      # scheduling a million jobs.
+      Sidekiq::Client.push_bulk(
+        'queue' => 'authorized_projects',
+        'args' => user_ids.zip,
+        'class' => 'AuthorizedProjectsWorker',
+        'at' => start.to_i
+      )
+
+      start += 5.minutes
+    end
+  end
+
+  def down
+  end
+end
diff --git a/db/migrate/20170503140202_turn_nested_groups_into_regular_groups_for_mysql.rb b/db/migrate/20170503140202_turn_nested_groups_into_regular_groups_for_mysql.rb
new file mode 100644
index 00000000000..c67690642c9
--- /dev/null
+++ b/db/migrate/20170503140202_turn_nested_groups_into_regular_groups_for_mysql.rb
@@ -0,0 +1,123 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+# This migration depends on code external to it. For example, it relies on
+# updating a namespace to also rename directories (uploads, GitLab pages, etc).
+# The alternative is to copy hundreds of lines of code into this migration,
+# adjust them where needed, etc; something which doesn't work well at all.
+class TurnNestedGroupsIntoRegularGroupsForMysql < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def run_migration?
+    Gitlab::Database.mysql?
+  end
+
+  def up
+    return unless run_migration?
+
+    # For all sub-groups we need to give the right people access. We do this as
+    # follows:
+    #
+    # 1. Get all the ancestors for the current namespace
+    # 2. Get all the members of these namespaces, along with their higher access
+    #    level
+    # 3. Give these members access to the current namespace
+    Namespace.unscoped.where('parent_id IS NOT NULL').find_each do |namespace|
+      rows = []
+      existing = namespace.members.pluck(:user_id)
+
+      all_members_for(namespace).each do |member|
+        next if existing.include?(member[:user_id])
+
+        rows << {
+          access_level: member[:access_level],
+          source_id: namespace.id,
+          source_type: 'Namespace',
+          user_id: member[:user_id],
+          notification_level: 3, # global
+          type: 'GroupMember',
+          created_at: Time.current,
+          updated_at: Time.current
+        }
+      end
+
+      bulk_insert_members(rows)
+
+      # This method relies on the parent to determine the proper path.
+      # Because we reset "parent_id" this method will not return the right path
+      # when moving namespaces.
+      full_path_was = namespace.send(:full_path_was)
+
+      namespace.define_singleton_method(:full_path_was) { full_path_was }
+
+      namespace.update!(parent_id: nil, path: new_path_for(namespace))
+    end
+  end
+
+  def down
+    # There is no way to go back from regular groups to nested groups.
+  end
+
+  # Generates a new (unique) path for a namespace.
+  def new_path_for(namespace)
+    counter = 1
+    base = namespace.full_path.tr('/', '-')
+    new_path = base
+
+    while Namespace.unscoped.where(path: new_path).exists?
+      new_path = base + "-#{counter}"
+      counter += 1
+    end
+
+    new_path
+  end
+
+  # Returns an Array containing all the ancestors of the current namespace.
+  #
+  # This method is not particularly efficient, but it's probably still faster
+  # than using the "routes" table. Most importantly of all, it _only_ depends
+  # on the namespaces table and the "parent_id" column.
+  def ancestors_for(namespace)
+    ancestors = []
+    current = namespace
+
+    while current&.parent_id
+      # We're using find_by(id: ...) here to deal with cases where the
+      # parent_id may point to a missing row.
+      current = Namespace.unscoped.select([:id, :parent_id]).
+        find_by(id: current.parent_id)
+
+      ancestors << current.id if current
+    end
+
+    ancestors
+  end
+
+  # Returns a relation containing all the members that have access to any of
+  # the current namespace's parent namespaces.
+  def all_members_for(namespace)
+    Member.
+      unscoped.
+      select(['user_id', 'MAX(access_level) AS access_level']).
+      where(source_type: 'Namespace', source_id: ancestors_for(namespace)).
+      group(:user_id)
+  end
+
+  def bulk_insert_members(rows)
+    return if rows.empty?
+
+    keys = rows.first.keys
+
+    tuples = rows.map do |row|
+      row.map { |(_, value)| connection.quote(value) }
+    end
+
+    execute <<-EOF.strip_heredoc
+    INSERT INTO members (#{keys.join(', ')})
+    VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')}
+    EOF
+  end
+end
diff --git a/db/migrate/20170504182103_add_index_project_group_links_group_id.rb b/db/migrate/20170504182103_add_index_project_group_links_group_id.rb
new file mode 100644
index 00000000000..62bf641daa6
--- /dev/null
+++ b/db/migrate/20170504182103_add_index_project_group_links_group_id.rb
@@ -0,0 +1,19 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddIndexProjectGroupLinksGroupId < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :project_group_links, :group_id
+  end
+
+  def down
+    remove_concurrent_index :project_group_links, :group_id
+  end
+end
diff --git a/db/migrate/20170524125940_add_source_to_ci_pipeline.rb b/db/migrate/20170524125940_add_source_to_ci_pipeline.rb
new file mode 100644
index 00000000000..1fa3d48037b
--- /dev/null
+++ b/db/migrate/20170524125940_add_source_to_ci_pipeline.rb
@@ -0,0 +1,9 @@
+class AddSourceToCiPipeline < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :ci_pipelines, :source, :integer
+  end
+end
diff --git a/db/migrate/20170525174156_create_feature_tables.rb b/db/migrate/20170525174156_create_feature_tables.rb
new file mode 100644
index 00000000000..a083c89c85f
--- /dev/null
+++ b/db/migrate/20170525174156_create_feature_tables.rb
@@ -0,0 +1,26 @@
+class CreateFeatureTables < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def self.up
+    create_table :features do |t|
+      t.string :key, null: false
+      t.timestamps null: false
+    end
+    add_index :features, :key, unique: true
+
+    create_table :feature_gates do |t|
+      t.string :feature_key, null: false
+      t.string :key, null: false
+      t.string :value
+      t.timestamps null: false
+    end
+    add_index :feature_gates, [:feature_key, :key, :value], unique: true
+  end
+
+  def self.down
+    drop_table :feature_gates
+    drop_table :features
+  end
+end
diff --git a/db/post_migrate/20170131214021_reset_users_authorized_projects_populated.rb b/db/post_migrate/20170131214021_reset_users_authorized_projects_populated.rb
index b518038e93a..82f8147547e 100644
--- a/db/post_migrate/20170131214021_reset_users_authorized_projects_populated.rb
+++ b/db/post_migrate/20170131214021_reset_users_authorized_projects_populated.rb
@@ -1,6 +1,4 @@
-# See http://doc.gitlab.com/ce/development/migration_style_guide.html
-# for more information on how to write migrations for GitLab.
-
+# rubocop:disable Migration/UpdateColumnInBatches
 class ResetUsersAuthorizedProjectsPopulated < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/post_migrate/20170309171644_reset_relative_position_for_issue.rb b/db/post_migrate/20170309171644_reset_relative_position_for_issue.rb
index b61dd7cfc61..b1c9eed1148 100644
--- a/db/post_migrate/20170309171644_reset_relative_position_for_issue.rb
+++ b/db/post_migrate/20170309171644_reset_relative_position_for_issue.rb
@@ -1,6 +1,4 @@
-# See http://doc.gitlab.com/ce/development/migration_style_guide.html
-# for more information on how to write migrations for GitLab.
-
+# rubocop:disable Migration/UpdateColumnInBatches
 class ResetRelativePositionForIssue < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/post_migrate/20170502070007_enable_auto_cancel_pending_pipelines_for_all.rb b/db/post_migrate/20170502070007_enable_auto_cancel_pending_pipelines_for_all.rb
index a19b73fc114..3c13a3d2518 100644
--- a/db/post_migrate/20170502070007_enable_auto_cancel_pending_pipelines_for_all.rb
+++ b/db/post_migrate/20170502070007_enable_auto_cancel_pending_pipelines_for_all.rb
@@ -1,3 +1,4 @@
+# rubocop:disable Migration/UpdateColumnInBatches
 class EnableAutoCancelPendingPipelinesForAll < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
diff --git a/db/post_migrate/20170503120310_remove_users_authorized_projects_populated.rb b/db/post_migrate/20170503120310_remove_users_authorized_projects_populated.rb
new file mode 100644
index 00000000000..1b44334395f
--- /dev/null
+++ b/db/post_migrate/20170503120310_remove_users_authorized_projects_populated.rb
@@ -0,0 +1,15 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class RemoveUsersAuthorizedProjectsPopulated < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def change
+    remove_column :users, :authorized_projects_populated, :boolean
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 275e7a4a7af..255da64d1a1 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20170524195203) do
+ActiveRecord::Schema.define(version: 20170525174156) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -284,6 +284,7 @@ ActiveRecord::Schema.define(version: 20170524195203) do
     t.integer "lock_version"
     t.integer "auto_canceled_by_id"
     t.integer "pipeline_schedule_id"
+    t.integer "source"
   end
 
   add_index "ci_pipelines", ["auto_canceled_by_id"], name: "index_ci_pipelines_on_auto_canceled_by_id", using: :btree
@@ -441,6 +442,24 @@ ActiveRecord::Schema.define(version: 20170524195203) do
   add_index "events", ["target_id"], name: "index_events_on_target_id", using: :btree
   add_index "events", ["target_type"], name: "index_events_on_target_type", using: :btree
 
+  create_table "feature_gates", force: :cascade do |t|
+    t.string "feature_key", null: false
+    t.string "key", null: false
+    t.string "value"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  add_index "feature_gates", ["feature_key", "key", "value"], name: "index_feature_gates_on_feature_key_and_key_and_value", unique: true, using: :btree
+
+  create_table "features", force: :cascade do |t|
+    t.string "key", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  add_index "features", ["key"], name: "index_features_on_key", unique: true, using: :btree
+
   create_table "forked_project_links", force: :cascade do |t|
     t.integer "forked_to_project_id", null: false
     t.integer "forked_from_project_id", null: false
@@ -929,6 +948,8 @@ ActiveRecord::Schema.define(version: 20170524195203) do
     t.date "expires_at"
   end
 
+  add_index "project_group_links", ["group_id"], name: "index_project_group_links_on_group_id", using: :btree
+
   create_table "project_import_data", force: :cascade do |t|
     t.integer "project_id"
     t.text "data"
@@ -1356,7 +1377,6 @@ ActiveRecord::Schema.define(version: 20170524195203) do
     t.boolean "external", default: false
     t.string "incoming_email_token"
     t.string "organization"
-    t.boolean "authorized_projects_populated"
     t.boolean "require_two_factor_authentication_from_group", default: false, null: false
     t.integer "two_factor_grace_period", default: 48, null: false
     t.boolean "ghost"
@@ -1473,4 +1493,4 @@ ActiveRecord::Schema.define(version: 20170524195203) do
   add_foreign_key "trending_projects", "projects", on_delete: :cascade
   add_foreign_key "u2f_registrations", "users"
   add_foreign_key "web_hook_logs", "web_hooks", on_delete: :cascade
-end
\ No newline at end of file
+end