CE port for pipelines for external pull requests

Detect if pipeline runs for a GitHub pull request

When using a mirror for CI/CD only we register a pull_request
webhook. When a pull_request webhook is received, if the
source branch SHA matches the actual head of the branch in the
repository we create immediately a new pipeline for the
external pull request. Otherwise we store the
pull request info for when the push webhook is received.

When using "only/except: external_pull_requests" we can detect
if the pipeline has a open pull request on GitHub and create or
not the job based on that.

5 files changed, 93 insertions, 0 deletions

diff --git a/db/migrate/20190829131130_create_external_pull_requests.rb b/db/migrate/20190829131130_create_external_pull_requests.rb
new file mode 100644
index 00000000000..0c3168807ec
--- /dev/null
+++ b/db/migrate/20190829131130_create_external_pull_requests.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class CreateExternalPullRequests < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX = 'index_external_pull_requests_on_project_and_branches'
+
+  def change
+    create_table :external_pull_requests do |t|
+      t.timestamps_with_timezone null: false
+      t.references :project, null: false, foreign_key: { on_delete: :cascade }, index: false
+      t.integer :pull_request_iid, null: false
+      t.integer :status, null: false, limit: 2
+      t.string :source_branch, null: false, limit: 255
+      t.string :target_branch, null: false, limit: 255
+      t.string :source_repository, null: false, limit: 255
+      t.string :target_repository, null: false, limit: 255
+      t.binary :source_sha, null: false
+      t.binary :target_sha, null: false
+
+      t.index [:project_id, :source_branch, :target_branch], unique: true, name: INDEX
+    end
+  end
+end
diff --git a/db/migrate/20190830075508_add_external_pull_request_id_to_ci_pipelines.rb b/db/migrate/20190830075508_add_external_pull_request_id_to_ci_pipelines.rb
new file mode 100644
index 00000000000..5abf56742b1
--- /dev/null
+++ b/db/migrate/20190830075508_add_external_pull_request_id_to_ci_pipelines.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class AddExternalPullRequestIdToCiPipelines < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    add_column :ci_pipelines, :external_pull_request_id, :bigint
+  end
+
+  def down
+    remove_column :ci_pipelines, :external_pull_request_id
+  end
+end
diff --git a/db/migrate/20190830080123_add_index_to_ci_pipelines_external_pull_request.rb b/db/migrate/20190830080123_add_index_to_ci_pipelines_external_pull_request.rb
new file mode 100644
index 00000000000..d2f5ad7a420
--- /dev/null
+++ b/db/migrate/20190830080123_add_index_to_ci_pipelines_external_pull_request.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class AddIndexToCiPipelinesExternalPullRequest < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :ci_pipelines, :external_pull_request_id, where: 'external_pull_request_id IS NOT NULL'
+  end
+
+  def down
+    remove_concurrent_index :ci_pipelines, :external_pull_request_id
+  end
+end
diff --git a/db/migrate/20190830080626_add_foreign_key_to_ci_pipelines_external_pull_request.rb b/db/migrate/20190830080626_add_foreign_key_to_ci_pipelines_external_pull_request.rb
new file mode 100644
index 00000000000..b38fda83047
--- /dev/null
+++ b/db/migrate/20190830080626_add_foreign_key_to_ci_pipelines_external_pull_request.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class AddForeignKeyToCiPipelinesExternalPullRequest < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :ci_pipelines, :external_pull_requests, column: :external_pull_request_id, on_delete: :nullify
+  end
+
+  def down
+    remove_foreign_key :ci_pipelines, :external_pull_requests
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 98c4403efe1..9301aa5a8de 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -752,7 +752,9 @@ ActiveRecord::Schema.define(version: 2019_09_04_173203) do
     t.integer "merge_request_id"
     t.binary "source_sha"
     t.binary "target_sha"
+    t.bigint "external_pull_request_id"
     t.index ["auto_canceled_by_id"], name: "index_ci_pipelines_on_auto_canceled_by_id"
+    t.index ["external_pull_request_id"], name: "index_ci_pipelines_on_external_pull_request_id", where: "(external_pull_request_id IS NOT NULL)"
     t.index ["merge_request_id"], name: "index_ci_pipelines_on_merge_request_id", where: "(merge_request_id IS NOT NULL)"
     t.index ["pipeline_schedule_id"], name: "index_ci_pipelines_on_pipeline_schedule_id"
     t.index ["project_id", "iid"], name: "index_ci_pipelines_on_project_id_and_iid", unique: true, where: "(iid IS NOT NULL)"
@@ -1321,6 +1323,21 @@ ActiveRecord::Schema.define(version: 2019_09_04_173203) do
     t.index ["target_type", "target_id"], name: "index_events_on_target_type_and_target_id"
   end
 
+  create_table "external_pull_requests", force: :cascade do |t|
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.bigint "project_id", null: false
+    t.integer "pull_request_iid", null: false
+    t.integer "status", limit: 2, null: false
+    t.string "source_branch", limit: 255, null: false
+    t.string "target_branch", limit: 255, null: false
+    t.string "source_repository", limit: 255, null: false
+    t.string "target_repository", limit: 255, null: false
+    t.binary "source_sha", null: false
+    t.binary "target_sha", null: false
+    t.index ["project_id", "source_branch", "target_branch"], name: "index_external_pull_requests_on_project_and_branches", unique: true
+  end
+
   create_table "feature_gates", id: :serial, force: :cascade do |t|
     t.string "feature_key", null: false
     t.string "key", null: false
@@ -3781,6 +3798,7 @@ ActiveRecord::Schema.define(version: 2019_09_04_173203) do
   add_foreign_key "ci_pipeline_variables", "ci_pipelines", column: "pipeline_id", name: "fk_f29c5f4380", on_delete: :cascade
   add_foreign_key "ci_pipelines", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_3d34ab2e06", on_delete: :nullify
   add_foreign_key "ci_pipelines", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_262d4c2d19", on_delete: :nullify
+  add_foreign_key "ci_pipelines", "external_pull_requests", name: "fk_190998ef09", on_delete: :nullify
   add_foreign_key "ci_pipelines", "merge_requests", name: "fk_a23be95014", on_delete: :cascade
   add_foreign_key "ci_pipelines", "projects", name: "fk_86635dbd80", on_delete: :cascade
   add_foreign_key "ci_runner_namespaces", "ci_runners", column: "runner_id", on_delete: :cascade
@@ -3845,6 +3863,7 @@ ActiveRecord::Schema.define(version: 2019_09_04_173203) do
   add_foreign_key "events", "namespaces", column: "group_id", on_delete: :cascade
   add_foreign_key "events", "projects", on_delete: :cascade
   add_foreign_key "events", "users", column: "author_id", name: "fk_edfd187b6f", on_delete: :cascade
+  add_foreign_key "external_pull_requests", "projects", on_delete: :cascade
   add_foreign_key "fork_network_members", "fork_networks", on_delete: :cascade
   add_foreign_key "fork_network_members", "projects", column: "forked_from_project_id", name: "fk_b01280dae4", on_delete: :nullify
   add_foreign_key "fork_network_members", "projects", on_delete: :cascade