Merge branch 'refactor-clusters' into 38464-k8s-apps

6 files changed, 266 insertions, 12 deletions

diff --git a/db/migrate/20171012101043_add_circuit_breaker_properties_to_application_settings.rb b/db/migrate/20171012101043_add_circuit_breaker_properties_to_application_settings.rb
new file mode 100644
index 00000000000..bcf7dbd8e64
--- /dev/null
+++ b/db/migrate/20171012101043_add_circuit_breaker_properties_to_application_settings.rb
@@ -0,0 +1,27 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddCircuitBreakerPropertiesToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :application_settings,
+               :circuitbreaker_failure_count_threshold,
+               :integer,
+               default: 160
+    add_column :application_settings,
+               :circuitbreaker_failure_wait_time,
+               :integer,
+               default: 30
+    add_column :application_settings,
+               :circuitbreaker_failure_reset_time,
+               :integer,
+               default: 1800
+    add_column :application_settings,
+               :circuitbreaker_storage_timeout,
+               :integer,
+               default: 30
+  end
+end
diff --git a/db/migrate/20171013094327_create_new_clusters_architectures.rb b/db/migrate/20171013094327_create_new_clusters_architectures.rb
new file mode 100644
index 00000000000..b196aa1949c
--- /dev/null
+++ b/db/migrate/20171013094327_create_new_clusters_architectures.rb
@@ -0,0 +1,68 @@
+class CreateNewClustersArchitectures < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    create_table :clusters do |t|
+      t.references :user, null: false, index: true, foreign_key: { on_delete: :nullify }
+
+      t.integer :provider_type
+      t.integer :platform_type
+
+      t.datetime_with_timezone :created_at, null: false
+      t.datetime_with_timezone :updated_at, null: false
+
+      t.boolean :enabled, index: true, default: true
+
+      t.string :name, null: false # If manual, read-write. If gcp, read-only.
+    end
+
+    create_table :cluster_projects do |t|
+      t.references :project, null: false, index: true, foreign_key: { on_delete: :cascade }
+      t.references :cluster, null: false, index: true, foreign_key: { on_delete: :cascade }
+
+      t.datetime_with_timezone :created_at, null: false
+      t.datetime_with_timezone :updated_at, null: false
+    end
+
+    create_table :cluster_platforms_kubernetes do |t|
+      t.references :cluster, null: false, index: { unique: true }, foreign_key: { on_delete: :cascade }
+
+      t.datetime_with_timezone :created_at, null: false
+      t.datetime_with_timezone :updated_at, null: false
+
+      t.text :api_url
+      t.text :ca_cert
+
+      t.string :namespace
+
+      t.string :username
+      t.text :encrypted_password
+      t.string :encrypted_password_iv
+
+      t.text :encrypted_token
+      t.string :encrypted_token_iv
+    end
+
+    create_table :cluster_providers_gcp do |t|
+      t.references :cluster, null: false, index: { unique: true }, foreign_key: { on_delete: :cascade }
+
+      t.integer :status
+      t.integer :num_nodes, null: false
+
+      t.datetime_with_timezone :created_at, null: false
+      t.datetime_with_timezone :updated_at, null: false
+
+      t.text :status_reason
+
+      t.string :gcp_project_id, null: false
+      t.string :zone, null: false
+      t.string :machine_type
+      t.string :operation_id
+
+      t.string :endpoint
+
+      t.text :encrypted_access_token
+      t.string :encrypted_access_token_iv
+    end
+  end
+end
diff --git a/db/migrate/20171013104327_migrate_gcp_clusters_to_new_clusters_architectures.rb b/db/migrate/20171013104327_migrate_gcp_clusters_to_new_clusters_architectures.rb
new file mode 100644
index 00000000000..6ff98899bcb
--- /dev/null
+++ b/db/migrate/20171013104327_migrate_gcp_clusters_to_new_clusters_architectures.rb
@@ -0,0 +1,86 @@
+class MigrateGcpClustersToNewClustersArchitectures < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def up
+    gcp_clusters = ActiveRecord::Base.connection.select_all('SELECT * from gcp_clusters;')
+
+    rows_for_clusters = Array.new
+    rows_for_cluster_projects = Array.new
+    rows_for_cluster_providers_gcp = Array.new
+    rows_for_cluster_platforms_kubernetes = Array.new
+
+    gcp_clusters.each do |gcp_cluster|
+      rows_for_clusters << params_for_clusters(gcp_cluster)
+      rows_for_cluster_projects << params_for_cluster_projects(gcp_cluster)
+      rows_for_cluster_providers_gcp << params_for_cluster_providers_gcp(gcp_cluster)
+      rows_for_cluster_platforms_kubernetes << params_for_cluster_platforms_kubernetes(gcp_cluster)
+    end
+
+    Gitlab::Database.bulk_insert('clusters', rows_for_clusters)
+    Gitlab::Database.bulk_insert('cluster_projects', rows_for_cluster_projects)
+    Gitlab::Database.bulk_insert('cluster_providers_gcp', rows_for_cluster_providers_gcp)
+    Gitlab::Database.bulk_insert('cluster_platforms_kubernetes', rows_for_cluster_platforms_kubernetes)
+  end
+
+  def down
+    execute('DELETE FROM clusters')
+  end
+
+  private
+
+  def params_for_clusters(gcp_cluster)
+    {
+      id: gcp_cluster['id'],
+      user_id: gcp_cluster['user_id'],
+      enabled: gcp_cluster['enabled'],
+      name: gcp_cluster['gcp_cluster_name'],
+      provider_type: Clusters::Cluster.provider_types[:gcp],
+      platform_type: Clusters::Cluster.platform_types[:kubernetes],
+      created_at: gcp_cluster['created_at'],
+      updated_at: gcp_cluster['updated_at']
+    }
+  end
+
+  def params_for_cluster_projects(gcp_cluster)
+    {
+      cluster_id: gcp_cluster['id'],
+      project_id: gcp_cluster['project_id'],
+      created_at: gcp_cluster['created_at'],
+      updated_at: gcp_cluster['updated_at']
+    }
+  end
+
+  def params_for_cluster_providers_gcp(gcp_cluster)
+    {
+      cluster_id: gcp_cluster['id'],
+      status: gcp_cluster['status'],
+      status_reason: gcp_cluster['status_reason'],
+      gcp_project_id: gcp_cluster['gcp_project_id'],
+      zone: gcp_cluster['gcp_cluster_zone'],
+      num_nodes: gcp_cluster['gcp_cluster_size'],
+      machine_type: gcp_cluster['gcp_machine_type'],
+      operation_id: gcp_cluster['gcp_operation_id'],
+      endpoint: gcp_cluster['endpoint'],
+      encrypted_access_token: gcp_cluster['encrypted_gcp_token'],
+      encrypted_access_token_iv: gcp_cluster['encrypted_gcp_token_iv'],
+      created_at: gcp_cluster['created_at'],
+      updated_at: gcp_cluster['updated_at']
+    }
+  end
+
+  def params_for_cluster_platforms_kubernetes(gcp_cluster)
+    {
+      cluster_id: gcp_cluster['id'],
+      api_url: 'https://' + gcp_cluster['endpoint'],
+      ca_cert: gcp_cluster['ca_cert'],
+      namespace: gcp_cluster['project_namespace'],
+      username: gcp_cluster['username'],
+      encrypted_password: gcp_cluster['encrypted_password'],
+      encrypted_password_iv: gcp_cluster['encrypted_password_iv'],
+      encrypted_token: gcp_cluster['encrypted_kubernetes_token'],
+      encrypted_token_iv: gcp_cluster['encrypted_kubernetes_token_iv'],
+      created_at: gcp_cluster['created_at'],
+      updated_at: gcp_cluster['updated_at']
+    }
+  end
+end
diff --git a/db/migrate/20171017145932_add_new_circuitbreaker_settings_to_application_settings.rb b/db/migrate/20171017145932_add_new_circuitbreaker_settings_to_application_settings.rb
new file mode 100644
index 00000000000..07eb25c0b0f
--- /dev/null
+++ b/db/migrate/20171017145932_add_new_circuitbreaker_settings_to_application_settings.rb
@@ -0,0 +1,16 @@
+class AddNewCircuitbreakerSettingsToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :application_settings,
+               :circuitbreaker_access_retries,
+               :integer,
+               default: 3
+    add_column :application_settings,
+               :circuitbreaker_backoff_threshold,
+               :integer,
+               default: 80
+  end
+end
diff --git a/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb b/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb
index 9441b236c8d..2125cc046e5 100644
--- a/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb
+++ b/db/post_migrate/20170525140254_rename_all_reserved_paths_again.rb
@@ -13,7 +13,6 @@ class RenameAllReservedPathsAgain < ActiveRecord::Migration
       .well-known
       abuse_reports
       admin
-      all
       api
       assets
       autocomplete
@@ -24,29 +23,20 @@ class RenameAllReservedPathsAgain < ActiveRecord::Migration
       groups
       health_check
       help
-      hooks
       import
       invites
-      issues
       jwt
       koding
-      member
-      merge_requests
-      new
-      notes
       notification_settings
       oauth
       profile
       projects
       public
-      repository
       robots.txt
       s
       search
       sent_notifications
-      services
       snippets
-      teams
       u
       unicorn_test
       unsubscribes
@@ -94,7 +84,6 @@ class RenameAllReservedPathsAgain < ActiveRecord::Migration
       notification_setting
       pipeline_quota
       projects
-      subgroups
   ].freeze
 
   def up
diff --git a/db/schema.rb b/db/schema.rb
index 8aadcfeb7d1..d76977d45f2 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20171006091000) do
+ActiveRecord::Schema.define(version: 20171017145932) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -134,6 +134,12 @@ ActiveRecord::Schema.define(version: 20171006091000) do
     t.boolean "hashed_storage_enabled", default: false, null: false
     t.boolean "project_export_enabled", default: true, null: false
     t.boolean "auto_devops_enabled", default: false, null: false
+    t.integer "circuitbreaker_failure_count_threshold", default: 160
+    t.integer "circuitbreaker_failure_wait_time", default: 30
+    t.integer "circuitbreaker_failure_reset_time", default: 1800
+    t.integer "circuitbreaker_storage_timeout", default: 30
+    t.integer "circuitbreaker_access_retries", default: 3
+    t.integer "circuitbreaker_backoff_threshold", default: 80
   end
 
   create_table "audit_events", force: :cascade do |t|
@@ -456,6 +462,63 @@ ActiveRecord::Schema.define(version: 20171006091000) do
 
   add_index "ci_variables", ["project_id", "key", "environment_scope"], name: "index_ci_variables_on_project_id_and_key_and_environment_scope", unique: true, using: :btree
 
+  create_table "cluster_platforms_kubernetes", force: :cascade do |t|
+    t.integer "cluster_id", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.text "api_url"
+    t.text "ca_cert"
+    t.string "namespace"
+    t.string "username"
+    t.text "encrypted_password"
+    t.string "encrypted_password_iv"
+    t.text "encrypted_token"
+    t.string "encrypted_token_iv"
+  end
+
+  add_index "cluster_platforms_kubernetes", ["cluster_id"], name: "index_cluster_platforms_kubernetes_on_cluster_id", unique: true, using: :btree
+
+  create_table "cluster_projects", force: :cascade do |t|
+    t.integer "project_id", null: false
+    t.integer "cluster_id", null: false
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+  end
+
+  add_index "cluster_projects", ["cluster_id"], name: "index_cluster_projects_on_cluster_id", using: :btree
+  add_index "cluster_projects", ["project_id"], name: "index_cluster_projects_on_project_id", using: :btree
+
+  create_table "cluster_providers_gcp", force: :cascade do |t|
+    t.integer "cluster_id", null: false
+    t.integer "status"
+    t.integer "num_nodes", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.text "status_reason"
+    t.string "gcp_project_id", null: false
+    t.string "zone", null: false
+    t.string "machine_type"
+    t.string "operation_id"
+    t.string "endpoint"
+    t.text "encrypted_access_token"
+    t.string "encrypted_access_token_iv"
+  end
+
+  add_index "cluster_providers_gcp", ["cluster_id"], name: "index_cluster_providers_gcp_on_cluster_id", unique: true, using: :btree
+
+  create_table "clusters", force: :cascade do |t|
+    t.integer "user_id", null: false
+    t.integer "provider_type"
+    t.integer "platform_type"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.boolean "enabled", default: true
+    t.string "name", null: false
+  end
+
+  add_index "clusters", ["enabled"], name: "index_clusters_on_enabled", using: :btree
+  add_index "clusters", ["user_id"], name: "index_clusters_on_user_id", using: :btree
+
   create_table "container_repositories", force: :cascade do |t|
     t.integer "project_id", null: false
     t.string "name", null: false
@@ -1804,6 +1867,11 @@ ActiveRecord::Schema.define(version: 20171006091000) do
   add_foreign_key "ci_triggers", "projects", name: "fk_e3e63f966e", on_delete: :cascade
   add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade
   add_foreign_key "ci_variables", "projects", name: "fk_ada5eb64b3", on_delete: :cascade
+  add_foreign_key "cluster_platforms_kubernetes", "clusters", on_delete: :cascade
+  add_foreign_key "cluster_projects", "clusters", on_delete: :cascade
+  add_foreign_key "cluster_projects", "projects", on_delete: :cascade
+  add_foreign_key "cluster_providers_gcp", "clusters", on_delete: :cascade
+  add_foreign_key "clusters", "users", on_delete: :nullify
   add_foreign_key "container_repositories", "projects"
   add_foreign_key "deploy_keys_projects", "projects", name: "fk_58a901ca7e", on_delete: :cascade
   add_foreign_key "deployments", "projects", name: "fk_b9a3851b82", on_delete: :cascade