Add latest changes from gitlab-org/gitlab@master

7 files changed, 181 insertions, 29 deletions

diff --git a/db/migrate/20200123090839_remove_analytics_repository_table_fks_on_projects.rb b/db/migrate/20200123090839_remove_analytics_repository_table_fks_on_projects.rb
new file mode 100644
index 00000000000..a591596c74c
--- /dev/null
+++ b/db/migrate/20200123090839_remove_analytics_repository_table_fks_on_projects.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class RemoveAnalyticsRepositoryTableFksOnProjects < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    with_lock_retries do
+      # Requires ExclusiveLock on all tables. analytics_* tables are empty
+      remove_foreign_key :analytics_repository_files, :projects
+      remove_foreign_key :analytics_repository_file_edits, :projects
+      remove_foreign_key :analytics_repository_file_commits, :projects
+    end
+  end
+
+  def down
+    with_lock_retries do
+      # rubocop:disable Migration/AddConcurrentForeignKey
+      add_foreign_key :analytics_repository_files, :projects, on_delete: :cascade
+      add_foreign_key :analytics_repository_file_edits, :projects, on_delete: :cascade
+      add_foreign_key :analytics_repository_file_commits, :projects, on_delete: :cascade
+      # rubocop:enable Migration/AddConcurrentForeignKey
+    end
+  end
+end
diff --git a/db/migrate/20200123091422_remove_analytics_repository_files_fk_on_other_analytics_tables.rb b/db/migrate/20200123091422_remove_analytics_repository_files_fk_on_other_analytics_tables.rb
new file mode 100644
index 00000000000..60de2e9cb7f
--- /dev/null
+++ b/db/migrate/20200123091422_remove_analytics_repository_files_fk_on_other_analytics_tables.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class RemoveAnalyticsRepositoryFilesFkOnOtherAnalyticsTables < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    with_lock_retries do
+      # Requires ExclusiveLock on all tables. analytics_* tables are empty
+      remove_foreign_key :analytics_repository_file_edits, :analytics_repository_files
+      remove_foreign_key :analytics_repository_file_commits, :analytics_repository_files
+    end
+  end
+
+  def down
+    with_lock_retries do
+      # rubocop:disable Migration/AddConcurrentForeignKey
+      add_foreign_key :analytics_repository_file_edits, :analytics_repository_files, on_delete: :cascade
+      add_foreign_key :analytics_repository_file_commits, :analytics_repository_files, on_delete: :cascade
+      # rubocop:enable Migration/AddConcurrentForeignKey
+    end
+  end
+end
diff --git a/db/migrate/20200123091622_drop_analytics_repository_files_table.rb b/db/migrate/20200123091622_drop_analytics_repository_files_table.rb
new file mode 100644
index 00000000000..aa31d23920a
--- /dev/null
+++ b/db/migrate/20200123091622_drop_analytics_repository_files_table.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class DropAnalyticsRepositoryFilesTable < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    # Requires ExclusiveLock on the table. Not in use, no records, no FKs.
+    drop_table :analytics_repository_files
+  end
+
+  def down
+    create_table :analytics_repository_files do |t|
+      t.bigint :project_id, null: false
+      t.string :file_path, limit: 4096, null: false
+    end
+
+    add_index :analytics_repository_files, [:project_id, :file_path], unique: true
+  end
+end
diff --git a/db/migrate/20200123091734_drop_analytics_repository_file_commits_table.rb b/db/migrate/20200123091734_drop_analytics_repository_file_commits_table.rb
new file mode 100644
index 00000000000..2d3c1c9a817
--- /dev/null
+++ b/db/migrate/20200123091734_drop_analytics_repository_file_commits_table.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class DropAnalyticsRepositoryFileCommitsTable < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    # Requires ExclusiveLock on the table. Not in use, no records, no FKs.
+    drop_table :analytics_repository_file_commits
+  end
+
+  def down
+    create_table :analytics_repository_file_commits do |t|
+      t.bigint :analytics_repository_file_id, null: false
+      t.index :analytics_repository_file_id, name: 'index_analytics_repository_file_commits_file_id'
+      t.bigint :project_id, null: false
+      t.date :committed_date, null: false
+      t.integer :commit_count, limit: 2, null: false
+    end
+
+    add_index :analytics_repository_file_commits,
+      [:project_id, :committed_date, :analytics_repository_file_id],
+      name: 'index_file_commits_on_committed_date_file_id_and_project_id',
+      unique: true
+  end
+end
diff --git a/db/migrate/20200123091854_drop_analytics_repository_file_edits_table.rb b/db/migrate/20200123091854_drop_analytics_repository_file_edits_table.rb
new file mode 100644
index 00000000000..3bb026727f4
--- /dev/null
+++ b/db/migrate/20200123091854_drop_analytics_repository_file_edits_table.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class DropAnalyticsRepositoryFileEditsTable < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    # Requires ExclusiveLock on the table. Not in use, no records, no FKs.
+    drop_table :analytics_repository_file_edits
+  end
+
+  def down
+    create_table :analytics_repository_file_edits do |t|
+      t.bigint :project_id, null: false
+      t.index :project_id
+      t.bigint :analytics_repository_file_id, null: false
+      t.date :committed_date, null: false
+      t.integer :num_edits, null: false, default: 0
+    end
+
+    add_index :analytics_repository_file_edits,
+      [:analytics_repository_file_id, :committed_date, :project_id],
+      name: 'index_file_edits_on_committed_date_file_id_and_project_id',
+      unique: true
+  end
+end
diff --git a/db/post_migrate/20200110121314_schedule_update_existing_subgroup_to_match_visibility_level_of_parent.rb b/db/post_migrate/20200110121314_schedule_update_existing_subgroup_to_match_visibility_level_of_parent.rb
new file mode 100644
index 00000000000..813cd600ddc
--- /dev/null
+++ b/db/post_migrate/20200110121314_schedule_update_existing_subgroup_to_match_visibility_level_of_parent.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class ScheduleUpdateExistingSubgroupToMatchVisibilityLevelOfParent < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  MIGRATION = 'UpdateExistingSubgroupToMatchVisibilityLevelOfParent'
+  DELAY_INTERVAL = 5.minutes.to_i
+  BATCH_SIZE = 1000
+  VISIBILITY_LEVELS = {
+    internal: 10,
+    private: 0
+  }
+
+  disable_ddl_transaction!
+
+  def up
+    offset = update_groups(VISIBILITY_LEVELS[:internal])
+    update_groups(VISIBILITY_LEVELS[:private], offset: offset)
+  end
+
+  def down
+    # no-op
+  end
+
+  private
+
+  def update_groups(level, offset: 0)
+    groups = exec_query <<~SQL
+      SELECT id
+      FROM namespaces
+      WHERE visibility_level = #{level}
+      AND type = 'Group'
+      AND EXISTS (SELECT 1
+                  FROM namespaces AS children
+                  WHERE children.parent_id = namespaces.id)
+    SQL
+
+    ids = groups.rows.flatten
+
+    iterator = 1
+
+    ids.in_groups_of(BATCH_SIZE, false) do |batch_of_ids|
+      delay = DELAY_INTERVAL * (iterator + offset)
+      BackgroundMigrationWorker.perform_in(delay, MIGRATION, [batch_of_ids, level])
+      iterator += 1
+    end
+
+    say("Background jobs for visibility level #{level} scheduled in #{iterator} iterations")
+
+    offset + iterator
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 80e7af66fb9..70e93a63148 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -94,30 +94,6 @@ ActiveRecord::Schema.define(version: 2020_01_27_090233) do
     t.index ["project_id"], name: "analytics_repository_languages_on_project_id"
   end
 
-  create_table "analytics_repository_file_commits", force: :cascade do |t|
-    t.bigint "analytics_repository_file_id", null: false
-    t.bigint "project_id", null: false
-    t.date "committed_date", null: false
-    t.integer "commit_count", limit: 2, null: false
-    t.index ["analytics_repository_file_id"], name: "index_analytics_repository_file_commits_file_id"
-    t.index ["project_id", "committed_date", "analytics_repository_file_id"], name: "index_file_commits_on_committed_date_file_id_and_project_id", unique: true
-  end
-
-  create_table "analytics_repository_file_edits", force: :cascade do |t|
-    t.bigint "project_id", null: false
-    t.bigint "analytics_repository_file_id", null: false
-    t.date "committed_date", null: false
-    t.integer "num_edits", default: 0, null: false
-    t.index ["analytics_repository_file_id", "committed_date", "project_id"], name: "index_file_edits_on_committed_date_file_id_and_project_id", unique: true
-    t.index ["project_id"], name: "index_analytics_repository_file_edits_on_project_id"
-  end
-
-  create_table "analytics_repository_files", force: :cascade do |t|
-    t.bigint "project_id", null: false
-    t.string "file_path", limit: 4096, null: false
-    t.index ["project_id", "file_path"], name: "index_analytics_repository_files_on_project_id_and_file_path", unique: true
-  end
-
   create_table "appearances", id: :serial, force: :cascade do |t|
     t.string "title", null: false
     t.text "description", null: false
@@ -4476,11 +4452,6 @@ ActiveRecord::Schema.define(version: 2020_01_27_090233) do
   add_foreign_key "analytics_cycle_analytics_project_stages", "projects", on_delete: :cascade
   add_foreign_key "analytics_language_trend_repository_languages", "programming_languages", on_delete: :cascade
   add_foreign_key "analytics_language_trend_repository_languages", "projects", on_delete: :cascade
-  add_foreign_key "analytics_repository_file_commits", "analytics_repository_files", on_delete: :cascade
-  add_foreign_key "analytics_repository_file_commits", "projects", on_delete: :cascade
-  add_foreign_key "analytics_repository_file_edits", "analytics_repository_files", on_delete: :cascade
-  add_foreign_key "analytics_repository_file_edits", "projects", on_delete: :cascade
-  add_foreign_key "analytics_repository_files", "projects", on_delete: :cascade
   add_foreign_key "application_settings", "namespaces", column: "custom_project_templates_group_id", on_delete: :nullify
   add_foreign_key "application_settings", "namespaces", column: "instance_administrators_group_id", name: "fk_e8a145f3a7", on_delete: :nullify
   add_foreign_key "application_settings", "projects", column: "file_template_project_id", name: "fk_ec757bd087", on_delete: :nullify