Merge branch 'add-background-migration-for-legacy-traces' into 'master'

Migrate job artifacts data from `ci_builds` to `ci_job_artifacts` table (with Background migrations)

Closes #46652

See merge request gitlab-org/gitlab-ce!18615

4 files changed, 59 insertions, 0 deletions

diff --git a/db/migrate/20180815160409_add_file_location_to_ci_job_artifacts.rb b/db/migrate/20180815160409_add_file_location_to_ci_job_artifacts.rb
new file mode 100644
index 00000000000..620342005fe
--- /dev/null
+++ b/db/migrate/20180815160409_add_file_location_to_ci_job_artifacts.rb
@@ -0,0 +1,9 @@
+class AddFileLocationToCiJobArtifacts < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :ci_job_artifacts, :file_location, :integer, limit: 2
+  end
+end
diff --git a/db/migrate/20180815170510_add_partial_index_to_ci_builds_artifacts_file.rb b/db/migrate/20180815170510_add_partial_index_to_ci_builds_artifacts_file.rb
new file mode 100644
index 00000000000..5e041ea6559
--- /dev/null
+++ b/db/migrate/20180815170510_add_partial_index_to_ci_builds_artifacts_file.rb
@@ -0,0 +1,16 @@
+class AddPartialIndexToCiBuildsArtifactsFile < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX_NAME = 'partial_index_ci_builds_on_id_with_legacy_artifacts'.freeze
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(:ci_builds, :id, where: "artifacts_file <> ''", name: INDEX_NAME)
+  end
+
+  def down
+    remove_concurrent_index_by_name(:ci_builds, INDEX_NAME)
+  end
+end
diff --git a/db/post_migrate/20180816161409_migrate_legacy_artifacts_to_job_artifacts.rb b/db/post_migrate/20180816161409_migrate_legacy_artifacts_to_job_artifacts.rb
new file mode 100644
index 00000000000..2dd711e9c10
--- /dev/null
+++ b/db/post_migrate/20180816161409_migrate_legacy_artifacts_to_job_artifacts.rb
@@ -0,0 +1,32 @@
+class MigrateLegacyArtifactsToJobArtifacts < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  MIGRATION = 'MigrateLegacyArtifacts'.freeze
+  BATCH_SIZE = 100
+
+  disable_ddl_transaction!
+
+  class Build < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'ci_builds'
+    self.inheritance_column = :_type_disabled
+
+    scope :with_legacy_artifacts, -> { where("artifacts_file <> ''") }
+  end
+
+  def up
+    MigrateLegacyArtifactsToJobArtifacts::Build
+      .with_legacy_artifacts.tap do |relation|
+      queue_background_migration_jobs_by_range_at_intervals(relation,
+                                                            MIGRATION,
+                                                            5.minutes,
+                                                            batch_size: BATCH_SIZE)
+    end
+  end
+
+  def down
+    # no-op
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 02e545bec7d..56c7265119d 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -342,6 +342,7 @@ ActiveRecord::Schema.define(version: 20180826111825) do
   add_index "ci_builds", ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree
   add_index "ci_builds", ["commit_id", "type", "name", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_name_and_ref", using: :btree
   add_index "ci_builds", ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree
+  add_index "ci_builds", ["id"], name: "partial_index_ci_builds_on_id_with_legacy_artifacts", where: "(artifacts_file <> ''::text)", using: :btree
   add_index "ci_builds", ["project_id", "id"], name: "index_ci_builds_on_project_id_and_id", using: :btree
   add_index "ci_builds", ["protected"], name: "index_ci_builds_on_protected", using: :btree
   add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree
@@ -397,6 +398,7 @@ ActiveRecord::Schema.define(version: 20180826111825) do
     t.string "file"
     t.binary "file_sha256"
     t.integer "file_format", limit: 2
+    t.integer "file_location", limit: 2
   end
 
   add_index "ci_job_artifacts", ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id", using: :btree