Add FilkeIterator

7 files changed, 146 insertions, 12 deletions

diff --git a/lib/gitlab/ci/trace/file_iterator.rb b/lib/gitlab/ci/trace/file_iterator.rb
new file mode 100644
index 00000000000..c01cd2c724f
--- /dev/null
+++ b/lib/gitlab/ci/trace/file_iterator.rb
@@ -0,0 +1,53 @@
+module Gitlab
+  module Ci
+    class Trace
+      class FileIterator
+        attr_reader :relative_path
+
+        def initialize(relative_path)
+          @relative_path = relative_path
+        end
+
+        def trace_files
+          Dir.chdir(Settings.gitlab_ci.builds_path) do
+            unless Dir.exist?(relative_path)
+              return yield relative_path
+            end
+
+            recursive(relative_path) do |path|
+              yield sanitized_path(path)
+            end
+          end
+        end
+
+        private
+
+        def recursive(pos, &block)
+          Dir.entries(pos).each do |entry|
+            if yyyy_mm?(entry) || project_id?(entry)
+              recursive(File.join(pos, entry), &block)
+            elsif trace_file?(entry)
+              yield File.join(pos, entry)
+            end
+          end
+        end
+
+        def yyyy_mm?(entry)
+          /^\d{4}_\d{2}$/ =~ entry
+        end
+
+        def project_id?(entry)
+          /^\d{1,}$/ =~ entry
+        end
+
+        def trace_file?(entry)
+          /^\d{1,}.log$/ =~ entry
+        end
+
+        def sanitized_path(path)
+          path.sub(/^\./, '').sub(%/^\//, '')
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/trace/migrator.rb b/lib/gitlab/ci/trace/migrator.rb
index 49af53c7da7..9af6f0081fb 100644
--- a/lib/gitlab/ci/trace/migrator.rb
+++ b/lib/gitlab/ci/trace/migrator.rb
@@ -14,7 +14,7 @@ module Gitlab
         end
 
         def perform
-          raise ArgumentError, "Trace file not found" unless File.exists?(path)
+          raise ArgumentError, "Trace file not found" unless File.exist?(path)
           raise ArgumentError, "Invalid trace path format" unless trace_path?
 
           backup!
@@ -24,7 +24,7 @@ module Gitlab
         private
 
         def trace_path?
-          /#{Settings.gitlab_ci.builds_path}\/\d{4}_\d{2}\/\d{1,}\/\d{1,}.log/ =~ path
+          %r{#{Settings.gitlab_ci.builds_path}\/\d{4}_\d{2}\/\d{1,}\/\d{1,}.log} =~ path
         end
 
         def status
diff --git a/lib/tasks/gitlab/traces.rake b/lib/tasks/gitlab/traces.rake
index ff1c1885c1c..6c5a86e4ad3 100644
--- a/lib/tasks/gitlab/traces.rake
+++ b/lib/tasks/gitlab/traces.rake
@@ -8,9 +8,11 @@ namespace :gitlab do
       logger = Logger.new(STDOUT)
       logger.info('Starting migration for trace files')
 
-      logger.info("args.relative_path: #{args.relative_path}")
+      Gitlab::Ci::Trace::FileIterator.new(args.relative_path).trace_files do |trace_path|
+        logger.info("Migrating... #{trace_path}")
 
-      Gitlab::Ci::Trace::Migrater.new(args.relative_path).perform
+        Gitlab::Ci::Trace::Migrator.new(trace_path).perform
+      end
     end
   end
 end
diff --git a/spec/lib/gitlab/ci/trace/file_iterator_spec.rb b/spec/lib/gitlab/ci/trace/file_iterator_spec.rb
new file mode 100644
index 00000000000..2f05645526d
--- /dev/null
+++ b/spec/lib/gitlab/ci/trace/file_iterator_spec.rb
@@ -0,0 +1,57 @@
+require 'spec_helper'
+
+describe Gitlab::Ci::Trace::FileIterator do
+  include TraceHelpers
+
+  describe '#trace_files' do
+    let(:builds_path) { Rails.root.join('tmp/tests/builds').to_s }
+
+    before do
+      create_trace_file(builds_path, '2018_02', '19', 1, '')
+      create_trace_file(builds_path, '2018_02', '19', 2, '')
+      create_trace_file(builds_path, '2018_02', '19', 3, '')
+      create_trace_file(builds_path, '2018_02', '19', 4, '')
+      create_trace_file(builds_path, '2018_02', '22', 5, '')
+      create_trace_file(builds_path, '2018_02', '22', 6, '')
+      create_trace_file(builds_path, '2018_03', '2', 7, '')
+      create_trace_file(builds_path, '2018_03', '2', 8, '')
+      create_trace_file(builds_path, '2018_03', '3', 9, '')
+    end
+
+    context 'when relative path points root' do
+      let(:relative_path) { '.' }
+
+      it 'iterates' do
+        expect { |b| described_class.new(relative_path).trace_files(&b) }
+          .to yield_control.exactly(9).times
+      end
+    end
+
+    context 'when relative path points yyyy_mm' do
+      let(:relative_path) { '2018_02' }
+
+      it 'iterates' do
+        expect { |b| described_class.new(relative_path).trace_files(&b) }
+          .to yield_control.exactly(6).times
+      end
+    end
+
+    context 'when relative path points project_id' do
+      let(:relative_path) { '2018_03/2' }
+
+      it 'iterates' do
+        expect { |b| described_class.new(relative_path).trace_files(&b) }
+          .to yield_control.exactly(2).times
+      end
+    end
+
+    context 'when relative path points trace file' do
+      let(:relative_path) { '2018_03/3/9.log' }
+
+      it 'iterates' do
+        expect { |b| described_class.new(relative_path).trace_files(&b) }
+          .to yield_control.exactly(1).times
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/ci/trace/migrator_spec.rb b/spec/lib/gitlab/ci/trace/migrator_spec.rb
index 0358a276bf1..224d7cfd80d 100644
--- a/spec/lib/gitlab/ci/trace/migrator_spec.rb
+++ b/spec/lib/gitlab/ci/trace/migrator_spec.rb
@@ -60,8 +60,8 @@ describe Gitlab::Ci::Trace::Migrator do
             create_trace_file(builds_path, yyyy_mm, project_id, job_id, trace_content) do |path|
               described_class.new(path.remove(builds_path)).perform
 
-              expect(File.exists?(path)).to be_falsy
-              expect(File.exists?(simulate_backup_path(path, :not_found))).to be_truthy
+              expect(File.exist?(path)).to be_falsy
+              expect(File.exist?(simulate_backup_path(path, :not_found))).to be_truthy
             end
           end
         end
@@ -78,8 +78,8 @@ describe Gitlab::Ci::Trace::Migrator do
           create_trace_file(builds_path, yyyy_mm, project_id, job_id, trace_content) do |path|
             described_class.new(path.remove(builds_path)).perform
 
-            expect(File.exists?(path)).to be_falsy
-            expect(File.exists?(simulate_backup_path(path, :not_found))).to be_truthy
+            expect(File.exist?(path)).to be_falsy
+            expect(File.exist?(simulate_backup_path(path, :not_found))).to be_truthy
           end
         end
       end
diff --git a/spec/support/trace/trace_helpers.rb b/spec/support/trace/trace_helpers.rb
index cfb941e6647..46ea5847c05 100644
--- a/spec/support/trace/trace_helpers.rb
+++ b/spec/support/trace/trace_helpers.rb
@@ -5,15 +5,15 @@ module TraceHelpers
 
     FileUtils.mkdir_p(trace_dir)
 
-    File.open(File.join(trace_dir, "#{job_id}.log"), 'w') do
-      |file| file.write(trace_content)
+    File.open(File.join(trace_dir, "#{job_id}.log"), 'w') do |file|
+      file.write(trace_content)
     end
 
-    yield trace_path
+    yield trace_path if block_given?
   end
 
   def artifacts_path?(path)
-    /.{2}\/.{2}\/.{64}\/\d{4}_\d{2}_\d{2}\/\d{1,}\/\d{1,}\/\d{1,}.log/ =~ path
+    %r{.{2}\/.{2}\/.{64}\/\d{4}_\d{2}_\d{2}\/\d{1,}\/\d{1,}\/\d{1,}.log} =~ path
   end
 
   def simulate_backup_path(path, status)
diff --git a/spec/tasks/gitlab/traces_rake_spec.rb b/spec/tasks/gitlab/traces_rake_spec.rb
new file mode 100644
index 00000000000..09096deb7c2
--- /dev/null
+++ b/spec/tasks/gitlab/traces_rake_spec.rb
@@ -0,0 +1,22 @@
+require 'rake_helper'
+
+describe 'gitlab:traces namespace rake task' do
+  before(:context) do
+    Rake.application.rake_require 'tasks/gitlab/traces'
+  end
+
+  subject { run_rake_task('gitlab:traces:migrate', relative_path) }
+
+  context 'when relative path points root' do
+    let(:relative_path) { '.' }
+
+    let!(:job1) { create(:ci_build, :trace_live, :success) }
+    let!(:job2) { create(:ci_build, :trace_artifact, :success) }
+
+    it 'migrates' do
+      expect { subject }.to change { Ci::JobArtifact.count }.by(1)
+
+      expect(job1.job_artifacts_trace).not_to be_nil
+    end
+  end
+end