Add latest changes from gitlab-org/gitlab@master

5 files changed, 327 insertions, 56 deletions

diff --git a/spec/lib/gitlab/git/changed_path_spec.rb b/spec/lib/gitlab/git/changed_path_spec.rb
index ef51021ba4c..50cdddc9e8a 100644
--- a/spec/lib/gitlab/git/changed_path_spec.rb
+++ b/spec/lib/gitlab/git/changed_path_spec.rb
@@ -3,9 +3,12 @@
 require 'fast_spec_helper'
 
 RSpec.describe Gitlab::Git::ChangedPath do
-  subject(:changed_path) { described_class.new(path: path, status: status) }
+  subject(:changed_path) { described_class.new(path: path, status: status, old_mode: old_mode, new_mode: new_mode) }
 
   let(:path) { 'test_path' }
+  let(:status) { :MODIFIED }
+  let(:old_mode) { '100644' }
+  let(:new_mode) { '100644' }
 
   describe '#new_file?' do
     subject(:new_file?) { changed_path.new_file? }
@@ -19,11 +22,33 @@ RSpec.describe Gitlab::Git::ChangedPath do
     end
 
     context 'when it is not a new file' do
-      let(:status) { :MODIFIED }
-
       it 'returns false' do
         expect(new_file?).to eq(false)
       end
     end
   end
+
+  describe '#submodule_change?' do
+    subject(:submodule_change?) { changed_path.submodule_change? }
+
+    context 'with a regular file change' do
+      it { is_expected.to eq false }
+    end
+
+    context 'with a submodule addition' do
+      let(:status) { :ADDED }
+      let(:old_mode) { '0' }
+      let(:new_mode) { '160000' }
+
+      it { is_expected.to eq true }
+    end
+
+    context 'with a submodule deletion' do
+      let(:status) { :MODIFIED }
+      let(:old_mode) { '160000' }
+      let(:new_mode) { '0' }
+
+      it { is_expected.to eq true }
+    end
+  end
 end
diff --git a/spec/lib/gitlab/git/compare_spec.rb b/spec/lib/gitlab/git/compare_spec.rb
index 5ee5e18d5af..521d4588d84 100644
--- a/spec/lib/gitlab/git/compare_spec.rb
+++ b/spec/lib/gitlab/git/compare_spec.rb
@@ -116,22 +116,20 @@ RSpec.describe Gitlab::Git::Compare, feature_category: :source_code_management d
   describe '#generated_files' do
     subject(:generated_files) { compare.generated_files }
 
-    context 'with a detected generated file' do
-      let_it_be(:project) { create(:project, :repository) }
-      let_it_be(:repository) { project.repository.raw }
-      let_it_be(:branch) { 'generated-file-test' }
-      let_it_be(:base) do
-        project
-          .repository
-          .create_file(
-            project.creator,
-            '.gitattributes',
-            "*.txt gitlab-generated\n",
-            branch_name: branch,
-            message: 'Add .gitattributes file')
-      end
+    let(:project) do
+      create(:project, :custom_repo, files: {
+        '.gitattributes' => '*.txt gitlab-generated'
+      })
+    end
+
+    let(:repository) { project.repository.raw }
+    let(:branch) { 'generated-file-test' }
+    let(:base) { project.default_branch }
+    let(:head) { branch }
 
-      let_it_be(:head) do
+    context 'with a detected generated file' do
+      before do
+        project.repository.create_branch(branch, project.default_branch)
         project
           .repository
           .create_file(
@@ -150,7 +148,7 @@ RSpec.describe Gitlab::Git::Compare, feature_category: :source_code_management d
             message: 'Add file2')
       end
 
-      it 'sets the diff as generated' do
+      it 'returns a set that incldues the generated file' do
         expect(generated_files).to eq Set.new(['file1.txt'])
       end
 
@@ -175,19 +173,16 @@ RSpec.describe Gitlab::Git::Compare, feature_category: :source_code_management d
       end
     end
 
-    context 'with updated .gitattributes in the HEAD' do
-      let_it_be(:project) { create(:project, :repository) }
-      let_it_be(:repository) { project.repository.raw }
-      let_it_be(:branch) { 'generated-file-test' }
-      let_it_be(:head) do
+    context 'with deleted .gitattributes in the HEAD' do
+      before do
+        project.repository.create_branch(branch, project.default_branch)
         project
           .repository
-          .create_file(
+          .delete_file(
             project.creator,
             '.gitattributes',
-            "*.txt gitlab-generated\n",
             branch_name: branch,
-            message: 'Add .gitattributes file')
+            message: 'Delete .gitattributes file')
         project
           .repository
           .create_file(
@@ -206,8 +201,8 @@ RSpec.describe Gitlab::Git::Compare, feature_category: :source_code_management d
             message: 'Add file2')
       end
 
-      it 'does not set any files as generated' do
-        expect(generated_files).to eq Set.new
+      it 'ignores the .gitattributes changes in the HEAD' do
+        expect(generated_files).to eq Set.new(['file1.txt'])
       end
     end
   end
diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb
index ca3c058968a..7c6a54161ae 100644
--- a/spec/lib/gitlab/git/repository_spec.rb
+++ b/spec/lib/gitlab/git/repository_spec.rb
@@ -1702,25 +1702,25 @@ RSpec.describe Gitlab::Git::Repository, feature_category: :source_code_managemen
     let_it_be(:diff_tree) { Gitlab::Git::DiffTree.from_commit(initial_commit) }
 
     let(:commit_1_files) do
-      [Gitlab::Git::ChangedPath.new(status: :ADDED, path: "files/executables/ls")]
+      [Gitlab::Git::ChangedPath.new(status: :ADDED, path: "files/executables/ls", old_mode: "0", new_mode: "100755")]
     end
 
     let(:commit_2_files) do
-      [Gitlab::Git::ChangedPath.new(status: :ADDED, path: "bar/branch-test.txt")]
+      [Gitlab::Git::ChangedPath.new(status: :ADDED, path: "bar/branch-test.txt", old_mode: "0", new_mode: "100644")]
     end
 
     let(:commit_3_files) do
       [
-        Gitlab::Git::ChangedPath.new(status: :MODIFIED, path: ".gitmodules"),
-        Gitlab::Git::ChangedPath.new(status: :ADDED, path: "gitlab-shell")
+        Gitlab::Git::ChangedPath.new(status: :MODIFIED, path: ".gitmodules", old_mode: "100644", new_mode: "100644"),
+        Gitlab::Git::ChangedPath.new(status: :ADDED, path: "gitlab-shell", old_mode: "0", new_mode: "160000")
       ]
     end
 
     let(:diff_tree_files) do
       [
-        Gitlab::Git::ChangedPath.new(status: :ADDED, path: ".gitignore"),
-        Gitlab::Git::ChangedPath.new(status: :ADDED, path: "LICENSE"),
-        Gitlab::Git::ChangedPath.new(status: :ADDED, path: "README.md")
+        Gitlab::Git::ChangedPath.new(status: :ADDED, path: ".gitignore", old_mode: "0", new_mode: "100644"),
+        Gitlab::Git::ChangedPath.new(status: :ADDED, path: "LICENSE", old_mode: "0", new_mode: "100644"),
+        Gitlab::Git::ChangedPath.new(status: :ADDED, path: "README.md", old_mode: "0", new_mode: "100644")
       ]
     end
 
@@ -2822,11 +2822,37 @@ RSpec.describe Gitlab::Git::Repository, feature_category: :source_code_managemen
       })
     end
 
+    let(:gitattr_content) { "" }
+
     let(:repository) { project.repository.raw }
-    let(:rev) { 'master' }
-    let(:paths) { ['file1.txt', 'file2.txt'] }
+    let(:base) { project.default_branch }
+    let(:branch) { 'detect-generated-files-test' }
+    let(:head) { branch }
+    let(:paths) do
+      [
+        Gitlab::Git::ChangedPath.new(status: :MODIFIED, path: 'file1.txt', old_mode: '100644', new_mode: '100644'),
+        Gitlab::Git::ChangedPath.new(status: :DELETED, path: 'file2.txt', old_mode: '100644', new_mode: '8')
+      ]
+    end
+
+    before do
+      project.repository.create_branch(branch, project.default_branch)
 
-    subject(:generated_files) { repository.detect_generated_files(rev, paths) }
+      project.repository.update_file(
+        project.creator,
+        'file1.txt',
+        'updated first file',
+        message: 'Update file',
+        branch_name: branch)
+
+      project.repository.delete_file(
+        project.creator,
+        'file2.txt',
+        message: 'Delete file',
+        branch_name: branch)
+    end
+
+    subject(:generated_files) { repository.detect_generated_files(base, head, paths) }
 
     context 'when the linguist-generated attribute is used' do
       let(:gitattr_content) { "*.txt text\nfile1.txt linguist-generated\n" }
@@ -2852,11 +2878,99 @@ RSpec.describe Gitlab::Git::Repository, feature_category: :source_code_managemen
       end
     end
 
-    context 'when the all files are generated' do
+    context 'when the gitlab-generated attribute is used to unset' do
+      let(:gitattr_content) { "file1.txt -gitlab-generated\n" }
+
+      it 'returns an empty set' do
+        expect(generated_files).to eq Set.new
+      end
+    end
+
+    context 'with an automatically detected file' do
+      before do
+        project.repository.create_file(
+          project.creator,
+          'package-lock.json',
+          'generated file content',
+          message: 'Add generated file',
+          branch_name: branch)
+      end
+
+      let(:paths) do
+        [
+          Gitlab::Git::ChangedPath.new(status: :MODIFIED, path: 'file1.txt', old_mode: '100644', new_mode: '100644'),
+          Gitlab::Git::ChangedPath.new(status: :DELETED, path: 'file2.txt', old_mode: '100644', new_mode: '100644'),
+          Gitlab::Git::ChangedPath.new(status: :ADDED, path: 'package-lock.json', old_mode: '0', new_mode: '100644')
+        ]
+      end
+
+      context 'when the manual override is used on non-detectable file' do
+        let(:gitattr_content) { "file1.txt gitlab-generated\n" }
+
+        it 'returns both manually overriden file and the detected file' do
+          expect(generated_files).to contain_exactly('file1.txt', 'package-lock.json')
+        end
+      end
+
+      context 'when the manual override is used on the detectable file' do
+        let(:gitattr_content) { "package-lock.json gitlab-generated\n" }
+
+        it 'returns the overriden file' do
+          expect(generated_files).to contain_exactly('package-lock.json')
+        end
+      end
+
+      context 'when the manual override is used on the detectable file to unset' do
+        let(:gitattr_content) { "package-lock.json -gitlab-generated\n" }
+
+        it 'returns an empty set' do
+          expect(generated_files).to eq Set.new
+        end
+      end
+
+      shared_examples 'an invalid request' do
+        it 'returns an empty set' do
+          expect(generated_files).to eq Set.new
+        end
+
+        it 'reports the exception' do
+          expect(Gitlab::ErrorTracking)
+            .to receive(:track_exception)
+            .with(
+              instance_of(Gitlab::Git::CommandError),
+              gl_project_path: repository.gl_project_path,
+              base: base,
+              head: head,
+              paths: paths.map(&:path)
+            )
+
+          generated_files
+        end
+      end
+
+      context 'when an unknown revision is given' do
+        let(:head) { 'unknownrevision' }
+
+        it_behaves_like 'an invalid request'
+      end
+
+      context 'when an unknown path is given' do
+        let(:paths) do
+          [
+            Gitlab::Git::ChangedPath.new(status: :MODIFIED, path: 'file1.txt', old_mode: '100644', new_mode: '100644'),
+            Gitlab::Git::ChangedPath.new(status: :MODIFIED, path: 'unknownpath', old_mode: '100644', new_mode: '100644')
+          ]
+        end
+
+        it_behaves_like 'an invalid request'
+      end
+    end
+
+    context 'when all files are marked as generated' do
       let(:gitattr_content) { "*.txt gitlab-generated\n" }
 
       it 'returns all generated files' do
-        expect(generated_files).to eq paths.to_set
+        expect(generated_files).to eq paths.map(&:path).to_set
       end
     end
 
diff --git a/spec/lib/gitlab/gitaly_client/analysis_service_spec.rb b/spec/lib/gitlab/gitaly_client/analysis_service_spec.rb
new file mode 100644
index 00000000000..c57d9b9592c
--- /dev/null
+++ b/spec/lib/gitlab/gitaly_client/analysis_service_spec.rb
@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::GitalyClient::AnalysisService, feature_category: :gitaly do
+  let_it_be(:project) do
+    create(:project, :repository)
+  end
+
+  let(:repository) { project.repository.raw }
+  let(:base) { project.default_branch }
+  let(:head) { branch }
+  let(:branch) { 'test-check-blobs-generated' }
+
+  let(:client) { described_class.new(repository) }
+
+  describe '#check_blobs_generated' do
+    subject(:check_blobs_generated) { client.check_blobs_generated(base, head, changed_paths) }
+
+    before do
+      project.repository.create_branch(branch, project.default_branch)
+
+      project.repository.create_file(
+        project.creator,
+        'file1.txt',
+        'new file content',
+        message: 'Add new file',
+        branch_name: branch)
+
+      project.repository.create_file(
+        project.creator,
+        'package-lock.json',
+        'new file content',
+        message: 'Add new file',
+        branch_name: branch)
+
+      project.repository.delete_file(
+        project.creator,
+        'README',
+        message: 'Delete README',
+        branch_name: branch)
+    end
+
+    context 'when valid changed_paths are given' do
+      let(:changed_paths) do
+        [
+          Gitlab::Git::ChangedPath.new(status: :DELETED, path: 'README', old_mode: '100644', new_mode: '0'),
+          Gitlab::Git::ChangedPath.new(status: :ADDED, path: 'file1.txt', old_mode: '0', new_mode: '100644'),
+          Gitlab::Git::ChangedPath.new(status: :ADDED, path: 'package-lock.json', old_mode: '0', new_mode: '100644')
+        ]
+      end
+
+      it 'returns an expected array' do
+        expect(check_blobs_generated).to contain_exactly(
+          { generated: false, path: 'README' },
+          { generated: false, path: 'file1.txt' },
+          { generated: true, path: 'package-lock.json' }
+        )
+      end
+
+      context 'when changed_paths includes a submodule' do
+        let(:changed_paths) do
+          [
+            Gitlab::Git::ChangedPath.new(status: :ADDED, path: 'package-lock.json', old_mode: '0', new_mode: '100644'),
+            Gitlab::Git::ChangedPath.new(status: :DELETED, path: 'gitlab-shell', old_mode: '160000', new_mode: '0')
+          ]
+        end
+
+        it 'returns an array wihout the submodule change' do
+          expect(check_blobs_generated).to contain_exactly(
+            { generated: true, path: 'package-lock.json' }
+          )
+        end
+      end
+
+      context 'when changed_paths only has a submodule' do
+        let(:changed_paths) do
+          [
+            Gitlab::Git::ChangedPath.new(status: :ADDED, path: 'gitlab-shell', old_mode: '0', new_mode: '160000')
+          ]
+        end
+
+        it 'returns an empty array' do
+          expect(check_blobs_generated).to eq([])
+        end
+      end
+    end
+
+    context 'when changed_paths includes a path with :' do
+      before do
+        project.repository.create_file(
+          project.creator,
+          'abc:def',
+          'new file content',
+          message: 'Add new file',
+          branch_name: branch)
+      end
+
+      let(:changed_paths) do
+        [
+          Gitlab::Git::ChangedPath.new(status: :ADDED, path: 'abc:def', old_mode: '0', new_mode: '100644')
+        ]
+      end
+
+      it 'returns an expected array' do
+        expect(check_blobs_generated).to contain_exactly(
+          { generated: false, path: 'abc:def' }
+        )
+      end
+    end
+
+    context 'when an unknown revision is given' do
+      let(:head)  { 'unknownrevision' }
+      let(:changed_paths) do
+        [
+          Gitlab::Git::ChangedPath.new(status: :ADDED, path: 'file1.txt', old_mode: '0', new_mode: '100644')
+        ]
+      end
+
+      it 'raises an error' do
+        expect { check_blobs_generated }.to raise_error(GRPC::Internal)
+      end
+    end
+
+    context 'when an unknown path is given' do
+      let(:changed_paths) do
+        [
+          Gitlab::Git::ChangedPath.new(status: :ADDED, path: 'unknownpath', old_mode: '0', new_mode: '100644')
+        ]
+      end
+
+      it 'raises an error' do
+        expect { check_blobs_generated }.to raise_error(GRPC::Internal)
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/gitaly_client/commit_service_spec.rb b/spec/lib/gitlab/gitaly_client/commit_service_spec.rb
index 02c7abadd99..9b924440989 100644
--- a/spec/lib/gitlab/gitaly_client/commit_service_spec.rb
+++ b/spec/lib/gitlab/gitaly_client/commit_service_spec.rb
@@ -203,15 +203,15 @@ RSpec.describe Gitlab::GitalyClient::CommitService, feature_category: :gitaly do
     shared_examples 'includes paths different in any parent' do
       let(:changed_paths) do
         [
-          { path: 'files/locked/foo.lfs', status: 'ADDED' },
-          { path: 'files/locked/foo.lfs', status: 'MODIFIED' },
-          { path: 'files/locked/bar.lfs', status: 'ADDED' },
-          { path: 'files/locked/foo.lfs', status: 'MODIFIED' },
-          { path: 'files/locked/bar.lfs', status: 'ADDED' },
-          { path: 'files/locked/bar.lfs', status: 'MODIFIED' },
-          { path: 'files/locked/bar.lfs', status: 'MODIFIED' },
-          { path: 'files/locked/baz.lfs', status: 'ADDED' },
-          { path: 'files/locked/baz.lfs', status: 'ADDED' }
+          { path: 'files/locked/foo.lfs', status: 'ADDED', old_mode: '0', new_mode: '100644' },
+          { path: 'files/locked/foo.lfs', status: 'MODIFIED', old_mode: '100644', new_mode: '100644' },
+          { path: 'files/locked/bar.lfs', status: 'ADDED', old_mode: '0', new_mode: '100644' },
+          { path: 'files/locked/foo.lfs', status: 'MODIFIED', old_mode: '100644', new_mode: '100644' },
+          { path: 'files/locked/bar.lfs', status: 'ADDED', old_mode: '0', new_mode: '100644' },
+          { path: 'files/locked/bar.lfs', status: 'MODIFIED', old_mode: '100644', new_mode: '100644' },
+          { path: 'files/locked/bar.lfs', status: 'MODIFIED', old_mode: '100644', new_mode: '100644' },
+          { path: 'files/locked/baz.lfs', status: 'ADDED', old_mode: '0', new_mode: '100644' },
+          { path: 'files/locked/baz.lfs', status: 'ADDED', old_mode: '0', new_mode: '100644' }
         ].as_json
       end
 
@@ -223,12 +223,12 @@ RSpec.describe Gitlab::GitalyClient::CommitService, feature_category: :gitaly do
     shared_examples 'includes paths different in all parents' do
       let(:changed_paths) do
         [
-          { path: 'files/locked/foo.lfs', status: 'ADDED' },
-          { path: 'files/locked/foo.lfs', status: 'MODIFIED' },
-          { path: 'files/locked/bar.lfs', status: 'ADDED' },
-          { path: 'files/locked/bar.lfs', status: 'MODIFIED' },
-          { path: 'files/locked/baz.lfs', status: 'ADDED' },
-          { path: 'files/locked/baz.lfs', status: 'ADDED' }
+          { path: 'files/locked/foo.lfs', status: 'ADDED', old_mode: '0', new_mode: '100644' },
+          { path: 'files/locked/foo.lfs', status: 'MODIFIED', old_mode: '100644', new_mode: '100644' },
+          { path: 'files/locked/bar.lfs', status: 'ADDED', old_mode: '0', new_mode: '100644' },
+          { path: 'files/locked/bar.lfs', status: 'MODIFIED', old_mode: '100644', new_mode: '100644' },
+          { path: 'files/locked/baz.lfs', status: 'ADDED', old_mode: '0', new_mode: '100644' },
+          { path: 'files/locked/baz.lfs', status: 'ADDED', old_mode: '0', new_mode: '100644' }
         ].as_json
       end