Add latest changes from gitlab-org/gitlab@master

9 files changed, 322 insertions, 61 deletions

diff --git a/spec/controllers/projects/jobs_controller_spec.rb b/spec/controllers/projects/jobs_controller_spec.rb
index e9e7c3c3bb3..06c29e767ad 100644
--- a/spec/controllers/projects/jobs_controller_spec.rb
+++ b/spec/controllers/projects/jobs_controller_spec.rb
@@ -755,23 +755,52 @@ RSpec.describe Projects::JobsController, :clean_gitlab_redis_shared_state do
     before do
       project.add_developer(user)
       sign_in(user)
-
-      post_retry
     end
 
     context 'when job is retryable' do
       let(:job) { create(:ci_build, :retryable, pipeline: pipeline) }
 
       it 'redirects to the retried job page' do
+        post_retry
+
         expect(response).to have_gitlab_http_status(:found)
         expect(response).to redirect_to(namespace_project_job_path(id: Ci::Build.last.id))
       end
+
+      shared_examples_for 'retried job has the same attributes' do
+        it 'creates a new build has the same attributes from the previous build' do
+          expect { post_retry }.to change { Ci::Build.count }.by(1)
+
+          retried_build = Ci::Build.last
+
+          Ci::RetryBuildService.clone_accessors.each do |accessor|
+            expect(job.read_attribute(accessor))
+              .to eq(retried_build.read_attribute(accessor)),
+              "Mismatched attribute on \"#{accessor}\". " \
+              "It was \"#{job.read_attribute(accessor)}\" but changed to \"#{retried_build.read_attribute(accessor)}\""
+          end
+        end
+      end
+
+      context 'with branch pipeline' do
+        let!(:job) { create(:ci_build, :retryable, tag: true, when: 'on_success', pipeline: pipeline) }
+
+        it_behaves_like 'retried job has the same attributes'
+      end
+
+      context 'with tag pipeline' do
+        let!(:job) { create(:ci_build, :retryable, tag: false, when: 'on_success', pipeline: pipeline) }
+
+        it_behaves_like 'retried job has the same attributes'
+      end
     end
 
     context 'when job is not retryable' do
       let(:job) { create(:ci_build, pipeline: pipeline) }
 
       it 'renders unprocessable_entity' do
+        post_retry
+
         expect(response).to have_gitlab_http_status(:unprocessable_entity)
       end
     end
diff --git a/spec/frontend/dropzone_input_spec.js b/spec/frontend/dropzone_input_spec.js
index 5e6ccbd7cda..acf7d0780cd 100644
--- a/spec/frontend/dropzone_input_spec.js
+++ b/spec/frontend/dropzone_input_spec.js
@@ -1,9 +1,12 @@
+import MockAdapter from 'axios-mock-adapter';
 import $ from 'jquery';
 import mock from 'xhr-mock';
 import waitForPromises from 'helpers/wait_for_promises';
 import { TEST_HOST } from 'spec/test_constants';
 import PasteMarkdownTable from '~/behaviors/markdown/paste_markdown_table';
 import dropzoneInput from '~/dropzone_input';
+import axios from '~/lib/utils/axios_utils';
+import httpStatusCodes from '~/lib/utils/http_status';
 
 const TEST_FILE = new File([], 'somefile.jpg');
 TEST_FILE.upload = {};
@@ -29,6 +32,16 @@ describe('dropzone_input', () => {
   });
 
   describe('handlePaste', () => {
+    const triggerPasteEvent = (clipboardData = {}) => {
+      const event = $.Event('paste');
+      const origEvent = new Event('paste');
+
+      origEvent.clipboardData = clipboardData;
+      event.originalEvent = origEvent;
+
+      $('.js-gfm-input').trigger(event);
+    };
+
     beforeEach(() => {
       loadFixtures('issues/new-issue.html');
 
@@ -38,24 +51,39 @@ describe('dropzone_input', () => {
     });
 
     it('pastes Markdown tables', () => {
-      const event = $.Event('paste');
-      const origEvent = new Event('paste');
+      jest.spyOn(PasteMarkdownTable.prototype, 'isTable');
+      jest.spyOn(PasteMarkdownTable.prototype, 'convertToTableMarkdown');
 
-      origEvent.clipboardData = {
+      triggerPasteEvent({
         types: ['text/plain', 'text/html'],
         getData: () => '<table><tr><td>Hello World</td></tr></table>',
         items: [],
-      };
-      event.originalEvent = origEvent;
-
-      jest.spyOn(PasteMarkdownTable.prototype, 'isTable');
-      jest.spyOn(PasteMarkdownTable.prototype, 'convertToTableMarkdown');
-
-      $('.js-gfm-input').trigger(event);
+      });
 
       expect(PasteMarkdownTable.prototype.isTable).toHaveBeenCalled();
       expect(PasteMarkdownTable.prototype.convertToTableMarkdown).toHaveBeenCalled();
     });
+
+    it('passes truncated long filename to post request', async () => {
+      const axiosMock = new MockAdapter(axios);
+      const longFileName = 'a'.repeat(300);
+
+      triggerPasteEvent({
+        types: ['text/plain', 'text/html', 'text/rtf', 'Files'],
+        getData: () => longFileName,
+        items: [
+          {
+            kind: 'file',
+            type: 'image/png',
+            getAsFile: () => new Blob(),
+          },
+        ],
+      });
+
+      axiosMock.onPost().reply(httpStatusCodes.OK, { link: { markdown: 'foo' } });
+      await waitForPromises();
+      expect(axiosMock.history.post[0].data.get('file').name).toHaveLength(246);
+    });
   });
 
   describe('shows error message', () => {
diff --git a/spec/frontend/fixtures/static/pipeline_graph.html b/spec/frontend/fixtures/static/pipeline_graph.html
deleted file mode 100644
index d2c30ff9211..00000000000
--- a/spec/frontend/fixtures/static/pipeline_graph.html
+++ /dev/null
@@ -1,24 +0,0 @@
-<div class="pipeline-visualization js-pipeline-graph">
-<ul class="stage-column-list">
-<li class="stage-column">
-<div class="stage-name">
-<a href="/">
-Test
-<div class="builds-container">
-<ul>
-<li class="build">
-<div class="curve"></div>
-<a>
-<svg></svg>
-<div>
-stop_review
-</div>
-</a>
-</li>
-</ul>
-</div>
-</a>
-</div>
-</li>
-</ul>
-</div>
diff --git a/spec/frontend/pipelines_spec.js b/spec/frontend/pipelines_spec.js
deleted file mode 100644
index add91fbcc23..00000000000
--- a/spec/frontend/pipelines_spec.js
+++ /dev/null
@@ -1,17 +0,0 @@
-import Pipelines from '~/pipelines';
-
-describe('Pipelines', () => {
-  beforeEach(() => {
-    loadFixtures('static/pipeline_graph.html');
-  });
-
-  it('should be defined', () => {
-    expect(Pipelines).toBeDefined();
-  });
-
-  it('should create a `Pipelines` instance without options', () => {
-    expect(() => {
-      new Pipelines(); // eslint-disable-line no-new
-    }).not.toThrow();
-  });
-});
diff --git a/spec/lib/gitlab/rack_attack/request_spec.rb b/spec/lib/gitlab/rack_attack/request_spec.rb
new file mode 100644
index 00000000000..3be7ec17e45
--- /dev/null
+++ b/spec/lib/gitlab/rack_attack/request_spec.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::RackAttack::Request do
+  describe 'FILES_PATH_REGEX' do
+    subject { described_class::FILES_PATH_REGEX }
+
+    it { is_expected.to match('/api/v4/projects/1/repository/files/README') }
+    it { is_expected.to match('/api/v4/projects/1/repository/files/README?ref=master') }
+    it { is_expected.to match('/api/v4/projects/1/repository/files/README/blame') }
+    it { is_expected.to match('/api/v4/projects/1/repository/files/README/raw') }
+    it { is_expected.to match('/api/v4/projects/some%2Fnested%2Frepo/repository/files/README') }
+    it { is_expected.not_to match('/api/v4/projects/some/nested/repo/repository/files/README') }
+  end
+end
diff --git a/spec/lib/gitlab/tracking_spec.rb b/spec/lib/gitlab/tracking_spec.rb
index 90109db6db2..02e66458f46 100644
--- a/spec/lib/gitlab/tracking_spec.rb
+++ b/spec/lib/gitlab/tracking_spec.rb
@@ -11,7 +11,7 @@ RSpec.describe Gitlab::Tracking do
     described_class.instance_variable_set("@snowplow", nil)
   end
 
-  describe '.snowplow_options' do
+  describe '.options' do
     it 'returns useful client options' do
       expected_fields = {
         namespace: 'gl',
@@ -22,13 +22,13 @@ RSpec.describe Gitlab::Tracking do
         linkClickTracking: true
       }
 
-      expect(subject.snowplow_options(nil)).to match(expected_fields)
+      expect(subject.options(nil)).to match(expected_fields)
     end
 
     it 'when feature flag is disabled' do
       stub_feature_flags(additional_snowplow_tracking: false)
 
-      expect(subject.snowplow_options(nil)).to include(
+      expect(subject.options(nil)).to include(
         formTracking: false,
         linkClickTracking: false
       )
diff --git a/spec/requests/rack_attack_global_spec.rb b/spec/requests/rack_attack_global_spec.rb
index 32fa1381869..046b0246636 100644
--- a/spec/requests/rack_attack_global_spec.rb
+++ b/spec/requests/rack_attack_global_spec.rb
@@ -24,7 +24,11 @@ RSpec.describe 'Rack Attack global throttles', :use_clean_rails_memory_store_cac
       throttle_authenticated_packages_api_requests_per_period: 100,
       throttle_authenticated_packages_api_period_in_seconds: 1,
       throttle_authenticated_git_lfs_requests_per_period: 100,
-      throttle_authenticated_git_lfs_period_in_seconds: 1
+      throttle_authenticated_git_lfs_period_in_seconds: 1,
+      throttle_unauthenticated_files_api_requests_per_period: 100,
+      throttle_unauthenticated_files_api_period_in_seconds: 1,
+      throttle_authenticated_files_api_requests_per_period: 100,
+      throttle_authenticated_files_api_period_in_seconds: 1
     }
   end
 
@@ -711,6 +715,212 @@ RSpec.describe 'Rack Attack global throttles', :use_clean_rails_memory_store_cac
     end
   end
 
+  describe 'Files API' do
+    let(:request_method) { 'GET' }
+
+    context 'unauthenticated' do
+      let_it_be(:project) { create(:project, :public, :custom_repo, files: { 'README' => 'foo' }) }
+
+      let(:throttle_setting_prefix) { 'throttle_unauthenticated_files_api' }
+      let(:files_path_that_does_not_require_authentication) { "/api/v4/projects/#{project.id}/repository/files/README?ref=master" }
+
+      def do_request
+        get files_path_that_does_not_require_authentication
+      end
+
+      before do
+        settings_to_set[:throttle_unauthenticated_files_api_requests_per_period] = requests_per_period
+        settings_to_set[:throttle_unauthenticated_files_api_period_in_seconds] = period_in_seconds
+      end
+
+      context 'when unauthenticated files api throttle is disabled' do
+        before do
+          settings_to_set[:throttle_unauthenticated_files_api_enabled] = false
+          stub_application_setting(settings_to_set)
+        end
+
+        it 'allows requests over the rate limit' do
+          (1 + requests_per_period).times do
+            do_request
+            expect(response).to have_gitlab_http_status(:ok)
+          end
+        end
+
+        context 'when unauthenticated api throttle is enabled' do
+          before do
+            settings_to_set[:throttle_unauthenticated_requests_per_period] = requests_per_period
+            settings_to_set[:throttle_unauthenticated_period_in_seconds] = period_in_seconds
+            settings_to_set[:throttle_unauthenticated_enabled] = true
+            stub_application_setting(settings_to_set)
+          end
+
+          it 'rejects requests over the unauthenticated api rate limit' do
+            requests_per_period.times do
+              do_request
+              expect(response).to have_gitlab_http_status(:ok)
+            end
+
+            expect_rejection { do_request }
+          end
+        end
+      end
+
+      context 'when unauthenticated files api throttle is enabled' do
+        before do
+          settings_to_set[:throttle_unauthenticated_files_api_requests_per_period] = requests_per_period # 1
+          settings_to_set[:throttle_unauthenticated_files_api_period_in_seconds] = period_in_seconds # 10_000
+          settings_to_set[:throttle_unauthenticated_files_api_enabled] = true
+          stub_application_setting(settings_to_set)
+        end
+
+        it 'rejects requests over the rate limit' do
+          requests_per_period.times do
+            do_request
+            expect(response).to have_gitlab_http_status(:ok)
+          end
+
+          expect_rejection { do_request }
+        end
+
+        context 'when feature flag is off' do
+          before do
+            stub_feature_flags(files_api_throttling: false)
+          end
+
+          it 'allows requests over the rate limit' do
+            (1 + requests_per_period).times do
+              do_request
+              expect(response).to have_gitlab_http_status(:ok)
+            end
+          end
+        end
+
+        context 'when unauthenticated api throttle is lower' do
+          before do
+            settings_to_set[:throttle_unauthenticated_requests_per_period] = 0
+            settings_to_set[:throttle_unauthenticated_period_in_seconds] = period_in_seconds
+            settings_to_set[:throttle_unauthenticated_enabled] = true
+            stub_application_setting(settings_to_set)
+          end
+
+          it 'ignores unauthenticated api throttle' do
+            requests_per_period.times do
+              do_request
+              expect(response).to have_gitlab_http_status(:ok)
+            end
+
+            expect_rejection { do_request }
+          end
+        end
+
+        it_behaves_like 'tracking when dry-run mode is set' do
+          let(:throttle_name) { 'throttle_unauthenticated_files_api' }
+        end
+      end
+    end
+
+    context 'authenticated', :api do
+      let_it_be(:project) { create(:project, :internal, :custom_repo, files: { 'README' => 'foo' }) }
+      let_it_be(:user) { create(:user) }
+      let_it_be(:token) { create(:personal_access_token, user: user) }
+      let_it_be(:other_user) { create(:user) }
+      let_it_be(:other_user_token) { create(:personal_access_token, user: other_user) }
+
+      let(:throttle_setting_prefix) { 'throttle_authenticated_files_api' }
+      let(:api_partial_url) { "/projects/#{project.id}/repository/files/README?ref=master" }
+
+      before do
+        stub_application_setting(settings_to_set)
+      end
+
+      context 'with the token in the query string' do
+        let(:request_args) { [api(api_partial_url, personal_access_token: token), {}] }
+        let(:other_user_request_args) { [api(api_partial_url, personal_access_token: other_user_token), {}] }
+
+        it_behaves_like 'rate-limited token-authenticated requests'
+      end
+
+      context 'with the token in the headers' do
+        let(:request_args) { api_get_args_with_token_headers(api_partial_url, personal_access_token_headers(token)) }
+        let(:other_user_request_args) { api_get_args_with_token_headers(api_partial_url, personal_access_token_headers(other_user_token)) }
+
+        it_behaves_like 'rate-limited token-authenticated requests'
+      end
+
+      context 'precedence over authenticated api throttle' do
+        before do
+          settings_to_set[:throttle_authenticated_files_api_requests_per_period] = requests_per_period
+          settings_to_set[:throttle_authenticated_files_api_period_in_seconds] = period_in_seconds
+        end
+
+        def do_request
+          get api(api_partial_url, personal_access_token: token)
+        end
+
+        context 'when authenticated files api throttle is enabled' do
+          before do
+            settings_to_set[:throttle_authenticated_files_api_enabled] = true
+          end
+
+          context 'when authenticated api throttle is lower' do
+            before do
+              settings_to_set[:throttle_authenticated_api_requests_per_period] = 0
+              settings_to_set[:throttle_authenticated_api_period_in_seconds] = period_in_seconds
+              settings_to_set[:throttle_authenticated_api_enabled] = true
+              stub_application_setting(settings_to_set)
+            end
+
+            it 'ignores authenticated api throttle' do
+              requests_per_period.times do
+                do_request
+                expect(response).to have_gitlab_http_status(:ok)
+              end
+
+              expect_rejection { do_request }
+            end
+          end
+
+          context 'when feature flag is off' do
+            before do
+              stub_feature_flags(files_api_throttling: false)
+            end
+
+            it 'allows requests over the rate limit' do
+              (1 + requests_per_period).times do
+                do_request
+                expect(response).to have_gitlab_http_status(:ok)
+              end
+            end
+          end
+        end
+
+        context 'when authenticated files api throttle is disabled' do
+          before do
+            settings_to_set[:throttle_authenticated_files_api_enabled] = false
+          end
+
+          context 'when authenticated api throttle is enabled' do
+            before do
+              settings_to_set[:throttle_authenticated_api_requests_per_period] = requests_per_period
+              settings_to_set[:throttle_authenticated_api_period_in_seconds] = period_in_seconds
+              settings_to_set[:throttle_authenticated_api_enabled] = true
+              stub_application_setting(settings_to_set)
+            end
+
+            it 'rejects requests over the authenticated api rate limit' do
+              requests_per_period.times do
+                do_request
+                expect(response).to have_gitlab_http_status(:ok)
+              end
+
+              expect_rejection { do_request }
+            end
+          end
+        end
+      end
+    end
+  end
+
   describe 'throttle bypass header' do
     let(:headers) { {} }
     let(:bypass_header) { 'gitlab-bypass-rate-limiting' }
diff --git a/spec/support/shared_examples/requests/rack_attack_shared_examples.rb b/spec/support/shared_examples/requests/rack_attack_shared_examples.rb
index 9d87a231bfa..8ddd2d900de 100644
--- a/spec/support/shared_examples/requests/rack_attack_shared_examples.rb
+++ b/spec/support/shared_examples/requests/rack_attack_shared_examples.rb
@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 #
 # Requires let variables:
-# * throttle_setting_prefix: "throttle_authenticated_api", "throttle_authenticated_web", "throttle_protected_paths", "throttle_authenticated_packages_api", "throttle_authenticated_git_lfs"
+# * throttle_setting_prefix: "throttle_authenticated_api", "throttle_authenticated_web", "throttle_protected_paths", "throttle_authenticated_packages_api", "throttle_authenticated_git_lfs", "throttle_authenticated_files_api"
 # * request_method
 # * request_args
 # * other_user_request_args
@@ -15,7 +15,8 @@ RSpec.shared_examples 'rate-limited token-authenticated requests' do
       "throttle_authenticated_api" => "throttle_authenticated_api",
       "throttle_authenticated_web" => "throttle_authenticated_web",
       "throttle_authenticated_packages_api" => "throttle_authenticated_packages_api",
-      "throttle_authenticated_git_lfs" => "throttle_authenticated_git_lfs"
+      "throttle_authenticated_git_lfs" => "throttle_authenticated_git_lfs",
+      "throttle_authenticated_files_api" => "throttle_authenticated_files_api"
     }
   end
 
diff --git a/spec/workers/background_migration_worker_spec.rb b/spec/workers/background_migration_worker_spec.rb
index 4575c270042..7892eb89e80 100644
--- a/spec/workers/background_migration_worker_spec.rb
+++ b/spec/workers/background_migration_worker_spec.rb
@@ -14,7 +14,17 @@ RSpec.describe BackgroundMigrationWorker, :clean_gitlab_redis_shared_state do
   describe '#perform' do
     before do
       allow(worker).to receive(:jid).and_return(1)
-      expect(worker).to receive(:always_perform?).and_return(false)
+      allow(worker).to receive(:always_perform?).and_return(false)
+    end
+
+    it 'can run scheduled job and retried job concurrently' do
+      expect(Gitlab::BackgroundMigration)
+        .to receive(:perform)
+        .with('Foo', [10, 20])
+        .exactly(2).time
+
+      worker.perform('Foo', [10, 20])
+      worker.perform('Foo', [10, 20], described_class::MAX_LEASE_ATTEMPTS - 1)
     end
 
     context 'when lease can be obtained' do
@@ -39,7 +49,7 @@ RSpec.describe BackgroundMigrationWorker, :clean_gitlab_redis_shared_state do
       before do
         expect(Gitlab::BackgroundMigration).not_to receive(:perform)
 
-        worker.lease_for('Foo').try_obtain
+        worker.lease_for('Foo', false).try_obtain
       end
 
       it 'reschedules the migration and decrements the lease_attempts' do
@@ -51,6 +61,10 @@ RSpec.describe BackgroundMigrationWorker, :clean_gitlab_redis_shared_state do
       end
 
       context 'when lease_attempts is 1' do
+        before do
+          worker.lease_for('Foo', true).try_obtain
+        end
+
         it 'reschedules the migration and decrements the lease_attempts' do
           expect(described_class)
             .to receive(:perform_in)
@@ -61,6 +75,10 @@ RSpec.describe BackgroundMigrationWorker, :clean_gitlab_redis_shared_state do
       end
 
       context 'when lease_attempts is 0' do
+        before do
+          worker.lease_for('Foo', true).try_obtain
+        end
+
         it 'gives up performing the migration' do
           expect(described_class).not_to receive(:perform_in)
           expect(Sidekiq.logger).to receive(:warn).with(