Merge branch 'master' into 'feature/gb/variables-expressions-in-only-except'

 Conflicts:
   app/models/ci/build.rb

17 files changed, 280 insertions, 71 deletions

diff --git a/lib/gitlab/background_migration/migrate_build_stage.rb b/lib/gitlab/background_migration/migrate_build_stage.rb
index 8fe4f1a2289..242e3143e71 100644
--- a/lib/gitlab/background_migration/migrate_build_stage.rb
+++ b/lib/gitlab/background_migration/migrate_build_stage.rb
@@ -12,6 +12,7 @@ module Gitlab
 
         class Build < ActiveRecord::Base
           self.table_name = 'ci_builds'
+          self.inheritance_column = :_type_disabled
 
           def ensure_stage!(attempts: 2)
             find_stage || create_stage!
diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb
index 884a3de8f62..1a25138e7d6 100644
--- a/lib/gitlab/bare_repository_import/importer.rb
+++ b/lib/gitlab/bare_repository_import/importer.rb
@@ -63,7 +63,7 @@ module Gitlab
           log " * Created #{project.name} (#{project_full_path})".color(:green)
 
           project.write_repository_config
-          project.repository.create_hooks
+          Gitlab::Git::Repository.create_hooks(project.repository.path_to_repo, Gitlab.config.gitlab_shell.hooks_path)
 
           ProjectCacheWorker.perform_async(project.id)
         else
diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb
index 3263790a876..3a197078d08 100644
--- a/lib/gitlab/checks/project_moved.rb
+++ b/lib/gitlab/checks/project_moved.rb
@@ -9,20 +9,16 @@ module Gitlab
         super(project, user, protocol)
       end
 
-      def message(rejected: false)
+      def message
         <<~MESSAGE
         Project '#{redirected_path}' was moved to '#{project.full_path}'.
 
         Please update your Git remote:
 
-          #{remote_url_message(rejected)}
+          git remote set-url origin #{url_to_repo}
         MESSAGE
       end
 
-      def permanent_redirect?
-        RedirectRoute.permanent.exists?(path: redirected_path)
-      end
-
       private
 
       attr_reader :redirected_path
@@ -30,18 +26,6 @@ module Gitlab
       def self.message_key(user_id, project_id)
         "#{REDIRECT_NAMESPACE}:#{user_id}:#{project_id}"
       end
-
-      def remote_url_message(rejected)
-        if rejected
-          "git remote set-url origin #{url_to_repo} and try again."
-        else
-          "git remote set-url origin #{url_to_repo}"
-        end
-      end
-
-      def url
-        protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo
-      end
     end
   end
 end
diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb
index 411f67f8ce7..0b1ebe4e048 100644
--- a/lib/gitlab/ci/build/step.rb
+++ b/lib/gitlab/ci/build/step.rb
@@ -14,7 +14,7 @@ module Gitlab
             self.new(:script).tap do |step|
               step.script = job.options[:before_script].to_a + job.options[:script].to_a
               step.script = job.commands.split("\n") if step.script.empty?
-              step.timeout = job.timeout
+              step.timeout = job.metadata_timeout
               step.when = WHEN_ON_SUCCESS
             end
           end
@@ -25,7 +25,7 @@ module Gitlab
 
             self.new(:after_script).tap do |step|
               step.script = after_script
-              step.timeout = job.timeout
+              step.timeout = job.metadata_timeout
               step.when = WHEN_ALWAYS
               step.allow_failure = true
             end
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index f7ff7ea212e..66ac4a40616 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -4,7 +4,8 @@ module Gitlab
     # Base GitLab CI Configuration facade
     #
     class Config
-      def initialize(config)
+      # EE would override this and utilize opts argument
+      def initialize(config, opts = {})
         @config = Loader.new(config).load!
 
         @global = Entry::Global.new(@config)
diff --git a/lib/gitlab/ci/trace/http_io.rb b/lib/gitlab/ci/trace/http_io.rb
new file mode 100644
index 00000000000..ac4308f4e2c
--- /dev/null
+++ b/lib/gitlab/ci/trace/http_io.rb
@@ -0,0 +1,187 @@
+##
+# This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html)
+# source: https://gitlab.com/snippets/1685610
+module Gitlab
+  module Ci
+    class Trace
+      class HttpIO
+        BUFFER_SIZE = 128.kilobytes
+
+        InvalidURLError = Class.new(StandardError)
+        FailedToGetChunkError = Class.new(StandardError)
+
+        attr_reader :uri, :size
+        attr_reader :tell
+        attr_reader :chunk, :chunk_range
+
+        alias_method :pos, :tell
+
+        def initialize(url, size)
+          raise InvalidURLError unless ::Gitlab::UrlSanitizer.valid?(url)
+
+          @uri = URI(url)
+          @size = size
+          @tell = 0
+        end
+
+        def close
+          # no-op
+        end
+
+        def binmode
+          # no-op
+        end
+
+        def binmode?
+          true
+        end
+
+        def path
+          nil
+        end
+
+        def url
+          @uri.to_s
+        end
+
+        def seek(pos, where = IO::SEEK_SET)
+          new_pos =
+            case where
+            when IO::SEEK_END
+              size + pos
+            when IO::SEEK_SET
+              pos
+            when IO::SEEK_CUR
+              tell + pos
+            else
+              -1
+            end
+
+          raise 'new position is outside of file' if new_pos < 0 || new_pos > size
+
+          @tell = new_pos
+        end
+
+        def eof?
+          tell == size
+        end
+
+        def each_line
+          until eof?
+            line = readline
+            break if line.nil?
+
+            yield(line)
+          end
+        end
+
+        def read(length = nil)
+          out = ""
+
+          until eof? || (length && out.length >= length)
+            data = get_chunk
+            break if data.empty?
+
+            out << data
+            @tell += data.bytesize
+          end
+
+          out = out[0, length] if length && out.length > length
+
+          out
+        end
+
+        def readline
+          out = ""
+
+          until eof?
+            data = get_chunk
+            new_line = data.index("\n")
+
+            if !new_line.nil?
+              out << data[0..new_line]
+              @tell += new_line + 1
+              break
+            else
+              out << data
+              @tell += data.bytesize
+            end
+          end
+
+          out
+        end
+
+        def write(data)
+          raise NotImplementedError
+        end
+
+        def truncate(offset)
+          raise NotImplementedError
+        end
+
+        def flush
+          raise NotImplementedError
+        end
+
+        def present?
+          true
+        end
+
+        private
+
+        ##
+        # The below methods are not implemented in IO class
+        #
+        def in_range?
+          @chunk_range&.include?(tell)
+        end
+
+        def get_chunk
+          unless in_range?
+            response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
+              http.request(request)
+            end
+
+            raise FailedToGetChunkError unless response.code == '200' || response.code == '206'
+
+            @chunk = response.body.force_encoding(Encoding::BINARY)
+            @chunk_range = response.content_range
+
+            ##
+            # Note: If provider does not return content_range, then we set it as we requested
+            # Provider: minio
+            # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
+            # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
+            # Provider: AWS
+            # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
+            # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
+            # Provider: GCS
+            # - When the file size is larger than requested Content-range, the Content-range is included in responces with Net::HTTPPartialContent 206
+            # - When the file size is smaller than requested Content-range, the Content-range is included in responces with Net::HTTPOK 200
+            @chunk_range ||= (chunk_start...(chunk_start + @chunk.length))
+          end
+
+          @chunk[chunk_offset..BUFFER_SIZE]
+        end
+
+        def request
+          Net::HTTP::Get.new(uri).tap do |request|
+            request.set_range(chunk_start, BUFFER_SIZE)
+          end
+        end
+
+        def chunk_offset
+          tell % BUFFER_SIZE
+        end
+
+        def chunk_start
+          (tell / BUFFER_SIZE) * BUFFER_SIZE
+        end
+
+        def chunk_end
+          [chunk_start + BUFFER_SIZE, size].min
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb
index d52194f688b..b3fe3ef1c4d 100644
--- a/lib/gitlab/ci/trace/stream.rb
+++ b/lib/gitlab/ci/trace/stream.rb
@@ -8,7 +8,7 @@ module Gitlab
 
         attr_reader :stream
 
-        delegate :close, :tell, :seek, :size, :path, :truncate, to: :stream, allow_nil: true
+        delegate :close, :tell, :seek, :size, :path, :url, :truncate, to: :stream, allow_nil: true
 
         delegate :valid?, to: :stream, as: :present?, allow_nil: true
 
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index bc2a6f98dae..e829f2a95f8 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -7,8 +7,8 @@ module Gitlab
 
       attr_reader :cache, :stages, :jobs
 
-      def initialize(config)
-        @ci_config = Gitlab::Ci::Config.new(config)
+      def initialize(config, opts = {})
+        @ci_config = Gitlab::Ci::Config.new(config, opts)
         @config = @ci_config.to_hash
 
         unless @ci_config.valid?
@@ -73,11 +73,11 @@ module Gitlab
         end
       end
 
-      def self.validation_message(content)
+      def self.validation_message(content, opts = {})
         return 'Please provide content of .gitlab-ci.yml' if content.blank?
 
         begin
-          Gitlab::Ci::YamlProcessor.new(content)
+          Gitlab::Ci::YamlProcessor.new(content, opts)
           nil
         rescue ValidationError => e
           e.message
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 44ca434056f..1634fe4e9cb 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -900,11 +900,42 @@ into similar problems in the future (e.g. when new tables are created).
         end
       end
 
-      # Rails' index_exists? doesn't work when you only give it a table and index
-      # name. As such we have to use some extra code to check if an index exists for
-      # a given name.
+      # Fetches indexes on a column by name for postgres.
+      #
+      # This will include indexes using an expression on the column, for example:
+      # `CREATE INDEX CONCURRENTLY index_name ON table (LOWER(column));`
+      #
+      # For mysql, it falls back to the default ActiveRecord implementation that
+      # will not find custom indexes. But it will select by name without passing
+      # a column.
+      #
+      # We can remove this when upgrading to Rails 5 with an updated `index_exists?`:
+      # - https://github.com/rails/rails/commit/edc2b7718725016e988089b5fb6d6fb9d6e16882
+      #
+      # Or this can be removed when we no longer support postgres < 9.5, so we
+      # can use `CREATE INDEX IF NOT EXISTS`.
       def index_exists_by_name?(table, index)
-        indexes(table).map(&:name).include?(index)
+        # We can't fall back to the normal `index_exists?` method because that
+        # does not find indexes without passing a column name.
+        if indexes(table).map(&:name).include?(index.to_s)
+          true
+        elsif Gitlab::Database.postgresql?
+          postgres_exists_by_name?(table, index)
+        else
+          false
+        end
+      end
+
+      def postgres_exists_by_name?(table, name)
+        index_sql = <<~SQL
+          SELECT COUNT(*)
+          FROM pg_index
+          JOIN pg_class i ON (indexrelid=i.oid)
+          JOIN pg_class t ON (indrelid=t.oid)
+          WHERE i.relname = '#{name}' AND t.relname = '#{table}'
+        SQL
+
+        connection.select_value(index_sql).to_i > 0
       end
     end
   end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 20b0647fce9..2d16a81c888 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -8,6 +8,7 @@ module Gitlab
     class Repository
       include Gitlab::Git::RepositoryMirroring
       include Gitlab::Git::Popen
+      include Gitlab::EncodingHelper
 
       ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[
         GIT_OBJECT_DIRECTORY
@@ -1479,7 +1480,7 @@ module Gitlab
         names.lines.each do |line|
           next unless line.start_with?(refs_prefix)
 
-          refs << line.rstrip[left_slice_count..-1]
+          refs << encode_utf8(line.rstrip[left_slice_count..-1])
         end
 
         refs
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index 6400089a22f..ed0644f6cf1 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -53,7 +53,7 @@ module Gitlab
       ensure_project_on_push!(cmd, changes)
 
       check_project_accessibility!
-      check_project_moved!
+      add_project_moved_message!
       check_repository_existence!
 
       case cmd
@@ -99,8 +99,6 @@ module Gitlab
     end
 
     def check_active_user!
-      return if deploy_key?
-
       if user && !user_access.allowed?
         raise UnauthorizedError, ERROR_MESSAGES[:account_blocked]
       end
@@ -125,16 +123,12 @@ module Gitlab
       end
     end
 
-    def check_project_moved!
+    def add_project_moved_message!
       return if redirected_path.nil?
 
       project_moved = Checks::ProjectMoved.new(project, user, protocol, redirected_path)
 
-      if project_moved.permanent_redirect?
-        project_moved.add_message
-      else
-        raise ProjectMovedError, project_moved.message(rejected: true)
-      end
+      project_moved.add_message
     end
 
     def check_command_disabled!(cmd)
@@ -219,7 +213,7 @@ module Gitlab
         raise UnauthorizedError, ERROR_MESSAGES[:read_only]
       end
 
-      if deploy_key
+      if deploy_key?
         unless deploy_key.can_push_to?(project)
           raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload]
         end
@@ -309,8 +303,10 @@ module Gitlab
         case actor
         when User
           actor
+        when DeployKey
+          nil
         when Key
-          actor.user unless actor.is_a?(DeployKey)
+          actor.user
         when :ci
           nil
         end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 8ca30ffc232..0abae70c443 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -83,6 +83,10 @@ module Gitlab
       end
     end
 
+    def self.random_storage
+      Gitlab.config.repositories.storages.keys.sample
+    end
+
     def self.address(storage)
       params = Gitlab.config.repositories.storages[storage]
       raise "storage not found: #{storage.inspect}" if params.nil?
diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb
index 58c356edfd1..f2d699d9dfb 100644
--- a/lib/gitlab/gitaly_client/remote_service.rb
+++ b/lib/gitlab/gitaly_client/remote_service.rb
@@ -3,6 +3,17 @@ module Gitlab
     class RemoteService
       MAX_MSG_SIZE = 128.kilobytes.freeze
 
+      def self.exists?(remote_url)
+        request = Gitaly::FindRemoteRepositoryRequest.new(remote: remote_url)
+
+        response = GitalyClient.call(GitalyClient.random_storage,
+                                     :remote_service,
+                                     :find_remote_repository, request,
+                                     timeout: GitalyClient.medium_timeout)
+
+        response.exists
+      end
+
       def initialize(repository)
         @repository = repository
         @gitaly_repo = repository.gitaly_repository
diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb
index ab0b751fe24..b1b283e98b5 100644
--- a/lib/gitlab/github_import/importer/repository_importer.rb
+++ b/lib/gitlab/github_import/importer/repository_importer.rb
@@ -16,7 +16,8 @@ module Gitlab
         # Returns true if we should import the wiki for the project.
         def import_wiki?
           client.repository(project.import_source)&.has_wiki &&
-            !project.wiki_repository_exists?
+            !project.wiki_repository_exists? &&
+            Gitlab::GitalyClient::RemoteService.exists?(wiki_url)
         end
 
         # Imports the repository data.
@@ -55,7 +56,6 @@ module Gitlab
 
         def import_wiki_repository
           wiki_path = "#{project.disk_path}.wiki"
-          wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git')
           storage_path = project.repository_storage_path
 
           gitlab_shell.import_repository(storage_path, wiki_path, wiki_url)
@@ -70,6 +70,10 @@ module Gitlab
           end
         end
 
+        def wiki_url
+          project.import_url.sub(/\.git\z/, '.wiki.git')
+        end
+
         def update_clone_time
           project.update_column(:last_repository_updated_at, Time.zone.now)
         end
diff --git a/lib/gitlab/verify/lfs_objects.rb b/lib/gitlab/verify/lfs_objects.rb
index fe51edbdeeb..970e2a7b718 100644
--- a/lib/gitlab/verify/lfs_objects.rb
+++ b/lib/gitlab/verify/lfs_objects.rb
@@ -12,7 +12,7 @@ module Gitlab
       private
 
       def relation
-        LfsObject.all
+        LfsObject.with_files_stored_locally
       end
 
       def expected_checksum(lfs_object)
diff --git a/lib/gitlab/verify/uploads.rb b/lib/gitlab/verify/uploads.rb
index 6972e517ea5..0ffa71a6d72 100644
--- a/lib/gitlab/verify/uploads.rb
+++ b/lib/gitlab/verify/uploads.rb
@@ -12,7 +12,7 @@ module Gitlab
       private
 
       def relation
-        Upload.all
+        Upload.with_files_stored_locally
       end
 
       def expected_checksum(upload)
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 0b0d667d4fd..b102812ec12 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -21,29 +21,18 @@ module Gitlab
         raise "Unsupported action: #{action}" unless ALLOWED_GIT_HTTP_ACTIONS.include?(action.to_s)
 
         project = repository.project
-        repo_path = repository.path_to_repo
-        params = {
+
+        {
           GL_ID: Gitlab::GlId.gl_id(user),
           GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki),
           GL_USERNAME: user&.username,
-          RepoPath: repo_path,
-          ShowAllRefs: show_all_refs
-        }
-        server = {
-          address: Gitlab::GitalyClient.address(project.repository_storage),
-          token: Gitlab::GitalyClient.token(project.repository_storage)
-        }
-        params[:Repository] = repository.gitaly_repository.to_h
-        params[:GitalyServer] = server
-
-        params
-      end
-
-      def lfs_upload_ok(oid, size)
-        {
-          StoreLFSPath: LfsObjectUploader.workhorse_upload_path,
-          LfsOid: oid,
-          LfsSize: size
+          ShowAllRefs: show_all_refs,
+          Repository: repository.gitaly_repository.to_h,
+          RepoPath: 'ignored but not allowed to be empty in gitlab-workhorse',
+          GitalyServer: {
+            address: Gitlab::GitalyClient.address(project.repository_storage),
+            token: Gitlab::GitalyClient.token(project.repository_storage)
+          }
         }
       end
 
@@ -52,7 +41,7 @@ module Gitlab
       end
 
       def send_git_blob(repository, blob)
-        params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show)
+        params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT)
                    {
                      'GitalyServer' => gitaly_server_hash(repository),
                      'GetBlobRequest' => {
@@ -80,7 +69,7 @@ module Gitlab
         params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format)
         raise "Repository or ref not found" if params.empty?
 
-        if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive)
+        if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT)
           params.merge!(
             'GitalyServer' => gitaly_server_hash(repository),
             'GitalyRepository' => repository.gitaly_repository.to_h
@@ -97,7 +86,7 @@ module Gitlab
       end
 
       def send_git_diff(repository, diff_refs)
-        params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff)
+        params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_diff, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT)
                    {
                      'GitalyServer' => gitaly_server_hash(repository),
                      'RawDiffRequest' => Gitaly::RawDiffRequest.new(
@@ -115,7 +104,7 @@ module Gitlab
       end
 
       def send_git_patch(repository, diff_refs)
-        params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_patch)
+        params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_send_git_patch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT)
                    {
                      'GitalyServer' => gitaly_server_hash(repository),
                      'RawPatchRequest' => Gitaly::RawPatchRequest.new(