Add latest changes from gitlab-org/gitlab@master

6 files changed, 562 insertions, 0 deletions

diff --git a/app/controllers/repositories/application_controller.rb b/app/controllers/repositories/application_controller.rb
new file mode 100644
index 00000000000..528cc310038
--- /dev/null
+++ b/app/controllers/repositories/application_controller.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Repositories
+  class ApplicationController < ::ApplicationController
+    skip_before_action :authenticate_user!
+  end
+end
diff --git a/app/controllers/repositories/git_http_client_controller.rb b/app/controllers/repositories/git_http_client_controller.rb
new file mode 100644
index 00000000000..76eb7c67205
--- /dev/null
+++ b/app/controllers/repositories/git_http_client_controller.rb
@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Repositories
+  class GitHttpClientController < Repositories::ApplicationController
+    include ActionController::HttpAuthentication::Basic
+    include KerberosSpnegoHelper
+    include Gitlab::Utils::StrongMemoize
+
+    attr_reader :authentication_result, :redirected_path
+
+    delegate :actor, :authentication_abilities, to: :authentication_result, allow_nil: true
+    delegate :type, to: :authentication_result, allow_nil: true, prefix: :auth_result
+
+    alias_method :user, :actor
+    alias_method :authenticated_user, :actor
+
+    # Git clients will not know what authenticity token to send along
+    skip_around_action :set_session_storage
+    skip_before_action :verify_authenticity_token
+
+    before_action :parse_repo_path
+    before_action :authenticate_user
+
+    private
+
+    def download_request?
+      raise NotImplementedError
+    end
+
+    def upload_request?
+      raise NotImplementedError
+    end
+
+    def authenticate_user
+      @authentication_result = Gitlab::Auth::Result.new
+
+      if allow_basic_auth? && basic_auth_provided?
+        login, password = user_name_and_password(request)
+
+        if handle_basic_authentication(login, password)
+          return # Allow access
+        end
+      elsif allow_kerberos_spnego_auth? && spnego_provided?
+        kerberos_user = find_kerberos_user
+
+        if kerberos_user
+          @authentication_result = Gitlab::Auth::Result.new(
+            kerberos_user, nil, :kerberos, Gitlab::Auth.full_authentication_abilities)
+
+          send_final_spnego_response
+          return # Allow access
+        end
+      elsif http_download_allowed?
+
+        @authentication_result = Gitlab::Auth::Result.new(nil, project, :none, [:download_code])
+
+        return # Allow access
+      end
+
+      send_challenges
+      render plain: "HTTP Basic: Access denied\n", status: :unauthorized
+    rescue Gitlab::Auth::MissingPersonalAccessTokenError
+      render_missing_personal_access_token
+    end
+
+    def basic_auth_provided?
+      has_basic_credentials?(request)
+    end
+
+    def send_challenges
+      challenges = []
+      challenges << 'Basic realm="GitLab"' if allow_basic_auth?
+      challenges << spnego_challenge if allow_kerberos_spnego_auth?
+      headers['Www-Authenticate'] = challenges.join("\n") if challenges.any?
+    end
+
+    def project
+      parse_repo_path unless defined?(@project)
+
+      @project
+    end
+
+    def parse_repo_path
+      @project, @repo_type, @redirected_path = Gitlab::RepoPath.parse("#{params[:namespace_id]}/#{params[:repository_id]}")
+    end
+
+    def render_missing_personal_access_token
+      render plain: "HTTP Basic: Access denied\n" \
+                    "You must use a personal access token with 'read_repository' or 'write_repository' scope for Git over HTTP.\n" \
+                    "You can generate one at #{profile_personal_access_tokens_url}",
+            status: :unauthorized
+    end
+
+    def repository
+      strong_memoize(:repository) do
+        repo_type.repository_for(project)
+      end
+    end
+
+    def repo_type
+      parse_repo_path unless defined?(@repo_type)
+
+      @repo_type
+    end
+
+    def handle_basic_authentication(login, password)
+      @authentication_result = Gitlab::Auth.find_for_git_client(
+        login, password, project: project, ip: request.ip)
+
+      @authentication_result.success?
+    end
+
+    def ci?
+      authentication_result.ci?(project)
+    end
+
+    def http_download_allowed?
+      Gitlab::ProtocolAccess.allowed?('http') &&
+      download_request? &&
+      project && Guest.can?(:download_code, project)
+    end
+  end
+end
+
+Repositories::GitHttpClientController.prepend_if_ee('EE::Repositories::GitHttpClientController')
diff --git a/app/controllers/repositories/git_http_controller.rb b/app/controllers/repositories/git_http_controller.rb
new file mode 100644
index 00000000000..82431ae286a
--- /dev/null
+++ b/app/controllers/repositories/git_http_controller.rb
@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module Repositories
+  class GitHttpController < Repositories::GitHttpClientController
+    include WorkhorseRequest
+
+    before_action :access_check
+    prepend_before_action :deny_head_requests, only: [:info_refs]
+
+    rescue_from Gitlab::GitAccess::UnauthorizedError, with: :render_403_with_exception
+    rescue_from Gitlab::GitAccess::NotFoundError, with: :render_404_with_exception
+    rescue_from Gitlab::GitAccess::ProjectCreationError, with: :render_422_with_exception
+    rescue_from Gitlab::GitAccess::TimeoutError, with: :render_503_with_exception
+
+    # GET /foo/bar.git/info/refs?service=git-upload-pack (git pull)
+    # GET /foo/bar.git/info/refs?service=git-receive-pack (git push)
+    def info_refs
+      log_user_activity if upload_pack?
+
+      render_ok
+    end
+
+    # POST /foo/bar.git/git-upload-pack (git pull)
+    def git_upload_pack
+      enqueue_fetch_statistics_update
+
+      render_ok
+    end
+
+    # POST /foo/bar.git/git-receive-pack" (git push)
+    def git_receive_pack
+      render_ok
+    end
+
+    private
+
+    def deny_head_requests
+      head :forbidden if request.head?
+    end
+
+    def download_request?
+      upload_pack?
+    end
+
+    def upload_pack?
+      git_command == 'git-upload-pack'
+    end
+
+    def git_command
+      if action_name == 'info_refs'
+        params[:service]
+      else
+        action_name.dasherize
+      end
+    end
+
+    def render_ok
+      set_workhorse_internal_api_content_type
+      render json: Gitlab::Workhorse.git_http_ok(repository, repo_type, user, action_name)
+    end
+
+    def render_403_with_exception(exception)
+      render plain: exception.message, status: :forbidden
+    end
+
+    def render_404_with_exception(exception)
+      render plain: exception.message, status: :not_found
+    end
+
+    def render_422_with_exception(exception)
+      render plain: exception.message, status: :unprocessable_entity
+    end
+
+    def render_503_with_exception(exception)
+      render plain: exception.message, status: :service_unavailable
+    end
+
+    def enqueue_fetch_statistics_update
+      return if Gitlab::Database.read_only?
+      return unless repo_type.project?
+      return unless project&.daily_statistics_enabled?
+
+      ProjectDailyStatisticsWorker.perform_async(project.id)
+    end
+
+    def access
+      @access ||= access_klass.new(access_actor, project, 'http',
+        authentication_abilities: authentication_abilities,
+        namespace_path: params[:namespace_id],
+        project_path: project_path,
+        redirected_path: redirected_path,
+        auth_result_type: auth_result_type)
+    end
+
+    def access_actor
+      return user if user
+      return :ci if ci?
+    end
+
+    def access_check
+      access.check(git_command, Gitlab::GitAccess::ANY)
+      @project ||= access.project
+    end
+
+    def access_klass
+      @access_klass ||= repo_type.access_checker_class
+    end
+
+    def project_path
+      @project_path ||= params[:repository_id].sub(/\.git$/, '')
+    end
+
+    def log_user_activity
+      Users::ActivityService.new(user).execute
+    end
+  end
+end
+
+Repositories::GitHttpController.prepend_if_ee('EE::Repositories::GitHttpController')
diff --git a/app/controllers/repositories/lfs_api_controller.rb b/app/controllers/repositories/lfs_api_controller.rb
new file mode 100644
index 00000000000..b1e0d1848d7
--- /dev/null
+++ b/app/controllers/repositories/lfs_api_controller.rb
@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module Repositories
+  class LfsApiController < Repositories::GitHttpClientController
+    include LfsRequest
+    include Gitlab::Utils::StrongMemoize
+
+    LFS_TRANSFER_CONTENT_TYPE = 'application/octet-stream'
+
+    skip_before_action :lfs_check_access!, only: [:deprecated]
+    before_action :lfs_check_batch_operation!, only: [:batch]
+
+    def batch
+      unless objects.present?
+        render_lfs_not_found
+        return
+      end
+
+      if download_request?
+        render json: { objects: download_objects! }
+      elsif upload_request?
+        render json: { objects: upload_objects! }
+      else
+        raise "Never reached"
+      end
+    end
+
+    def deprecated
+      render(
+        json: {
+          message: _('Server supports batch API only, please update your Git LFS client to version 1.0.1 and up.'),
+          documentation_url: "#{Gitlab.config.gitlab.url}/help"
+        },
+        status: :not_implemented
+      )
+    end
+
+    private
+
+    def download_request?
+      params[:operation] == 'download'
+    end
+
+    def upload_request?
+      params[:operation] == 'upload'
+    end
+
+    # rubocop: disable CodeReuse/ActiveRecord
+    def existing_oids
+      @existing_oids ||= begin
+        project.all_lfs_objects.where(oid: objects.map { |o| o['oid'].to_s }).pluck(:oid)
+      end
+    end
+    # rubocop: enable CodeReuse/ActiveRecord
+
+    def download_objects!
+      objects.each do |object|
+        if existing_oids.include?(object[:oid])
+          object[:actions] = download_actions(object)
+
+          if Guest.can?(:download_code, project)
+            object[:authenticated] = true
+          end
+        else
+          object[:error] = {
+            code: 404,
+            message: _("Object does not exist on the server or you don't have permissions to access it")
+          }
+        end
+      end
+      objects
+    end
+
+    def upload_objects!
+      objects.each do |object|
+        object[:actions] = upload_actions(object) unless existing_oids.include?(object[:oid])
+      end
+      objects
+    end
+
+    def download_actions(object)
+      {
+        download: {
+          href: "#{project.http_url_to_repo}/gitlab-lfs/objects/#{object[:oid]}",
+          header: {
+            Authorization: authorization_header
+          }.compact
+        }
+      }
+    end
+
+    def upload_actions(object)
+      {
+        upload: {
+          href: "#{project.http_url_to_repo}/gitlab-lfs/objects/#{object[:oid]}/#{object[:size]}",
+          header: {
+            Authorization: authorization_header,
+            # git-lfs v2.5.0 sets the Content-Type based on the uploaded file. This
+            # ensures that Workhorse can intercept the request.
+            'Content-Type': LFS_TRANSFER_CONTENT_TYPE
+          }.compact
+        }
+      }
+    end
+
+    def lfs_check_batch_operation!
+      if batch_operation_disallowed?
+        render(
+          json: {
+            message: lfs_read_only_message
+          },
+          content_type: LfsRequest::CONTENT_TYPE,
+          status: :forbidden
+        )
+      end
+    end
+
+    # Overridden in EE
+    def batch_operation_disallowed?
+      upload_request? && Gitlab::Database.read_only?
+    end
+
+    # Overridden in EE
+    def lfs_read_only_message
+      _('You cannot write to this read-only GitLab instance.')
+    end
+
+    def authorization_header
+      strong_memoize(:authorization_header) do
+        lfs_auth_header || request.headers['Authorization']
+      end
+    end
+
+    def lfs_auth_header
+      return unless user.is_a?(User)
+
+      Gitlab::LfsToken.new(user).basic_encoding
+    end
+  end
+end
+
+Repositories::LfsApiController.prepend_if_ee('EE::Repositories::LfsApiController')
diff --git a/app/controllers/repositories/lfs_locks_api_controller.rb b/app/controllers/repositories/lfs_locks_api_controller.rb
new file mode 100644
index 00000000000..19fc09ad4de
--- /dev/null
+++ b/app/controllers/repositories/lfs_locks_api_controller.rb
@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Repositories
+  class LfsLocksApiController < Repositories::GitHttpClientController
+    include LfsRequest
+
+    def create
+      @result = Lfs::LockFileService.new(project, user, lfs_params).execute
+
+      render_json(@result[:lock])
+    end
+
+    def unlock
+      @result = Lfs::UnlockFileService.new(project, user, lfs_params).execute
+
+      render_json(@result[:lock])
+    end
+
+    def index
+      @result = Lfs::LocksFinderService.new(project, user, lfs_params).execute
+
+      render_json(@result[:locks])
+    end
+
+    def verify
+      @result = Lfs::LocksFinderService.new(project, user, {}).execute
+
+      ours, theirs = split_by_owner(@result[:locks])
+
+      render_json({ ours: ours, theirs: theirs }, false)
+    end
+
+    private
+
+    def render_json(data, process = true)
+      render json: build_payload(data, process),
+            content_type: LfsRequest::CONTENT_TYPE,
+            status: @result[:http_status]
+    end
+
+    def build_payload(data, process)
+      data = LfsFileLockSerializer.new.represent(data) if process
+
+      return data if @result[:status] == :success
+
+      # When the locking failed due to an existent Lock, the existent record
+      # is returned in `@result[:lock]`
+      error_payload(@result[:message], @result[:lock] ? data : {})
+    end
+
+    def error_payload(message, custom_attrs = {})
+      custom_attrs.merge({
+        message: message,
+        documentation_url: help_url
+      })
+    end
+
+    def split_by_owner(locks)
+      groups = locks.partition { |lock| lock.user_id == user.id }
+
+      groups.map! do |records|
+        LfsFileLockSerializer.new.represent(records, root: false)
+      end
+    end
+
+    def download_request?
+      params[:action] == 'index'
+    end
+
+    def upload_request?
+      %w(create unlock verify).include?(params[:action])
+    end
+
+    def lfs_params
+      params.permit(:id, :path, :force)
+    end
+  end
+end
diff --git a/app/controllers/repositories/lfs_storage_controller.rb b/app/controllers/repositories/lfs_storage_controller.rb
new file mode 100644
index 00000000000..58f496e16d3
--- /dev/null
+++ b/app/controllers/repositories/lfs_storage_controller.rb
@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Repositories
+  class LfsStorageController < Repositories::GitHttpClientController
+    include LfsRequest
+    include WorkhorseRequest
+    include SendFileUpload
+
+    skip_before_action :verify_workhorse_api!, only: :download
+
+    def download
+      lfs_object = LfsObject.find_by_oid(oid)
+      unless lfs_object && lfs_object.file.exists?
+        render_lfs_not_found
+        return
+      end
+
+      send_upload(lfs_object.file, send_params: { content_type: "application/octet-stream" })
+    end
+
+    def upload_authorize
+      set_workhorse_internal_api_content_type
+
+      authorized = LfsObjectUploader.workhorse_authorize(has_length: true)
+      authorized.merge!(LfsOid: oid, LfsSize: size)
+
+      render json: authorized
+    end
+
+    def upload_finalize
+      if store_file!(oid, size)
+        head 200
+      else
+        render plain: 'Unprocessable entity', status: :unprocessable_entity
+      end
+    rescue ActiveRecord::RecordInvalid
+      render_lfs_forbidden
+    rescue UploadedFile::InvalidPathError
+      render_lfs_forbidden
+    rescue ObjectStorage::RemoteStoreError
+      render_lfs_forbidden
+    end
+
+    private
+
+    def download_request?
+      action_name == 'download'
+    end
+
+    def upload_request?
+      %w[upload_authorize upload_finalize].include? action_name
+    end
+
+    def oid
+      params[:oid].to_s
+    end
+
+    def size
+      params[:size].to_i
+    end
+
+    # rubocop: disable CodeReuse/ActiveRecord
+    def store_file!(oid, size)
+      object = LfsObject.find_by(oid: oid, size: size)
+      unless object&.file&.exists?
+        object = create_file!(oid, size)
+      end
+
+      return unless object
+
+      link_to_project!(object)
+    end
+    # rubocop: enable CodeReuse/ActiveRecord
+
+    def create_file!(oid, size)
+      uploaded_file = UploadedFile.from_params(
+        params, :file, LfsObjectUploader.workhorse_local_upload_path)
+      return unless uploaded_file
+
+      LfsObject.create!(oid: oid, size: size, file: uploaded_file)
+    end
+
+    # rubocop: disable CodeReuse/ActiveRecord
+    def link_to_project!(object)
+      if object && !object.projects.exists?(storage_project.id)
+        object.lfs_objects_projects.create!(project: storage_project)
+      end
+    end
+    # rubocop: enable CodeReuse/ActiveRecord
+  end
+end